1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright(c) 2015 - 2018 Intel Corporation. 4 */ 5 6 #include <linux/spinlock.h> 7 #include <linux/seqlock.h> 8 #include <linux/netdevice.h> 9 #include <linux/moduleparam.h> 10 #include <linux/bitops.h> 11 #include <linux/timer.h> 12 #include <linux/vmalloc.h> 13 #include <linux/highmem.h> 14 #include <linux/sysfs.h> 15 16 #include "hfi.h" 17 #include "common.h" 18 #include "qp.h" 19 #include "sdma.h" 20 #include "iowait.h" 21 #include "trace.h" 22 23 /* must be a power of 2 >= 64 <= 32768 */ 24 #define SDMA_DESCQ_CNT 2048 25 #define SDMA_DESC_INTR 64 26 #define INVALID_TAIL 0xffff 27 #define SDMA_PAD max_t(size_t, MAX_16B_PADDING, sizeof(u32)) 28 29 static uint sdma_descq_cnt = SDMA_DESCQ_CNT; 30 module_param(sdma_descq_cnt, uint, S_IRUGO); 31 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries"); 32 33 static uint sdma_idle_cnt = 250; 34 module_param(sdma_idle_cnt, uint, S_IRUGO); 35 MODULE_PARM_DESC(sdma_idle_cnt, "sdma interrupt idle delay (ns,default 250)"); 36 37 uint mod_num_sdma; 38 module_param_named(num_sdma, mod_num_sdma, uint, S_IRUGO); 39 MODULE_PARM_DESC(num_sdma, "Set max number SDMA engines to use"); 40 41 static uint sdma_desct_intr = SDMA_DESC_INTR; 42 module_param_named(desct_intr, sdma_desct_intr, uint, S_IRUGO | S_IWUSR); 43 MODULE_PARM_DESC(desct_intr, "Number of SDMA descriptor before interrupt"); 44 45 #define SDMA_WAIT_BATCH_SIZE 20 46 /* max wait time for a SDMA engine to indicate it has halted */ 47 #define SDMA_ERR_HALT_TIMEOUT 10 /* ms */ 48 /* all SDMA engine errors that cause a halt */ 49 50 #define SD(name) SEND_DMA_##name 51 #define ALL_SDMA_ENG_HALT_ERRS \ 52 (SD(ENG_ERR_STATUS_SDMA_WRONG_DW_ERR_SMASK) \ 53 | SD(ENG_ERR_STATUS_SDMA_GEN_MISMATCH_ERR_SMASK) \ 54 | SD(ENG_ERR_STATUS_SDMA_TOO_LONG_ERR_SMASK) \ 55 | SD(ENG_ERR_STATUS_SDMA_TAIL_OUT_OF_BOUNDS_ERR_SMASK) \ 56 | SD(ENG_ERR_STATUS_SDMA_FIRST_DESC_ERR_SMASK) \ 57 | SD(ENG_ERR_STATUS_SDMA_MEM_READ_ERR_SMASK) \ 58 | SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK) \ 59 | SD(ENG_ERR_STATUS_SDMA_LENGTH_MISMATCH_ERR_SMASK) \ 60 | SD(ENG_ERR_STATUS_SDMA_PACKET_DESC_OVERFLOW_ERR_SMASK) \ 61 | SD(ENG_ERR_STATUS_SDMA_HEADER_SELECT_ERR_SMASK) \ 62 | SD(ENG_ERR_STATUS_SDMA_HEADER_ADDRESS_ERR_SMASK) \ 63 | SD(ENG_ERR_STATUS_SDMA_HEADER_LENGTH_ERR_SMASK) \ 64 | SD(ENG_ERR_STATUS_SDMA_TIMEOUT_ERR_SMASK) \ 65 | SD(ENG_ERR_STATUS_SDMA_DESC_TABLE_UNC_ERR_SMASK) \ 66 | SD(ENG_ERR_STATUS_SDMA_ASSEMBLY_UNC_ERR_SMASK) \ 67 | SD(ENG_ERR_STATUS_SDMA_PACKET_TRACKING_UNC_ERR_SMASK) \ 68 | SD(ENG_ERR_STATUS_SDMA_HEADER_STORAGE_UNC_ERR_SMASK) \ 69 | SD(ENG_ERR_STATUS_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SMASK)) 70 71 /* sdma_sendctrl operations */ 72 #define SDMA_SENDCTRL_OP_ENABLE BIT(0) 73 #define SDMA_SENDCTRL_OP_INTENABLE BIT(1) 74 #define SDMA_SENDCTRL_OP_HALT BIT(2) 75 #define SDMA_SENDCTRL_OP_CLEANUP BIT(3) 76 77 /* handle long defines */ 78 #define SDMA_EGRESS_PACKET_OCCUPANCY_SMASK \ 79 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SMASK 80 #define SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT \ 81 SEND_EGRESS_SEND_DMA_STATUS_SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT 82 83 static const char * const sdma_state_names[] = { 84 [sdma_state_s00_hw_down] = "s00_HwDown", 85 [sdma_state_s10_hw_start_up_halt_wait] = "s10_HwStartUpHaltWait", 86 [sdma_state_s15_hw_start_up_clean_wait] = "s15_HwStartUpCleanWait", 87 [sdma_state_s20_idle] = "s20_Idle", 88 [sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait", 89 [sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait", 90 [sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait", 91 [sdma_state_s60_idle_halt_wait] = "s60_IdleHaltWait", 92 [sdma_state_s80_hw_freeze] = "s80_HwFreeze", 93 [sdma_state_s82_freeze_sw_clean] = "s82_FreezeSwClean", 94 [sdma_state_s99_running] = "s99_Running", 95 }; 96 97 #ifdef CONFIG_SDMA_VERBOSITY 98 static const char * const sdma_event_names[] = { 99 [sdma_event_e00_go_hw_down] = "e00_GoHwDown", 100 [sdma_event_e10_go_hw_start] = "e10_GoHwStart", 101 [sdma_event_e15_hw_halt_done] = "e15_HwHaltDone", 102 [sdma_event_e25_hw_clean_up_done] = "e25_HwCleanUpDone", 103 [sdma_event_e30_go_running] = "e30_GoRunning", 104 [sdma_event_e40_sw_cleaned] = "e40_SwCleaned", 105 [sdma_event_e50_hw_cleaned] = "e50_HwCleaned", 106 [sdma_event_e60_hw_halted] = "e60_HwHalted", 107 [sdma_event_e70_go_idle] = "e70_GoIdle", 108 [sdma_event_e80_hw_freeze] = "e80_HwFreeze", 109 [sdma_event_e81_hw_frozen] = "e81_HwFrozen", 110 [sdma_event_e82_hw_unfreeze] = "e82_HwUnfreeze", 111 [sdma_event_e85_link_down] = "e85_LinkDown", 112 [sdma_event_e90_sw_halted] = "e90_SwHalted", 113 }; 114 #endif 115 116 static const struct sdma_set_state_action sdma_action_table[] = { 117 [sdma_state_s00_hw_down] = { 118 .go_s99_running_tofalse = 1, 119 .op_enable = 0, 120 .op_intenable = 0, 121 .op_halt = 0, 122 .op_cleanup = 0, 123 }, 124 [sdma_state_s10_hw_start_up_halt_wait] = { 125 .op_enable = 0, 126 .op_intenable = 0, 127 .op_halt = 1, 128 .op_cleanup = 0, 129 }, 130 [sdma_state_s15_hw_start_up_clean_wait] = { 131 .op_enable = 0, 132 .op_intenable = 1, 133 .op_halt = 0, 134 .op_cleanup = 1, 135 }, 136 [sdma_state_s20_idle] = { 137 .op_enable = 0, 138 .op_intenable = 1, 139 .op_halt = 0, 140 .op_cleanup = 0, 141 }, 142 [sdma_state_s30_sw_clean_up_wait] = { 143 .op_enable = 0, 144 .op_intenable = 0, 145 .op_halt = 0, 146 .op_cleanup = 0, 147 }, 148 [sdma_state_s40_hw_clean_up_wait] = { 149 .op_enable = 0, 150 .op_intenable = 0, 151 .op_halt = 0, 152 .op_cleanup = 1, 153 }, 154 [sdma_state_s50_hw_halt_wait] = { 155 .op_enable = 0, 156 .op_intenable = 0, 157 .op_halt = 0, 158 .op_cleanup = 0, 159 }, 160 [sdma_state_s60_idle_halt_wait] = { 161 .go_s99_running_tofalse = 1, 162 .op_enable = 0, 163 .op_intenable = 0, 164 .op_halt = 1, 165 .op_cleanup = 0, 166 }, 167 [sdma_state_s80_hw_freeze] = { 168 .op_enable = 0, 169 .op_intenable = 0, 170 .op_halt = 0, 171 .op_cleanup = 0, 172 }, 173 [sdma_state_s82_freeze_sw_clean] = { 174 .op_enable = 0, 175 .op_intenable = 0, 176 .op_halt = 0, 177 .op_cleanup = 0, 178 }, 179 [sdma_state_s99_running] = { 180 .op_enable = 1, 181 .op_intenable = 1, 182 .op_halt = 0, 183 .op_cleanup = 0, 184 .go_s99_running_totrue = 1, 185 }, 186 }; 187 188 #define SDMA_TAIL_UPDATE_THRESH 0x1F 189 190 /* declare all statics here rather than keep sorting */ 191 static void sdma_complete(struct kref *); 192 static void sdma_finalput(struct sdma_state *); 193 static void sdma_get(struct sdma_state *); 194 static void sdma_hw_clean_up_task(struct tasklet_struct *); 195 static void sdma_put(struct sdma_state *); 196 static void sdma_set_state(struct sdma_engine *, enum sdma_states); 197 static void sdma_start_hw_clean_up(struct sdma_engine *); 198 static void sdma_sw_clean_up_task(struct tasklet_struct *); 199 static void sdma_sendctrl(struct sdma_engine *, unsigned); 200 static void init_sdma_regs(struct sdma_engine *, u32, uint); 201 static void sdma_process_event( 202 struct sdma_engine *sde, 203 enum sdma_events event); 204 static void __sdma_process_event( 205 struct sdma_engine *sde, 206 enum sdma_events event); 207 static void dump_sdma_state(struct sdma_engine *sde); 208 static void sdma_make_progress(struct sdma_engine *sde, u64 status); 209 static void sdma_desc_avail(struct sdma_engine *sde, uint avail); 210 static void sdma_flush_descq(struct sdma_engine *sde); 211 212 /** 213 * sdma_state_name() - return state string from enum 214 * @state: state 215 */ 216 static const char *sdma_state_name(enum sdma_states state) 217 { 218 return sdma_state_names[state]; 219 } 220 221 static void sdma_get(struct sdma_state *ss) 222 { 223 kref_get(&ss->kref); 224 } 225 226 static void sdma_complete(struct kref *kref) 227 { 228 struct sdma_state *ss = 229 container_of(kref, struct sdma_state, kref); 230 231 complete(&ss->comp); 232 } 233 234 static void sdma_put(struct sdma_state *ss) 235 { 236 kref_put(&ss->kref, sdma_complete); 237 } 238 239 static void sdma_finalput(struct sdma_state *ss) 240 { 241 sdma_put(ss); 242 wait_for_completion(&ss->comp); 243 } 244 245 static inline void write_sde_csr( 246 struct sdma_engine *sde, 247 u32 offset0, 248 u64 value) 249 { 250 write_kctxt_csr(sde->dd, sde->this_idx, offset0, value); 251 } 252 253 static inline u64 read_sde_csr( 254 struct sdma_engine *sde, 255 u32 offset0) 256 { 257 return read_kctxt_csr(sde->dd, sde->this_idx, offset0); 258 } 259 260 /* 261 * sdma_wait_for_packet_egress() - wait for the VL FIFO occupancy for 262 * sdma engine 'sde' to drop to 0. 263 */ 264 static void sdma_wait_for_packet_egress(struct sdma_engine *sde, 265 int pause) 266 { 267 u64 off = 8 * sde->this_idx; 268 struct hfi1_devdata *dd = sde->dd; 269 int lcnt = 0; 270 u64 reg_prev; 271 u64 reg = 0; 272 273 while (1) { 274 reg_prev = reg; 275 reg = read_csr(dd, off + SEND_EGRESS_SEND_DMA_STATUS); 276 277 reg &= SDMA_EGRESS_PACKET_OCCUPANCY_SMASK; 278 reg >>= SDMA_EGRESS_PACKET_OCCUPANCY_SHIFT; 279 if (reg == 0) 280 break; 281 /* counter is reest if accupancy count changes */ 282 if (reg != reg_prev) 283 lcnt = 0; 284 if (lcnt++ > 500) { 285 /* timed out - bounce the link */ 286 dd_dev_err(dd, "%s: engine %u timeout waiting for packets to egress, remaining count %u, bouncing link\n", 287 __func__, sde->this_idx, (u32)reg); 288 queue_work(dd->pport->link_wq, 289 &dd->pport->link_bounce_work); 290 break; 291 } 292 udelay(1); 293 } 294 } 295 296 /* 297 * sdma_wait() - wait for packet egress to complete for all SDMA engines, 298 * and pause for credit return. 299 */ 300 void sdma_wait(struct hfi1_devdata *dd) 301 { 302 int i; 303 304 for (i = 0; i < dd->num_sdma; i++) { 305 struct sdma_engine *sde = &dd->per_sdma[i]; 306 307 sdma_wait_for_packet_egress(sde, 0); 308 } 309 } 310 311 static inline void sdma_set_desc_cnt(struct sdma_engine *sde, unsigned cnt) 312 { 313 u64 reg; 314 315 if (!(sde->dd->flags & HFI1_HAS_SDMA_TIMEOUT)) 316 return; 317 reg = cnt; 318 reg &= SD(DESC_CNT_CNT_MASK); 319 reg <<= SD(DESC_CNT_CNT_SHIFT); 320 write_sde_csr(sde, SD(DESC_CNT), reg); 321 } 322 323 static inline void complete_tx(struct sdma_engine *sde, 324 struct sdma_txreq *tx, 325 int res) 326 { 327 /* protect against complete modifying */ 328 struct iowait *wait = tx->wait; 329 callback_t complete = tx->complete; 330 331 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 332 trace_hfi1_sdma_out_sn(sde, tx->sn); 333 if (WARN_ON_ONCE(sde->head_sn != tx->sn)) 334 dd_dev_err(sde->dd, "expected %llu got %llu\n", 335 sde->head_sn, tx->sn); 336 sde->head_sn++; 337 #endif 338 __sdma_txclean(sde->dd, tx); 339 if (complete) 340 (*complete)(tx, res); 341 if (iowait_sdma_dec(wait)) 342 iowait_drain_wakeup(wait); 343 } 344 345 /* 346 * Complete all the sdma requests with a SDMA_TXREQ_S_ABORTED status 347 * 348 * Depending on timing there can be txreqs in two places: 349 * - in the descq ring 350 * - in the flush list 351 * 352 * To avoid ordering issues the descq ring needs to be flushed 353 * first followed by the flush list. 354 * 355 * This routine is called from two places 356 * - From a work queue item 357 * - Directly from the state machine just before setting the 358 * state to running 359 * 360 * Must be called with head_lock held 361 * 362 */ 363 static void sdma_flush(struct sdma_engine *sde) 364 { 365 struct sdma_txreq *txp, *txp_next; 366 LIST_HEAD(flushlist); 367 unsigned long flags; 368 uint seq; 369 370 /* flush from head to tail */ 371 sdma_flush_descq(sde); 372 spin_lock_irqsave(&sde->flushlist_lock, flags); 373 /* copy flush list */ 374 list_splice_init(&sde->flushlist, &flushlist); 375 spin_unlock_irqrestore(&sde->flushlist_lock, flags); 376 /* flush from flush list */ 377 list_for_each_entry_safe(txp, txp_next, &flushlist, list) 378 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED); 379 /* wakeup QPs orphaned on the dmawait list */ 380 do { 381 struct iowait *w, *nw; 382 383 seq = read_seqbegin(&sde->waitlock); 384 if (!list_empty(&sde->dmawait)) { 385 write_seqlock(&sde->waitlock); 386 list_for_each_entry_safe(w, nw, &sde->dmawait, list) { 387 if (w->wakeup) { 388 w->wakeup(w, SDMA_AVAIL_REASON); 389 list_del_init(&w->list); 390 } 391 } 392 write_sequnlock(&sde->waitlock); 393 } 394 } while (read_seqretry(&sde->waitlock, seq)); 395 } 396 397 /* 398 * Fields a work request for flushing the descq ring 399 * and the flush list 400 * 401 * If the engine has been brought to running during 402 * the scheduling delay, the flush is ignored, assuming 403 * that the process of bringing the engine to running 404 * would have done this flush prior to going to running. 405 * 406 */ 407 static void sdma_field_flush(struct work_struct *work) 408 { 409 unsigned long flags; 410 struct sdma_engine *sde = 411 container_of(work, struct sdma_engine, flush_worker); 412 413 write_seqlock_irqsave(&sde->head_lock, flags); 414 if (!__sdma_running(sde)) 415 sdma_flush(sde); 416 write_sequnlock_irqrestore(&sde->head_lock, flags); 417 } 418 419 static void sdma_err_halt_wait(struct work_struct *work) 420 { 421 struct sdma_engine *sde = container_of(work, struct sdma_engine, 422 err_halt_worker); 423 u64 statuscsr; 424 unsigned long timeout; 425 426 timeout = jiffies + msecs_to_jiffies(SDMA_ERR_HALT_TIMEOUT); 427 while (1) { 428 statuscsr = read_sde_csr(sde, SD(STATUS)); 429 statuscsr &= SD(STATUS_ENG_HALTED_SMASK); 430 if (statuscsr) 431 break; 432 if (time_after(jiffies, timeout)) { 433 dd_dev_err(sde->dd, 434 "SDMA engine %d - timeout waiting for engine to halt\n", 435 sde->this_idx); 436 /* 437 * Continue anyway. This could happen if there was 438 * an uncorrectable error in the wrong spot. 439 */ 440 break; 441 } 442 usleep_range(80, 120); 443 } 444 445 sdma_process_event(sde, sdma_event_e15_hw_halt_done); 446 } 447 448 static void sdma_err_progress_check_schedule(struct sdma_engine *sde) 449 { 450 if (!is_bx(sde->dd) && HFI1_CAP_IS_KSET(SDMA_AHG)) { 451 unsigned index; 452 struct hfi1_devdata *dd = sde->dd; 453 454 for (index = 0; index < dd->num_sdma; index++) { 455 struct sdma_engine *curr_sdma = &dd->per_sdma[index]; 456 457 if (curr_sdma != sde) 458 curr_sdma->progress_check_head = 459 curr_sdma->descq_head; 460 } 461 dd_dev_err(sde->dd, 462 "SDMA engine %d - check scheduled\n", 463 sde->this_idx); 464 mod_timer(&sde->err_progress_check_timer, jiffies + 10); 465 } 466 } 467 468 static void sdma_err_progress_check(struct timer_list *t) 469 { 470 unsigned index; 471 struct sdma_engine *sde = timer_container_of(sde, t, 472 err_progress_check_timer); 473 474 dd_dev_err(sde->dd, "SDE progress check event\n"); 475 for (index = 0; index < sde->dd->num_sdma; index++) { 476 struct sdma_engine *curr_sde = &sde->dd->per_sdma[index]; 477 unsigned long flags; 478 479 /* check progress on each engine except the current one */ 480 if (curr_sde == sde) 481 continue; 482 /* 483 * We must lock interrupts when acquiring sde->lock, 484 * to avoid a deadlock if interrupt triggers and spins on 485 * the same lock on same CPU 486 */ 487 spin_lock_irqsave(&curr_sde->tail_lock, flags); 488 write_seqlock(&curr_sde->head_lock); 489 490 /* skip non-running queues */ 491 if (curr_sde->state.current_state != sdma_state_s99_running) { 492 write_sequnlock(&curr_sde->head_lock); 493 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); 494 continue; 495 } 496 497 if ((curr_sde->descq_head != curr_sde->descq_tail) && 498 (curr_sde->descq_head == 499 curr_sde->progress_check_head)) 500 __sdma_process_event(curr_sde, 501 sdma_event_e90_sw_halted); 502 write_sequnlock(&curr_sde->head_lock); 503 spin_unlock_irqrestore(&curr_sde->tail_lock, flags); 504 } 505 schedule_work(&sde->err_halt_worker); 506 } 507 508 static void sdma_hw_clean_up_task(struct tasklet_struct *t) 509 { 510 struct sdma_engine *sde = from_tasklet(sde, t, 511 sdma_hw_clean_up_task); 512 u64 statuscsr; 513 514 while (1) { 515 #ifdef CONFIG_SDMA_VERBOSITY 516 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 517 sde->this_idx, slashstrip(__FILE__), __LINE__, 518 __func__); 519 #endif 520 statuscsr = read_sde_csr(sde, SD(STATUS)); 521 statuscsr &= SD(STATUS_ENG_CLEANED_UP_SMASK); 522 if (statuscsr) 523 break; 524 udelay(10); 525 } 526 527 sdma_process_event(sde, sdma_event_e25_hw_clean_up_done); 528 } 529 530 static inline struct sdma_txreq *get_txhead(struct sdma_engine *sde) 531 { 532 return sde->tx_ring[sde->tx_head & sde->sdma_mask]; 533 } 534 535 /* 536 * flush ring for recovery 537 */ 538 static void sdma_flush_descq(struct sdma_engine *sde) 539 { 540 u16 head, tail; 541 int progress = 0; 542 struct sdma_txreq *txp = get_txhead(sde); 543 544 /* The reason for some of the complexity of this code is that 545 * not all descriptors have corresponding txps. So, we have to 546 * be able to skip over descs until we wander into the range of 547 * the next txp on the list. 548 */ 549 head = sde->descq_head & sde->sdma_mask; 550 tail = sde->descq_tail & sde->sdma_mask; 551 while (head != tail) { 552 /* advance head, wrap if needed */ 553 head = ++sde->descq_head & sde->sdma_mask; 554 /* if now past this txp's descs, do the callback */ 555 if (txp && txp->next_descq_idx == head) { 556 /* remove from list */ 557 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; 558 complete_tx(sde, txp, SDMA_TXREQ_S_ABORTED); 559 trace_hfi1_sdma_progress(sde, head, tail, txp); 560 txp = get_txhead(sde); 561 } 562 progress++; 563 } 564 if (progress) 565 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 566 } 567 568 static void sdma_sw_clean_up_task(struct tasklet_struct *t) 569 { 570 struct sdma_engine *sde = from_tasklet(sde, t, sdma_sw_clean_up_task); 571 unsigned long flags; 572 573 spin_lock_irqsave(&sde->tail_lock, flags); 574 write_seqlock(&sde->head_lock); 575 576 /* 577 * At this point, the following should always be true: 578 * - We are halted, so no more descriptors are getting retired. 579 * - We are not running, so no one is submitting new work. 580 * - Only we can send the e40_sw_cleaned, so we can't start 581 * running again until we say so. So, the active list and 582 * descq are ours to play with. 583 */ 584 585 /* 586 * In the error clean up sequence, software clean must be called 587 * before the hardware clean so we can use the hardware head in 588 * the progress routine. A hardware clean or SPC unfreeze will 589 * reset the hardware head. 590 * 591 * Process all retired requests. The progress routine will use the 592 * latest physical hardware head - we are not running so speed does 593 * not matter. 594 */ 595 sdma_make_progress(sde, 0); 596 597 sdma_flush(sde); 598 599 /* 600 * Reset our notion of head and tail. 601 * Note that the HW registers have been reset via an earlier 602 * clean up. 603 */ 604 sde->descq_tail = 0; 605 sde->descq_head = 0; 606 sde->desc_avail = sdma_descq_freecnt(sde); 607 *sde->head_dma = 0; 608 609 __sdma_process_event(sde, sdma_event_e40_sw_cleaned); 610 611 write_sequnlock(&sde->head_lock); 612 spin_unlock_irqrestore(&sde->tail_lock, flags); 613 } 614 615 static void sdma_sw_tear_down(struct sdma_engine *sde) 616 { 617 struct sdma_state *ss = &sde->state; 618 619 /* Releasing this reference means the state machine has stopped. */ 620 sdma_put(ss); 621 622 /* stop waiting for all unfreeze events to complete */ 623 atomic_set(&sde->dd->sdma_unfreeze_count, -1); 624 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 625 } 626 627 static void sdma_start_hw_clean_up(struct sdma_engine *sde) 628 { 629 tasklet_hi_schedule(&sde->sdma_hw_clean_up_task); 630 } 631 632 static void sdma_set_state(struct sdma_engine *sde, 633 enum sdma_states next_state) 634 { 635 struct sdma_state *ss = &sde->state; 636 const struct sdma_set_state_action *action = sdma_action_table; 637 unsigned op = 0; 638 639 trace_hfi1_sdma_state( 640 sde, 641 sdma_state_names[ss->current_state], 642 sdma_state_names[next_state]); 643 644 /* debugging bookkeeping */ 645 ss->previous_state = ss->current_state; 646 ss->previous_op = ss->current_op; 647 ss->current_state = next_state; 648 649 if (ss->previous_state != sdma_state_s99_running && 650 next_state == sdma_state_s99_running) 651 sdma_flush(sde); 652 653 if (action[next_state].op_enable) 654 op |= SDMA_SENDCTRL_OP_ENABLE; 655 656 if (action[next_state].op_intenable) 657 op |= SDMA_SENDCTRL_OP_INTENABLE; 658 659 if (action[next_state].op_halt) 660 op |= SDMA_SENDCTRL_OP_HALT; 661 662 if (action[next_state].op_cleanup) 663 op |= SDMA_SENDCTRL_OP_CLEANUP; 664 665 if (action[next_state].go_s99_running_tofalse) 666 ss->go_s99_running = 0; 667 668 if (action[next_state].go_s99_running_totrue) 669 ss->go_s99_running = 1; 670 671 ss->current_op = op; 672 sdma_sendctrl(sde, ss->current_op); 673 } 674 675 /** 676 * sdma_get_descq_cnt() - called when device probed 677 * 678 * Return a validated descq count. 679 * 680 * This is currently only used in the verbs initialization to build the tx 681 * list. 682 * 683 * This will probably be deleted in favor of a more scalable approach to 684 * alloc tx's. 685 * 686 */ 687 u16 sdma_get_descq_cnt(void) 688 { 689 u16 count = sdma_descq_cnt; 690 691 if (!count) 692 return SDMA_DESCQ_CNT; 693 /* count must be a power of 2 greater than 64 and less than 694 * 32768. Otherwise return default. 695 */ 696 if (!is_power_of_2(count)) 697 return SDMA_DESCQ_CNT; 698 if (count < 64 || count > 32768) 699 return SDMA_DESCQ_CNT; 700 return count; 701 } 702 703 /** 704 * sdma_engine_get_vl() - return vl for a given sdma engine 705 * @sde: sdma engine 706 * 707 * This function returns the vl mapped to a given engine, or an error if 708 * the mapping can't be found. The mapping fields are protected by RCU. 709 */ 710 int sdma_engine_get_vl(struct sdma_engine *sde) 711 { 712 struct hfi1_devdata *dd = sde->dd; 713 struct sdma_vl_map *m; 714 u8 vl; 715 716 if (sde->this_idx >= TXE_NUM_SDMA_ENGINES) 717 return -EINVAL; 718 719 rcu_read_lock(); 720 m = rcu_dereference(dd->sdma_map); 721 if (unlikely(!m)) { 722 rcu_read_unlock(); 723 return -EINVAL; 724 } 725 vl = m->engine_to_vl[sde->this_idx]; 726 rcu_read_unlock(); 727 728 return vl; 729 } 730 731 /** 732 * sdma_select_engine_vl() - select sdma engine 733 * @dd: devdata 734 * @selector: a spreading factor 735 * @vl: this vl 736 * 737 * 738 * This function returns an engine based on the selector and a vl. The 739 * mapping fields are protected by RCU. 740 */ 741 struct sdma_engine *sdma_select_engine_vl( 742 struct hfi1_devdata *dd, 743 u32 selector, 744 u8 vl) 745 { 746 struct sdma_vl_map *m; 747 struct sdma_map_elem *e; 748 struct sdma_engine *rval; 749 750 /* NOTE This should only happen if SC->VL changed after the initial 751 * checks on the QP/AH 752 * Default will return engine 0 below 753 */ 754 if (vl >= num_vls) { 755 rval = NULL; 756 goto done; 757 } 758 759 rcu_read_lock(); 760 m = rcu_dereference(dd->sdma_map); 761 if (unlikely(!m)) { 762 rcu_read_unlock(); 763 return &dd->per_sdma[0]; 764 } 765 e = m->map[vl & m->mask]; 766 rval = e->sde[selector & e->mask]; 767 rcu_read_unlock(); 768 769 done: 770 rval = !rval ? &dd->per_sdma[0] : rval; 771 trace_hfi1_sdma_engine_select(dd, selector, vl, rval->this_idx); 772 return rval; 773 } 774 775 /** 776 * sdma_select_engine_sc() - select sdma engine 777 * @dd: devdata 778 * @selector: a spreading factor 779 * @sc5: the 5 bit sc 780 * 781 * 782 * This function returns an engine based on the selector and an sc. 783 */ 784 struct sdma_engine *sdma_select_engine_sc( 785 struct hfi1_devdata *dd, 786 u32 selector, 787 u8 sc5) 788 { 789 u8 vl = sc_to_vlt(dd, sc5); 790 791 return sdma_select_engine_vl(dd, selector, vl); 792 } 793 794 struct sdma_rht_map_elem { 795 u32 mask; 796 u8 ctr; 797 struct sdma_engine *sde[]; 798 }; 799 800 struct sdma_rht_node { 801 unsigned long cpu_id; 802 struct sdma_rht_map_elem *map[HFI1_MAX_VLS_SUPPORTED]; 803 struct rhash_head node; 804 }; 805 806 #define NR_CPUS_HINT 192 807 808 static const struct rhashtable_params sdma_rht_params = { 809 .nelem_hint = NR_CPUS_HINT, 810 .head_offset = offsetof(struct sdma_rht_node, node), 811 .key_offset = offsetof(struct sdma_rht_node, cpu_id), 812 .key_len = sizeof_field(struct sdma_rht_node, cpu_id), 813 .max_size = NR_CPUS, 814 .min_size = 8, 815 .automatic_shrinking = true, 816 }; 817 818 /* 819 * sdma_select_user_engine() - select sdma engine based on user setup 820 * @dd: devdata 821 * @selector: a spreading factor 822 * @vl: this vl 823 * 824 * This function returns an sdma engine for a user sdma request. 825 * User defined sdma engine affinity setting is honored when applicable, 826 * otherwise system default sdma engine mapping is used. To ensure correct 827 * ordering, the mapping from <selector, vl> to sde must remain unchanged. 828 */ 829 struct sdma_engine *sdma_select_user_engine(struct hfi1_devdata *dd, 830 u32 selector, u8 vl) 831 { 832 struct sdma_rht_node *rht_node; 833 struct sdma_engine *sde = NULL; 834 unsigned long cpu_id; 835 836 /* 837 * To ensure that always the same sdma engine(s) will be 838 * selected make sure the process is pinned to this CPU only. 839 */ 840 if (current->nr_cpus_allowed != 1) 841 goto out; 842 843 rcu_read_lock(); 844 cpu_id = smp_processor_id(); 845 rht_node = rhashtable_lookup(dd->sdma_rht, &cpu_id, 846 sdma_rht_params); 847 848 if (rht_node && rht_node->map[vl]) { 849 struct sdma_rht_map_elem *map = rht_node->map[vl]; 850 851 sde = map->sde[selector & map->mask]; 852 } 853 rcu_read_unlock(); 854 855 if (sde) 856 return sde; 857 858 out: 859 return sdma_select_engine_vl(dd, selector, vl); 860 } 861 862 static void sdma_populate_sde_map(struct sdma_rht_map_elem *map) 863 { 864 int i; 865 866 for (i = 0; i < roundup_pow_of_two(map->ctr ? : 1) - map->ctr; i++) 867 map->sde[map->ctr + i] = map->sde[i]; 868 } 869 870 static void sdma_cleanup_sde_map(struct sdma_rht_map_elem *map, 871 struct sdma_engine *sde) 872 { 873 unsigned int i, pow; 874 875 /* only need to check the first ctr entries for a match */ 876 for (i = 0; i < map->ctr; i++) { 877 if (map->sde[i] == sde) { 878 memmove(&map->sde[i], &map->sde[i + 1], 879 (map->ctr - i - 1) * sizeof(map->sde[0])); 880 map->ctr--; 881 pow = roundup_pow_of_two(map->ctr ? : 1); 882 map->mask = pow - 1; 883 sdma_populate_sde_map(map); 884 break; 885 } 886 } 887 } 888 889 /* 890 * Prevents concurrent reads and writes of the sdma engine cpu_mask 891 */ 892 static DEFINE_MUTEX(process_to_sde_mutex); 893 894 ssize_t sdma_set_cpu_to_sde_map(struct sdma_engine *sde, const char *buf, 895 size_t count) 896 { 897 struct hfi1_devdata *dd = sde->dd; 898 cpumask_var_t mask, new_mask; 899 unsigned long cpu; 900 int ret, vl, sz; 901 struct sdma_rht_node *rht_node; 902 903 vl = sdma_engine_get_vl(sde); 904 if (unlikely(vl < 0 || vl >= ARRAY_SIZE(rht_node->map))) 905 return -EINVAL; 906 907 ret = zalloc_cpumask_var(&mask, GFP_KERNEL); 908 if (!ret) 909 return -ENOMEM; 910 911 ret = zalloc_cpumask_var(&new_mask, GFP_KERNEL); 912 if (!ret) { 913 free_cpumask_var(mask); 914 return -ENOMEM; 915 } 916 ret = cpulist_parse(buf, mask); 917 if (ret) 918 goto out_free; 919 920 if (!cpumask_subset(mask, cpu_online_mask)) { 921 dd_dev_warn(sde->dd, "Invalid CPU mask\n"); 922 ret = -EINVAL; 923 goto out_free; 924 } 925 926 sz = sizeof(struct sdma_rht_map_elem) + 927 (TXE_NUM_SDMA_ENGINES * sizeof(struct sdma_engine *)); 928 929 mutex_lock(&process_to_sde_mutex); 930 931 for_each_cpu(cpu, mask) { 932 /* Check if we have this already mapped */ 933 if (cpumask_test_cpu(cpu, &sde->cpu_mask)) { 934 cpumask_set_cpu(cpu, new_mask); 935 continue; 936 } 937 938 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, 939 sdma_rht_params); 940 if (!rht_node) { 941 rht_node = kzalloc_obj(*rht_node); 942 if (!rht_node) { 943 ret = -ENOMEM; 944 goto out; 945 } 946 947 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); 948 if (!rht_node->map[vl]) { 949 kfree(rht_node); 950 ret = -ENOMEM; 951 goto out; 952 } 953 rht_node->cpu_id = cpu; 954 rht_node->map[vl]->mask = 0; 955 rht_node->map[vl]->ctr = 1; 956 rht_node->map[vl]->sde[0] = sde; 957 958 ret = rhashtable_insert_fast(dd->sdma_rht, 959 &rht_node->node, 960 sdma_rht_params); 961 if (ret) { 962 kfree(rht_node->map[vl]); 963 kfree(rht_node); 964 dd_dev_err(sde->dd, "Failed to set process to sde affinity for cpu %lu\n", 965 cpu); 966 goto out; 967 } 968 969 } else { 970 int ctr, pow; 971 972 /* Add new user mappings */ 973 if (!rht_node->map[vl]) 974 rht_node->map[vl] = kzalloc(sz, GFP_KERNEL); 975 976 if (!rht_node->map[vl]) { 977 ret = -ENOMEM; 978 goto out; 979 } 980 981 rht_node->map[vl]->ctr++; 982 ctr = rht_node->map[vl]->ctr; 983 rht_node->map[vl]->sde[ctr - 1] = sde; 984 pow = roundup_pow_of_two(ctr); 985 rht_node->map[vl]->mask = pow - 1; 986 987 /* Populate the sde map table */ 988 sdma_populate_sde_map(rht_node->map[vl]); 989 } 990 cpumask_set_cpu(cpu, new_mask); 991 } 992 993 /* Clean up old mappings */ 994 for_each_online_cpu(cpu) { 995 struct sdma_rht_node *rht_node; 996 997 /* Don't cleanup sdes that are set in the new mask */ 998 if (cpumask_test_cpu(cpu, mask)) 999 continue; 1000 1001 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpu, 1002 sdma_rht_params); 1003 if (rht_node) { 1004 bool empty = true; 1005 int i; 1006 1007 /* Remove mappings for old sde */ 1008 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1009 if (rht_node->map[i]) 1010 sdma_cleanup_sde_map(rht_node->map[i], 1011 sde); 1012 1013 /* Free empty hash table entries */ 1014 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) { 1015 if (!rht_node->map[i]) 1016 continue; 1017 1018 if (rht_node->map[i]->ctr) { 1019 empty = false; 1020 break; 1021 } 1022 } 1023 1024 if (empty) { 1025 ret = rhashtable_remove_fast(dd->sdma_rht, 1026 &rht_node->node, 1027 sdma_rht_params); 1028 WARN_ON(ret); 1029 1030 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1031 kfree(rht_node->map[i]); 1032 1033 kfree(rht_node); 1034 } 1035 } 1036 } 1037 1038 cpumask_copy(&sde->cpu_mask, new_mask); 1039 out: 1040 mutex_unlock(&process_to_sde_mutex); 1041 out_free: 1042 free_cpumask_var(mask); 1043 free_cpumask_var(new_mask); 1044 return ret ? : strnlen(buf, PAGE_SIZE); 1045 } 1046 1047 ssize_t sdma_get_cpu_to_sde_map(struct sdma_engine *sde, char *buf) 1048 { 1049 mutex_lock(&process_to_sde_mutex); 1050 if (cpumask_empty(&sde->cpu_mask)) 1051 snprintf(buf, PAGE_SIZE, "%s\n", "empty"); 1052 else 1053 sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(&sde->cpu_mask)); 1054 mutex_unlock(&process_to_sde_mutex); 1055 return strnlen(buf, PAGE_SIZE); 1056 } 1057 1058 static void sdma_rht_free(void *ptr, void *arg) 1059 { 1060 struct sdma_rht_node *rht_node = ptr; 1061 int i; 1062 1063 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) 1064 kfree(rht_node->map[i]); 1065 1066 kfree(rht_node); 1067 } 1068 1069 /** 1070 * sdma_seqfile_dump_cpu_list() - debugfs dump the cpu to sdma mappings 1071 * @s: seq file 1072 * @dd: hfi1_devdata 1073 * @cpuid: cpu id 1074 * 1075 * This routine dumps the process to sde mappings per cpu 1076 */ 1077 void sdma_seqfile_dump_cpu_list(struct seq_file *s, 1078 struct hfi1_devdata *dd, 1079 unsigned long cpuid) 1080 { 1081 struct sdma_rht_node *rht_node; 1082 int i, j; 1083 1084 rht_node = rhashtable_lookup_fast(dd->sdma_rht, &cpuid, 1085 sdma_rht_params); 1086 if (!rht_node) 1087 return; 1088 1089 seq_printf(s, "cpu%3lu: ", cpuid); 1090 for (i = 0; i < HFI1_MAX_VLS_SUPPORTED; i++) { 1091 if (!rht_node->map[i] || !rht_node->map[i]->ctr) 1092 continue; 1093 1094 seq_printf(s, " vl%d: [", i); 1095 1096 for (j = 0; j < rht_node->map[i]->ctr; j++) { 1097 if (!rht_node->map[i]->sde[j]) 1098 continue; 1099 1100 if (j > 0) 1101 seq_puts(s, ","); 1102 1103 seq_printf(s, " sdma%2d", 1104 rht_node->map[i]->sde[j]->this_idx); 1105 } 1106 seq_puts(s, " ]"); 1107 } 1108 1109 seq_puts(s, "\n"); 1110 } 1111 1112 /* 1113 * Free the indicated map struct 1114 */ 1115 static void sdma_map_free(struct sdma_vl_map *m) 1116 { 1117 int i; 1118 1119 for (i = 0; m && i < m->actual_vls; i++) 1120 kfree(m->map[i]); 1121 kfree(m); 1122 } 1123 1124 /* 1125 * Handle RCU callback 1126 */ 1127 static void sdma_map_rcu_callback(struct rcu_head *list) 1128 { 1129 struct sdma_vl_map *m = container_of(list, struct sdma_vl_map, list); 1130 1131 sdma_map_free(m); 1132 } 1133 1134 /** 1135 * sdma_map_init - called when # vls change 1136 * @dd: hfi1_devdata 1137 * @port: port number 1138 * @num_vls: number of vls 1139 * @vl_engines: per vl engine mapping (optional) 1140 * 1141 * This routine changes the mapping based on the number of vls. 1142 * 1143 * vl_engines is used to specify a non-uniform vl/engine loading. NULL 1144 * implies auto computing the loading and giving each VLs a uniform 1145 * distribution of engines per VL. 1146 * 1147 * The auto algorithm computes the sde_per_vl and the number of extra 1148 * engines. Any extra engines are added from the last VL on down. 1149 * 1150 * rcu locking is used here to control access to the mapping fields. 1151 * 1152 * If either the num_vls or num_sdma are non-power of 2, the array sizes 1153 * in the struct sdma_vl_map and the struct sdma_map_elem are rounded 1154 * up to the next highest power of 2 and the first entry is reused 1155 * in a round robin fashion. 1156 * 1157 * If an error occurs the map change is not done and the mapping is 1158 * not changed. 1159 * 1160 */ 1161 int sdma_map_init(struct hfi1_devdata *dd, u8 port, u8 num_vls, u8 *vl_engines) 1162 { 1163 int i, j; 1164 int extra, sde_per_vl; 1165 int engine = 0; 1166 u8 lvl_engines[OPA_MAX_VLS]; 1167 struct sdma_vl_map *oldmap, *newmap; 1168 1169 if (!(dd->flags & HFI1_HAS_SEND_DMA)) 1170 return 0; 1171 1172 if (!vl_engines) { 1173 /* truncate divide */ 1174 sde_per_vl = dd->num_sdma / num_vls; 1175 /* extras */ 1176 extra = dd->num_sdma % num_vls; 1177 vl_engines = lvl_engines; 1178 /* add extras from last vl down */ 1179 for (i = num_vls - 1; i >= 0; i--, extra--) 1180 vl_engines[i] = sde_per_vl + (extra > 0 ? 1 : 0); 1181 } 1182 /* build new map */ 1183 newmap = kzalloc( 1184 sizeof(struct sdma_vl_map) + 1185 roundup_pow_of_two(num_vls) * 1186 sizeof(struct sdma_map_elem *), 1187 GFP_KERNEL); 1188 if (!newmap) 1189 goto bail; 1190 newmap->actual_vls = num_vls; 1191 newmap->vls = roundup_pow_of_two(num_vls); 1192 newmap->mask = (1 << ilog2(newmap->vls)) - 1; 1193 /* initialize back-map */ 1194 for (i = 0; i < TXE_NUM_SDMA_ENGINES; i++) 1195 newmap->engine_to_vl[i] = -1; 1196 for (i = 0; i < newmap->vls; i++) { 1197 /* save for wrap around */ 1198 int first_engine = engine; 1199 1200 if (i < newmap->actual_vls) { 1201 int sz = roundup_pow_of_two(vl_engines[i]); 1202 1203 /* only allocate once */ 1204 newmap->map[i] = kzalloc( 1205 sizeof(struct sdma_map_elem) + 1206 sz * sizeof(struct sdma_engine *), 1207 GFP_KERNEL); 1208 if (!newmap->map[i]) 1209 goto bail; 1210 newmap->map[i]->mask = (1 << ilog2(sz)) - 1; 1211 /* assign engines */ 1212 for (j = 0; j < sz; j++) { 1213 newmap->map[i]->sde[j] = 1214 &dd->per_sdma[engine]; 1215 if (++engine >= first_engine + vl_engines[i]) 1216 /* wrap back to first engine */ 1217 engine = first_engine; 1218 } 1219 /* assign back-map */ 1220 for (j = 0; j < vl_engines[i]; j++) 1221 newmap->engine_to_vl[first_engine + j] = i; 1222 } else { 1223 /* just re-use entry without allocating */ 1224 newmap->map[i] = newmap->map[i % num_vls]; 1225 } 1226 engine = first_engine + vl_engines[i]; 1227 } 1228 /* newmap in hand, save old map */ 1229 spin_lock_irq(&dd->sde_map_lock); 1230 oldmap = rcu_dereference_protected(dd->sdma_map, 1231 lockdep_is_held(&dd->sde_map_lock)); 1232 1233 /* publish newmap */ 1234 rcu_assign_pointer(dd->sdma_map, newmap); 1235 1236 spin_unlock_irq(&dd->sde_map_lock); 1237 /* success, free any old map after grace period */ 1238 if (oldmap) 1239 call_rcu(&oldmap->list, sdma_map_rcu_callback); 1240 return 0; 1241 bail: 1242 /* free any partial allocation */ 1243 sdma_map_free(newmap); 1244 return -ENOMEM; 1245 } 1246 1247 /** 1248 * sdma_clean - Clean up allocated memory 1249 * @dd: struct hfi1_devdata 1250 * @num_engines: num sdma engines 1251 * 1252 * This routine can be called regardless of the success of 1253 * sdma_init() 1254 */ 1255 void sdma_clean(struct hfi1_devdata *dd, size_t num_engines) 1256 { 1257 size_t i; 1258 struct sdma_engine *sde; 1259 struct sdma_vl_map *map; 1260 1261 if (dd->sdma_pad_dma) { 1262 dma_free_coherent(&dd->pcidev->dev, SDMA_PAD, 1263 (void *)dd->sdma_pad_dma, 1264 dd->sdma_pad_phys); 1265 dd->sdma_pad_dma = NULL; 1266 dd->sdma_pad_phys = 0; 1267 } 1268 if (dd->sdma_heads_dma) { 1269 dma_free_coherent(&dd->pcidev->dev, dd->sdma_heads_size, 1270 (void *)dd->sdma_heads_dma, 1271 dd->sdma_heads_phys); 1272 dd->sdma_heads_dma = NULL; 1273 dd->sdma_heads_phys = 0; 1274 } 1275 for (i = 0; dd->per_sdma && i < num_engines; ++i) { 1276 sde = &dd->per_sdma[i]; 1277 1278 sde->head_dma = NULL; 1279 sde->head_phys = 0; 1280 1281 if (sde->descq) { 1282 dma_free_coherent( 1283 &dd->pcidev->dev, 1284 sde->descq_cnt * sizeof(u64[2]), 1285 sde->descq, 1286 sde->descq_phys 1287 ); 1288 sde->descq = NULL; 1289 sde->descq_phys = 0; 1290 } 1291 kvfree(sde->tx_ring); 1292 sde->tx_ring = NULL; 1293 } 1294 if (rcu_access_pointer(dd->sdma_map)) { 1295 spin_lock_irq(&dd->sde_map_lock); 1296 map = rcu_access_pointer(dd->sdma_map); 1297 RCU_INIT_POINTER(dd->sdma_map, NULL); 1298 spin_unlock_irq(&dd->sde_map_lock); 1299 synchronize_rcu(); 1300 sdma_map_free(map); 1301 } 1302 kfree(dd->per_sdma); 1303 dd->per_sdma = NULL; 1304 1305 if (dd->sdma_rht) { 1306 rhashtable_free_and_destroy(dd->sdma_rht, sdma_rht_free, NULL); 1307 kfree(dd->sdma_rht); 1308 dd->sdma_rht = NULL; 1309 } 1310 } 1311 1312 /** 1313 * sdma_init() - called when device probed 1314 * @dd: hfi1_devdata 1315 * @port: port number (currently only zero) 1316 * 1317 * Initializes each sde and its csrs. 1318 * Interrupts are not required to be enabled. 1319 * 1320 * Returns: 1321 * 0 - success, -errno on failure 1322 */ 1323 int sdma_init(struct hfi1_devdata *dd, u8 port) 1324 { 1325 unsigned this_idx; 1326 struct sdma_engine *sde; 1327 struct rhashtable *tmp_sdma_rht; 1328 u16 descq_cnt; 1329 void *curr_head; 1330 struct hfi1_pportdata *ppd = dd->pport + port; 1331 u32 per_sdma_credits; 1332 uint idle_cnt = sdma_idle_cnt; 1333 size_t num_engines = chip_sdma_engines(dd); 1334 int ret = -ENOMEM; 1335 1336 if (!HFI1_CAP_IS_KSET(SDMA)) { 1337 HFI1_CAP_CLEAR(SDMA_AHG); 1338 return 0; 1339 } 1340 if (mod_num_sdma && 1341 /* can't exceed chip support */ 1342 mod_num_sdma <= chip_sdma_engines(dd) && 1343 /* count must be >= vls */ 1344 mod_num_sdma >= num_vls) 1345 num_engines = mod_num_sdma; 1346 1347 dd_dev_info(dd, "SDMA mod_num_sdma: %u\n", mod_num_sdma); 1348 dd_dev_info(dd, "SDMA chip_sdma_engines: %u\n", chip_sdma_engines(dd)); 1349 dd_dev_info(dd, "SDMA chip_sdma_mem_size: %u\n", 1350 chip_sdma_mem_size(dd)); 1351 1352 per_sdma_credits = 1353 chip_sdma_mem_size(dd) / (num_engines * SDMA_BLOCK_SIZE); 1354 1355 /* set up freeze waitqueue */ 1356 init_waitqueue_head(&dd->sdma_unfreeze_wq); 1357 atomic_set(&dd->sdma_unfreeze_count, 0); 1358 1359 descq_cnt = sdma_get_descq_cnt(); 1360 dd_dev_info(dd, "SDMA engines %zu descq_cnt %u\n", 1361 num_engines, descq_cnt); 1362 1363 /* alloc memory for array of send engines */ 1364 dd->per_sdma = kcalloc_node(num_engines, sizeof(*dd->per_sdma), 1365 GFP_KERNEL, dd->node); 1366 if (!dd->per_sdma) 1367 return ret; 1368 1369 idle_cnt = ns_to_cclock(dd, idle_cnt); 1370 if (idle_cnt) 1371 dd->default_desc1 = 1372 SDMA_DESC1_HEAD_TO_HOST_FLAG; 1373 else 1374 dd->default_desc1 = 1375 SDMA_DESC1_INT_REQ_FLAG; 1376 1377 if (!sdma_desct_intr) 1378 sdma_desct_intr = SDMA_DESC_INTR; 1379 1380 /* Allocate memory for SendDMA descriptor FIFOs */ 1381 for (this_idx = 0; this_idx < num_engines; ++this_idx) { 1382 sde = &dd->per_sdma[this_idx]; 1383 sde->dd = dd; 1384 sde->ppd = ppd; 1385 sde->this_idx = this_idx; 1386 sde->descq_cnt = descq_cnt; 1387 sde->desc_avail = sdma_descq_freecnt(sde); 1388 sde->sdma_shift = ilog2(descq_cnt); 1389 sde->sdma_mask = (1 << sde->sdma_shift) - 1; 1390 1391 /* Create a mask specifically for each interrupt source */ 1392 sde->int_mask = (u64)1 << (0 * TXE_NUM_SDMA_ENGINES + 1393 this_idx); 1394 sde->progress_mask = (u64)1 << (1 * TXE_NUM_SDMA_ENGINES + 1395 this_idx); 1396 sde->idle_mask = (u64)1 << (2 * TXE_NUM_SDMA_ENGINES + 1397 this_idx); 1398 /* Create a combined mask to cover all 3 interrupt sources */ 1399 sde->imask = sde->int_mask | sde->progress_mask | 1400 sde->idle_mask; 1401 1402 spin_lock_init(&sde->tail_lock); 1403 seqlock_init(&sde->head_lock); 1404 spin_lock_init(&sde->senddmactrl_lock); 1405 spin_lock_init(&sde->flushlist_lock); 1406 seqlock_init(&sde->waitlock); 1407 /* insure there is always a zero bit */ 1408 sde->ahg_bits = 0xfffffffe00000000ULL; 1409 1410 sdma_set_state(sde, sdma_state_s00_hw_down); 1411 1412 /* set up reference counting */ 1413 kref_init(&sde->state.kref); 1414 init_completion(&sde->state.comp); 1415 1416 INIT_LIST_HEAD(&sde->flushlist); 1417 INIT_LIST_HEAD(&sde->dmawait); 1418 1419 sde->tail_csr = 1420 get_kctxt_csr_addr(dd, this_idx, SD(TAIL)); 1421 1422 tasklet_setup(&sde->sdma_hw_clean_up_task, 1423 sdma_hw_clean_up_task); 1424 tasklet_setup(&sde->sdma_sw_clean_up_task, 1425 sdma_sw_clean_up_task); 1426 INIT_WORK(&sde->err_halt_worker, sdma_err_halt_wait); 1427 INIT_WORK(&sde->flush_worker, sdma_field_flush); 1428 1429 sde->progress_check_head = 0; 1430 1431 timer_setup(&sde->err_progress_check_timer, 1432 sdma_err_progress_check, 0); 1433 1434 sde->descq = dma_alloc_coherent(&dd->pcidev->dev, 1435 descq_cnt * sizeof(u64[2]), 1436 &sde->descq_phys, GFP_KERNEL); 1437 if (!sde->descq) 1438 goto bail; 1439 sde->tx_ring = 1440 kvzalloc_node(array_size(descq_cnt, 1441 sizeof(struct sdma_txreq *)), 1442 GFP_KERNEL, dd->node); 1443 if (!sde->tx_ring) 1444 goto bail; 1445 } 1446 1447 dd->sdma_heads_size = L1_CACHE_BYTES * num_engines; 1448 /* Allocate memory for DMA of head registers to memory */ 1449 dd->sdma_heads_dma = dma_alloc_coherent(&dd->pcidev->dev, 1450 dd->sdma_heads_size, 1451 &dd->sdma_heads_phys, 1452 GFP_KERNEL); 1453 if (!dd->sdma_heads_dma) { 1454 dd_dev_err(dd, "failed to allocate SendDMA head memory\n"); 1455 goto bail; 1456 } 1457 1458 /* Allocate memory for pad */ 1459 dd->sdma_pad_dma = dma_alloc_coherent(&dd->pcidev->dev, SDMA_PAD, 1460 &dd->sdma_pad_phys, GFP_KERNEL); 1461 if (!dd->sdma_pad_dma) { 1462 dd_dev_err(dd, "failed to allocate SendDMA pad memory\n"); 1463 goto bail; 1464 } 1465 1466 /* assign each engine to different cacheline and init registers */ 1467 curr_head = (void *)dd->sdma_heads_dma; 1468 for (this_idx = 0; this_idx < num_engines; ++this_idx) { 1469 unsigned long phys_offset; 1470 1471 sde = &dd->per_sdma[this_idx]; 1472 1473 sde->head_dma = curr_head; 1474 curr_head += L1_CACHE_BYTES; 1475 phys_offset = (unsigned long)sde->head_dma - 1476 (unsigned long)dd->sdma_heads_dma; 1477 sde->head_phys = dd->sdma_heads_phys + phys_offset; 1478 init_sdma_regs(sde, per_sdma_credits, idle_cnt); 1479 } 1480 dd->flags |= HFI1_HAS_SEND_DMA; 1481 dd->flags |= idle_cnt ? HFI1_HAS_SDMA_TIMEOUT : 0; 1482 dd->num_sdma = num_engines; 1483 ret = sdma_map_init(dd, port, ppd->vls_operational, NULL); 1484 if (ret < 0) 1485 goto bail; 1486 1487 tmp_sdma_rht = kzalloc_obj(*tmp_sdma_rht); 1488 if (!tmp_sdma_rht) { 1489 ret = -ENOMEM; 1490 goto bail; 1491 } 1492 1493 ret = rhashtable_init(tmp_sdma_rht, &sdma_rht_params); 1494 if (ret < 0) { 1495 kfree(tmp_sdma_rht); 1496 goto bail; 1497 } 1498 1499 dd->sdma_rht = tmp_sdma_rht; 1500 1501 dd_dev_info(dd, "SDMA num_sdma: %u\n", dd->num_sdma); 1502 return 0; 1503 1504 bail: 1505 sdma_clean(dd, num_engines); 1506 return ret; 1507 } 1508 1509 /** 1510 * sdma_all_running() - called when the link goes up 1511 * @dd: hfi1_devdata 1512 * 1513 * This routine moves all engines to the running state. 1514 */ 1515 void sdma_all_running(struct hfi1_devdata *dd) 1516 { 1517 struct sdma_engine *sde; 1518 unsigned int i; 1519 1520 /* move all engines to running */ 1521 for (i = 0; i < dd->num_sdma; ++i) { 1522 sde = &dd->per_sdma[i]; 1523 sdma_process_event(sde, sdma_event_e30_go_running); 1524 } 1525 } 1526 1527 /** 1528 * sdma_start() - called to kick off state processing for all engines 1529 * @dd: hfi1_devdata 1530 * 1531 * This routine is for kicking off the state processing for all required 1532 * sdma engines. Interrupts need to be working at this point. 1533 * 1534 */ 1535 void sdma_start(struct hfi1_devdata *dd) 1536 { 1537 unsigned i; 1538 struct sdma_engine *sde; 1539 1540 /* kick off the engines state processing */ 1541 for (i = 0; i < dd->num_sdma; ++i) { 1542 sde = &dd->per_sdma[i]; 1543 sdma_process_event(sde, sdma_event_e10_go_hw_start); 1544 } 1545 } 1546 1547 /** 1548 * sdma_exit() - used when module is removed 1549 * @dd: hfi1_devdata 1550 */ 1551 void sdma_exit(struct hfi1_devdata *dd) 1552 { 1553 unsigned this_idx; 1554 struct sdma_engine *sde; 1555 1556 for (this_idx = 0; dd->per_sdma && this_idx < dd->num_sdma; 1557 ++this_idx) { 1558 sde = &dd->per_sdma[this_idx]; 1559 if (!list_empty(&sde->dmawait)) 1560 dd_dev_err(dd, "sde %u: dmawait list not empty!\n", 1561 sde->this_idx); 1562 sdma_process_event(sde, sdma_event_e00_go_hw_down); 1563 1564 timer_delete_sync(&sde->err_progress_check_timer); 1565 1566 /* 1567 * This waits for the state machine to exit so it is not 1568 * necessary to kill the sdma_sw_clean_up_task to make sure 1569 * it is not running. 1570 */ 1571 sdma_finalput(&sde->state); 1572 cancel_work_sync(&sde->err_halt_worker); 1573 tasklet_kill(&sde->sdma_hw_clean_up_task); 1574 cancel_work_sync(&sde->flush_worker); 1575 } 1576 } 1577 1578 /* 1579 * unmap the indicated descriptor 1580 */ 1581 static inline void sdma_unmap_desc( 1582 struct hfi1_devdata *dd, 1583 struct sdma_desc *descp) 1584 { 1585 switch (sdma_mapping_type(descp)) { 1586 case SDMA_MAP_SINGLE: 1587 dma_unmap_single(&dd->pcidev->dev, sdma_mapping_addr(descp), 1588 sdma_mapping_len(descp), DMA_TO_DEVICE); 1589 break; 1590 case SDMA_MAP_PAGE: 1591 dma_unmap_page(&dd->pcidev->dev, sdma_mapping_addr(descp), 1592 sdma_mapping_len(descp), DMA_TO_DEVICE); 1593 break; 1594 } 1595 1596 if (descp->pinning_ctx && descp->ctx_put) 1597 descp->ctx_put(descp->pinning_ctx); 1598 descp->pinning_ctx = NULL; 1599 } 1600 1601 /* 1602 * return the mode as indicated by the first 1603 * descriptor in the tx. 1604 */ 1605 static inline u8 ahg_mode(struct sdma_txreq *tx) 1606 { 1607 return (tx->descp[0].qw[1] & SDMA_DESC1_HEADER_MODE_SMASK) 1608 >> SDMA_DESC1_HEADER_MODE_SHIFT; 1609 } 1610 1611 /** 1612 * __sdma_txclean() - clean tx of mappings, descp *kmalloc's 1613 * @dd: hfi1_devdata for unmapping 1614 * @tx: tx request to clean 1615 * 1616 * This is used in the progress routine to clean the tx or 1617 * by the ULP to toss an in-process tx build. 1618 * 1619 * The code can be called multiple times without issue. 1620 * 1621 */ 1622 void __sdma_txclean( 1623 struct hfi1_devdata *dd, 1624 struct sdma_txreq *tx) 1625 { 1626 u16 i; 1627 1628 if (tx->num_desc) { 1629 u8 skip = 0, mode = ahg_mode(tx); 1630 1631 /* unmap first */ 1632 sdma_unmap_desc(dd, &tx->descp[0]); 1633 /* determine number of AHG descriptors to skip */ 1634 if (mode > SDMA_AHG_APPLY_UPDATE1) 1635 skip = mode >> 1; 1636 for (i = 1 + skip; i < tx->num_desc; i++) 1637 sdma_unmap_desc(dd, &tx->descp[i]); 1638 tx->num_desc = 0; 1639 } 1640 kfree(tx->coalesce_buf); 1641 tx->coalesce_buf = NULL; 1642 /* kmalloc'ed descp */ 1643 if (unlikely(tx->desc_limit > ARRAY_SIZE(tx->descs))) { 1644 tx->desc_limit = ARRAY_SIZE(tx->descs); 1645 kfree(tx->descp); 1646 } 1647 } 1648 1649 static inline u16 sdma_gethead(struct sdma_engine *sde) 1650 { 1651 struct hfi1_devdata *dd = sde->dd; 1652 int use_dmahead; 1653 u16 hwhead; 1654 1655 #ifdef CONFIG_SDMA_VERBOSITY 1656 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1657 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1658 #endif 1659 1660 retry: 1661 use_dmahead = HFI1_CAP_IS_KSET(USE_SDMA_HEAD) && __sdma_running(sde) && 1662 (dd->flags & HFI1_HAS_SDMA_TIMEOUT); 1663 hwhead = use_dmahead ? 1664 (u16)le64_to_cpu(*sde->head_dma) : 1665 (u16)read_sde_csr(sde, SD(HEAD)); 1666 1667 if (unlikely(HFI1_CAP_IS_KSET(SDMA_HEAD_CHECK))) { 1668 u16 cnt; 1669 u16 swtail; 1670 u16 swhead; 1671 int sane; 1672 1673 swhead = sde->descq_head & sde->sdma_mask; 1674 /* this code is really bad for cache line trading */ 1675 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 1676 cnt = sde->descq_cnt; 1677 1678 if (swhead < swtail) 1679 /* not wrapped */ 1680 sane = (hwhead >= swhead) & (hwhead <= swtail); 1681 else if (swhead > swtail) 1682 /* wrapped around */ 1683 sane = ((hwhead >= swhead) && (hwhead < cnt)) || 1684 (hwhead <= swtail); 1685 else 1686 /* empty */ 1687 sane = (hwhead == swhead); 1688 1689 if (unlikely(!sane)) { 1690 dd_dev_err(dd, "SDMA(%u) bad head (%s) hwhd=%u swhd=%u swtl=%u cnt=%u\n", 1691 sde->this_idx, 1692 use_dmahead ? "dma" : "kreg", 1693 hwhead, swhead, swtail, cnt); 1694 if (use_dmahead) { 1695 /* try one more time, using csr */ 1696 use_dmahead = 0; 1697 goto retry; 1698 } 1699 /* proceed as if no progress */ 1700 hwhead = swhead; 1701 } 1702 } 1703 return hwhead; 1704 } 1705 1706 /* 1707 * This is called when there are send DMA descriptors that might be 1708 * available. 1709 * 1710 * This is called with head_lock held. 1711 */ 1712 static void sdma_desc_avail(struct sdma_engine *sde, uint avail) 1713 { 1714 struct iowait *wait, *nw, *twait; 1715 struct iowait *waits[SDMA_WAIT_BATCH_SIZE]; 1716 uint i, n = 0, seq, tidx = 0; 1717 1718 #ifdef CONFIG_SDMA_VERBOSITY 1719 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 1720 slashstrip(__FILE__), __LINE__, __func__); 1721 dd_dev_err(sde->dd, "avail: %u\n", avail); 1722 #endif 1723 1724 do { 1725 seq = read_seqbegin(&sde->waitlock); 1726 if (!list_empty(&sde->dmawait)) { 1727 /* at least one item */ 1728 write_seqlock(&sde->waitlock); 1729 /* Harvest waiters wanting DMA descriptors */ 1730 list_for_each_entry_safe( 1731 wait, 1732 nw, 1733 &sde->dmawait, 1734 list) { 1735 u32 num_desc; 1736 1737 if (!wait->wakeup) 1738 continue; 1739 if (n == ARRAY_SIZE(waits)) 1740 break; 1741 iowait_init_priority(wait); 1742 num_desc = iowait_get_all_desc(wait); 1743 if (num_desc > avail) 1744 break; 1745 avail -= num_desc; 1746 /* Find the top-priority wait memeber */ 1747 if (n) { 1748 twait = waits[tidx]; 1749 tidx = 1750 iowait_priority_update_top(wait, 1751 twait, 1752 n, 1753 tidx); 1754 } 1755 list_del_init(&wait->list); 1756 waits[n++] = wait; 1757 } 1758 write_sequnlock(&sde->waitlock); 1759 break; 1760 } 1761 } while (read_seqretry(&sde->waitlock, seq)); 1762 1763 /* Schedule the top-priority entry first */ 1764 if (n) 1765 waits[tidx]->wakeup(waits[tidx], SDMA_AVAIL_REASON); 1766 1767 for (i = 0; i < n; i++) 1768 if (i != tidx) 1769 waits[i]->wakeup(waits[i], SDMA_AVAIL_REASON); 1770 } 1771 1772 /* head_lock must be held */ 1773 static void sdma_make_progress(struct sdma_engine *sde, u64 status) 1774 { 1775 struct sdma_txreq *txp = NULL; 1776 int progress = 0; 1777 u16 hwhead, swhead; 1778 int idle_check_done = 0; 1779 1780 hwhead = sdma_gethead(sde); 1781 1782 /* The reason for some of the complexity of this code is that 1783 * not all descriptors have corresponding txps. So, we have to 1784 * be able to skip over descs until we wander into the range of 1785 * the next txp on the list. 1786 */ 1787 1788 retry: 1789 txp = get_txhead(sde); 1790 swhead = sde->descq_head & sde->sdma_mask; 1791 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp); 1792 while (swhead != hwhead) { 1793 /* advance head, wrap if needed */ 1794 swhead = ++sde->descq_head & sde->sdma_mask; 1795 1796 /* if now past this txp's descs, do the callback */ 1797 if (txp && txp->next_descq_idx == swhead) { 1798 /* remove from list */ 1799 sde->tx_ring[sde->tx_head++ & sde->sdma_mask] = NULL; 1800 complete_tx(sde, txp, SDMA_TXREQ_S_OK); 1801 /* see if there is another txp */ 1802 txp = get_txhead(sde); 1803 } 1804 trace_hfi1_sdma_progress(sde, hwhead, swhead, txp); 1805 progress++; 1806 } 1807 1808 /* 1809 * The SDMA idle interrupt is not guaranteed to be ordered with respect 1810 * to updates to the dma_head location in host memory. The head 1811 * value read might not be fully up to date. If there are pending 1812 * descriptors and the SDMA idle interrupt fired then read from the 1813 * CSR SDMA head instead to get the latest value from the hardware. 1814 * The hardware SDMA head should be read at most once in this invocation 1815 * of sdma_make_progress(..) which is ensured by idle_check_done flag 1816 */ 1817 if ((status & sde->idle_mask) && !idle_check_done) { 1818 u16 swtail; 1819 1820 swtail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 1821 if (swtail != hwhead) { 1822 hwhead = (u16)read_sde_csr(sde, SD(HEAD)); 1823 idle_check_done = 1; 1824 goto retry; 1825 } 1826 } 1827 1828 sde->last_status = status; 1829 if (progress) 1830 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 1831 } 1832 1833 /* 1834 * sdma_engine_interrupt() - interrupt handler for engine 1835 * @sde: sdma engine 1836 * @status: sdma interrupt reason 1837 * 1838 * Status is a mask of the 3 possible interrupts for this engine. It will 1839 * contain bits _only_ for this SDMA engine. It will contain at least one 1840 * bit, it may contain more. 1841 */ 1842 void sdma_engine_interrupt(struct sdma_engine *sde, u64 status) 1843 { 1844 trace_hfi1_sdma_engine_interrupt(sde, status); 1845 write_seqlock(&sde->head_lock); 1846 sdma_set_desc_cnt(sde, sdma_desct_intr); 1847 if (status & sde->idle_mask) 1848 sde->idle_int_cnt++; 1849 else if (status & sde->progress_mask) 1850 sde->progress_int_cnt++; 1851 else if (status & sde->int_mask) 1852 sde->sdma_int_cnt++; 1853 sdma_make_progress(sde, status); 1854 write_sequnlock(&sde->head_lock); 1855 } 1856 1857 /** 1858 * sdma_engine_error() - error handler for engine 1859 * @sde: sdma engine 1860 * @status: sdma interrupt reason 1861 */ 1862 void sdma_engine_error(struct sdma_engine *sde, u64 status) 1863 { 1864 unsigned long flags; 1865 1866 #ifdef CONFIG_SDMA_VERBOSITY 1867 dd_dev_err(sde->dd, "CONFIG SDMA(%u) error status 0x%llx state %s\n", 1868 sde->this_idx, 1869 (unsigned long long)status, 1870 sdma_state_names[sde->state.current_state]); 1871 #endif 1872 spin_lock_irqsave(&sde->tail_lock, flags); 1873 write_seqlock(&sde->head_lock); 1874 if (status & ALL_SDMA_ENG_HALT_ERRS) 1875 __sdma_process_event(sde, sdma_event_e60_hw_halted); 1876 if (status & ~SD(ENG_ERR_STATUS_SDMA_HALT_ERR_SMASK)) { 1877 dd_dev_err(sde->dd, 1878 "SDMA (%u) engine error: 0x%llx state %s\n", 1879 sde->this_idx, 1880 (unsigned long long)status, 1881 sdma_state_names[sde->state.current_state]); 1882 dump_sdma_state(sde); 1883 } 1884 write_sequnlock(&sde->head_lock); 1885 spin_unlock_irqrestore(&sde->tail_lock, flags); 1886 } 1887 1888 static void sdma_sendctrl(struct sdma_engine *sde, unsigned op) 1889 { 1890 u64 set_senddmactrl = 0; 1891 u64 clr_senddmactrl = 0; 1892 unsigned long flags; 1893 1894 #ifdef CONFIG_SDMA_VERBOSITY 1895 dd_dev_err(sde->dd, "CONFIG SDMA(%u) senddmactrl E=%d I=%d H=%d C=%d\n", 1896 sde->this_idx, 1897 (op & SDMA_SENDCTRL_OP_ENABLE) ? 1 : 0, 1898 (op & SDMA_SENDCTRL_OP_INTENABLE) ? 1 : 0, 1899 (op & SDMA_SENDCTRL_OP_HALT) ? 1 : 0, 1900 (op & SDMA_SENDCTRL_OP_CLEANUP) ? 1 : 0); 1901 #endif 1902 1903 if (op & SDMA_SENDCTRL_OP_ENABLE) 1904 set_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK); 1905 else 1906 clr_senddmactrl |= SD(CTRL_SDMA_ENABLE_SMASK); 1907 1908 if (op & SDMA_SENDCTRL_OP_INTENABLE) 1909 set_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK); 1910 else 1911 clr_senddmactrl |= SD(CTRL_SDMA_INT_ENABLE_SMASK); 1912 1913 if (op & SDMA_SENDCTRL_OP_HALT) 1914 set_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK); 1915 else 1916 clr_senddmactrl |= SD(CTRL_SDMA_HALT_SMASK); 1917 1918 spin_lock_irqsave(&sde->senddmactrl_lock, flags); 1919 1920 sde->p_senddmactrl |= set_senddmactrl; 1921 sde->p_senddmactrl &= ~clr_senddmactrl; 1922 1923 if (op & SDMA_SENDCTRL_OP_CLEANUP) 1924 write_sde_csr(sde, SD(CTRL), 1925 sde->p_senddmactrl | 1926 SD(CTRL_SDMA_CLEANUP_SMASK)); 1927 else 1928 write_sde_csr(sde, SD(CTRL), sde->p_senddmactrl); 1929 1930 spin_unlock_irqrestore(&sde->senddmactrl_lock, flags); 1931 1932 #ifdef CONFIG_SDMA_VERBOSITY 1933 sdma_dumpstate(sde); 1934 #endif 1935 } 1936 1937 static void sdma_setlengen(struct sdma_engine *sde) 1938 { 1939 #ifdef CONFIG_SDMA_VERBOSITY 1940 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1941 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1942 #endif 1943 1944 /* 1945 * Set SendDmaLenGen and clear-then-set the MSB of the generation 1946 * count to enable generation checking and load the internal 1947 * generation counter. 1948 */ 1949 write_sde_csr(sde, SD(LEN_GEN), 1950 (sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)); 1951 write_sde_csr(sde, SD(LEN_GEN), 1952 ((sde->descq_cnt / 64) << SD(LEN_GEN_LENGTH_SHIFT)) | 1953 (4ULL << SD(LEN_GEN_GENERATION_SHIFT))); 1954 } 1955 1956 static inline void sdma_update_tail(struct sdma_engine *sde, u16 tail) 1957 { 1958 /* Commit writes to memory and advance the tail on the chip */ 1959 smp_wmb(); /* see get_txhead() */ 1960 writeq(tail, sde->tail_csr); 1961 } 1962 1963 /* 1964 * This is called when changing to state s10_hw_start_up_halt_wait as 1965 * a result of send buffer errors or send DMA descriptor errors. 1966 */ 1967 static void sdma_hw_start_up(struct sdma_engine *sde) 1968 { 1969 u64 reg; 1970 1971 #ifdef CONFIG_SDMA_VERBOSITY 1972 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", 1973 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 1974 #endif 1975 1976 sdma_setlengen(sde); 1977 sdma_update_tail(sde, 0); /* Set SendDmaTail */ 1978 *sde->head_dma = 0; 1979 1980 reg = SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_MASK) << 1981 SD(ENG_ERR_CLEAR_SDMA_HEADER_REQUEST_FIFO_UNC_ERR_SHIFT); 1982 write_sde_csr(sde, SD(ENG_ERR_CLEAR), reg); 1983 } 1984 1985 /* 1986 * set_sdma_integrity 1987 * 1988 * Set the SEND_DMA_CHECK_ENABLE register for send DMA engine 'sde'. 1989 */ 1990 static void set_sdma_integrity(struct sdma_engine *sde) 1991 { 1992 struct hfi1_devdata *dd = sde->dd; 1993 1994 write_sde_csr(sde, SD(CHECK_ENABLE), 1995 hfi1_pkt_base_sdma_integrity(dd)); 1996 } 1997 1998 static void init_sdma_regs( 1999 struct sdma_engine *sde, 2000 u32 credits, 2001 uint idle_cnt) 2002 { 2003 u8 opval, opmask; 2004 #ifdef CONFIG_SDMA_VERBOSITY 2005 struct hfi1_devdata *dd = sde->dd; 2006 2007 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", 2008 sde->this_idx, slashstrip(__FILE__), __LINE__, __func__); 2009 #endif 2010 2011 write_sde_csr(sde, SD(BASE_ADDR), sde->descq_phys); 2012 sdma_setlengen(sde); 2013 sdma_update_tail(sde, 0); /* Set SendDmaTail */ 2014 write_sde_csr(sde, SD(RELOAD_CNT), idle_cnt); 2015 write_sde_csr(sde, SD(DESC_CNT), 0); 2016 write_sde_csr(sde, SD(HEAD_ADDR), sde->head_phys); 2017 write_sde_csr(sde, SD(MEMORY), 2018 ((u64)credits << SD(MEMORY_SDMA_MEMORY_CNT_SHIFT)) | 2019 ((u64)(credits * sde->this_idx) << 2020 SD(MEMORY_SDMA_MEMORY_INDEX_SHIFT))); 2021 write_sde_csr(sde, SD(ENG_ERR_MASK), ~0ull); 2022 set_sdma_integrity(sde); 2023 opmask = OPCODE_CHECK_MASK_DISABLED; 2024 opval = OPCODE_CHECK_VAL_DISABLED; 2025 write_sde_csr(sde, SD(CHECK_OPCODE), 2026 (opmask << SEND_CTXT_CHECK_OPCODE_MASK_SHIFT) | 2027 (opval << SEND_CTXT_CHECK_OPCODE_VALUE_SHIFT)); 2028 } 2029 2030 #ifdef CONFIG_SDMA_VERBOSITY 2031 2032 #define sdma_dumpstate_helper0(reg) do { \ 2033 csr = read_csr(sde->dd, reg); \ 2034 dd_dev_err(sde->dd, "%36s 0x%016llx\n", #reg, csr); \ 2035 } while (0) 2036 2037 #define sdma_dumpstate_helper(reg) do { \ 2038 csr = read_sde_csr(sde, reg); \ 2039 dd_dev_err(sde->dd, "%36s[%02u] 0x%016llx\n", \ 2040 #reg, sde->this_idx, csr); \ 2041 } while (0) 2042 2043 #define sdma_dumpstate_helper2(reg) do { \ 2044 csr = read_csr(sde->dd, reg + (8 * i)); \ 2045 dd_dev_err(sde->dd, "%33s_%02u 0x%016llx\n", \ 2046 #reg, i, csr); \ 2047 } while (0) 2048 2049 void sdma_dumpstate(struct sdma_engine *sde) 2050 { 2051 u64 csr; 2052 unsigned i; 2053 2054 sdma_dumpstate_helper(SD(CTRL)); 2055 sdma_dumpstate_helper(SD(STATUS)); 2056 sdma_dumpstate_helper0(SD(ERR_STATUS)); 2057 sdma_dumpstate_helper0(SD(ERR_MASK)); 2058 sdma_dumpstate_helper(SD(ENG_ERR_STATUS)); 2059 sdma_dumpstate_helper(SD(ENG_ERR_MASK)); 2060 2061 for (i = 0; i < CCE_NUM_INT_CSRS; ++i) { 2062 sdma_dumpstate_helper2(CCE_INT_STATUS); 2063 sdma_dumpstate_helper2(CCE_INT_MASK); 2064 sdma_dumpstate_helper2(CCE_INT_BLOCKED); 2065 } 2066 2067 sdma_dumpstate_helper(SD(TAIL)); 2068 sdma_dumpstate_helper(SD(HEAD)); 2069 sdma_dumpstate_helper(SD(PRIORITY_THLD)); 2070 sdma_dumpstate_helper(SD(IDLE_CNT)); 2071 sdma_dumpstate_helper(SD(RELOAD_CNT)); 2072 sdma_dumpstate_helper(SD(DESC_CNT)); 2073 sdma_dumpstate_helper(SD(DESC_FETCHED_CNT)); 2074 sdma_dumpstate_helper(SD(MEMORY)); 2075 sdma_dumpstate_helper0(SD(ENGINES)); 2076 sdma_dumpstate_helper0(SD(MEM_SIZE)); 2077 /* sdma_dumpstate_helper(SEND_EGRESS_SEND_DMA_STATUS); */ 2078 sdma_dumpstate_helper(SD(BASE_ADDR)); 2079 sdma_dumpstate_helper(SD(LEN_GEN)); 2080 sdma_dumpstate_helper(SD(HEAD_ADDR)); 2081 sdma_dumpstate_helper(SD(CHECK_ENABLE)); 2082 sdma_dumpstate_helper(SD(CHECK_VL)); 2083 sdma_dumpstate_helper(SD(CHECK_JOB_KEY)); 2084 sdma_dumpstate_helper(SD(CHECK_PARTITION_KEY)); 2085 sdma_dumpstate_helper(SD(CHECK_SLID)); 2086 sdma_dumpstate_helper(SD(CHECK_OPCODE)); 2087 } 2088 #endif 2089 2090 static void dump_sdma_state(struct sdma_engine *sde) 2091 { 2092 struct hw_sdma_desc *descqp; 2093 u64 desc[2]; 2094 u64 addr; 2095 u8 gen; 2096 u16 len; 2097 u16 head, tail, cnt; 2098 2099 head = sde->descq_head & sde->sdma_mask; 2100 tail = sde->descq_tail & sde->sdma_mask; 2101 cnt = sdma_descq_freecnt(sde); 2102 2103 dd_dev_err(sde->dd, 2104 "SDMA (%u) descq_head: %u descq_tail: %u freecnt: %u FLE %d\n", 2105 sde->this_idx, head, tail, cnt, 2106 !list_empty(&sde->flushlist)); 2107 2108 /* print info for each entry in the descriptor queue */ 2109 while (head != tail) { 2110 char flags[6] = { 'x', 'x', 'x', 'x', 0 }; 2111 2112 descqp = &sde->descq[head]; 2113 desc[0] = le64_to_cpu(descqp->qw[0]); 2114 desc[1] = le64_to_cpu(descqp->qw[1]); 2115 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; 2116 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ? 2117 'H' : '-'; 2118 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; 2119 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; 2120 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT) 2121 & SDMA_DESC0_PHY_ADDR_MASK; 2122 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT) 2123 & SDMA_DESC1_GENERATION_MASK; 2124 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT) 2125 & SDMA_DESC0_BYTE_COUNT_MASK; 2126 dd_dev_err(sde->dd, 2127 "SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n", 2128 head, flags, addr, gen, len); 2129 dd_dev_err(sde->dd, 2130 "\tdesc0:0x%016llx desc1 0x%016llx\n", 2131 desc[0], desc[1]); 2132 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) 2133 dd_dev_err(sde->dd, 2134 "\taidx: %u amode: %u alen: %u\n", 2135 (u8)((desc[1] & 2136 SDMA_DESC1_HEADER_INDEX_SMASK) >> 2137 SDMA_DESC1_HEADER_INDEX_SHIFT), 2138 (u8)((desc[1] & 2139 SDMA_DESC1_HEADER_MODE_SMASK) >> 2140 SDMA_DESC1_HEADER_MODE_SHIFT), 2141 (u8)((desc[1] & 2142 SDMA_DESC1_HEADER_DWS_SMASK) >> 2143 SDMA_DESC1_HEADER_DWS_SHIFT)); 2144 head++; 2145 head &= sde->sdma_mask; 2146 } 2147 } 2148 2149 #define SDE_FMT \ 2150 "SDE %u CPU %d STE %s C 0x%llx S 0x%016llx E 0x%llx T(HW) 0x%llx T(SW) 0x%x H(HW) 0x%llx H(SW) 0x%x H(D) 0x%llx DM 0x%llx GL 0x%llx R 0x%llx LIS 0x%llx AHGI 0x%llx TXT %u TXH %u DT %u DH %u FLNE %d DQF %u SLC 0x%llx\n" 2151 /** 2152 * sdma_seqfile_dump_sde() - debugfs dump of sde 2153 * @s: seq file 2154 * @sde: send dma engine to dump 2155 * 2156 * This routine dumps the sde to the indicated seq file. 2157 */ 2158 void sdma_seqfile_dump_sde(struct seq_file *s, struct sdma_engine *sde) 2159 { 2160 u16 head, tail; 2161 struct hw_sdma_desc *descqp; 2162 u64 desc[2]; 2163 u64 addr; 2164 u8 gen; 2165 u16 len; 2166 2167 head = sde->descq_head & sde->sdma_mask; 2168 tail = READ_ONCE(sde->descq_tail) & sde->sdma_mask; 2169 seq_printf(s, SDE_FMT, sde->this_idx, 2170 sde->cpu, 2171 sdma_state_name(sde->state.current_state), 2172 (unsigned long long)read_sde_csr(sde, SD(CTRL)), 2173 (unsigned long long)read_sde_csr(sde, SD(STATUS)), 2174 (unsigned long long)read_sde_csr(sde, SD(ENG_ERR_STATUS)), 2175 (unsigned long long)read_sde_csr(sde, SD(TAIL)), tail, 2176 (unsigned long long)read_sde_csr(sde, SD(HEAD)), head, 2177 (unsigned long long)le64_to_cpu(*sde->head_dma), 2178 (unsigned long long)read_sde_csr(sde, SD(MEMORY)), 2179 (unsigned long long)read_sde_csr(sde, SD(LEN_GEN)), 2180 (unsigned long long)read_sde_csr(sde, SD(RELOAD_CNT)), 2181 (unsigned long long)sde->last_status, 2182 (unsigned long long)sde->ahg_bits, 2183 sde->tx_tail, 2184 sde->tx_head, 2185 sde->descq_tail, 2186 sde->descq_head, 2187 !list_empty(&sde->flushlist), 2188 sde->descq_full_count, 2189 (unsigned long long)read_sde_csr(sde, SEND_DMA_CHECK_SLID)); 2190 2191 /* print info for each entry in the descriptor queue */ 2192 while (head != tail) { 2193 char flags[6] = { 'x', 'x', 'x', 'x', 0 }; 2194 2195 descqp = &sde->descq[head]; 2196 desc[0] = le64_to_cpu(descqp->qw[0]); 2197 desc[1] = le64_to_cpu(descqp->qw[1]); 2198 flags[0] = (desc[1] & SDMA_DESC1_INT_REQ_FLAG) ? 'I' : '-'; 2199 flags[1] = (desc[1] & SDMA_DESC1_HEAD_TO_HOST_FLAG) ? 2200 'H' : '-'; 2201 flags[2] = (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) ? 'F' : '-'; 2202 flags[3] = (desc[0] & SDMA_DESC0_LAST_DESC_FLAG) ? 'L' : '-'; 2203 addr = (desc[0] >> SDMA_DESC0_PHY_ADDR_SHIFT) 2204 & SDMA_DESC0_PHY_ADDR_MASK; 2205 gen = (desc[1] >> SDMA_DESC1_GENERATION_SHIFT) 2206 & SDMA_DESC1_GENERATION_MASK; 2207 len = (desc[0] >> SDMA_DESC0_BYTE_COUNT_SHIFT) 2208 & SDMA_DESC0_BYTE_COUNT_MASK; 2209 seq_printf(s, 2210 "\tdesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes\n", 2211 head, flags, addr, gen, len); 2212 if (desc[0] & SDMA_DESC0_FIRST_DESC_FLAG) 2213 seq_printf(s, "\t\tahgidx: %u ahgmode: %u\n", 2214 (u8)((desc[1] & 2215 SDMA_DESC1_HEADER_INDEX_SMASK) >> 2216 SDMA_DESC1_HEADER_INDEX_SHIFT), 2217 (u8)((desc[1] & 2218 SDMA_DESC1_HEADER_MODE_SMASK) >> 2219 SDMA_DESC1_HEADER_MODE_SHIFT)); 2220 head = (head + 1) & sde->sdma_mask; 2221 } 2222 } 2223 2224 /* 2225 * add the generation number into 2226 * the qw1 and return 2227 */ 2228 static inline u64 add_gen(struct sdma_engine *sde, u64 qw1) 2229 { 2230 u8 generation = (sde->descq_tail >> sde->sdma_shift) & 3; 2231 2232 qw1 &= ~SDMA_DESC1_GENERATION_SMASK; 2233 qw1 |= ((u64)generation & SDMA_DESC1_GENERATION_MASK) 2234 << SDMA_DESC1_GENERATION_SHIFT; 2235 return qw1; 2236 } 2237 2238 /* 2239 * This routine submits the indicated tx 2240 * 2241 * Space has already been guaranteed and 2242 * tail side of ring is locked. 2243 * 2244 * The hardware tail update is done 2245 * in the caller and that is facilitated 2246 * by returning the new tail. 2247 * 2248 * There is special case logic for ahg 2249 * to not add the generation number for 2250 * up to 2 descriptors that follow the 2251 * first descriptor. 2252 * 2253 */ 2254 static inline u16 submit_tx(struct sdma_engine *sde, struct sdma_txreq *tx) 2255 { 2256 int i; 2257 u16 tail; 2258 struct sdma_desc *descp = tx->descp; 2259 u8 skip = 0, mode = ahg_mode(tx); 2260 2261 tail = sde->descq_tail & sde->sdma_mask; 2262 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); 2263 sde->descq[tail].qw[1] = cpu_to_le64(add_gen(sde, descp->qw[1])); 2264 trace_hfi1_sdma_descriptor(sde, descp->qw[0], descp->qw[1], 2265 tail, &sde->descq[tail]); 2266 tail = ++sde->descq_tail & sde->sdma_mask; 2267 descp++; 2268 if (mode > SDMA_AHG_APPLY_UPDATE1) 2269 skip = mode >> 1; 2270 for (i = 1; i < tx->num_desc; i++, descp++) { 2271 u64 qw1; 2272 2273 sde->descq[tail].qw[0] = cpu_to_le64(descp->qw[0]); 2274 if (skip) { 2275 /* edits don't have generation */ 2276 qw1 = descp->qw[1]; 2277 skip--; 2278 } else { 2279 /* replace generation with real one for non-edits */ 2280 qw1 = add_gen(sde, descp->qw[1]); 2281 } 2282 sde->descq[tail].qw[1] = cpu_to_le64(qw1); 2283 trace_hfi1_sdma_descriptor(sde, descp->qw[0], qw1, 2284 tail, &sde->descq[tail]); 2285 tail = ++sde->descq_tail & sde->sdma_mask; 2286 } 2287 tx->next_descq_idx = tail; 2288 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2289 tx->sn = sde->tail_sn++; 2290 trace_hfi1_sdma_in_sn(sde, tx->sn); 2291 WARN_ON_ONCE(sde->tx_ring[sde->tx_tail & sde->sdma_mask]); 2292 #endif 2293 sde->tx_ring[sde->tx_tail++ & sde->sdma_mask] = tx; 2294 sde->desc_avail -= tx->num_desc; 2295 return tail; 2296 } 2297 2298 /* 2299 * Check for progress 2300 */ 2301 static int sdma_check_progress( 2302 struct sdma_engine *sde, 2303 struct iowait_work *wait, 2304 struct sdma_txreq *tx, 2305 bool pkts_sent) 2306 { 2307 int ret; 2308 2309 sde->desc_avail = sdma_descq_freecnt(sde); 2310 if (tx->num_desc <= sde->desc_avail) 2311 return -EAGAIN; 2312 /* pulse the head_lock */ 2313 if (wait && iowait_ioww_to_iow(wait)->sleep) { 2314 unsigned seq; 2315 2316 seq = raw_seqcount_begin( 2317 (const seqcount_t *)&sde->head_lock.seqcount); 2318 ret = wait->iow->sleep(sde, wait, tx, seq, pkts_sent); 2319 if (ret == -EAGAIN) 2320 sde->desc_avail = sdma_descq_freecnt(sde); 2321 } else { 2322 ret = -EBUSY; 2323 } 2324 return ret; 2325 } 2326 2327 /** 2328 * sdma_send_txreq() - submit a tx req to ring 2329 * @sde: sdma engine to use 2330 * @wait: SE wait structure to use when full (may be NULL) 2331 * @tx: sdma_txreq to submit 2332 * @pkts_sent: has any packet been sent yet? 2333 * 2334 * The call submits the tx into the ring. If a iowait structure is non-NULL 2335 * the packet will be queued to the list in wait. 2336 * 2337 * Return: 2338 * 0 - Success, -EINVAL - sdma_txreq incomplete, -EBUSY - no space in 2339 * ring (wait == NULL) 2340 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state 2341 */ 2342 int sdma_send_txreq(struct sdma_engine *sde, 2343 struct iowait_work *wait, 2344 struct sdma_txreq *tx, 2345 bool pkts_sent) 2346 { 2347 int ret = 0; 2348 u16 tail; 2349 unsigned long flags; 2350 2351 /* user should have supplied entire packet */ 2352 if (unlikely(tx->tlen)) 2353 return -EINVAL; 2354 tx->wait = iowait_ioww_to_iow(wait); 2355 spin_lock_irqsave(&sde->tail_lock, flags); 2356 retry: 2357 if (unlikely(!__sdma_running(sde))) 2358 goto unlock_noconn; 2359 if (unlikely(tx->num_desc > sde->desc_avail)) 2360 goto nodesc; 2361 tail = submit_tx(sde, tx); 2362 if (wait) 2363 iowait_sdma_inc(iowait_ioww_to_iow(wait)); 2364 sdma_update_tail(sde, tail); 2365 unlock: 2366 spin_unlock_irqrestore(&sde->tail_lock, flags); 2367 return ret; 2368 unlock_noconn: 2369 if (wait) 2370 iowait_sdma_inc(iowait_ioww_to_iow(wait)); 2371 tx->next_descq_idx = 0; 2372 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2373 tx->sn = sde->tail_sn++; 2374 trace_hfi1_sdma_in_sn(sde, tx->sn); 2375 #endif 2376 spin_lock(&sde->flushlist_lock); 2377 list_add_tail(&tx->list, &sde->flushlist); 2378 spin_unlock(&sde->flushlist_lock); 2379 iowait_inc_wait_count(wait, tx->num_desc); 2380 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); 2381 ret = -ECOMM; 2382 goto unlock; 2383 nodesc: 2384 ret = sdma_check_progress(sde, wait, tx, pkts_sent); 2385 if (ret == -EAGAIN) { 2386 ret = 0; 2387 goto retry; 2388 } 2389 sde->descq_full_count++; 2390 goto unlock; 2391 } 2392 2393 /** 2394 * sdma_send_txlist() - submit a list of tx req to ring 2395 * @sde: sdma engine to use 2396 * @wait: SE wait structure to use when full (may be NULL) 2397 * @tx_list: list of sdma_txreqs to submit 2398 * @count_out: pointer to a u16 which, after return will contain the total number of 2399 * sdma_txreqs removed from the tx_list. This will include sdma_txreqs 2400 * whose SDMA descriptors are submitted to the ring and the sdma_txreqs 2401 * which are added to SDMA engine flush list if the SDMA engine state is 2402 * not running. 2403 * 2404 * The call submits the list into the ring. 2405 * 2406 * If the iowait structure is non-NULL and not equal to the iowait list 2407 * the unprocessed part of the list will be appended to the list in wait. 2408 * 2409 * In all cases, the tx_list will be updated so the head of the tx_list is 2410 * the list of descriptors that have yet to be transmitted. 2411 * 2412 * The intent of this call is to provide a more efficient 2413 * way of submitting multiple packets to SDMA while holding the tail 2414 * side locking. 2415 * 2416 * Return: 2417 * 0 - Success, 2418 * -EINVAL - sdma_txreq incomplete, -EBUSY - no space in ring (wait == NULL) 2419 * -EIOCBQUEUED - tx queued to iowait, -ECOMM bad sdma state 2420 */ 2421 int sdma_send_txlist(struct sdma_engine *sde, struct iowait_work *wait, 2422 struct list_head *tx_list, u16 *count_out) 2423 { 2424 struct sdma_txreq *tx, *tx_next; 2425 int ret = 0; 2426 unsigned long flags; 2427 u16 tail = INVALID_TAIL; 2428 u32 submit_count = 0, flush_count = 0, total_count; 2429 2430 spin_lock_irqsave(&sde->tail_lock, flags); 2431 retry: 2432 list_for_each_entry_safe(tx, tx_next, tx_list, list) { 2433 tx->wait = iowait_ioww_to_iow(wait); 2434 if (unlikely(!__sdma_running(sde))) 2435 goto unlock_noconn; 2436 if (unlikely(tx->num_desc > sde->desc_avail)) 2437 goto nodesc; 2438 if (unlikely(tx->tlen)) { 2439 ret = -EINVAL; 2440 goto update_tail; 2441 } 2442 list_del_init(&tx->list); 2443 tail = submit_tx(sde, tx); 2444 submit_count++; 2445 if (tail != INVALID_TAIL && 2446 (submit_count & SDMA_TAIL_UPDATE_THRESH) == 0) { 2447 sdma_update_tail(sde, tail); 2448 tail = INVALID_TAIL; 2449 } 2450 } 2451 update_tail: 2452 total_count = submit_count + flush_count; 2453 if (wait) { 2454 iowait_sdma_add(iowait_ioww_to_iow(wait), total_count); 2455 iowait_starve_clear(submit_count > 0, 2456 iowait_ioww_to_iow(wait)); 2457 } 2458 if (tail != INVALID_TAIL) 2459 sdma_update_tail(sde, tail); 2460 spin_unlock_irqrestore(&sde->tail_lock, flags); 2461 *count_out = total_count; 2462 return ret; 2463 unlock_noconn: 2464 spin_lock(&sde->flushlist_lock); 2465 list_for_each_entry_safe(tx, tx_next, tx_list, list) { 2466 tx->wait = iowait_ioww_to_iow(wait); 2467 list_del_init(&tx->list); 2468 tx->next_descq_idx = 0; 2469 #ifdef CONFIG_HFI1_DEBUG_SDMA_ORDER 2470 tx->sn = sde->tail_sn++; 2471 trace_hfi1_sdma_in_sn(sde, tx->sn); 2472 #endif 2473 list_add_tail(&tx->list, &sde->flushlist); 2474 flush_count++; 2475 iowait_inc_wait_count(wait, tx->num_desc); 2476 } 2477 spin_unlock(&sde->flushlist_lock); 2478 queue_work_on(sde->cpu, system_highpri_wq, &sde->flush_worker); 2479 ret = -ECOMM; 2480 goto update_tail; 2481 nodesc: 2482 ret = sdma_check_progress(sde, wait, tx, submit_count > 0); 2483 if (ret == -EAGAIN) { 2484 ret = 0; 2485 goto retry; 2486 } 2487 sde->descq_full_count++; 2488 goto update_tail; 2489 } 2490 2491 static void sdma_process_event(struct sdma_engine *sde, enum sdma_events event) 2492 { 2493 unsigned long flags; 2494 2495 spin_lock_irqsave(&sde->tail_lock, flags); 2496 write_seqlock(&sde->head_lock); 2497 2498 __sdma_process_event(sde, event); 2499 2500 if (sde->state.current_state == sdma_state_s99_running) 2501 sdma_desc_avail(sde, sdma_descq_freecnt(sde)); 2502 2503 write_sequnlock(&sde->head_lock); 2504 spin_unlock_irqrestore(&sde->tail_lock, flags); 2505 } 2506 2507 static void __sdma_process_event(struct sdma_engine *sde, 2508 enum sdma_events event) 2509 { 2510 struct sdma_state *ss = &sde->state; 2511 int need_progress = 0; 2512 2513 /* CONFIG SDMA temporary */ 2514 #ifdef CONFIG_SDMA_VERBOSITY 2515 dd_dev_err(sde->dd, "CONFIG SDMA(%u) [%s] %s\n", sde->this_idx, 2516 sdma_state_names[ss->current_state], 2517 sdma_event_names[event]); 2518 #endif 2519 2520 switch (ss->current_state) { 2521 case sdma_state_s00_hw_down: 2522 switch (event) { 2523 case sdma_event_e00_go_hw_down: 2524 break; 2525 case sdma_event_e30_go_running: 2526 /* 2527 * If down, but running requested (usually result 2528 * of link up, then we need to start up. 2529 * This can happen when hw down is requested while 2530 * bringing the link up with traffic active on 2531 * 7220, e.g. 2532 */ 2533 ss->go_s99_running = 1; 2534 fallthrough; /* and start dma engine */ 2535 case sdma_event_e10_go_hw_start: 2536 /* This reference means the state machine is started */ 2537 sdma_get(&sde->state); 2538 sdma_set_state(sde, 2539 sdma_state_s10_hw_start_up_halt_wait); 2540 break; 2541 case sdma_event_e15_hw_halt_done: 2542 break; 2543 case sdma_event_e25_hw_clean_up_done: 2544 break; 2545 case sdma_event_e40_sw_cleaned: 2546 sdma_sw_tear_down(sde); 2547 break; 2548 case sdma_event_e50_hw_cleaned: 2549 break; 2550 case sdma_event_e60_hw_halted: 2551 break; 2552 case sdma_event_e70_go_idle: 2553 break; 2554 case sdma_event_e80_hw_freeze: 2555 break; 2556 case sdma_event_e81_hw_frozen: 2557 break; 2558 case sdma_event_e82_hw_unfreeze: 2559 break; 2560 case sdma_event_e85_link_down: 2561 break; 2562 case sdma_event_e90_sw_halted: 2563 break; 2564 } 2565 break; 2566 2567 case sdma_state_s10_hw_start_up_halt_wait: 2568 switch (event) { 2569 case sdma_event_e00_go_hw_down: 2570 sdma_set_state(sde, sdma_state_s00_hw_down); 2571 sdma_sw_tear_down(sde); 2572 break; 2573 case sdma_event_e10_go_hw_start: 2574 break; 2575 case sdma_event_e15_hw_halt_done: 2576 sdma_set_state(sde, 2577 sdma_state_s15_hw_start_up_clean_wait); 2578 sdma_start_hw_clean_up(sde); 2579 break; 2580 case sdma_event_e25_hw_clean_up_done: 2581 break; 2582 case sdma_event_e30_go_running: 2583 ss->go_s99_running = 1; 2584 break; 2585 case sdma_event_e40_sw_cleaned: 2586 break; 2587 case sdma_event_e50_hw_cleaned: 2588 break; 2589 case sdma_event_e60_hw_halted: 2590 schedule_work(&sde->err_halt_worker); 2591 break; 2592 case sdma_event_e70_go_idle: 2593 ss->go_s99_running = 0; 2594 break; 2595 case sdma_event_e80_hw_freeze: 2596 break; 2597 case sdma_event_e81_hw_frozen: 2598 break; 2599 case sdma_event_e82_hw_unfreeze: 2600 break; 2601 case sdma_event_e85_link_down: 2602 break; 2603 case sdma_event_e90_sw_halted: 2604 break; 2605 } 2606 break; 2607 2608 case sdma_state_s15_hw_start_up_clean_wait: 2609 switch (event) { 2610 case sdma_event_e00_go_hw_down: 2611 sdma_set_state(sde, sdma_state_s00_hw_down); 2612 sdma_sw_tear_down(sde); 2613 break; 2614 case sdma_event_e10_go_hw_start: 2615 break; 2616 case sdma_event_e15_hw_halt_done: 2617 break; 2618 case sdma_event_e25_hw_clean_up_done: 2619 sdma_hw_start_up(sde); 2620 sdma_set_state(sde, ss->go_s99_running ? 2621 sdma_state_s99_running : 2622 sdma_state_s20_idle); 2623 break; 2624 case sdma_event_e30_go_running: 2625 ss->go_s99_running = 1; 2626 break; 2627 case sdma_event_e40_sw_cleaned: 2628 break; 2629 case sdma_event_e50_hw_cleaned: 2630 break; 2631 case sdma_event_e60_hw_halted: 2632 break; 2633 case sdma_event_e70_go_idle: 2634 ss->go_s99_running = 0; 2635 break; 2636 case sdma_event_e80_hw_freeze: 2637 break; 2638 case sdma_event_e81_hw_frozen: 2639 break; 2640 case sdma_event_e82_hw_unfreeze: 2641 break; 2642 case sdma_event_e85_link_down: 2643 break; 2644 case sdma_event_e90_sw_halted: 2645 break; 2646 } 2647 break; 2648 2649 case sdma_state_s20_idle: 2650 switch (event) { 2651 case sdma_event_e00_go_hw_down: 2652 sdma_set_state(sde, sdma_state_s00_hw_down); 2653 sdma_sw_tear_down(sde); 2654 break; 2655 case sdma_event_e10_go_hw_start: 2656 break; 2657 case sdma_event_e15_hw_halt_done: 2658 break; 2659 case sdma_event_e25_hw_clean_up_done: 2660 break; 2661 case sdma_event_e30_go_running: 2662 sdma_set_state(sde, sdma_state_s99_running); 2663 ss->go_s99_running = 1; 2664 break; 2665 case sdma_event_e40_sw_cleaned: 2666 break; 2667 case sdma_event_e50_hw_cleaned: 2668 break; 2669 case sdma_event_e60_hw_halted: 2670 sdma_set_state(sde, sdma_state_s50_hw_halt_wait); 2671 schedule_work(&sde->err_halt_worker); 2672 break; 2673 case sdma_event_e70_go_idle: 2674 break; 2675 case sdma_event_e85_link_down: 2676 case sdma_event_e80_hw_freeze: 2677 sdma_set_state(sde, sdma_state_s80_hw_freeze); 2678 atomic_dec(&sde->dd->sdma_unfreeze_count); 2679 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2680 break; 2681 case sdma_event_e81_hw_frozen: 2682 break; 2683 case sdma_event_e82_hw_unfreeze: 2684 break; 2685 case sdma_event_e90_sw_halted: 2686 break; 2687 } 2688 break; 2689 2690 case sdma_state_s30_sw_clean_up_wait: 2691 switch (event) { 2692 case sdma_event_e00_go_hw_down: 2693 sdma_set_state(sde, sdma_state_s00_hw_down); 2694 break; 2695 case sdma_event_e10_go_hw_start: 2696 break; 2697 case sdma_event_e15_hw_halt_done: 2698 break; 2699 case sdma_event_e25_hw_clean_up_done: 2700 break; 2701 case sdma_event_e30_go_running: 2702 ss->go_s99_running = 1; 2703 break; 2704 case sdma_event_e40_sw_cleaned: 2705 sdma_set_state(sde, sdma_state_s40_hw_clean_up_wait); 2706 sdma_start_hw_clean_up(sde); 2707 break; 2708 case sdma_event_e50_hw_cleaned: 2709 break; 2710 case sdma_event_e60_hw_halted: 2711 break; 2712 case sdma_event_e70_go_idle: 2713 ss->go_s99_running = 0; 2714 break; 2715 case sdma_event_e80_hw_freeze: 2716 break; 2717 case sdma_event_e81_hw_frozen: 2718 break; 2719 case sdma_event_e82_hw_unfreeze: 2720 break; 2721 case sdma_event_e85_link_down: 2722 ss->go_s99_running = 0; 2723 break; 2724 case sdma_event_e90_sw_halted: 2725 break; 2726 } 2727 break; 2728 2729 case sdma_state_s40_hw_clean_up_wait: 2730 switch (event) { 2731 case sdma_event_e00_go_hw_down: 2732 sdma_set_state(sde, sdma_state_s00_hw_down); 2733 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2734 break; 2735 case sdma_event_e10_go_hw_start: 2736 break; 2737 case sdma_event_e15_hw_halt_done: 2738 break; 2739 case sdma_event_e25_hw_clean_up_done: 2740 sdma_hw_start_up(sde); 2741 sdma_set_state(sde, ss->go_s99_running ? 2742 sdma_state_s99_running : 2743 sdma_state_s20_idle); 2744 break; 2745 case sdma_event_e30_go_running: 2746 ss->go_s99_running = 1; 2747 break; 2748 case sdma_event_e40_sw_cleaned: 2749 break; 2750 case sdma_event_e50_hw_cleaned: 2751 break; 2752 case sdma_event_e60_hw_halted: 2753 break; 2754 case sdma_event_e70_go_idle: 2755 ss->go_s99_running = 0; 2756 break; 2757 case sdma_event_e80_hw_freeze: 2758 break; 2759 case sdma_event_e81_hw_frozen: 2760 break; 2761 case sdma_event_e82_hw_unfreeze: 2762 break; 2763 case sdma_event_e85_link_down: 2764 ss->go_s99_running = 0; 2765 break; 2766 case sdma_event_e90_sw_halted: 2767 break; 2768 } 2769 break; 2770 2771 case sdma_state_s50_hw_halt_wait: 2772 switch (event) { 2773 case sdma_event_e00_go_hw_down: 2774 sdma_set_state(sde, sdma_state_s00_hw_down); 2775 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2776 break; 2777 case sdma_event_e10_go_hw_start: 2778 break; 2779 case sdma_event_e15_hw_halt_done: 2780 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait); 2781 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2782 break; 2783 case sdma_event_e25_hw_clean_up_done: 2784 break; 2785 case sdma_event_e30_go_running: 2786 ss->go_s99_running = 1; 2787 break; 2788 case sdma_event_e40_sw_cleaned: 2789 break; 2790 case sdma_event_e50_hw_cleaned: 2791 break; 2792 case sdma_event_e60_hw_halted: 2793 schedule_work(&sde->err_halt_worker); 2794 break; 2795 case sdma_event_e70_go_idle: 2796 ss->go_s99_running = 0; 2797 break; 2798 case sdma_event_e80_hw_freeze: 2799 break; 2800 case sdma_event_e81_hw_frozen: 2801 break; 2802 case sdma_event_e82_hw_unfreeze: 2803 break; 2804 case sdma_event_e85_link_down: 2805 ss->go_s99_running = 0; 2806 break; 2807 case sdma_event_e90_sw_halted: 2808 break; 2809 } 2810 break; 2811 2812 case sdma_state_s60_idle_halt_wait: 2813 switch (event) { 2814 case sdma_event_e00_go_hw_down: 2815 sdma_set_state(sde, sdma_state_s00_hw_down); 2816 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2817 break; 2818 case sdma_event_e10_go_hw_start: 2819 break; 2820 case sdma_event_e15_hw_halt_done: 2821 sdma_set_state(sde, sdma_state_s30_sw_clean_up_wait); 2822 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2823 break; 2824 case sdma_event_e25_hw_clean_up_done: 2825 break; 2826 case sdma_event_e30_go_running: 2827 ss->go_s99_running = 1; 2828 break; 2829 case sdma_event_e40_sw_cleaned: 2830 break; 2831 case sdma_event_e50_hw_cleaned: 2832 break; 2833 case sdma_event_e60_hw_halted: 2834 schedule_work(&sde->err_halt_worker); 2835 break; 2836 case sdma_event_e70_go_idle: 2837 ss->go_s99_running = 0; 2838 break; 2839 case sdma_event_e80_hw_freeze: 2840 break; 2841 case sdma_event_e81_hw_frozen: 2842 break; 2843 case sdma_event_e82_hw_unfreeze: 2844 break; 2845 case sdma_event_e85_link_down: 2846 break; 2847 case sdma_event_e90_sw_halted: 2848 break; 2849 } 2850 break; 2851 2852 case sdma_state_s80_hw_freeze: 2853 switch (event) { 2854 case sdma_event_e00_go_hw_down: 2855 sdma_set_state(sde, sdma_state_s00_hw_down); 2856 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2857 break; 2858 case sdma_event_e10_go_hw_start: 2859 break; 2860 case sdma_event_e15_hw_halt_done: 2861 break; 2862 case sdma_event_e25_hw_clean_up_done: 2863 break; 2864 case sdma_event_e30_go_running: 2865 ss->go_s99_running = 1; 2866 break; 2867 case sdma_event_e40_sw_cleaned: 2868 break; 2869 case sdma_event_e50_hw_cleaned: 2870 break; 2871 case sdma_event_e60_hw_halted: 2872 break; 2873 case sdma_event_e70_go_idle: 2874 ss->go_s99_running = 0; 2875 break; 2876 case sdma_event_e80_hw_freeze: 2877 break; 2878 case sdma_event_e81_hw_frozen: 2879 sdma_set_state(sde, sdma_state_s82_freeze_sw_clean); 2880 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2881 break; 2882 case sdma_event_e82_hw_unfreeze: 2883 break; 2884 case sdma_event_e85_link_down: 2885 break; 2886 case sdma_event_e90_sw_halted: 2887 break; 2888 } 2889 break; 2890 2891 case sdma_state_s82_freeze_sw_clean: 2892 switch (event) { 2893 case sdma_event_e00_go_hw_down: 2894 sdma_set_state(sde, sdma_state_s00_hw_down); 2895 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2896 break; 2897 case sdma_event_e10_go_hw_start: 2898 break; 2899 case sdma_event_e15_hw_halt_done: 2900 break; 2901 case sdma_event_e25_hw_clean_up_done: 2902 break; 2903 case sdma_event_e30_go_running: 2904 ss->go_s99_running = 1; 2905 break; 2906 case sdma_event_e40_sw_cleaned: 2907 /* notify caller this engine is done cleaning */ 2908 atomic_dec(&sde->dd->sdma_unfreeze_count); 2909 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2910 break; 2911 case sdma_event_e50_hw_cleaned: 2912 break; 2913 case sdma_event_e60_hw_halted: 2914 break; 2915 case sdma_event_e70_go_idle: 2916 ss->go_s99_running = 0; 2917 break; 2918 case sdma_event_e80_hw_freeze: 2919 break; 2920 case sdma_event_e81_hw_frozen: 2921 break; 2922 case sdma_event_e82_hw_unfreeze: 2923 sdma_hw_start_up(sde); 2924 sdma_set_state(sde, ss->go_s99_running ? 2925 sdma_state_s99_running : 2926 sdma_state_s20_idle); 2927 break; 2928 case sdma_event_e85_link_down: 2929 break; 2930 case sdma_event_e90_sw_halted: 2931 break; 2932 } 2933 break; 2934 2935 case sdma_state_s99_running: 2936 switch (event) { 2937 case sdma_event_e00_go_hw_down: 2938 sdma_set_state(sde, sdma_state_s00_hw_down); 2939 tasklet_hi_schedule(&sde->sdma_sw_clean_up_task); 2940 break; 2941 case sdma_event_e10_go_hw_start: 2942 break; 2943 case sdma_event_e15_hw_halt_done: 2944 break; 2945 case sdma_event_e25_hw_clean_up_done: 2946 break; 2947 case sdma_event_e30_go_running: 2948 break; 2949 case sdma_event_e40_sw_cleaned: 2950 break; 2951 case sdma_event_e50_hw_cleaned: 2952 break; 2953 case sdma_event_e60_hw_halted: 2954 need_progress = 1; 2955 sdma_err_progress_check_schedule(sde); 2956 fallthrough; 2957 case sdma_event_e90_sw_halted: 2958 /* 2959 * SW initiated halt does not perform engines 2960 * progress check 2961 */ 2962 sdma_set_state(sde, sdma_state_s50_hw_halt_wait); 2963 schedule_work(&sde->err_halt_worker); 2964 break; 2965 case sdma_event_e70_go_idle: 2966 sdma_set_state(sde, sdma_state_s60_idle_halt_wait); 2967 break; 2968 case sdma_event_e85_link_down: 2969 ss->go_s99_running = 0; 2970 fallthrough; 2971 case sdma_event_e80_hw_freeze: 2972 sdma_set_state(sde, sdma_state_s80_hw_freeze); 2973 atomic_dec(&sde->dd->sdma_unfreeze_count); 2974 wake_up_interruptible(&sde->dd->sdma_unfreeze_wq); 2975 break; 2976 case sdma_event_e81_hw_frozen: 2977 break; 2978 case sdma_event_e82_hw_unfreeze: 2979 break; 2980 } 2981 break; 2982 } 2983 2984 ss->last_event = event; 2985 if (need_progress) 2986 sdma_make_progress(sde, 0); 2987 } 2988 2989 /* 2990 * _extend_sdma_tx_descs() - helper to extend txreq 2991 * 2992 * This is called once the initial nominal allocation 2993 * of descriptors in the sdma_txreq is exhausted. 2994 * 2995 * The code will bump the allocation up to the max 2996 * of MAX_DESC (64) descriptors. There doesn't seem 2997 * much point in an interim step. The last descriptor 2998 * is reserved for coalesce buffer in order to support 2999 * cases where input packet has >MAX_DESC iovecs. 3000 * 3001 */ 3002 static int _extend_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) 3003 { 3004 int i; 3005 struct sdma_desc *descp; 3006 3007 /* Handle last descriptor */ 3008 if (unlikely((tx->num_desc == (MAX_DESC - 1)))) { 3009 /* if tlen is 0, it is for padding, release last descriptor */ 3010 if (!tx->tlen) { 3011 tx->desc_limit = MAX_DESC; 3012 } else if (!tx->coalesce_buf) { 3013 /* allocate coalesce buffer with space for padding */ 3014 tx->coalesce_buf = kmalloc(tx->tlen + sizeof(u32), 3015 GFP_ATOMIC); 3016 if (!tx->coalesce_buf) 3017 goto enomem; 3018 tx->coalesce_idx = 0; 3019 } 3020 return 0; 3021 } 3022 3023 if (unlikely(tx->num_desc == MAX_DESC)) 3024 goto enomem; 3025 3026 descp = kmalloc_objs(struct sdma_desc, MAX_DESC, GFP_ATOMIC); 3027 if (!descp) 3028 goto enomem; 3029 tx->descp = descp; 3030 3031 /* reserve last descriptor for coalescing */ 3032 tx->desc_limit = MAX_DESC - 1; 3033 /* copy ones already built */ 3034 for (i = 0; i < tx->num_desc; i++) 3035 tx->descp[i] = tx->descs[i]; 3036 return 0; 3037 enomem: 3038 __sdma_txclean(dd, tx); 3039 return -ENOMEM; 3040 } 3041 3042 /* 3043 * ext_coal_sdma_tx_descs() - extend or coalesce sdma tx descriptors 3044 * 3045 * This is called once the initial nominal allocation of descriptors 3046 * in the sdma_txreq is exhausted. 3047 * 3048 * This function calls _extend_sdma_tx_descs to extend or allocate 3049 * coalesce buffer. If there is a allocated coalesce buffer, it will 3050 * copy the input packet data into the coalesce buffer. It also adds 3051 * coalesce buffer descriptor once when whole packet is received. 3052 * 3053 * Return: 3054 * <0 - error 3055 * 0 - coalescing, don't populate descriptor 3056 * 1 - continue with populating descriptor 3057 */ 3058 int ext_coal_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx, 3059 int type, void *kvaddr, struct page *page, 3060 unsigned long offset, u16 len) 3061 { 3062 int pad_len, rval; 3063 dma_addr_t addr; 3064 3065 rval = _extend_sdma_tx_descs(dd, tx); 3066 if (rval) { 3067 __sdma_txclean(dd, tx); 3068 return rval; 3069 } 3070 3071 /* If coalesce buffer is allocated, copy data into it */ 3072 if (tx->coalesce_buf) { 3073 if (type == SDMA_MAP_NONE) { 3074 __sdma_txclean(dd, tx); 3075 return -EINVAL; 3076 } 3077 3078 if (type == SDMA_MAP_PAGE) { 3079 kvaddr = kmap_local_page(page); 3080 kvaddr += offset; 3081 } else if (WARN_ON(!kvaddr)) { 3082 __sdma_txclean(dd, tx); 3083 return -EINVAL; 3084 } 3085 3086 memcpy(tx->coalesce_buf + tx->coalesce_idx, kvaddr, len); 3087 tx->coalesce_idx += len; 3088 if (type == SDMA_MAP_PAGE) 3089 kunmap_local(kvaddr); 3090 3091 /* If there is more data, return */ 3092 if (tx->tlen - tx->coalesce_idx) 3093 return 0; 3094 3095 /* Whole packet is received; add any padding */ 3096 pad_len = tx->packet_len & (sizeof(u32) - 1); 3097 if (pad_len) { 3098 pad_len = sizeof(u32) - pad_len; 3099 memset(tx->coalesce_buf + tx->coalesce_idx, 0, pad_len); 3100 /* padding is taken care of for coalescing case */ 3101 tx->packet_len += pad_len; 3102 tx->tlen += pad_len; 3103 } 3104 3105 /* dma map the coalesce buffer */ 3106 addr = dma_map_single(&dd->pcidev->dev, 3107 tx->coalesce_buf, 3108 tx->tlen, 3109 DMA_TO_DEVICE); 3110 3111 if (unlikely(dma_mapping_error(&dd->pcidev->dev, addr))) { 3112 __sdma_txclean(dd, tx); 3113 return -ENOSPC; 3114 } 3115 3116 /* Add descriptor for coalesce buffer */ 3117 tx->desc_limit = MAX_DESC; 3118 return _sdma_txadd_daddr(dd, SDMA_MAP_SINGLE, tx, 3119 addr, tx->tlen, NULL, NULL, NULL); 3120 } 3121 3122 return 1; 3123 } 3124 3125 /* Update sdes when the lmc changes */ 3126 void sdma_update_lmc(struct hfi1_devdata *dd, u64 mask, u32 lid) 3127 { 3128 struct sdma_engine *sde; 3129 int i; 3130 u64 sreg; 3131 3132 sreg = ((mask & SD(CHECK_SLID_MASK_MASK)) << 3133 SD(CHECK_SLID_MASK_SHIFT)) | 3134 (((lid & mask) & SD(CHECK_SLID_VALUE_MASK)) << 3135 SD(CHECK_SLID_VALUE_SHIFT)); 3136 3137 for (i = 0; i < dd->num_sdma; i++) { 3138 hfi1_cdbg(LINKVERB, "SendDmaEngine[%d].SLID_CHECK = 0x%x", 3139 i, (u32)sreg); 3140 sde = &dd->per_sdma[i]; 3141 write_sde_csr(sde, SD(CHECK_SLID), sreg); 3142 } 3143 } 3144 3145 /* tx not dword sized - pad */ 3146 int _pad_sdma_tx_descs(struct hfi1_devdata *dd, struct sdma_txreq *tx) 3147 { 3148 int rval = 0; 3149 3150 if ((unlikely(tx->num_desc == tx->desc_limit))) { 3151 rval = _extend_sdma_tx_descs(dd, tx); 3152 if (rval) { 3153 __sdma_txclean(dd, tx); 3154 return rval; 3155 } 3156 } 3157 3158 /* finish the one just added */ 3159 make_tx_sdma_desc( 3160 tx, 3161 SDMA_MAP_NONE, 3162 dd->sdma_pad_phys, 3163 sizeof(u32) - (tx->packet_len & (sizeof(u32) - 1)), 3164 NULL, NULL, NULL); 3165 tx->num_desc++; 3166 _sdma_close_tx(dd, tx); 3167 return rval; 3168 } 3169 3170 /* 3171 * Add ahg to the sdma_txreq 3172 * 3173 * The logic will consume up to 3 3174 * descriptors at the beginning of 3175 * sdma_txreq. 3176 */ 3177 void _sdma_txreq_ahgadd( 3178 struct sdma_txreq *tx, 3179 u8 num_ahg, 3180 u8 ahg_entry, 3181 u32 *ahg, 3182 u8 ahg_hlen) 3183 { 3184 u32 i, shift = 0, desc = 0; 3185 u8 mode; 3186 3187 WARN_ON_ONCE(num_ahg > 9 || (ahg_hlen & 3) || ahg_hlen == 4); 3188 /* compute mode */ 3189 if (num_ahg == 1) 3190 mode = SDMA_AHG_APPLY_UPDATE1; 3191 else if (num_ahg <= 5) 3192 mode = SDMA_AHG_APPLY_UPDATE2; 3193 else 3194 mode = SDMA_AHG_APPLY_UPDATE3; 3195 tx->num_desc++; 3196 /* initialize to consumed descriptors to zero */ 3197 switch (mode) { 3198 case SDMA_AHG_APPLY_UPDATE3: 3199 tx->num_desc++; 3200 tx->descs[2].qw[0] = 0; 3201 tx->descs[2].qw[1] = 0; 3202 fallthrough; 3203 case SDMA_AHG_APPLY_UPDATE2: 3204 tx->num_desc++; 3205 tx->descs[1].qw[0] = 0; 3206 tx->descs[1].qw[1] = 0; 3207 break; 3208 } 3209 ahg_hlen >>= 2; 3210 tx->descs[0].qw[1] |= 3211 (((u64)ahg_entry & SDMA_DESC1_HEADER_INDEX_MASK) 3212 << SDMA_DESC1_HEADER_INDEX_SHIFT) | 3213 (((u64)ahg_hlen & SDMA_DESC1_HEADER_DWS_MASK) 3214 << SDMA_DESC1_HEADER_DWS_SHIFT) | 3215 (((u64)mode & SDMA_DESC1_HEADER_MODE_MASK) 3216 << SDMA_DESC1_HEADER_MODE_SHIFT) | 3217 (((u64)ahg[0] & SDMA_DESC1_HEADER_UPDATE1_MASK) 3218 << SDMA_DESC1_HEADER_UPDATE1_SHIFT); 3219 for (i = 0; i < (num_ahg - 1); i++) { 3220 if (!shift && !(i & 2)) 3221 desc++; 3222 tx->descs[desc].qw[!!(i & 2)] |= 3223 (((u64)ahg[i + 1]) 3224 << shift); 3225 shift = (shift + 32) & 63; 3226 } 3227 } 3228 3229 /** 3230 * sdma_ahg_alloc - allocate an AHG entry 3231 * @sde: engine to allocate from 3232 * 3233 * Return: 3234 * 0-31 when successful, -EOPNOTSUPP if AHG is not enabled, 3235 * -ENOSPC if an entry is not available 3236 */ 3237 int sdma_ahg_alloc(struct sdma_engine *sde) 3238 { 3239 int nr; 3240 int oldbit; 3241 3242 if (!sde) { 3243 trace_hfi1_ahg_allocate(sde, -EINVAL); 3244 return -EINVAL; 3245 } 3246 while (1) { 3247 nr = ffz(READ_ONCE(sde->ahg_bits)); 3248 if (nr > 31) { 3249 trace_hfi1_ahg_allocate(sde, -ENOSPC); 3250 return -ENOSPC; 3251 } 3252 oldbit = test_and_set_bit(nr, &sde->ahg_bits); 3253 if (!oldbit) 3254 break; 3255 cpu_relax(); 3256 } 3257 trace_hfi1_ahg_allocate(sde, nr); 3258 return nr; 3259 } 3260 3261 /** 3262 * sdma_ahg_free - free an AHG entry 3263 * @sde: engine to return AHG entry 3264 * @ahg_index: index to free 3265 * 3266 * This routine frees the indicate AHG entry. 3267 */ 3268 void sdma_ahg_free(struct sdma_engine *sde, int ahg_index) 3269 { 3270 if (!sde) 3271 return; 3272 trace_hfi1_ahg_deallocate(sde, ahg_index); 3273 if (ahg_index < 0 || ahg_index > 31) 3274 return; 3275 clear_bit(ahg_index, &sde->ahg_bits); 3276 } 3277 3278 /* 3279 * SPC freeze handling for SDMA engines. Called when the driver knows 3280 * the SPC is going into a freeze but before the freeze is fully 3281 * settled. Generally an error interrupt. 3282 * 3283 * This event will pull the engine out of running so no more entries can be 3284 * added to the engine's queue. 3285 */ 3286 void sdma_freeze_notify(struct hfi1_devdata *dd, int link_down) 3287 { 3288 int i; 3289 enum sdma_events event = link_down ? sdma_event_e85_link_down : 3290 sdma_event_e80_hw_freeze; 3291 3292 /* set up the wait but do not wait here */ 3293 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); 3294 3295 /* tell all engines to stop running and wait */ 3296 for (i = 0; i < dd->num_sdma; i++) 3297 sdma_process_event(&dd->per_sdma[i], event); 3298 3299 /* sdma_freeze() will wait for all engines to have stopped */ 3300 } 3301 3302 /* 3303 * SPC freeze handling for SDMA engines. Called when the driver knows 3304 * the SPC is fully frozen. 3305 */ 3306 void sdma_freeze(struct hfi1_devdata *dd) 3307 { 3308 int i; 3309 int ret; 3310 3311 /* 3312 * Make sure all engines have moved out of the running state before 3313 * continuing. 3314 */ 3315 ret = wait_event_interruptible(dd->sdma_unfreeze_wq, 3316 atomic_read(&dd->sdma_unfreeze_count) <= 3317 0); 3318 /* interrupted or count is negative, then unloading - just exit */ 3319 if (ret || atomic_read(&dd->sdma_unfreeze_count) < 0) 3320 return; 3321 3322 /* set up the count for the next wait */ 3323 atomic_set(&dd->sdma_unfreeze_count, dd->num_sdma); 3324 3325 /* tell all engines that the SPC is frozen, they can start cleaning */ 3326 for (i = 0; i < dd->num_sdma; i++) 3327 sdma_process_event(&dd->per_sdma[i], sdma_event_e81_hw_frozen); 3328 3329 /* 3330 * Wait for everyone to finish software clean before exiting. The 3331 * software clean will read engine CSRs, so must be completed before 3332 * the next step, which will clear the engine CSRs. 3333 */ 3334 (void)wait_event_interruptible(dd->sdma_unfreeze_wq, 3335 atomic_read(&dd->sdma_unfreeze_count) <= 0); 3336 /* no need to check results - done no matter what */ 3337 } 3338 3339 /* 3340 * SPC freeze handling for the SDMA engines. Called after the SPC is unfrozen. 3341 * 3342 * The SPC freeze acts like a SDMA halt and a hardware clean combined. All 3343 * that is left is a software clean. We could do it after the SPC is fully 3344 * frozen, but then we'd have to add another state to wait for the unfreeze. 3345 * Instead, just defer the software clean until the unfreeze step. 3346 */ 3347 void sdma_unfreeze(struct hfi1_devdata *dd) 3348 { 3349 int i; 3350 3351 /* tell all engines start freeze clean up */ 3352 for (i = 0; i < dd->num_sdma; i++) 3353 sdma_process_event(&dd->per_sdma[i], 3354 sdma_event_e82_hw_unfreeze); 3355 } 3356 3357 /** 3358 * _sdma_engine_progress_schedule() - schedule progress on engine 3359 * @sde: sdma_engine to schedule progress 3360 * 3361 */ 3362 void _sdma_engine_progress_schedule( 3363 struct sdma_engine *sde) 3364 { 3365 trace_hfi1_sdma_engine_progress(sde, sde->progress_mask); 3366 /* assume we have selected a good cpu */ 3367 write_csr(sde->dd, 3368 CCE_INT_FORCE + (8 * (IS_SDMA_START / 64)), 3369 sde->progress_mask); 3370 } 3371