1 /* 2 * Copyright (c) 2007, 2008, 2009, 2010 QLogic Corporation. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/spinlock.h> 34 #include <linux/netdevice.h> 35 #include <linux/moduleparam.h> 36 37 #include "qib.h" 38 #include "qib_common.h" 39 40 /* default pio off, sdma on */ 41 static ushort sdma_descq_cnt = 256; 42 module_param_named(sdma_descq_cnt, sdma_descq_cnt, ushort, S_IRUGO); 43 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries"); 44 45 /* 46 * Bits defined in the send DMA descriptor. 47 */ 48 #define SDMA_DESC_LAST (1ULL << 11) 49 #define SDMA_DESC_FIRST (1ULL << 12) 50 #define SDMA_DESC_DMA_HEAD (1ULL << 13) 51 #define SDMA_DESC_USE_LARGE_BUF (1ULL << 14) 52 #define SDMA_DESC_INTR (1ULL << 15) 53 #define SDMA_DESC_COUNT_LSB 16 54 #define SDMA_DESC_GEN_LSB 30 55 56 char *qib_sdma_state_names[] = { 57 [qib_sdma_state_s00_hw_down] = "s00_HwDown", 58 [qib_sdma_state_s10_hw_start_up_wait] = "s10_HwStartUpWait", 59 [qib_sdma_state_s20_idle] = "s20_Idle", 60 [qib_sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait", 61 [qib_sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait", 62 [qib_sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait", 63 [qib_sdma_state_s99_running] = "s99_Running", 64 }; 65 66 char *qib_sdma_event_names[] = { 67 [qib_sdma_event_e00_go_hw_down] = "e00_GoHwDown", 68 [qib_sdma_event_e10_go_hw_start] = "e10_GoHwStart", 69 [qib_sdma_event_e20_hw_started] = "e20_HwStarted", 70 [qib_sdma_event_e30_go_running] = "e30_GoRunning", 71 [qib_sdma_event_e40_sw_cleaned] = "e40_SwCleaned", 72 [qib_sdma_event_e50_hw_cleaned] = "e50_HwCleaned", 73 [qib_sdma_event_e60_hw_halted] = "e60_HwHalted", 74 [qib_sdma_event_e70_go_idle] = "e70_GoIdle", 75 [qib_sdma_event_e7220_err_halted] = "e7220_ErrHalted", 76 [qib_sdma_event_e7322_err_halted] = "e7322_ErrHalted", 77 [qib_sdma_event_e90_timer_tick] = "e90_TimerTick", 78 }; 79 80 /* declare all statics here rather than keep sorting */ 81 static int alloc_sdma(struct qib_pportdata *); 82 static void sdma_complete(struct kref *); 83 static void sdma_finalput(struct qib_sdma_state *); 84 static void sdma_get(struct qib_sdma_state *); 85 static void sdma_put(struct qib_sdma_state *); 86 static void sdma_set_state(struct qib_pportdata *, enum qib_sdma_states); 87 static void sdma_start_sw_clean_up(struct qib_pportdata *); 88 static void sdma_sw_clean_up_task(unsigned long); 89 static void unmap_desc(struct qib_pportdata *, unsigned); 90 91 static void sdma_get(struct qib_sdma_state *ss) 92 { 93 kref_get(&ss->kref); 94 } 95 96 static void sdma_complete(struct kref *kref) 97 { 98 struct qib_sdma_state *ss = 99 container_of(kref, struct qib_sdma_state, kref); 100 101 complete(&ss->comp); 102 } 103 104 static void sdma_put(struct qib_sdma_state *ss) 105 { 106 kref_put(&ss->kref, sdma_complete); 107 } 108 109 static void sdma_finalput(struct qib_sdma_state *ss) 110 { 111 sdma_put(ss); 112 wait_for_completion(&ss->comp); 113 } 114 115 /* 116 * Complete all the sdma requests on the active list, in the correct 117 * order, and with appropriate processing. Called when cleaning up 118 * after sdma shutdown, and when new sdma requests are submitted for 119 * a link that is down. This matches what is done for requests 120 * that complete normally, it's just the full list. 121 * 122 * Must be called with sdma_lock held 123 */ 124 static void clear_sdma_activelist(struct qib_pportdata *ppd) 125 { 126 struct qib_sdma_txreq *txp, *txp_next; 127 128 list_for_each_entry_safe(txp, txp_next, &ppd->sdma_activelist, list) { 129 list_del_init(&txp->list); 130 if (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) { 131 unsigned idx; 132 133 idx = txp->start_idx; 134 while (idx != txp->next_descq_idx) { 135 unmap_desc(ppd, idx); 136 if (++idx == ppd->sdma_descq_cnt) 137 idx = 0; 138 } 139 } 140 if (txp->callback) 141 (*txp->callback)(txp, QIB_SDMA_TXREQ_S_ABORTED); 142 } 143 } 144 145 static void sdma_sw_clean_up_task(unsigned long opaque) 146 { 147 struct qib_pportdata *ppd = (struct qib_pportdata *) opaque; 148 unsigned long flags; 149 150 spin_lock_irqsave(&ppd->sdma_lock, flags); 151 152 /* 153 * At this point, the following should always be true: 154 * - We are halted, so no more descriptors are getting retired. 155 * - We are not running, so no one is submitting new work. 156 * - Only we can send the e40_sw_cleaned, so we can't start 157 * running again until we say so. So, the active list and 158 * descq are ours to play with. 159 */ 160 161 /* Process all retired requests. */ 162 qib_sdma_make_progress(ppd); 163 164 clear_sdma_activelist(ppd); 165 166 /* 167 * Resync count of added and removed. It is VERY important that 168 * sdma_descq_removed NEVER decrement - user_sdma depends on it. 169 */ 170 ppd->sdma_descq_removed = ppd->sdma_descq_added; 171 172 /* 173 * Reset our notion of head and tail. 174 * Note that the HW registers will be reset when switching states 175 * due to calling __qib_sdma_process_event() below. 176 */ 177 ppd->sdma_descq_tail = 0; 178 ppd->sdma_descq_head = 0; 179 ppd->sdma_head_dma[0] = 0; 180 ppd->sdma_generation = 0; 181 182 __qib_sdma_process_event(ppd, qib_sdma_event_e40_sw_cleaned); 183 184 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 185 } 186 187 /* 188 * This is called when changing to state qib_sdma_state_s10_hw_start_up_wait 189 * as a result of send buffer errors or send DMA descriptor errors. 190 * We want to disarm the buffers in these cases. 191 */ 192 static void sdma_hw_start_up(struct qib_pportdata *ppd) 193 { 194 struct qib_sdma_state *ss = &ppd->sdma_state; 195 unsigned bufno; 196 197 for (bufno = ss->first_sendbuf; bufno < ss->last_sendbuf; ++bufno) 198 ppd->dd->f_sendctrl(ppd, QIB_SENDCTRL_DISARM_BUF(bufno)); 199 200 ppd->dd->f_sdma_hw_start_up(ppd); 201 } 202 203 static void sdma_sw_tear_down(struct qib_pportdata *ppd) 204 { 205 struct qib_sdma_state *ss = &ppd->sdma_state; 206 207 /* Releasing this reference means the state machine has stopped. */ 208 sdma_put(ss); 209 } 210 211 static void sdma_start_sw_clean_up(struct qib_pportdata *ppd) 212 { 213 tasklet_hi_schedule(&ppd->sdma_sw_clean_up_task); 214 } 215 216 static void sdma_set_state(struct qib_pportdata *ppd, 217 enum qib_sdma_states next_state) 218 { 219 struct qib_sdma_state *ss = &ppd->sdma_state; 220 struct sdma_set_state_action *action = ss->set_state_action; 221 unsigned op = 0; 222 223 /* debugging bookkeeping */ 224 ss->previous_state = ss->current_state; 225 ss->previous_op = ss->current_op; 226 227 ss->current_state = next_state; 228 229 if (action[next_state].op_enable) 230 op |= QIB_SDMA_SENDCTRL_OP_ENABLE; 231 232 if (action[next_state].op_intenable) 233 op |= QIB_SDMA_SENDCTRL_OP_INTENABLE; 234 235 if (action[next_state].op_halt) 236 op |= QIB_SDMA_SENDCTRL_OP_HALT; 237 238 if (action[next_state].op_drain) 239 op |= QIB_SDMA_SENDCTRL_OP_DRAIN; 240 241 if (action[next_state].go_s99_running_tofalse) 242 ss->go_s99_running = 0; 243 244 if (action[next_state].go_s99_running_totrue) 245 ss->go_s99_running = 1; 246 247 ss->current_op = op; 248 249 ppd->dd->f_sdma_sendctrl(ppd, ss->current_op); 250 } 251 252 static void unmap_desc(struct qib_pportdata *ppd, unsigned head) 253 { 254 __le64 *descqp = &ppd->sdma_descq[head].qw[0]; 255 u64 desc[2]; 256 dma_addr_t addr; 257 size_t len; 258 259 desc[0] = le64_to_cpu(descqp[0]); 260 desc[1] = le64_to_cpu(descqp[1]); 261 262 addr = (desc[1] << 32) | (desc[0] >> 32); 263 len = (desc[0] >> 14) & (0x7ffULL << 2); 264 dma_unmap_single(&ppd->dd->pcidev->dev, addr, len, DMA_TO_DEVICE); 265 } 266 267 static int alloc_sdma(struct qib_pportdata *ppd) 268 { 269 ppd->sdma_descq_cnt = sdma_descq_cnt; 270 if (!ppd->sdma_descq_cnt) 271 ppd->sdma_descq_cnt = 256; 272 273 /* Allocate memory for SendDMA descriptor FIFO */ 274 ppd->sdma_descq = dma_alloc_coherent(&ppd->dd->pcidev->dev, 275 ppd->sdma_descq_cnt * sizeof(u64[2]), &ppd->sdma_descq_phys, 276 GFP_KERNEL); 277 278 if (!ppd->sdma_descq) { 279 qib_dev_err(ppd->dd, "failed to allocate SendDMA descriptor " 280 "FIFO memory\n"); 281 goto bail; 282 } 283 284 /* Allocate memory for DMA of head register to memory */ 285 ppd->sdma_head_dma = dma_alloc_coherent(&ppd->dd->pcidev->dev, 286 PAGE_SIZE, &ppd->sdma_head_phys, GFP_KERNEL); 287 if (!ppd->sdma_head_dma) { 288 qib_dev_err(ppd->dd, "failed to allocate SendDMA " 289 "head memory\n"); 290 goto cleanup_descq; 291 } 292 ppd->sdma_head_dma[0] = 0; 293 return 0; 294 295 cleanup_descq: 296 dma_free_coherent(&ppd->dd->pcidev->dev, 297 ppd->sdma_descq_cnt * sizeof(u64[2]), (void *)ppd->sdma_descq, 298 ppd->sdma_descq_phys); 299 ppd->sdma_descq = NULL; 300 ppd->sdma_descq_phys = 0; 301 bail: 302 ppd->sdma_descq_cnt = 0; 303 return -ENOMEM; 304 } 305 306 static void free_sdma(struct qib_pportdata *ppd) 307 { 308 struct qib_devdata *dd = ppd->dd; 309 310 if (ppd->sdma_head_dma) { 311 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE, 312 (void *)ppd->sdma_head_dma, 313 ppd->sdma_head_phys); 314 ppd->sdma_head_dma = NULL; 315 ppd->sdma_head_phys = 0; 316 } 317 318 if (ppd->sdma_descq) { 319 dma_free_coherent(&dd->pcidev->dev, 320 ppd->sdma_descq_cnt * sizeof(u64[2]), 321 ppd->sdma_descq, ppd->sdma_descq_phys); 322 ppd->sdma_descq = NULL; 323 ppd->sdma_descq_phys = 0; 324 } 325 } 326 327 static inline void make_sdma_desc(struct qib_pportdata *ppd, 328 u64 *sdmadesc, u64 addr, u64 dwlen, 329 u64 dwoffset) 330 { 331 332 WARN_ON(addr & 3); 333 /* SDmaPhyAddr[47:32] */ 334 sdmadesc[1] = addr >> 32; 335 /* SDmaPhyAddr[31:0] */ 336 sdmadesc[0] = (addr & 0xfffffffcULL) << 32; 337 /* SDmaGeneration[1:0] */ 338 sdmadesc[0] |= (ppd->sdma_generation & 3ULL) << 339 SDMA_DESC_GEN_LSB; 340 /* SDmaDwordCount[10:0] */ 341 sdmadesc[0] |= (dwlen & 0x7ffULL) << SDMA_DESC_COUNT_LSB; 342 /* SDmaBufOffset[12:2] */ 343 sdmadesc[0] |= dwoffset & 0x7ffULL; 344 } 345 346 /* sdma_lock must be held */ 347 int qib_sdma_make_progress(struct qib_pportdata *ppd) 348 { 349 struct list_head *lp = NULL; 350 struct qib_sdma_txreq *txp = NULL; 351 struct qib_devdata *dd = ppd->dd; 352 int progress = 0; 353 u16 hwhead; 354 u16 idx = 0; 355 356 hwhead = dd->f_sdma_gethead(ppd); 357 358 /* The reason for some of the complexity of this code is that 359 * not all descriptors have corresponding txps. So, we have to 360 * be able to skip over descs until we wander into the range of 361 * the next txp on the list. 362 */ 363 364 if (!list_empty(&ppd->sdma_activelist)) { 365 lp = ppd->sdma_activelist.next; 366 txp = list_entry(lp, struct qib_sdma_txreq, list); 367 idx = txp->start_idx; 368 } 369 370 while (ppd->sdma_descq_head != hwhead) { 371 /* if desc is part of this txp, unmap if needed */ 372 if (txp && (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) && 373 (idx == ppd->sdma_descq_head)) { 374 unmap_desc(ppd, ppd->sdma_descq_head); 375 if (++idx == ppd->sdma_descq_cnt) 376 idx = 0; 377 } 378 379 /* increment dequed desc count */ 380 ppd->sdma_descq_removed++; 381 382 /* advance head, wrap if needed */ 383 if (++ppd->sdma_descq_head == ppd->sdma_descq_cnt) 384 ppd->sdma_descq_head = 0; 385 386 /* if now past this txp's descs, do the callback */ 387 if (txp && txp->next_descq_idx == ppd->sdma_descq_head) { 388 /* remove from active list */ 389 list_del_init(&txp->list); 390 if (txp->callback) 391 (*txp->callback)(txp, QIB_SDMA_TXREQ_S_OK); 392 /* see if there is another txp */ 393 if (list_empty(&ppd->sdma_activelist)) 394 txp = NULL; 395 else { 396 lp = ppd->sdma_activelist.next; 397 txp = list_entry(lp, struct qib_sdma_txreq, 398 list); 399 idx = txp->start_idx; 400 } 401 } 402 progress = 1; 403 } 404 if (progress) 405 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd)); 406 return progress; 407 } 408 409 /* 410 * This is called from interrupt context. 411 */ 412 void qib_sdma_intr(struct qib_pportdata *ppd) 413 { 414 unsigned long flags; 415 416 spin_lock_irqsave(&ppd->sdma_lock, flags); 417 418 __qib_sdma_intr(ppd); 419 420 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 421 } 422 423 void __qib_sdma_intr(struct qib_pportdata *ppd) 424 { 425 if (__qib_sdma_running(ppd)) 426 qib_sdma_make_progress(ppd); 427 } 428 429 int qib_setup_sdma(struct qib_pportdata *ppd) 430 { 431 struct qib_devdata *dd = ppd->dd; 432 unsigned long flags; 433 int ret = 0; 434 435 ret = alloc_sdma(ppd); 436 if (ret) 437 goto bail; 438 439 /* set consistent sdma state */ 440 ppd->dd->f_sdma_init_early(ppd); 441 spin_lock_irqsave(&ppd->sdma_lock, flags); 442 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 443 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 444 445 /* set up reference counting */ 446 kref_init(&ppd->sdma_state.kref); 447 init_completion(&ppd->sdma_state.comp); 448 449 ppd->sdma_generation = 0; 450 ppd->sdma_descq_head = 0; 451 ppd->sdma_descq_removed = 0; 452 ppd->sdma_descq_added = 0; 453 454 INIT_LIST_HEAD(&ppd->sdma_activelist); 455 456 tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task, 457 (unsigned long)ppd); 458 459 ret = dd->f_init_sdma_regs(ppd); 460 if (ret) 461 goto bail_alloc; 462 463 qib_sdma_process_event(ppd, qib_sdma_event_e10_go_hw_start); 464 465 return 0; 466 467 bail_alloc: 468 qib_teardown_sdma(ppd); 469 bail: 470 return ret; 471 } 472 473 void qib_teardown_sdma(struct qib_pportdata *ppd) 474 { 475 qib_sdma_process_event(ppd, qib_sdma_event_e00_go_hw_down); 476 477 /* 478 * This waits for the state machine to exit so it is not 479 * necessary to kill the sdma_sw_clean_up_task to make sure 480 * it is not running. 481 */ 482 sdma_finalput(&ppd->sdma_state); 483 484 free_sdma(ppd); 485 } 486 487 int qib_sdma_running(struct qib_pportdata *ppd) 488 { 489 unsigned long flags; 490 int ret; 491 492 spin_lock_irqsave(&ppd->sdma_lock, flags); 493 ret = __qib_sdma_running(ppd); 494 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 495 496 return ret; 497 } 498 499 /* 500 * Complete a request when sdma not running; likely only request 501 * but to simplify the code, always queue it, then process the full 502 * activelist. We process the entire list to ensure that this particular 503 * request does get it's callback, but in the correct order. 504 * Must be called with sdma_lock held 505 */ 506 static void complete_sdma_err_req(struct qib_pportdata *ppd, 507 struct qib_verbs_txreq *tx) 508 { 509 atomic_inc(&tx->qp->s_dma_busy); 510 /* no sdma descriptors, so no unmap_desc */ 511 tx->txreq.start_idx = 0; 512 tx->txreq.next_descq_idx = 0; 513 list_add_tail(&tx->txreq.list, &ppd->sdma_activelist); 514 clear_sdma_activelist(ppd); 515 } 516 517 /* 518 * This function queues one IB packet onto the send DMA queue per call. 519 * The caller is responsible for checking: 520 * 1) The number of send DMA descriptor entries is less than the size of 521 * the descriptor queue. 522 * 2) The IB SGE addresses and lengths are 32-bit aligned 523 * (except possibly the last SGE's length) 524 * 3) The SGE addresses are suitable for passing to dma_map_single(). 525 */ 526 int qib_sdma_verbs_send(struct qib_pportdata *ppd, 527 struct qib_sge_state *ss, u32 dwords, 528 struct qib_verbs_txreq *tx) 529 { 530 unsigned long flags; 531 struct qib_sge *sge; 532 struct qib_qp *qp; 533 int ret = 0; 534 u16 tail; 535 __le64 *descqp; 536 u64 sdmadesc[2]; 537 u32 dwoffset; 538 dma_addr_t addr; 539 540 spin_lock_irqsave(&ppd->sdma_lock, flags); 541 542 retry: 543 if (unlikely(!__qib_sdma_running(ppd))) { 544 complete_sdma_err_req(ppd, tx); 545 goto unlock; 546 } 547 548 if (tx->txreq.sg_count > qib_sdma_descq_freecnt(ppd)) { 549 if (qib_sdma_make_progress(ppd)) 550 goto retry; 551 if (ppd->dd->flags & QIB_HAS_SDMA_TIMEOUT) 552 ppd->dd->f_sdma_set_desc_cnt(ppd, 553 ppd->sdma_descq_cnt / 2); 554 goto busy; 555 } 556 557 dwoffset = tx->hdr_dwords; 558 make_sdma_desc(ppd, sdmadesc, (u64) tx->txreq.addr, dwoffset, 0); 559 560 sdmadesc[0] |= SDMA_DESC_FIRST; 561 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF) 562 sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF; 563 564 /* write to the descq */ 565 tail = ppd->sdma_descq_tail; 566 descqp = &ppd->sdma_descq[tail].qw[0]; 567 *descqp++ = cpu_to_le64(sdmadesc[0]); 568 *descqp++ = cpu_to_le64(sdmadesc[1]); 569 570 /* increment the tail */ 571 if (++tail == ppd->sdma_descq_cnt) { 572 tail = 0; 573 descqp = &ppd->sdma_descq[0].qw[0]; 574 ++ppd->sdma_generation; 575 } 576 577 tx->txreq.start_idx = tail; 578 579 sge = &ss->sge; 580 while (dwords) { 581 u32 dw; 582 u32 len; 583 584 len = dwords << 2; 585 if (len > sge->length) 586 len = sge->length; 587 if (len > sge->sge_length) 588 len = sge->sge_length; 589 BUG_ON(len == 0); 590 dw = (len + 3) >> 2; 591 addr = dma_map_single(&ppd->dd->pcidev->dev, sge->vaddr, 592 dw << 2, DMA_TO_DEVICE); 593 if (dma_mapping_error(&ppd->dd->pcidev->dev, addr)) 594 goto unmap; 595 sdmadesc[0] = 0; 596 make_sdma_desc(ppd, sdmadesc, (u64) addr, dw, dwoffset); 597 /* SDmaUseLargeBuf has to be set in every descriptor */ 598 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF) 599 sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF; 600 /* write to the descq */ 601 *descqp++ = cpu_to_le64(sdmadesc[0]); 602 *descqp++ = cpu_to_le64(sdmadesc[1]); 603 604 /* increment the tail */ 605 if (++tail == ppd->sdma_descq_cnt) { 606 tail = 0; 607 descqp = &ppd->sdma_descq[0].qw[0]; 608 ++ppd->sdma_generation; 609 } 610 sge->vaddr += len; 611 sge->length -= len; 612 sge->sge_length -= len; 613 if (sge->sge_length == 0) { 614 if (--ss->num_sge) 615 *sge = *ss->sg_list++; 616 } else if (sge->length == 0 && sge->mr->lkey) { 617 if (++sge->n >= QIB_SEGSZ) { 618 if (++sge->m >= sge->mr->mapsz) 619 break; 620 sge->n = 0; 621 } 622 sge->vaddr = 623 sge->mr->map[sge->m]->segs[sge->n].vaddr; 624 sge->length = 625 sge->mr->map[sge->m]->segs[sge->n].length; 626 } 627 628 dwoffset += dw; 629 dwords -= dw; 630 } 631 632 if (!tail) 633 descqp = &ppd->sdma_descq[ppd->sdma_descq_cnt].qw[0]; 634 descqp -= 2; 635 descqp[0] |= cpu_to_le64(SDMA_DESC_LAST); 636 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_HEADTOHOST) 637 descqp[0] |= cpu_to_le64(SDMA_DESC_DMA_HEAD); 638 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_INTREQ) 639 descqp[0] |= cpu_to_le64(SDMA_DESC_INTR); 640 641 atomic_inc(&tx->qp->s_dma_busy); 642 tx->txreq.next_descq_idx = tail; 643 ppd->dd->f_sdma_update_tail(ppd, tail); 644 ppd->sdma_descq_added += tx->txreq.sg_count; 645 list_add_tail(&tx->txreq.list, &ppd->sdma_activelist); 646 goto unlock; 647 648 unmap: 649 for (;;) { 650 if (!tail) 651 tail = ppd->sdma_descq_cnt - 1; 652 else 653 tail--; 654 if (tail == ppd->sdma_descq_tail) 655 break; 656 unmap_desc(ppd, tail); 657 } 658 qp = tx->qp; 659 qib_put_txreq(tx); 660 spin_lock(&qp->r_lock); 661 spin_lock(&qp->s_lock); 662 if (qp->ibqp.qp_type == IB_QPT_RC) { 663 /* XXX what about error sending RDMA read responses? */ 664 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) 665 qib_error_qp(qp, IB_WC_GENERAL_ERR); 666 } else if (qp->s_wqe) 667 qib_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR); 668 spin_unlock(&qp->s_lock); 669 spin_unlock(&qp->r_lock); 670 /* return zero to process the next send work request */ 671 goto unlock; 672 673 busy: 674 qp = tx->qp; 675 spin_lock(&qp->s_lock); 676 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { 677 struct qib_ibdev *dev; 678 679 /* 680 * If we couldn't queue the DMA request, save the info 681 * and try again later rather than destroying the 682 * buffer and undoing the side effects of the copy. 683 */ 684 tx->ss = ss; 685 tx->dwords = dwords; 686 qp->s_tx = tx; 687 dev = &ppd->dd->verbs_dev; 688 spin_lock(&dev->pending_lock); 689 if (list_empty(&qp->iowait)) { 690 struct qib_ibport *ibp; 691 692 ibp = &ppd->ibport_data; 693 ibp->n_dmawait++; 694 qp->s_flags |= QIB_S_WAIT_DMA_DESC; 695 list_add_tail(&qp->iowait, &dev->dmawait); 696 } 697 spin_unlock(&dev->pending_lock); 698 qp->s_flags &= ~QIB_S_BUSY; 699 spin_unlock(&qp->s_lock); 700 ret = -EBUSY; 701 } else { 702 spin_unlock(&qp->s_lock); 703 qib_put_txreq(tx); 704 } 705 unlock: 706 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 707 return ret; 708 } 709 710 void qib_sdma_process_event(struct qib_pportdata *ppd, 711 enum qib_sdma_events event) 712 { 713 unsigned long flags; 714 715 spin_lock_irqsave(&ppd->sdma_lock, flags); 716 717 __qib_sdma_process_event(ppd, event); 718 719 if (ppd->sdma_state.current_state == qib_sdma_state_s99_running) 720 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd)); 721 722 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 723 } 724 725 void __qib_sdma_process_event(struct qib_pportdata *ppd, 726 enum qib_sdma_events event) 727 { 728 struct qib_sdma_state *ss = &ppd->sdma_state; 729 730 switch (ss->current_state) { 731 case qib_sdma_state_s00_hw_down: 732 switch (event) { 733 case qib_sdma_event_e00_go_hw_down: 734 break; 735 case qib_sdma_event_e30_go_running: 736 /* 737 * If down, but running requested (usually result 738 * of link up, then we need to start up. 739 * This can happen when hw down is requested while 740 * bringing the link up with traffic active on 741 * 7220, e.g. */ 742 ss->go_s99_running = 1; 743 /* fall through and start dma engine */ 744 case qib_sdma_event_e10_go_hw_start: 745 /* This reference means the state machine is started */ 746 sdma_get(&ppd->sdma_state); 747 sdma_set_state(ppd, 748 qib_sdma_state_s10_hw_start_up_wait); 749 break; 750 case qib_sdma_event_e20_hw_started: 751 break; 752 case qib_sdma_event_e40_sw_cleaned: 753 sdma_sw_tear_down(ppd); 754 break; 755 case qib_sdma_event_e50_hw_cleaned: 756 break; 757 case qib_sdma_event_e60_hw_halted: 758 break; 759 case qib_sdma_event_e70_go_idle: 760 break; 761 case qib_sdma_event_e7220_err_halted: 762 break; 763 case qib_sdma_event_e7322_err_halted: 764 break; 765 case qib_sdma_event_e90_timer_tick: 766 break; 767 } 768 break; 769 770 case qib_sdma_state_s10_hw_start_up_wait: 771 switch (event) { 772 case qib_sdma_event_e00_go_hw_down: 773 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 774 sdma_sw_tear_down(ppd); 775 break; 776 case qib_sdma_event_e10_go_hw_start: 777 break; 778 case qib_sdma_event_e20_hw_started: 779 sdma_set_state(ppd, ss->go_s99_running ? 780 qib_sdma_state_s99_running : 781 qib_sdma_state_s20_idle); 782 break; 783 case qib_sdma_event_e30_go_running: 784 ss->go_s99_running = 1; 785 break; 786 case qib_sdma_event_e40_sw_cleaned: 787 break; 788 case qib_sdma_event_e50_hw_cleaned: 789 break; 790 case qib_sdma_event_e60_hw_halted: 791 break; 792 case qib_sdma_event_e70_go_idle: 793 ss->go_s99_running = 0; 794 break; 795 case qib_sdma_event_e7220_err_halted: 796 break; 797 case qib_sdma_event_e7322_err_halted: 798 break; 799 case qib_sdma_event_e90_timer_tick: 800 break; 801 } 802 break; 803 804 case qib_sdma_state_s20_idle: 805 switch (event) { 806 case qib_sdma_event_e00_go_hw_down: 807 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 808 sdma_sw_tear_down(ppd); 809 break; 810 case qib_sdma_event_e10_go_hw_start: 811 break; 812 case qib_sdma_event_e20_hw_started: 813 break; 814 case qib_sdma_event_e30_go_running: 815 sdma_set_state(ppd, qib_sdma_state_s99_running); 816 ss->go_s99_running = 1; 817 break; 818 case qib_sdma_event_e40_sw_cleaned: 819 break; 820 case qib_sdma_event_e50_hw_cleaned: 821 break; 822 case qib_sdma_event_e60_hw_halted: 823 break; 824 case qib_sdma_event_e70_go_idle: 825 break; 826 case qib_sdma_event_e7220_err_halted: 827 break; 828 case qib_sdma_event_e7322_err_halted: 829 break; 830 case qib_sdma_event_e90_timer_tick: 831 break; 832 } 833 break; 834 835 case qib_sdma_state_s30_sw_clean_up_wait: 836 switch (event) { 837 case qib_sdma_event_e00_go_hw_down: 838 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 839 break; 840 case qib_sdma_event_e10_go_hw_start: 841 break; 842 case qib_sdma_event_e20_hw_started: 843 break; 844 case qib_sdma_event_e30_go_running: 845 ss->go_s99_running = 1; 846 break; 847 case qib_sdma_event_e40_sw_cleaned: 848 sdma_set_state(ppd, 849 qib_sdma_state_s10_hw_start_up_wait); 850 sdma_hw_start_up(ppd); 851 break; 852 case qib_sdma_event_e50_hw_cleaned: 853 break; 854 case qib_sdma_event_e60_hw_halted: 855 break; 856 case qib_sdma_event_e70_go_idle: 857 ss->go_s99_running = 0; 858 break; 859 case qib_sdma_event_e7220_err_halted: 860 break; 861 case qib_sdma_event_e7322_err_halted: 862 break; 863 case qib_sdma_event_e90_timer_tick: 864 break; 865 } 866 break; 867 868 case qib_sdma_state_s40_hw_clean_up_wait: 869 switch (event) { 870 case qib_sdma_event_e00_go_hw_down: 871 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 872 sdma_start_sw_clean_up(ppd); 873 break; 874 case qib_sdma_event_e10_go_hw_start: 875 break; 876 case qib_sdma_event_e20_hw_started: 877 break; 878 case qib_sdma_event_e30_go_running: 879 ss->go_s99_running = 1; 880 break; 881 case qib_sdma_event_e40_sw_cleaned: 882 break; 883 case qib_sdma_event_e50_hw_cleaned: 884 sdma_set_state(ppd, 885 qib_sdma_state_s30_sw_clean_up_wait); 886 sdma_start_sw_clean_up(ppd); 887 break; 888 case qib_sdma_event_e60_hw_halted: 889 break; 890 case qib_sdma_event_e70_go_idle: 891 ss->go_s99_running = 0; 892 break; 893 case qib_sdma_event_e7220_err_halted: 894 break; 895 case qib_sdma_event_e7322_err_halted: 896 break; 897 case qib_sdma_event_e90_timer_tick: 898 break; 899 } 900 break; 901 902 case qib_sdma_state_s50_hw_halt_wait: 903 switch (event) { 904 case qib_sdma_event_e00_go_hw_down: 905 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 906 sdma_start_sw_clean_up(ppd); 907 break; 908 case qib_sdma_event_e10_go_hw_start: 909 break; 910 case qib_sdma_event_e20_hw_started: 911 break; 912 case qib_sdma_event_e30_go_running: 913 ss->go_s99_running = 1; 914 break; 915 case qib_sdma_event_e40_sw_cleaned: 916 break; 917 case qib_sdma_event_e50_hw_cleaned: 918 break; 919 case qib_sdma_event_e60_hw_halted: 920 sdma_set_state(ppd, 921 qib_sdma_state_s40_hw_clean_up_wait); 922 ppd->dd->f_sdma_hw_clean_up(ppd); 923 break; 924 case qib_sdma_event_e70_go_idle: 925 ss->go_s99_running = 0; 926 break; 927 case qib_sdma_event_e7220_err_halted: 928 break; 929 case qib_sdma_event_e7322_err_halted: 930 break; 931 case qib_sdma_event_e90_timer_tick: 932 break; 933 } 934 break; 935 936 case qib_sdma_state_s99_running: 937 switch (event) { 938 case qib_sdma_event_e00_go_hw_down: 939 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 940 sdma_start_sw_clean_up(ppd); 941 break; 942 case qib_sdma_event_e10_go_hw_start: 943 break; 944 case qib_sdma_event_e20_hw_started: 945 break; 946 case qib_sdma_event_e30_go_running: 947 break; 948 case qib_sdma_event_e40_sw_cleaned: 949 break; 950 case qib_sdma_event_e50_hw_cleaned: 951 break; 952 case qib_sdma_event_e60_hw_halted: 953 sdma_set_state(ppd, 954 qib_sdma_state_s30_sw_clean_up_wait); 955 sdma_start_sw_clean_up(ppd); 956 break; 957 case qib_sdma_event_e70_go_idle: 958 sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait); 959 ss->go_s99_running = 0; 960 break; 961 case qib_sdma_event_e7220_err_halted: 962 sdma_set_state(ppd, 963 qib_sdma_state_s30_sw_clean_up_wait); 964 sdma_start_sw_clean_up(ppd); 965 break; 966 case qib_sdma_event_e7322_err_halted: 967 sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait); 968 break; 969 case qib_sdma_event_e90_timer_tick: 970 break; 971 } 972 break; 973 } 974 975 ss->last_event = event; 976 } 977