1 /* 2 * Copyright (c) 2012 Intel Corporation. All rights reserved. 3 * Copyright (c) 2007 - 2012 QLogic Corporation. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/spinlock.h> 35 #include <linux/netdevice.h> 36 #include <linux/moduleparam.h> 37 38 #include "qib.h" 39 #include "qib_common.h" 40 41 /* default pio off, sdma on */ 42 static ushort sdma_descq_cnt = 256; 43 module_param_named(sdma_descq_cnt, sdma_descq_cnt, ushort, S_IRUGO); 44 MODULE_PARM_DESC(sdma_descq_cnt, "Number of SDMA descq entries"); 45 46 /* 47 * Bits defined in the send DMA descriptor. 48 */ 49 #define SDMA_DESC_LAST (1ULL << 11) 50 #define SDMA_DESC_FIRST (1ULL << 12) 51 #define SDMA_DESC_DMA_HEAD (1ULL << 13) 52 #define SDMA_DESC_USE_LARGE_BUF (1ULL << 14) 53 #define SDMA_DESC_INTR (1ULL << 15) 54 #define SDMA_DESC_COUNT_LSB 16 55 #define SDMA_DESC_GEN_LSB 30 56 57 char *qib_sdma_state_names[] = { 58 [qib_sdma_state_s00_hw_down] = "s00_HwDown", 59 [qib_sdma_state_s10_hw_start_up_wait] = "s10_HwStartUpWait", 60 [qib_sdma_state_s20_idle] = "s20_Idle", 61 [qib_sdma_state_s30_sw_clean_up_wait] = "s30_SwCleanUpWait", 62 [qib_sdma_state_s40_hw_clean_up_wait] = "s40_HwCleanUpWait", 63 [qib_sdma_state_s50_hw_halt_wait] = "s50_HwHaltWait", 64 [qib_sdma_state_s99_running] = "s99_Running", 65 }; 66 67 char *qib_sdma_event_names[] = { 68 [qib_sdma_event_e00_go_hw_down] = "e00_GoHwDown", 69 [qib_sdma_event_e10_go_hw_start] = "e10_GoHwStart", 70 [qib_sdma_event_e20_hw_started] = "e20_HwStarted", 71 [qib_sdma_event_e30_go_running] = "e30_GoRunning", 72 [qib_sdma_event_e40_sw_cleaned] = "e40_SwCleaned", 73 [qib_sdma_event_e50_hw_cleaned] = "e50_HwCleaned", 74 [qib_sdma_event_e60_hw_halted] = "e60_HwHalted", 75 [qib_sdma_event_e70_go_idle] = "e70_GoIdle", 76 [qib_sdma_event_e7220_err_halted] = "e7220_ErrHalted", 77 [qib_sdma_event_e7322_err_halted] = "e7322_ErrHalted", 78 [qib_sdma_event_e90_timer_tick] = "e90_TimerTick", 79 }; 80 81 /* declare all statics here rather than keep sorting */ 82 static int alloc_sdma(struct qib_pportdata *); 83 static void sdma_complete(struct kref *); 84 static void sdma_finalput(struct qib_sdma_state *); 85 static void sdma_get(struct qib_sdma_state *); 86 static void sdma_put(struct qib_sdma_state *); 87 static void sdma_set_state(struct qib_pportdata *, enum qib_sdma_states); 88 static void sdma_start_sw_clean_up(struct qib_pportdata *); 89 static void sdma_sw_clean_up_task(unsigned long); 90 static void unmap_desc(struct qib_pportdata *, unsigned); 91 92 static void sdma_get(struct qib_sdma_state *ss) 93 { 94 kref_get(&ss->kref); 95 } 96 97 static void sdma_complete(struct kref *kref) 98 { 99 struct qib_sdma_state *ss = 100 container_of(kref, struct qib_sdma_state, kref); 101 102 complete(&ss->comp); 103 } 104 105 static void sdma_put(struct qib_sdma_state *ss) 106 { 107 kref_put(&ss->kref, sdma_complete); 108 } 109 110 static void sdma_finalput(struct qib_sdma_state *ss) 111 { 112 sdma_put(ss); 113 wait_for_completion(&ss->comp); 114 } 115 116 /* 117 * Complete all the sdma requests on the active list, in the correct 118 * order, and with appropriate processing. Called when cleaning up 119 * after sdma shutdown, and when new sdma requests are submitted for 120 * a link that is down. This matches what is done for requests 121 * that complete normally, it's just the full list. 122 * 123 * Must be called with sdma_lock held 124 */ 125 static void clear_sdma_activelist(struct qib_pportdata *ppd) 126 { 127 struct qib_sdma_txreq *txp, *txp_next; 128 129 list_for_each_entry_safe(txp, txp_next, &ppd->sdma_activelist, list) { 130 list_del_init(&txp->list); 131 if (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) { 132 unsigned idx; 133 134 idx = txp->start_idx; 135 while (idx != txp->next_descq_idx) { 136 unmap_desc(ppd, idx); 137 if (++idx == ppd->sdma_descq_cnt) 138 idx = 0; 139 } 140 } 141 if (txp->callback) 142 (*txp->callback)(txp, QIB_SDMA_TXREQ_S_ABORTED); 143 } 144 } 145 146 static void sdma_sw_clean_up_task(unsigned long opaque) 147 { 148 struct qib_pportdata *ppd = (struct qib_pportdata *) opaque; 149 unsigned long flags; 150 151 spin_lock_irqsave(&ppd->sdma_lock, flags); 152 153 /* 154 * At this point, the following should always be true: 155 * - We are halted, so no more descriptors are getting retired. 156 * - We are not running, so no one is submitting new work. 157 * - Only we can send the e40_sw_cleaned, so we can't start 158 * running again until we say so. So, the active list and 159 * descq are ours to play with. 160 */ 161 162 /* Process all retired requests. */ 163 qib_sdma_make_progress(ppd); 164 165 clear_sdma_activelist(ppd); 166 167 /* 168 * Resync count of added and removed. It is VERY important that 169 * sdma_descq_removed NEVER decrement - user_sdma depends on it. 170 */ 171 ppd->sdma_descq_removed = ppd->sdma_descq_added; 172 173 /* 174 * Reset our notion of head and tail. 175 * Note that the HW registers will be reset when switching states 176 * due to calling __qib_sdma_process_event() below. 177 */ 178 ppd->sdma_descq_tail = 0; 179 ppd->sdma_descq_head = 0; 180 ppd->sdma_head_dma[0] = 0; 181 ppd->sdma_generation = 0; 182 183 __qib_sdma_process_event(ppd, qib_sdma_event_e40_sw_cleaned); 184 185 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 186 } 187 188 /* 189 * This is called when changing to state qib_sdma_state_s10_hw_start_up_wait 190 * as a result of send buffer errors or send DMA descriptor errors. 191 * We want to disarm the buffers in these cases. 192 */ 193 static void sdma_hw_start_up(struct qib_pportdata *ppd) 194 { 195 struct qib_sdma_state *ss = &ppd->sdma_state; 196 unsigned bufno; 197 198 for (bufno = ss->first_sendbuf; bufno < ss->last_sendbuf; ++bufno) 199 ppd->dd->f_sendctrl(ppd, QIB_SENDCTRL_DISARM_BUF(bufno)); 200 201 ppd->dd->f_sdma_hw_start_up(ppd); 202 } 203 204 static void sdma_sw_tear_down(struct qib_pportdata *ppd) 205 { 206 struct qib_sdma_state *ss = &ppd->sdma_state; 207 208 /* Releasing this reference means the state machine has stopped. */ 209 sdma_put(ss); 210 } 211 212 static void sdma_start_sw_clean_up(struct qib_pportdata *ppd) 213 { 214 tasklet_hi_schedule(&ppd->sdma_sw_clean_up_task); 215 } 216 217 static void sdma_set_state(struct qib_pportdata *ppd, 218 enum qib_sdma_states next_state) 219 { 220 struct qib_sdma_state *ss = &ppd->sdma_state; 221 struct sdma_set_state_action *action = ss->set_state_action; 222 unsigned op = 0; 223 224 /* debugging bookkeeping */ 225 ss->previous_state = ss->current_state; 226 ss->previous_op = ss->current_op; 227 228 ss->current_state = next_state; 229 230 if (action[next_state].op_enable) 231 op |= QIB_SDMA_SENDCTRL_OP_ENABLE; 232 233 if (action[next_state].op_intenable) 234 op |= QIB_SDMA_SENDCTRL_OP_INTENABLE; 235 236 if (action[next_state].op_halt) 237 op |= QIB_SDMA_SENDCTRL_OP_HALT; 238 239 if (action[next_state].op_drain) 240 op |= QIB_SDMA_SENDCTRL_OP_DRAIN; 241 242 if (action[next_state].go_s99_running_tofalse) 243 ss->go_s99_running = 0; 244 245 if (action[next_state].go_s99_running_totrue) 246 ss->go_s99_running = 1; 247 248 ss->current_op = op; 249 250 ppd->dd->f_sdma_sendctrl(ppd, ss->current_op); 251 } 252 253 static void unmap_desc(struct qib_pportdata *ppd, unsigned head) 254 { 255 __le64 *descqp = &ppd->sdma_descq[head].qw[0]; 256 u64 desc[2]; 257 dma_addr_t addr; 258 size_t len; 259 260 desc[0] = le64_to_cpu(descqp[0]); 261 desc[1] = le64_to_cpu(descqp[1]); 262 263 addr = (desc[1] << 32) | (desc[0] >> 32); 264 len = (desc[0] >> 14) & (0x7ffULL << 2); 265 dma_unmap_single(&ppd->dd->pcidev->dev, addr, len, DMA_TO_DEVICE); 266 } 267 268 static int alloc_sdma(struct qib_pportdata *ppd) 269 { 270 ppd->sdma_descq_cnt = sdma_descq_cnt; 271 if (!ppd->sdma_descq_cnt) 272 ppd->sdma_descq_cnt = 256; 273 274 /* Allocate memory for SendDMA descriptor FIFO */ 275 ppd->sdma_descq = dma_alloc_coherent(&ppd->dd->pcidev->dev, 276 ppd->sdma_descq_cnt * sizeof(u64[2]), &ppd->sdma_descq_phys, 277 GFP_KERNEL); 278 279 if (!ppd->sdma_descq) { 280 qib_dev_err(ppd->dd, 281 "failed to allocate SendDMA descriptor FIFO memory\n"); 282 goto bail; 283 } 284 285 /* Allocate memory for DMA of head register to memory */ 286 ppd->sdma_head_dma = dma_alloc_coherent(&ppd->dd->pcidev->dev, 287 PAGE_SIZE, &ppd->sdma_head_phys, GFP_KERNEL); 288 if (!ppd->sdma_head_dma) { 289 qib_dev_err(ppd->dd, 290 "failed to allocate SendDMA head memory\n"); 291 goto cleanup_descq; 292 } 293 ppd->sdma_head_dma[0] = 0; 294 return 0; 295 296 cleanup_descq: 297 dma_free_coherent(&ppd->dd->pcidev->dev, 298 ppd->sdma_descq_cnt * sizeof(u64[2]), (void *)ppd->sdma_descq, 299 ppd->sdma_descq_phys); 300 ppd->sdma_descq = NULL; 301 ppd->sdma_descq_phys = 0; 302 bail: 303 ppd->sdma_descq_cnt = 0; 304 return -ENOMEM; 305 } 306 307 static void free_sdma(struct qib_pportdata *ppd) 308 { 309 struct qib_devdata *dd = ppd->dd; 310 311 if (ppd->sdma_head_dma) { 312 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE, 313 (void *)ppd->sdma_head_dma, 314 ppd->sdma_head_phys); 315 ppd->sdma_head_dma = NULL; 316 ppd->sdma_head_phys = 0; 317 } 318 319 if (ppd->sdma_descq) { 320 dma_free_coherent(&dd->pcidev->dev, 321 ppd->sdma_descq_cnt * sizeof(u64[2]), 322 ppd->sdma_descq, ppd->sdma_descq_phys); 323 ppd->sdma_descq = NULL; 324 ppd->sdma_descq_phys = 0; 325 } 326 } 327 328 static inline void make_sdma_desc(struct qib_pportdata *ppd, 329 u64 *sdmadesc, u64 addr, u64 dwlen, 330 u64 dwoffset) 331 { 332 333 WARN_ON(addr & 3); 334 /* SDmaPhyAddr[47:32] */ 335 sdmadesc[1] = addr >> 32; 336 /* SDmaPhyAddr[31:0] */ 337 sdmadesc[0] = (addr & 0xfffffffcULL) << 32; 338 /* SDmaGeneration[1:0] */ 339 sdmadesc[0] |= (ppd->sdma_generation & 3ULL) << 340 SDMA_DESC_GEN_LSB; 341 /* SDmaDwordCount[10:0] */ 342 sdmadesc[0] |= (dwlen & 0x7ffULL) << SDMA_DESC_COUNT_LSB; 343 /* SDmaBufOffset[12:2] */ 344 sdmadesc[0] |= dwoffset & 0x7ffULL; 345 } 346 347 /* sdma_lock must be held */ 348 int qib_sdma_make_progress(struct qib_pportdata *ppd) 349 { 350 struct list_head *lp = NULL; 351 struct qib_sdma_txreq *txp = NULL; 352 struct qib_devdata *dd = ppd->dd; 353 int progress = 0; 354 u16 hwhead; 355 u16 idx = 0; 356 357 hwhead = dd->f_sdma_gethead(ppd); 358 359 /* The reason for some of the complexity of this code is that 360 * not all descriptors have corresponding txps. So, we have to 361 * be able to skip over descs until we wander into the range of 362 * the next txp on the list. 363 */ 364 365 if (!list_empty(&ppd->sdma_activelist)) { 366 lp = ppd->sdma_activelist.next; 367 txp = list_entry(lp, struct qib_sdma_txreq, list); 368 idx = txp->start_idx; 369 } 370 371 while (ppd->sdma_descq_head != hwhead) { 372 /* if desc is part of this txp, unmap if needed */ 373 if (txp && (txp->flags & QIB_SDMA_TXREQ_F_FREEDESC) && 374 (idx == ppd->sdma_descq_head)) { 375 unmap_desc(ppd, ppd->sdma_descq_head); 376 if (++idx == ppd->sdma_descq_cnt) 377 idx = 0; 378 } 379 380 /* increment dequed desc count */ 381 ppd->sdma_descq_removed++; 382 383 /* advance head, wrap if needed */ 384 if (++ppd->sdma_descq_head == ppd->sdma_descq_cnt) 385 ppd->sdma_descq_head = 0; 386 387 /* if now past this txp's descs, do the callback */ 388 if (txp && txp->next_descq_idx == ppd->sdma_descq_head) { 389 /* remove from active list */ 390 list_del_init(&txp->list); 391 if (txp->callback) 392 (*txp->callback)(txp, QIB_SDMA_TXREQ_S_OK); 393 /* see if there is another txp */ 394 if (list_empty(&ppd->sdma_activelist)) 395 txp = NULL; 396 else { 397 lp = ppd->sdma_activelist.next; 398 txp = list_entry(lp, struct qib_sdma_txreq, 399 list); 400 idx = txp->start_idx; 401 } 402 } 403 progress = 1; 404 } 405 if (progress) 406 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd)); 407 return progress; 408 } 409 410 /* 411 * This is called from interrupt context. 412 */ 413 void qib_sdma_intr(struct qib_pportdata *ppd) 414 { 415 unsigned long flags; 416 417 spin_lock_irqsave(&ppd->sdma_lock, flags); 418 419 __qib_sdma_intr(ppd); 420 421 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 422 } 423 424 void __qib_sdma_intr(struct qib_pportdata *ppd) 425 { 426 if (__qib_sdma_running(ppd)) { 427 qib_sdma_make_progress(ppd); 428 if (!list_empty(&ppd->sdma_userpending)) 429 qib_user_sdma_send_desc(ppd, &ppd->sdma_userpending); 430 } 431 } 432 433 int qib_setup_sdma(struct qib_pportdata *ppd) 434 { 435 struct qib_devdata *dd = ppd->dd; 436 unsigned long flags; 437 int ret = 0; 438 439 ret = alloc_sdma(ppd); 440 if (ret) 441 goto bail; 442 443 /* set consistent sdma state */ 444 ppd->dd->f_sdma_init_early(ppd); 445 spin_lock_irqsave(&ppd->sdma_lock, flags); 446 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 447 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 448 449 /* set up reference counting */ 450 kref_init(&ppd->sdma_state.kref); 451 init_completion(&ppd->sdma_state.comp); 452 453 ppd->sdma_generation = 0; 454 ppd->sdma_descq_head = 0; 455 ppd->sdma_descq_removed = 0; 456 ppd->sdma_descq_added = 0; 457 458 ppd->sdma_intrequest = 0; 459 INIT_LIST_HEAD(&ppd->sdma_userpending); 460 461 INIT_LIST_HEAD(&ppd->sdma_activelist); 462 463 tasklet_init(&ppd->sdma_sw_clean_up_task, sdma_sw_clean_up_task, 464 (unsigned long)ppd); 465 466 ret = dd->f_init_sdma_regs(ppd); 467 if (ret) 468 goto bail_alloc; 469 470 qib_sdma_process_event(ppd, qib_sdma_event_e10_go_hw_start); 471 472 return 0; 473 474 bail_alloc: 475 qib_teardown_sdma(ppd); 476 bail: 477 return ret; 478 } 479 480 void qib_teardown_sdma(struct qib_pportdata *ppd) 481 { 482 qib_sdma_process_event(ppd, qib_sdma_event_e00_go_hw_down); 483 484 /* 485 * This waits for the state machine to exit so it is not 486 * necessary to kill the sdma_sw_clean_up_task to make sure 487 * it is not running. 488 */ 489 sdma_finalput(&ppd->sdma_state); 490 491 free_sdma(ppd); 492 } 493 494 int qib_sdma_running(struct qib_pportdata *ppd) 495 { 496 unsigned long flags; 497 int ret; 498 499 spin_lock_irqsave(&ppd->sdma_lock, flags); 500 ret = __qib_sdma_running(ppd); 501 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 502 503 return ret; 504 } 505 506 /* 507 * Complete a request when sdma not running; likely only request 508 * but to simplify the code, always queue it, then process the full 509 * activelist. We process the entire list to ensure that this particular 510 * request does get it's callback, but in the correct order. 511 * Must be called with sdma_lock held 512 */ 513 static void complete_sdma_err_req(struct qib_pportdata *ppd, 514 struct qib_verbs_txreq *tx) 515 { 516 atomic_inc(&tx->qp->s_dma_busy); 517 /* no sdma descriptors, so no unmap_desc */ 518 tx->txreq.start_idx = 0; 519 tx->txreq.next_descq_idx = 0; 520 list_add_tail(&tx->txreq.list, &ppd->sdma_activelist); 521 clear_sdma_activelist(ppd); 522 } 523 524 /* 525 * This function queues one IB packet onto the send DMA queue per call. 526 * The caller is responsible for checking: 527 * 1) The number of send DMA descriptor entries is less than the size of 528 * the descriptor queue. 529 * 2) The IB SGE addresses and lengths are 32-bit aligned 530 * (except possibly the last SGE's length) 531 * 3) The SGE addresses are suitable for passing to dma_map_single(). 532 */ 533 int qib_sdma_verbs_send(struct qib_pportdata *ppd, 534 struct qib_sge_state *ss, u32 dwords, 535 struct qib_verbs_txreq *tx) 536 { 537 unsigned long flags; 538 struct qib_sge *sge; 539 struct qib_qp *qp; 540 int ret = 0; 541 u16 tail; 542 __le64 *descqp; 543 u64 sdmadesc[2]; 544 u32 dwoffset; 545 dma_addr_t addr; 546 547 spin_lock_irqsave(&ppd->sdma_lock, flags); 548 549 retry: 550 if (unlikely(!__qib_sdma_running(ppd))) { 551 complete_sdma_err_req(ppd, tx); 552 goto unlock; 553 } 554 555 if (tx->txreq.sg_count > qib_sdma_descq_freecnt(ppd)) { 556 if (qib_sdma_make_progress(ppd)) 557 goto retry; 558 if (ppd->dd->flags & QIB_HAS_SDMA_TIMEOUT) 559 ppd->dd->f_sdma_set_desc_cnt(ppd, 560 ppd->sdma_descq_cnt / 2); 561 goto busy; 562 } 563 564 dwoffset = tx->hdr_dwords; 565 make_sdma_desc(ppd, sdmadesc, (u64) tx->txreq.addr, dwoffset, 0); 566 567 sdmadesc[0] |= SDMA_DESC_FIRST; 568 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF) 569 sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF; 570 571 /* write to the descq */ 572 tail = ppd->sdma_descq_tail; 573 descqp = &ppd->sdma_descq[tail].qw[0]; 574 *descqp++ = cpu_to_le64(sdmadesc[0]); 575 *descqp++ = cpu_to_le64(sdmadesc[1]); 576 577 /* increment the tail */ 578 if (++tail == ppd->sdma_descq_cnt) { 579 tail = 0; 580 descqp = &ppd->sdma_descq[0].qw[0]; 581 ++ppd->sdma_generation; 582 } 583 584 tx->txreq.start_idx = tail; 585 586 sge = &ss->sge; 587 while (dwords) { 588 u32 dw; 589 u32 len; 590 591 len = dwords << 2; 592 if (len > sge->length) 593 len = sge->length; 594 if (len > sge->sge_length) 595 len = sge->sge_length; 596 BUG_ON(len == 0); 597 dw = (len + 3) >> 2; 598 addr = dma_map_single(&ppd->dd->pcidev->dev, sge->vaddr, 599 dw << 2, DMA_TO_DEVICE); 600 if (dma_mapping_error(&ppd->dd->pcidev->dev, addr)) 601 goto unmap; 602 sdmadesc[0] = 0; 603 make_sdma_desc(ppd, sdmadesc, (u64) addr, dw, dwoffset); 604 /* SDmaUseLargeBuf has to be set in every descriptor */ 605 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_USELARGEBUF) 606 sdmadesc[0] |= SDMA_DESC_USE_LARGE_BUF; 607 /* write to the descq */ 608 *descqp++ = cpu_to_le64(sdmadesc[0]); 609 *descqp++ = cpu_to_le64(sdmadesc[1]); 610 611 /* increment the tail */ 612 if (++tail == ppd->sdma_descq_cnt) { 613 tail = 0; 614 descqp = &ppd->sdma_descq[0].qw[0]; 615 ++ppd->sdma_generation; 616 } 617 sge->vaddr += len; 618 sge->length -= len; 619 sge->sge_length -= len; 620 if (sge->sge_length == 0) { 621 if (--ss->num_sge) 622 *sge = *ss->sg_list++; 623 } else if (sge->length == 0 && sge->mr->lkey) { 624 if (++sge->n >= QIB_SEGSZ) { 625 if (++sge->m >= sge->mr->mapsz) 626 break; 627 sge->n = 0; 628 } 629 sge->vaddr = 630 sge->mr->map[sge->m]->segs[sge->n].vaddr; 631 sge->length = 632 sge->mr->map[sge->m]->segs[sge->n].length; 633 } 634 635 dwoffset += dw; 636 dwords -= dw; 637 } 638 639 if (!tail) 640 descqp = &ppd->sdma_descq[ppd->sdma_descq_cnt].qw[0]; 641 descqp -= 2; 642 descqp[0] |= cpu_to_le64(SDMA_DESC_LAST); 643 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_HEADTOHOST) 644 descqp[0] |= cpu_to_le64(SDMA_DESC_DMA_HEAD); 645 if (tx->txreq.flags & QIB_SDMA_TXREQ_F_INTREQ) 646 descqp[0] |= cpu_to_le64(SDMA_DESC_INTR); 647 648 atomic_inc(&tx->qp->s_dma_busy); 649 tx->txreq.next_descq_idx = tail; 650 ppd->dd->f_sdma_update_tail(ppd, tail); 651 ppd->sdma_descq_added += tx->txreq.sg_count; 652 list_add_tail(&tx->txreq.list, &ppd->sdma_activelist); 653 goto unlock; 654 655 unmap: 656 for (;;) { 657 if (!tail) 658 tail = ppd->sdma_descq_cnt - 1; 659 else 660 tail--; 661 if (tail == ppd->sdma_descq_tail) 662 break; 663 unmap_desc(ppd, tail); 664 } 665 qp = tx->qp; 666 qib_put_txreq(tx); 667 spin_lock(&qp->r_lock); 668 spin_lock(&qp->s_lock); 669 if (qp->ibqp.qp_type == IB_QPT_RC) { 670 /* XXX what about error sending RDMA read responses? */ 671 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) 672 qib_error_qp(qp, IB_WC_GENERAL_ERR); 673 } else if (qp->s_wqe) 674 qib_send_complete(qp, qp->s_wqe, IB_WC_GENERAL_ERR); 675 spin_unlock(&qp->s_lock); 676 spin_unlock(&qp->r_lock); 677 /* return zero to process the next send work request */ 678 goto unlock; 679 680 busy: 681 qp = tx->qp; 682 spin_lock(&qp->s_lock); 683 if (ib_qib_state_ops[qp->state] & QIB_PROCESS_RECV_OK) { 684 struct qib_ibdev *dev; 685 686 /* 687 * If we couldn't queue the DMA request, save the info 688 * and try again later rather than destroying the 689 * buffer and undoing the side effects of the copy. 690 */ 691 tx->ss = ss; 692 tx->dwords = dwords; 693 qp->s_tx = tx; 694 dev = &ppd->dd->verbs_dev; 695 spin_lock(&dev->pending_lock); 696 if (list_empty(&qp->iowait)) { 697 struct qib_ibport *ibp; 698 699 ibp = &ppd->ibport_data; 700 ibp->n_dmawait++; 701 qp->s_flags |= QIB_S_WAIT_DMA_DESC; 702 list_add_tail(&qp->iowait, &dev->dmawait); 703 } 704 spin_unlock(&dev->pending_lock); 705 qp->s_flags &= ~QIB_S_BUSY; 706 spin_unlock(&qp->s_lock); 707 ret = -EBUSY; 708 } else { 709 spin_unlock(&qp->s_lock); 710 qib_put_txreq(tx); 711 } 712 unlock: 713 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 714 return ret; 715 } 716 717 /* 718 * sdma_lock should be acquired before calling this routine 719 */ 720 void dump_sdma_state(struct qib_pportdata *ppd) 721 { 722 struct qib_sdma_desc *descq; 723 struct qib_sdma_txreq *txp, *txpnext; 724 __le64 *descqp; 725 u64 desc[2]; 726 u64 addr; 727 u16 gen, dwlen, dwoffset; 728 u16 head, tail, cnt; 729 730 head = ppd->sdma_descq_head; 731 tail = ppd->sdma_descq_tail; 732 cnt = qib_sdma_descq_freecnt(ppd); 733 descq = ppd->sdma_descq; 734 735 qib_dev_porterr(ppd->dd, ppd->port, 736 "SDMA ppd->sdma_descq_head: %u\n", head); 737 qib_dev_porterr(ppd->dd, ppd->port, 738 "SDMA ppd->sdma_descq_tail: %u\n", tail); 739 qib_dev_porterr(ppd->dd, ppd->port, 740 "SDMA sdma_descq_freecnt: %u\n", cnt); 741 742 /* print info for each entry in the descriptor queue */ 743 while (head != tail) { 744 char flags[6] = { 'x', 'x', 'x', 'x', 'x', 0 }; 745 746 descqp = &descq[head].qw[0]; 747 desc[0] = le64_to_cpu(descqp[0]); 748 desc[1] = le64_to_cpu(descqp[1]); 749 flags[0] = (desc[0] & 1<<15) ? 'I' : '-'; 750 flags[1] = (desc[0] & 1<<14) ? 'L' : 'S'; 751 flags[2] = (desc[0] & 1<<13) ? 'H' : '-'; 752 flags[3] = (desc[0] & 1<<12) ? 'F' : '-'; 753 flags[4] = (desc[0] & 1<<11) ? 'L' : '-'; 754 addr = (desc[1] << 32) | ((desc[0] >> 32) & 0xfffffffcULL); 755 gen = (desc[0] >> 30) & 3ULL; 756 dwlen = (desc[0] >> 14) & (0x7ffULL << 2); 757 dwoffset = (desc[0] & 0x7ffULL) << 2; 758 qib_dev_porterr(ppd->dd, ppd->port, 759 "SDMA sdmadesc[%u]: flags:%s addr:0x%016llx gen:%u len:%u bytes offset:%u bytes\n", 760 head, flags, addr, gen, dwlen, dwoffset); 761 if (++head == ppd->sdma_descq_cnt) 762 head = 0; 763 } 764 765 /* print dma descriptor indices from the TX requests */ 766 list_for_each_entry_safe(txp, txpnext, &ppd->sdma_activelist, 767 list) 768 qib_dev_porterr(ppd->dd, ppd->port, 769 "SDMA txp->start_idx: %u txp->next_descq_idx: %u\n", 770 txp->start_idx, txp->next_descq_idx); 771 } 772 773 void qib_sdma_process_event(struct qib_pportdata *ppd, 774 enum qib_sdma_events event) 775 { 776 unsigned long flags; 777 778 spin_lock_irqsave(&ppd->sdma_lock, flags); 779 780 __qib_sdma_process_event(ppd, event); 781 782 if (ppd->sdma_state.current_state == qib_sdma_state_s99_running) 783 qib_verbs_sdma_desc_avail(ppd, qib_sdma_descq_freecnt(ppd)); 784 785 spin_unlock_irqrestore(&ppd->sdma_lock, flags); 786 } 787 788 void __qib_sdma_process_event(struct qib_pportdata *ppd, 789 enum qib_sdma_events event) 790 { 791 struct qib_sdma_state *ss = &ppd->sdma_state; 792 793 switch (ss->current_state) { 794 case qib_sdma_state_s00_hw_down: 795 switch (event) { 796 case qib_sdma_event_e00_go_hw_down: 797 break; 798 case qib_sdma_event_e30_go_running: 799 /* 800 * If down, but running requested (usually result 801 * of link up, then we need to start up. 802 * This can happen when hw down is requested while 803 * bringing the link up with traffic active on 804 * 7220, e.g. */ 805 ss->go_s99_running = 1; 806 /* fall through and start dma engine */ 807 case qib_sdma_event_e10_go_hw_start: 808 /* This reference means the state machine is started */ 809 sdma_get(&ppd->sdma_state); 810 sdma_set_state(ppd, 811 qib_sdma_state_s10_hw_start_up_wait); 812 break; 813 case qib_sdma_event_e20_hw_started: 814 break; 815 case qib_sdma_event_e40_sw_cleaned: 816 sdma_sw_tear_down(ppd); 817 break; 818 case qib_sdma_event_e50_hw_cleaned: 819 break; 820 case qib_sdma_event_e60_hw_halted: 821 break; 822 case qib_sdma_event_e70_go_idle: 823 break; 824 case qib_sdma_event_e7220_err_halted: 825 break; 826 case qib_sdma_event_e7322_err_halted: 827 break; 828 case qib_sdma_event_e90_timer_tick: 829 break; 830 } 831 break; 832 833 case qib_sdma_state_s10_hw_start_up_wait: 834 switch (event) { 835 case qib_sdma_event_e00_go_hw_down: 836 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 837 sdma_sw_tear_down(ppd); 838 break; 839 case qib_sdma_event_e10_go_hw_start: 840 break; 841 case qib_sdma_event_e20_hw_started: 842 sdma_set_state(ppd, ss->go_s99_running ? 843 qib_sdma_state_s99_running : 844 qib_sdma_state_s20_idle); 845 break; 846 case qib_sdma_event_e30_go_running: 847 ss->go_s99_running = 1; 848 break; 849 case qib_sdma_event_e40_sw_cleaned: 850 break; 851 case qib_sdma_event_e50_hw_cleaned: 852 break; 853 case qib_sdma_event_e60_hw_halted: 854 break; 855 case qib_sdma_event_e70_go_idle: 856 ss->go_s99_running = 0; 857 break; 858 case qib_sdma_event_e7220_err_halted: 859 break; 860 case qib_sdma_event_e7322_err_halted: 861 break; 862 case qib_sdma_event_e90_timer_tick: 863 break; 864 } 865 break; 866 867 case qib_sdma_state_s20_idle: 868 switch (event) { 869 case qib_sdma_event_e00_go_hw_down: 870 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 871 sdma_sw_tear_down(ppd); 872 break; 873 case qib_sdma_event_e10_go_hw_start: 874 break; 875 case qib_sdma_event_e20_hw_started: 876 break; 877 case qib_sdma_event_e30_go_running: 878 sdma_set_state(ppd, qib_sdma_state_s99_running); 879 ss->go_s99_running = 1; 880 break; 881 case qib_sdma_event_e40_sw_cleaned: 882 break; 883 case qib_sdma_event_e50_hw_cleaned: 884 break; 885 case qib_sdma_event_e60_hw_halted: 886 break; 887 case qib_sdma_event_e70_go_idle: 888 break; 889 case qib_sdma_event_e7220_err_halted: 890 break; 891 case qib_sdma_event_e7322_err_halted: 892 break; 893 case qib_sdma_event_e90_timer_tick: 894 break; 895 } 896 break; 897 898 case qib_sdma_state_s30_sw_clean_up_wait: 899 switch (event) { 900 case qib_sdma_event_e00_go_hw_down: 901 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 902 break; 903 case qib_sdma_event_e10_go_hw_start: 904 break; 905 case qib_sdma_event_e20_hw_started: 906 break; 907 case qib_sdma_event_e30_go_running: 908 ss->go_s99_running = 1; 909 break; 910 case qib_sdma_event_e40_sw_cleaned: 911 sdma_set_state(ppd, 912 qib_sdma_state_s10_hw_start_up_wait); 913 sdma_hw_start_up(ppd); 914 break; 915 case qib_sdma_event_e50_hw_cleaned: 916 break; 917 case qib_sdma_event_e60_hw_halted: 918 break; 919 case qib_sdma_event_e70_go_idle: 920 ss->go_s99_running = 0; 921 break; 922 case qib_sdma_event_e7220_err_halted: 923 break; 924 case qib_sdma_event_e7322_err_halted: 925 break; 926 case qib_sdma_event_e90_timer_tick: 927 break; 928 } 929 break; 930 931 case qib_sdma_state_s40_hw_clean_up_wait: 932 switch (event) { 933 case qib_sdma_event_e00_go_hw_down: 934 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 935 sdma_start_sw_clean_up(ppd); 936 break; 937 case qib_sdma_event_e10_go_hw_start: 938 break; 939 case qib_sdma_event_e20_hw_started: 940 break; 941 case qib_sdma_event_e30_go_running: 942 ss->go_s99_running = 1; 943 break; 944 case qib_sdma_event_e40_sw_cleaned: 945 break; 946 case qib_sdma_event_e50_hw_cleaned: 947 sdma_set_state(ppd, 948 qib_sdma_state_s30_sw_clean_up_wait); 949 sdma_start_sw_clean_up(ppd); 950 break; 951 case qib_sdma_event_e60_hw_halted: 952 break; 953 case qib_sdma_event_e70_go_idle: 954 ss->go_s99_running = 0; 955 break; 956 case qib_sdma_event_e7220_err_halted: 957 break; 958 case qib_sdma_event_e7322_err_halted: 959 break; 960 case qib_sdma_event_e90_timer_tick: 961 break; 962 } 963 break; 964 965 case qib_sdma_state_s50_hw_halt_wait: 966 switch (event) { 967 case qib_sdma_event_e00_go_hw_down: 968 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 969 sdma_start_sw_clean_up(ppd); 970 break; 971 case qib_sdma_event_e10_go_hw_start: 972 break; 973 case qib_sdma_event_e20_hw_started: 974 break; 975 case qib_sdma_event_e30_go_running: 976 ss->go_s99_running = 1; 977 break; 978 case qib_sdma_event_e40_sw_cleaned: 979 break; 980 case qib_sdma_event_e50_hw_cleaned: 981 break; 982 case qib_sdma_event_e60_hw_halted: 983 sdma_set_state(ppd, 984 qib_sdma_state_s40_hw_clean_up_wait); 985 ppd->dd->f_sdma_hw_clean_up(ppd); 986 break; 987 case qib_sdma_event_e70_go_idle: 988 ss->go_s99_running = 0; 989 break; 990 case qib_sdma_event_e7220_err_halted: 991 break; 992 case qib_sdma_event_e7322_err_halted: 993 break; 994 case qib_sdma_event_e90_timer_tick: 995 break; 996 } 997 break; 998 999 case qib_sdma_state_s99_running: 1000 switch (event) { 1001 case qib_sdma_event_e00_go_hw_down: 1002 sdma_set_state(ppd, qib_sdma_state_s00_hw_down); 1003 sdma_start_sw_clean_up(ppd); 1004 break; 1005 case qib_sdma_event_e10_go_hw_start: 1006 break; 1007 case qib_sdma_event_e20_hw_started: 1008 break; 1009 case qib_sdma_event_e30_go_running: 1010 break; 1011 case qib_sdma_event_e40_sw_cleaned: 1012 break; 1013 case qib_sdma_event_e50_hw_cleaned: 1014 break; 1015 case qib_sdma_event_e60_hw_halted: 1016 sdma_set_state(ppd, 1017 qib_sdma_state_s30_sw_clean_up_wait); 1018 sdma_start_sw_clean_up(ppd); 1019 break; 1020 case qib_sdma_event_e70_go_idle: 1021 sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait); 1022 ss->go_s99_running = 0; 1023 break; 1024 case qib_sdma_event_e7220_err_halted: 1025 sdma_set_state(ppd, 1026 qib_sdma_state_s30_sw_clean_up_wait); 1027 sdma_start_sw_clean_up(ppd); 1028 break; 1029 case qib_sdma_event_e7322_err_halted: 1030 sdma_set_state(ppd, qib_sdma_state_s50_hw_halt_wait); 1031 break; 1032 case qib_sdma_event_e90_timer_tick: 1033 break; 1034 } 1035 break; 1036 } 1037 1038 ss->last_event = event; 1039 } 1040