1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #include <linux/interrupt.h> 5 #include <linux/module.h> 6 #include <linux/printk.h> 7 8 #include "ionic_fw.h" 9 #include "ionic_ibdev.h" 10 11 #define IONIC_EQ_COUNT_MIN 4 12 #define IONIC_AQ_COUNT_MIN 1 13 14 /* not a valid queue position or negative error status */ 15 #define IONIC_ADMIN_POSTED 0x10000 16 17 /* cpu can be held with irq disabled for COUNT * MS (for create/destroy_ah) */ 18 #define IONIC_ADMIN_BUSY_RETRY_COUNT 2000 19 #define IONIC_ADMIN_BUSY_RETRY_MS 1 20 21 /* admin queue will be considered failed if a command takes longer */ 22 #define IONIC_ADMIN_TIMEOUT (HZ * 2) 23 #define IONIC_ADMIN_WARN (HZ / 8) 24 25 /* will poll for admin cq to tolerate and report from missed event */ 26 #define IONIC_ADMIN_DELAY (HZ / 8) 27 28 /* work queue for polling the event queue and admin cq */ 29 struct workqueue_struct *ionic_evt_workq; 30 31 static void ionic_admin_timedout(struct ionic_aq *aq) 32 { 33 struct ionic_ibdev *dev = aq->dev; 34 unsigned long irqflags; 35 u16 pos; 36 37 spin_lock_irqsave(&aq->lock, irqflags); 38 if (ionic_queue_empty(&aq->q)) 39 goto out; 40 41 /* Reset ALL adminq if any one times out */ 42 if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) 43 queue_work(ionic_evt_workq, &dev->reset_work); 44 45 ibdev_err(&dev->ibdev, "admin command timed out, aq %d after: %ums\n", 46 aq->aqid, (u32)jiffies_to_msecs(jiffies - aq->stamp)); 47 48 pos = (aq->q.prod - 1) & aq->q.mask; 49 if (pos == aq->q.cons) 50 goto out; 51 52 ibdev_warn(&dev->ibdev, "admin pos %u (last posted)\n", pos); 53 print_hex_dump(KERN_WARNING, "cmd ", DUMP_PREFIX_OFFSET, 16, 1, 54 ionic_queue_at(&aq->q, pos), 55 BIT(aq->q.stride_log2), true); 56 57 out: 58 spin_unlock_irqrestore(&aq->lock, irqflags); 59 } 60 61 static void ionic_admin_reset_dwork(struct ionic_ibdev *dev) 62 { 63 if (atomic_read(&dev->admin_state) == IONIC_ADMIN_KILLED) 64 return; 65 66 queue_delayed_work(ionic_evt_workq, &dev->admin_dwork, 67 IONIC_ADMIN_DELAY); 68 } 69 70 static void ionic_admin_reset_wdog(struct ionic_aq *aq) 71 { 72 if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) 73 return; 74 75 aq->stamp = jiffies; 76 ionic_admin_reset_dwork(aq->dev); 77 } 78 79 static bool ionic_admin_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq, 80 struct ionic_v1_cqe **cqe) 81 { 82 struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q); 83 84 if (unlikely(cq->color != ionic_v1_cqe_color(qcqe))) 85 return false; 86 87 /* Prevent out-of-order reads of the CQE */ 88 dma_rmb(); 89 *cqe = qcqe; 90 91 return true; 92 } 93 94 static void ionic_admin_poll_locked(struct ionic_aq *aq) 95 { 96 struct ionic_cq *cq = &aq->vcq->cq[0]; 97 struct ionic_admin_wr *wr, *wr_next; 98 struct ionic_ibdev *dev = aq->dev; 99 u32 wr_strides, avlbl_strides; 100 struct ionic_v1_cqe *cqe; 101 u32 qtf, qid; 102 u16 old_prod; 103 u8 type; 104 105 lockdep_assert_held(&aq->lock); 106 107 if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) { 108 list_for_each_entry_safe(wr, wr_next, &aq->wr_prod, aq_ent) { 109 INIT_LIST_HEAD(&wr->aq_ent); 110 aq->q_wr[wr->status].wr = NULL; 111 wr->status = atomic_read(&aq->admin_state); 112 complete_all(&wr->work); 113 } 114 INIT_LIST_HEAD(&aq->wr_prod); 115 116 list_for_each_entry_safe(wr, wr_next, &aq->wr_post, aq_ent) { 117 INIT_LIST_HEAD(&wr->aq_ent); 118 wr->status = atomic_read(&aq->admin_state); 119 complete_all(&wr->work); 120 } 121 INIT_LIST_HEAD(&aq->wr_post); 122 123 return; 124 } 125 126 old_prod = cq->q.prod; 127 128 while (ionic_admin_next_cqe(dev, cq, &cqe)) { 129 qtf = ionic_v1_cqe_qtf(cqe); 130 qid = ionic_v1_cqe_qtf_qid(qtf); 131 type = ionic_v1_cqe_qtf_type(qtf); 132 133 if (unlikely(type != IONIC_V1_CQE_TYPE_ADMIN)) { 134 ibdev_warn_ratelimited(&dev->ibdev, 135 "bad cqe type %u\n", type); 136 goto cq_next; 137 } 138 139 if (unlikely(qid != aq->aqid)) { 140 ibdev_warn_ratelimited(&dev->ibdev, 141 "bad cqe qid %u\n", qid); 142 goto cq_next; 143 } 144 145 if (unlikely(be16_to_cpu(cqe->admin.cmd_idx) != aq->q.cons)) { 146 ibdev_warn_ratelimited(&dev->ibdev, 147 "bad idx %u cons %u qid %u\n", 148 be16_to_cpu(cqe->admin.cmd_idx), 149 aq->q.cons, qid); 150 goto cq_next; 151 } 152 153 if (unlikely(ionic_queue_empty(&aq->q))) { 154 ibdev_warn_ratelimited(&dev->ibdev, 155 "bad cqe for empty adminq\n"); 156 goto cq_next; 157 } 158 159 wr = aq->q_wr[aq->q.cons].wr; 160 if (wr) { 161 aq->q_wr[aq->q.cons].wr = NULL; 162 list_del_init(&wr->aq_ent); 163 164 wr->cqe = *cqe; 165 wr->status = atomic_read(&aq->admin_state); 166 complete_all(&wr->work); 167 } 168 169 ionic_queue_consume_entries(&aq->q, 170 aq->q_wr[aq->q.cons].wqe_strides); 171 172 cq_next: 173 ionic_queue_produce(&cq->q); 174 cq->color = ionic_color_wrap(cq->q.prod, cq->color); 175 } 176 177 if (old_prod != cq->q.prod) { 178 ionic_admin_reset_wdog(aq); 179 cq->q.cons = cq->q.prod; 180 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, 181 ionic_queue_dbell_val(&cq->q)); 182 queue_work(ionic_evt_workq, &aq->work); 183 } else if (!aq->armed) { 184 aq->armed = true; 185 cq->arm_any_prod = ionic_queue_next(&cq->q, cq->arm_any_prod); 186 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, 187 cq->q.dbell | IONIC_CQ_RING_ARM | 188 cq->arm_any_prod); 189 queue_work(ionic_evt_workq, &aq->work); 190 } 191 192 if (atomic_read(&aq->admin_state) != IONIC_ADMIN_ACTIVE) 193 return; 194 195 old_prod = aq->q.prod; 196 197 if (ionic_queue_empty(&aq->q) && !list_empty(&aq->wr_post)) 198 ionic_admin_reset_wdog(aq); 199 200 if (list_empty(&aq->wr_post)) 201 return; 202 203 do { 204 u8 *src; 205 int i, src_len; 206 size_t stride_len; 207 208 wr = list_first_entry(&aq->wr_post, struct ionic_admin_wr, 209 aq_ent); 210 wr_strides = (le16_to_cpu(wr->wqe.len) + ADMIN_WQE_HDR_LEN + 211 (ADMIN_WQE_STRIDE - 1)) >> aq->q.stride_log2; 212 avlbl_strides = ionic_queue_length_remaining(&aq->q); 213 214 if (wr_strides > avlbl_strides) 215 break; 216 217 list_move(&wr->aq_ent, &aq->wr_prod); 218 wr->status = aq->q.prod; 219 aq->q_wr[aq->q.prod].wr = wr; 220 aq->q_wr[aq->q.prod].wqe_strides = wr_strides; 221 222 src_len = le16_to_cpu(wr->wqe.len); 223 src = (uint8_t *)&wr->wqe.cmd; 224 225 /* First stride */ 226 memcpy(ionic_queue_at_prod(&aq->q), &wr->wqe, 227 ADMIN_WQE_HDR_LEN); 228 stride_len = ADMIN_WQE_STRIDE - ADMIN_WQE_HDR_LEN; 229 if (stride_len > src_len) 230 stride_len = src_len; 231 memcpy(ionic_queue_at_prod(&aq->q) + ADMIN_WQE_HDR_LEN, 232 src, stride_len); 233 ibdev_dbg(&dev->ibdev, "post admin prod %u (%u strides)\n", 234 aq->q.prod, wr_strides); 235 print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, 236 ionic_queue_at_prod(&aq->q), 237 BIT(aq->q.stride_log2), true); 238 ionic_queue_produce(&aq->q); 239 240 /* Remaining strides */ 241 for (i = stride_len; i < src_len; i += stride_len) { 242 stride_len = ADMIN_WQE_STRIDE; 243 244 if (i + stride_len > src_len) 245 stride_len = src_len - i; 246 247 memcpy(ionic_queue_at_prod(&aq->q), src + i, 248 stride_len); 249 print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, 250 ionic_queue_at_prod(&aq->q), 251 BIT(aq->q.stride_log2), true); 252 ionic_queue_produce(&aq->q); 253 } 254 } while (!list_empty(&aq->wr_post)); 255 256 if (old_prod != aq->q.prod) 257 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.aq_qtype, 258 ionic_queue_dbell_val(&aq->q)); 259 } 260 261 static void ionic_admin_dwork(struct work_struct *ws) 262 { 263 struct ionic_ibdev *dev = 264 container_of(ws, struct ionic_ibdev, admin_dwork.work); 265 struct ionic_aq *aq, *bad_aq = NULL; 266 bool do_reschedule = false; 267 unsigned long irqflags; 268 bool do_reset = false; 269 u16 pos; 270 int i; 271 272 for (i = 0; i < dev->lif_cfg.aq_count; i++) { 273 aq = dev->aq_vec[i]; 274 275 spin_lock_irqsave(&aq->lock, irqflags); 276 277 if (ionic_queue_empty(&aq->q)) 278 goto next_aq; 279 280 /* Reschedule if any queue has outstanding work */ 281 do_reschedule = true; 282 283 if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_WARN)) 284 /* Warning threshold not met, nothing to do */ 285 goto next_aq; 286 287 /* See if polling now makes some progress */ 288 pos = aq->q.cons; 289 ionic_admin_poll_locked(aq); 290 if (pos != aq->q.cons) { 291 ibdev_dbg(&dev->ibdev, 292 "missed event for acq %d\n", aq->cqid); 293 goto next_aq; 294 } 295 296 if (time_is_after_eq_jiffies(aq->stamp + 297 IONIC_ADMIN_TIMEOUT)) { 298 /* Timeout threshold not met */ 299 ibdev_dbg(&dev->ibdev, "no progress after %ums\n", 300 (u32)jiffies_to_msecs(jiffies - aq->stamp)); 301 goto next_aq; 302 } 303 304 /* Queue timed out */ 305 bad_aq = aq; 306 do_reset = true; 307 next_aq: 308 spin_unlock_irqrestore(&aq->lock, irqflags); 309 } 310 311 if (do_reset) 312 /* Reset RDMA lif on a timeout */ 313 ionic_admin_timedout(bad_aq); 314 else if (do_reschedule) 315 /* Try to poll again later */ 316 ionic_admin_reset_dwork(dev); 317 } 318 319 static void ionic_admin_work(struct work_struct *ws) 320 { 321 struct ionic_aq *aq = container_of(ws, struct ionic_aq, work); 322 unsigned long irqflags; 323 324 spin_lock_irqsave(&aq->lock, irqflags); 325 ionic_admin_poll_locked(aq); 326 spin_unlock_irqrestore(&aq->lock, irqflags); 327 } 328 329 static void ionic_admin_post_aq(struct ionic_aq *aq, struct ionic_admin_wr *wr) 330 { 331 unsigned long irqflags; 332 bool poll; 333 334 wr->status = IONIC_ADMIN_POSTED; 335 wr->aq = aq; 336 337 spin_lock_irqsave(&aq->lock, irqflags); 338 poll = list_empty(&aq->wr_post); 339 list_add(&wr->aq_ent, &aq->wr_post); 340 if (poll) 341 ionic_admin_poll_locked(aq); 342 spin_unlock_irqrestore(&aq->lock, irqflags); 343 } 344 345 void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr) 346 { 347 int aq_idx; 348 349 /* Use cpu id for the adminq selection */ 350 aq_idx = raw_smp_processor_id() % dev->lif_cfg.aq_count; 351 ionic_admin_post_aq(dev->aq_vec[aq_idx], wr); 352 } 353 354 static void ionic_admin_cancel(struct ionic_admin_wr *wr) 355 { 356 struct ionic_aq *aq = wr->aq; 357 unsigned long irqflags; 358 359 spin_lock_irqsave(&aq->lock, irqflags); 360 361 if (!list_empty(&wr->aq_ent)) { 362 list_del(&wr->aq_ent); 363 if (wr->status != IONIC_ADMIN_POSTED) 364 aq->q_wr[wr->status].wr = NULL; 365 } 366 367 spin_unlock_irqrestore(&aq->lock, irqflags); 368 } 369 370 static int ionic_admin_busy_wait(struct ionic_admin_wr *wr) 371 { 372 struct ionic_aq *aq = wr->aq; 373 unsigned long irqflags; 374 int try_i; 375 376 for (try_i = 0; try_i < IONIC_ADMIN_BUSY_RETRY_COUNT; ++try_i) { 377 if (completion_done(&wr->work)) 378 return 0; 379 380 mdelay(IONIC_ADMIN_BUSY_RETRY_MS); 381 382 spin_lock_irqsave(&aq->lock, irqflags); 383 ionic_admin_poll_locked(aq); 384 spin_unlock_irqrestore(&aq->lock, irqflags); 385 } 386 387 /* 388 * we timed out. Initiate RDMA LIF reset and indicate 389 * error to caller. 390 */ 391 ionic_admin_timedout(aq); 392 return -ETIMEDOUT; 393 } 394 395 int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, 396 enum ionic_admin_flags flags) 397 { 398 int rc, timo; 399 400 if (flags & IONIC_ADMIN_F_BUSYWAIT) { 401 /* Spin */ 402 rc = ionic_admin_busy_wait(wr); 403 } else if (flags & IONIC_ADMIN_F_INTERRUPT) { 404 /* 405 * Interruptible sleep, 1s timeout 406 * This is used for commands which are safe for the caller 407 * to clean up without killing and resetting the adminq. 408 */ 409 timo = wait_for_completion_interruptible_timeout(&wr->work, 410 HZ); 411 if (timo > 0) 412 rc = 0; 413 else if (timo == 0) 414 rc = -ETIMEDOUT; 415 else 416 rc = timo; 417 } else { 418 /* 419 * Uninterruptible sleep 420 * This is used for commands which are NOT safe for the 421 * caller to clean up. Cleanup must be handled by the 422 * adminq kill and reset process so that host memory is 423 * not corrupted by the device. 424 */ 425 wait_for_completion(&wr->work); 426 rc = 0; 427 } 428 429 if (rc) { 430 ibdev_warn(&dev->ibdev, "wait status %d\n", rc); 431 ionic_admin_cancel(wr); 432 } else if (wr->status == IONIC_ADMIN_KILLED) { 433 ibdev_dbg(&dev->ibdev, "admin killed\n"); 434 435 /* No error if admin already killed during teardown */ 436 rc = (flags & IONIC_ADMIN_F_TEARDOWN) ? 0 : -ENODEV; 437 } else if (ionic_v1_cqe_error(&wr->cqe)) { 438 ibdev_warn(&dev->ibdev, "opcode %u error %u\n", 439 wr->wqe.op, 440 be32_to_cpu(wr->cqe.status_length)); 441 rc = -EINVAL; 442 } 443 return rc; 444 } 445 446 static int ionic_rdma_devcmd(struct ionic_ibdev *dev, 447 struct ionic_admin_ctx *admin) 448 { 449 int rc; 450 451 rc = ionic_adminq_post_wait(dev->lif_cfg.lif, admin); 452 if (rc) 453 return rc; 454 455 return ionic_error_to_errno(admin->comp.comp.status); 456 } 457 458 int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev) 459 { 460 struct ionic_admin_ctx admin = { 461 .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), 462 .cmd.rdma_reset = { 463 .opcode = IONIC_CMD_RDMA_RESET_LIF, 464 .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), 465 }, 466 }; 467 468 return ionic_rdma_devcmd(dev, &admin); 469 } 470 471 static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev, 472 struct ionic_queue *q, 473 u32 qid, u32 cid, u16 opcode) 474 { 475 struct ionic_admin_ctx admin = { 476 .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), 477 .cmd.rdma_queue = { 478 .opcode = opcode, 479 .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), 480 .qid_ver = cpu_to_le32(qid), 481 .cid = cpu_to_le32(cid), 482 .dbid = cpu_to_le16(dev->lif_cfg.dbid), 483 .depth_log2 = q->depth_log2, 484 .stride_log2 = q->stride_log2, 485 .dma_addr = cpu_to_le64(q->dma), 486 }, 487 }; 488 489 return ionic_rdma_devcmd(dev, &admin); 490 } 491 492 static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context) 493 { 494 struct ionic_aq *aq = cq_context; 495 unsigned long irqflags; 496 497 spin_lock_irqsave(&aq->lock, irqflags); 498 aq->armed = false; 499 if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) 500 queue_work(ionic_evt_workq, &aq->work); 501 spin_unlock_irqrestore(&aq->lock, irqflags); 502 } 503 504 static void ionic_rdma_admincq_event(struct ib_event *event, void *cq_context) 505 { 506 struct ionic_aq *aq = cq_context; 507 508 ibdev_err(&aq->dev->ibdev, "admincq event %d\n", event->event); 509 } 510 511 static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev, 512 int comp_vector) 513 { 514 struct ib_cq_init_attr attr = { 515 .cqe = IONIC_AQ_DEPTH, 516 .comp_vector = comp_vector, 517 }; 518 struct ionic_tbl_buf buf = {}; 519 struct ionic_vcq *vcq; 520 struct ionic_cq *cq; 521 int rc; 522 523 vcq = kzalloc(sizeof(*vcq), GFP_KERNEL); 524 if (!vcq) 525 return ERR_PTR(-ENOMEM); 526 527 vcq->ibcq.device = &dev->ibdev; 528 vcq->ibcq.comp_handler = ionic_rdma_admincq_comp; 529 vcq->ibcq.event_handler = ionic_rdma_admincq_event; 530 atomic_set(&vcq->ibcq.usecnt, 0); 531 532 vcq->udma_mask = 1; 533 cq = &vcq->cq[0]; 534 535 rc = ionic_create_cq_common(vcq, &buf, &attr, NULL, NULL, 536 NULL, NULL, 0); 537 if (rc) 538 goto err_init; 539 540 rc = ionic_rdma_queue_devcmd(dev, &cq->q, cq->cqid, cq->eqid, 541 IONIC_CMD_RDMA_CREATE_CQ); 542 if (rc) 543 goto err_cmd; 544 545 return vcq; 546 547 err_cmd: 548 ionic_destroy_cq_common(dev, cq); 549 err_init: 550 kfree(vcq); 551 552 return ERR_PTR(rc); 553 } 554 555 static struct ionic_aq *__ionic_create_rdma_adminq(struct ionic_ibdev *dev, 556 u32 aqid, u32 cqid) 557 { 558 struct ionic_aq *aq; 559 int rc; 560 561 aq = kzalloc(sizeof(*aq), GFP_KERNEL); 562 if (!aq) 563 return ERR_PTR(-ENOMEM); 564 565 atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED); 566 aq->dev = dev; 567 aq->aqid = aqid; 568 aq->cqid = cqid; 569 spin_lock_init(&aq->lock); 570 571 rc = ionic_queue_init(&aq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH, 572 ADMIN_WQE_STRIDE); 573 if (rc) 574 goto err_q; 575 576 ionic_queue_dbell_init(&aq->q, aq->aqid); 577 578 aq->q_wr = kcalloc((u32)aq->q.mask + 1, sizeof(*aq->q_wr), GFP_KERNEL); 579 if (!aq->q_wr) { 580 rc = -ENOMEM; 581 goto err_wr; 582 } 583 584 INIT_LIST_HEAD(&aq->wr_prod); 585 INIT_LIST_HEAD(&aq->wr_post); 586 587 INIT_WORK(&aq->work, ionic_admin_work); 588 aq->armed = false; 589 590 return aq; 591 592 err_wr: 593 ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev); 594 err_q: 595 kfree(aq); 596 597 return ERR_PTR(rc); 598 } 599 600 static void __ionic_destroy_rdma_adminq(struct ionic_ibdev *dev, 601 struct ionic_aq *aq) 602 { 603 ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev); 604 kfree(aq); 605 } 606 607 static struct ionic_aq *ionic_create_rdma_adminq(struct ionic_ibdev *dev, 608 u32 aqid, u32 cqid) 609 { 610 struct ionic_aq *aq; 611 int rc; 612 613 aq = __ionic_create_rdma_adminq(dev, aqid, cqid); 614 if (IS_ERR(aq)) 615 return aq; 616 617 rc = ionic_rdma_queue_devcmd(dev, &aq->q, aq->aqid, aq->cqid, 618 IONIC_CMD_RDMA_CREATE_ADMINQ); 619 if (rc) 620 goto err_cmd; 621 622 return aq; 623 624 err_cmd: 625 __ionic_destroy_rdma_adminq(dev, aq); 626 627 return ERR_PTR(rc); 628 } 629 630 static void ionic_flush_qs(struct ionic_ibdev *dev) 631 { 632 struct ionic_qp *qp, *qp_tmp; 633 struct ionic_cq *cq, *cq_tmp; 634 LIST_HEAD(flush_list); 635 unsigned long index; 636 637 WARN_ON(!irqs_disabled()); 638 639 /* Flush qp send and recv */ 640 xa_lock(&dev->qp_tbl); 641 xa_for_each(&dev->qp_tbl, index, qp) { 642 kref_get(&qp->qp_kref); 643 list_add_tail(&qp->ibkill_flush_ent, &flush_list); 644 } 645 xa_unlock(&dev->qp_tbl); 646 647 list_for_each_entry_safe(qp, qp_tmp, &flush_list, ibkill_flush_ent) { 648 ionic_flush_qp(dev, qp); 649 kref_put(&qp->qp_kref, ionic_qp_complete); 650 list_del(&qp->ibkill_flush_ent); 651 } 652 653 /* Notify completions */ 654 xa_lock(&dev->cq_tbl); 655 xa_for_each(&dev->cq_tbl, index, cq) { 656 kref_get(&cq->cq_kref); 657 list_add_tail(&cq->ibkill_flush_ent, &flush_list); 658 } 659 xa_unlock(&dev->cq_tbl); 660 661 list_for_each_entry_safe(cq, cq_tmp, &flush_list, ibkill_flush_ent) { 662 ionic_notify_flush_cq(cq); 663 kref_put(&cq->cq_kref, ionic_cq_complete); 664 list_del(&cq->ibkill_flush_ent); 665 } 666 } 667 668 static void ionic_kill_ibdev(struct ionic_ibdev *dev, bool fatal_path) 669 { 670 unsigned long irqflags; 671 bool do_flush = false; 672 int i; 673 674 /* Mark AQs for drain and flush the QPs while irq is disabled */ 675 local_irq_save(irqflags); 676 677 /* Mark the admin queue, flushing at most once */ 678 for (i = 0; i < dev->lif_cfg.aq_count; i++) { 679 struct ionic_aq *aq = dev->aq_vec[i]; 680 681 spin_lock(&aq->lock); 682 if (atomic_read(&aq->admin_state) != IONIC_ADMIN_KILLED) { 683 atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED); 684 /* Flush incomplete admin commands */ 685 ionic_admin_poll_locked(aq); 686 do_flush = true; 687 } 688 spin_unlock(&aq->lock); 689 } 690 691 if (do_flush) 692 ionic_flush_qs(dev); 693 694 local_irq_restore(irqflags); 695 696 /* Post a fatal event if requested */ 697 if (fatal_path) { 698 struct ib_event ev; 699 700 ev.device = &dev->ibdev; 701 ev.element.port_num = 1; 702 ev.event = IB_EVENT_DEVICE_FATAL; 703 704 ib_dispatch_event(&ev); 705 } 706 707 atomic_set(&dev->admin_state, IONIC_ADMIN_KILLED); 708 } 709 710 void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path) 711 { 712 enum ionic_admin_state old_state; 713 unsigned long irqflags = 0; 714 int i, rc; 715 716 if (!dev->aq_vec) 717 return; 718 719 /* 720 * Admin queues are transitioned from active to paused to killed state. 721 * When in paused state, no new commands are issued to the device, 722 * nor are any completed locally. After resetting the lif, it will be 723 * safe to resume the rdma admin queues in the killed state. Commands 724 * will not be issued to the device, but will complete locally with status 725 * IONIC_ADMIN_KILLED. Handling completion will ensure that creating or 726 * modifying resources fails, but destroying resources succeeds. 727 * If there was a failure resetting the lif using this strategy, 728 * then the state of the device is unknown. 729 */ 730 old_state = atomic_cmpxchg(&dev->admin_state, IONIC_ADMIN_ACTIVE, 731 IONIC_ADMIN_PAUSED); 732 if (old_state != IONIC_ADMIN_ACTIVE) 733 return; 734 735 /* Pause all the AQs */ 736 local_irq_save(irqflags); 737 for (i = 0; i < dev->lif_cfg.aq_count; i++) { 738 struct ionic_aq *aq = dev->aq_vec[i]; 739 740 spin_lock(&aq->lock); 741 /* pause rdma admin queues to reset lif */ 742 if (atomic_read(&aq->admin_state) == IONIC_ADMIN_ACTIVE) 743 atomic_set(&aq->admin_state, IONIC_ADMIN_PAUSED); 744 spin_unlock(&aq->lock); 745 } 746 local_irq_restore(irqflags); 747 748 rc = ionic_rdma_reset_devcmd(dev); 749 if (unlikely(rc)) { 750 ibdev_err(&dev->ibdev, "failed to reset rdma %d\n", rc); 751 ionic_request_rdma_reset(dev->lif_cfg.lif); 752 } 753 754 ionic_kill_ibdev(dev, fatal_path); 755 } 756 757 static void ionic_reset_work(struct work_struct *ws) 758 { 759 struct ionic_ibdev *dev = 760 container_of(ws, struct ionic_ibdev, reset_work); 761 762 ionic_kill_rdma_admin(dev, true); 763 } 764 765 static bool ionic_next_eqe(struct ionic_eq *eq, struct ionic_v1_eqe *eqe) 766 { 767 struct ionic_v1_eqe *qeqe; 768 bool color; 769 770 qeqe = ionic_queue_at_prod(&eq->q); 771 color = ionic_v1_eqe_color(qeqe); 772 773 /* cons is color for eq */ 774 if (eq->q.cons != color) 775 return false; 776 777 /* Prevent out-of-order reads of the EQE */ 778 dma_rmb(); 779 780 ibdev_dbg(&eq->dev->ibdev, "poll eq prod %u\n", eq->q.prod); 781 print_hex_dump_debug("eqe ", DUMP_PREFIX_OFFSET, 16, 1, 782 qeqe, BIT(eq->q.stride_log2), true); 783 *eqe = *qeqe; 784 785 return true; 786 } 787 788 static void ionic_cq_event(struct ionic_ibdev *dev, u32 cqid, u8 code) 789 { 790 unsigned long irqflags; 791 struct ib_event ibev; 792 struct ionic_cq *cq; 793 794 xa_lock_irqsave(&dev->cq_tbl, irqflags); 795 cq = xa_load(&dev->cq_tbl, cqid); 796 if (cq) 797 kref_get(&cq->cq_kref); 798 xa_unlock_irqrestore(&dev->cq_tbl, irqflags); 799 800 if (!cq) { 801 ibdev_dbg(&dev->ibdev, 802 "missing cqid %#x code %u\n", cqid, code); 803 return; 804 } 805 806 switch (code) { 807 case IONIC_V1_EQE_CQ_NOTIFY: 808 if (cq->vcq->ibcq.comp_handler) 809 cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq, 810 cq->vcq->ibcq.cq_context); 811 break; 812 813 case IONIC_V1_EQE_CQ_ERR: 814 if (cq->vcq->ibcq.event_handler) { 815 ibev.event = IB_EVENT_CQ_ERR; 816 ibev.device = &dev->ibdev; 817 ibev.element.cq = &cq->vcq->ibcq; 818 819 cq->vcq->ibcq.event_handler(&ibev, 820 cq->vcq->ibcq.cq_context); 821 } 822 break; 823 824 default: 825 ibdev_dbg(&dev->ibdev, 826 "unrecognized cqid %#x code %u\n", cqid, code); 827 break; 828 } 829 830 kref_put(&cq->cq_kref, ionic_cq_complete); 831 } 832 833 static void ionic_qp_event(struct ionic_ibdev *dev, u32 qpid, u8 code) 834 { 835 unsigned long irqflags; 836 struct ib_event ibev; 837 struct ionic_qp *qp; 838 839 xa_lock_irqsave(&dev->qp_tbl, irqflags); 840 qp = xa_load(&dev->qp_tbl, qpid); 841 if (qp) 842 kref_get(&qp->qp_kref); 843 xa_unlock_irqrestore(&dev->qp_tbl, irqflags); 844 845 if (!qp) { 846 ibdev_dbg(&dev->ibdev, 847 "missing qpid %#x code %u\n", qpid, code); 848 return; 849 } 850 851 ibev.device = &dev->ibdev; 852 ibev.element.qp = &qp->ibqp; 853 854 switch (code) { 855 case IONIC_V1_EQE_SQ_DRAIN: 856 ibev.event = IB_EVENT_SQ_DRAINED; 857 break; 858 859 case IONIC_V1_EQE_QP_COMM_EST: 860 ibev.event = IB_EVENT_COMM_EST; 861 break; 862 863 case IONIC_V1_EQE_QP_LAST_WQE: 864 ibev.event = IB_EVENT_QP_LAST_WQE_REACHED; 865 break; 866 867 case IONIC_V1_EQE_QP_ERR: 868 ibev.event = IB_EVENT_QP_FATAL; 869 break; 870 871 case IONIC_V1_EQE_QP_ERR_REQUEST: 872 ibev.event = IB_EVENT_QP_REQ_ERR; 873 break; 874 875 case IONIC_V1_EQE_QP_ERR_ACCESS: 876 ibev.event = IB_EVENT_QP_ACCESS_ERR; 877 break; 878 879 default: 880 ibdev_dbg(&dev->ibdev, 881 "unrecognized qpid %#x code %u\n", qpid, code); 882 goto out; 883 } 884 885 if (qp->ibqp.event_handler) 886 qp->ibqp.event_handler(&ibev, qp->ibqp.qp_context); 887 888 out: 889 kref_put(&qp->qp_kref, ionic_qp_complete); 890 } 891 892 static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget) 893 { 894 struct ionic_ibdev *dev = eq->dev; 895 struct ionic_v1_eqe eqe; 896 u16 npolled = 0; 897 u8 type, code; 898 u32 evt, qid; 899 900 while (npolled < budget) { 901 if (!ionic_next_eqe(eq, &eqe)) 902 break; 903 904 ionic_queue_produce(&eq->q); 905 906 /* cons is color for eq */ 907 eq->q.cons = ionic_color_wrap(eq->q.prod, eq->q.cons); 908 909 ++npolled; 910 911 evt = ionic_v1_eqe_evt(&eqe); 912 type = ionic_v1_eqe_evt_type(evt); 913 code = ionic_v1_eqe_evt_code(evt); 914 qid = ionic_v1_eqe_evt_qid(evt); 915 916 switch (type) { 917 case IONIC_V1_EQE_TYPE_CQ: 918 ionic_cq_event(dev, qid, code); 919 break; 920 921 case IONIC_V1_EQE_TYPE_QP: 922 ionic_qp_event(dev, qid, code); 923 break; 924 925 default: 926 ibdev_dbg(&dev->ibdev, 927 "unknown event %#x type %u\n", evt, type); 928 } 929 } 930 931 return npolled; 932 } 933 934 static void ionic_poll_eq_work(struct work_struct *work) 935 { 936 struct ionic_eq *eq = container_of(work, struct ionic_eq, work); 937 u32 npolled; 938 939 if (unlikely(!eq->enable) || WARN_ON(eq->armed)) 940 return; 941 942 npolled = ionic_poll_eq(eq, IONIC_EQ_WORK_BUDGET); 943 if (npolled == IONIC_EQ_WORK_BUDGET) { 944 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 945 npolled, 0); 946 queue_work(ionic_evt_workq, &eq->work); 947 } else { 948 xchg(&eq->armed, 1); 949 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 950 0, IONIC_INTR_CRED_UNMASK); 951 } 952 } 953 954 static irqreturn_t ionic_poll_eq_isr(int irq, void *eqptr) 955 { 956 struct ionic_eq *eq = eqptr; 957 int was_armed; 958 u32 npolled; 959 960 was_armed = xchg(&eq->armed, 0); 961 962 if (unlikely(!eq->enable) || !was_armed) 963 return IRQ_HANDLED; 964 965 npolled = ionic_poll_eq(eq, IONIC_EQ_ISR_BUDGET); 966 if (npolled == IONIC_EQ_ISR_BUDGET) { 967 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 968 npolled, 0); 969 queue_work(ionic_evt_workq, &eq->work); 970 } else { 971 xchg(&eq->armed, 1); 972 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 973 0, IONIC_INTR_CRED_UNMASK); 974 } 975 976 return IRQ_HANDLED; 977 } 978 979 static struct ionic_eq *ionic_create_eq(struct ionic_ibdev *dev, int eqid) 980 { 981 struct ionic_intr_info intr_obj = { }; 982 struct ionic_eq *eq; 983 int rc; 984 985 eq = kzalloc(sizeof(*eq), GFP_KERNEL); 986 if (!eq) 987 return ERR_PTR(-ENOMEM); 988 989 eq->dev = dev; 990 991 rc = ionic_queue_init(&eq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH, 992 sizeof(struct ionic_v1_eqe)); 993 if (rc) 994 goto err_q; 995 996 eq->eqid = eqid; 997 998 eq->armed = true; 999 eq->enable = false; 1000 INIT_WORK(&eq->work, ionic_poll_eq_work); 1001 1002 rc = ionic_intr_alloc(dev->lif_cfg.lif, &intr_obj); 1003 if (rc < 0) 1004 goto err_intr; 1005 1006 eq->irq = intr_obj.vector; 1007 eq->intr = intr_obj.index; 1008 1009 ionic_queue_dbell_init(&eq->q, eq->eqid); 1010 1011 /* cons is color for eq */ 1012 eq->q.cons = true; 1013 1014 snprintf(eq->name, sizeof(eq->name), "%s-%d-%d-eq", 1015 "ionr", dev->lif_cfg.lif_index, eq->eqid); 1016 1017 ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET); 1018 ionic_intr_mask_assert(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET); 1019 ionic_intr_coal_init(dev->lif_cfg.intr_ctrl, eq->intr, 0); 1020 ionic_intr_clean(dev->lif_cfg.intr_ctrl, eq->intr); 1021 1022 eq->enable = true; 1023 1024 rc = request_irq(eq->irq, ionic_poll_eq_isr, 0, eq->name, eq); 1025 if (rc) 1026 goto err_irq; 1027 1028 rc = ionic_rdma_queue_devcmd(dev, &eq->q, eq->eqid, eq->intr, 1029 IONIC_CMD_RDMA_CREATE_EQ); 1030 if (rc) 1031 goto err_cmd; 1032 1033 ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_CLEAR); 1034 1035 return eq; 1036 1037 err_cmd: 1038 eq->enable = false; 1039 free_irq(eq->irq, eq); 1040 flush_work(&eq->work); 1041 err_irq: 1042 ionic_intr_free(dev->lif_cfg.lif, eq->intr); 1043 err_intr: 1044 ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev); 1045 err_q: 1046 kfree(eq); 1047 1048 return ERR_PTR(rc); 1049 } 1050 1051 static void ionic_destroy_eq(struct ionic_eq *eq) 1052 { 1053 struct ionic_ibdev *dev = eq->dev; 1054 1055 eq->enable = false; 1056 free_irq(eq->irq, eq); 1057 flush_work(&eq->work); 1058 1059 ionic_intr_free(dev->lif_cfg.lif, eq->intr); 1060 ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev); 1061 kfree(eq); 1062 } 1063 1064 int ionic_create_rdma_admin(struct ionic_ibdev *dev) 1065 { 1066 int eq_i = 0, aq_i = 0, rc = 0; 1067 struct ionic_vcq *vcq; 1068 struct ionic_aq *aq; 1069 struct ionic_eq *eq; 1070 1071 dev->eq_vec = NULL; 1072 dev->aq_vec = NULL; 1073 1074 INIT_WORK(&dev->reset_work, ionic_reset_work); 1075 INIT_DELAYED_WORK(&dev->admin_dwork, ionic_admin_dwork); 1076 atomic_set(&dev->admin_state, IONIC_ADMIN_KILLED); 1077 1078 if (dev->lif_cfg.aq_count > IONIC_AQ_COUNT) { 1079 ibdev_dbg(&dev->ibdev, "limiting adminq count to %d\n", 1080 IONIC_AQ_COUNT); 1081 dev->lif_cfg.aq_count = IONIC_AQ_COUNT; 1082 } 1083 1084 if (dev->lif_cfg.eq_count > IONIC_EQ_COUNT) { 1085 dev_dbg(&dev->ibdev.dev, "limiting eventq count to %d\n", 1086 IONIC_EQ_COUNT); 1087 dev->lif_cfg.eq_count = IONIC_EQ_COUNT; 1088 } 1089 1090 /* need at least two eq and one aq */ 1091 if (dev->lif_cfg.eq_count < IONIC_EQ_COUNT_MIN || 1092 dev->lif_cfg.aq_count < IONIC_AQ_COUNT_MIN) { 1093 rc = -EINVAL; 1094 goto out; 1095 } 1096 1097 dev->eq_vec = kmalloc_array(dev->lif_cfg.eq_count, sizeof(*dev->eq_vec), 1098 GFP_KERNEL); 1099 if (!dev->eq_vec) { 1100 rc = -ENOMEM; 1101 goto out; 1102 } 1103 1104 for (eq_i = 0; eq_i < dev->lif_cfg.eq_count; ++eq_i) { 1105 eq = ionic_create_eq(dev, eq_i + dev->lif_cfg.eq_base); 1106 if (IS_ERR(eq)) { 1107 rc = PTR_ERR(eq); 1108 1109 if (eq_i < IONIC_EQ_COUNT_MIN) { 1110 ibdev_err(&dev->ibdev, 1111 "fail create eq %d\n", rc); 1112 goto out; 1113 } 1114 1115 /* ok, just fewer eq than device supports */ 1116 ibdev_dbg(&dev->ibdev, "eq count %d want %d rc %d\n", 1117 eq_i, dev->lif_cfg.eq_count, rc); 1118 1119 rc = 0; 1120 break; 1121 } 1122 1123 dev->eq_vec[eq_i] = eq; 1124 } 1125 1126 dev->lif_cfg.eq_count = eq_i; 1127 1128 dev->aq_vec = kmalloc_array(dev->lif_cfg.aq_count, sizeof(*dev->aq_vec), 1129 GFP_KERNEL); 1130 if (!dev->aq_vec) { 1131 rc = -ENOMEM; 1132 goto out; 1133 } 1134 1135 /* Create one CQ per AQ */ 1136 for (aq_i = 0; aq_i < dev->lif_cfg.aq_count; ++aq_i) { 1137 vcq = ionic_create_rdma_admincq(dev, aq_i % eq_i); 1138 if (IS_ERR(vcq)) { 1139 rc = PTR_ERR(vcq); 1140 1141 if (!aq_i) { 1142 ibdev_err(&dev->ibdev, 1143 "failed to create acq %d\n", rc); 1144 goto out; 1145 } 1146 1147 /* ok, just fewer adminq than device supports */ 1148 ibdev_dbg(&dev->ibdev, "acq count %d want %d rc %d\n", 1149 aq_i, dev->lif_cfg.aq_count, rc); 1150 break; 1151 } 1152 1153 aq = ionic_create_rdma_adminq(dev, aq_i + dev->lif_cfg.aq_base, 1154 vcq->cq[0].cqid); 1155 if (IS_ERR(aq)) { 1156 /* Clean up the dangling CQ */ 1157 ionic_destroy_cq_common(dev, &vcq->cq[0]); 1158 kfree(vcq); 1159 1160 rc = PTR_ERR(aq); 1161 1162 if (!aq_i) { 1163 ibdev_err(&dev->ibdev, 1164 "failed to create aq %d\n", rc); 1165 goto out; 1166 } 1167 1168 /* ok, just fewer adminq than device supports */ 1169 ibdev_dbg(&dev->ibdev, "aq count %d want %d rc %d\n", 1170 aq_i, dev->lif_cfg.aq_count, rc); 1171 break; 1172 } 1173 1174 vcq->ibcq.cq_context = aq; 1175 aq->vcq = vcq; 1176 1177 atomic_set(&aq->admin_state, IONIC_ADMIN_ACTIVE); 1178 dev->aq_vec[aq_i] = aq; 1179 } 1180 1181 atomic_set(&dev->admin_state, IONIC_ADMIN_ACTIVE); 1182 out: 1183 dev->lif_cfg.eq_count = eq_i; 1184 dev->lif_cfg.aq_count = aq_i; 1185 1186 return rc; 1187 } 1188 1189 void ionic_destroy_rdma_admin(struct ionic_ibdev *dev) 1190 { 1191 struct ionic_vcq *vcq; 1192 struct ionic_aq *aq; 1193 struct ionic_eq *eq; 1194 1195 /* 1196 * Killing the admin before destroy makes sure all admin and 1197 * completions are flushed. admin_state = IONIC_ADMIN_KILLED 1198 * stops queueing up further works. 1199 */ 1200 cancel_delayed_work_sync(&dev->admin_dwork); 1201 cancel_work_sync(&dev->reset_work); 1202 1203 if (dev->aq_vec) { 1204 while (dev->lif_cfg.aq_count > 0) { 1205 aq = dev->aq_vec[--dev->lif_cfg.aq_count]; 1206 vcq = aq->vcq; 1207 1208 cancel_work_sync(&aq->work); 1209 1210 __ionic_destroy_rdma_adminq(dev, aq); 1211 if (vcq) { 1212 ionic_destroy_cq_common(dev, &vcq->cq[0]); 1213 kfree(vcq); 1214 } 1215 } 1216 1217 kfree(dev->aq_vec); 1218 } 1219 1220 if (dev->eq_vec) { 1221 while (dev->lif_cfg.eq_count > 0) { 1222 eq = dev->eq_vec[--dev->lif_cfg.eq_count]; 1223 ionic_destroy_eq(eq); 1224 } 1225 1226 kfree(dev->eq_vec); 1227 } 1228 } 1229