1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #include <linux/interrupt.h> 5 #include <linux/module.h> 6 #include <linux/printk.h> 7 8 #include "ionic_fw.h" 9 #include "ionic_ibdev.h" 10 11 #define IONIC_EQ_COUNT_MIN 4 12 #define IONIC_AQ_COUNT_MIN 1 13 14 /* not a valid queue position or negative error status */ 15 #define IONIC_ADMIN_POSTED 0x10000 16 17 /* cpu can be held with irq disabled for COUNT * MS (for create/destroy_ah) */ 18 #define IONIC_ADMIN_BUSY_RETRY_COUNT 2000 19 #define IONIC_ADMIN_BUSY_RETRY_MS 1 20 21 /* admin queue will be considered failed if a command takes longer */ 22 #define IONIC_ADMIN_TIMEOUT (HZ * 2) 23 #define IONIC_ADMIN_WARN (HZ / 8) 24 25 /* will poll for admin cq to tolerate and report from missed event */ 26 #define IONIC_ADMIN_DELAY (HZ / 8) 27 28 /* work queue for polling the event queue and admin cq */ 29 struct workqueue_struct *ionic_evt_workq; 30 31 static void ionic_admin_timedout(struct ionic_aq *aq) 32 { 33 struct ionic_ibdev *dev = aq->dev; 34 unsigned long irqflags; 35 u16 pos; 36 37 spin_lock_irqsave(&aq->lock, irqflags); 38 if (ionic_queue_empty(&aq->q)) 39 goto out; 40 41 /* Reset ALL adminq if any one times out */ 42 if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) 43 queue_work(ionic_evt_workq, &dev->reset_work); 44 45 ibdev_err(&dev->ibdev, "admin command timed out, aq %d after: %ums\n", 46 aq->aqid, (u32)jiffies_to_msecs(jiffies - aq->stamp)); 47 48 pos = (aq->q.prod - 1) & aq->q.mask; 49 if (pos == aq->q.cons) 50 goto out; 51 52 ibdev_warn(&dev->ibdev, "admin pos %u (last posted)\n", pos); 53 print_hex_dump(KERN_WARNING, "cmd ", DUMP_PREFIX_OFFSET, 16, 1, 54 ionic_queue_at(&aq->q, pos), 55 BIT(aq->q.stride_log2), true); 56 57 out: 58 spin_unlock_irqrestore(&aq->lock, irqflags); 59 } 60 61 static void ionic_admin_reset_dwork(struct ionic_ibdev *dev) 62 { 63 if (atomic_read(&dev->admin_state) == IONIC_ADMIN_KILLED) 64 return; 65 66 queue_delayed_work(ionic_evt_workq, &dev->admin_dwork, 67 IONIC_ADMIN_DELAY); 68 } 69 70 static void ionic_admin_reset_wdog(struct ionic_aq *aq) 71 { 72 if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) 73 return; 74 75 aq->stamp = jiffies; 76 ionic_admin_reset_dwork(aq->dev); 77 } 78 79 static bool ionic_admin_next_cqe(struct ionic_ibdev *dev, struct ionic_cq *cq, 80 struct ionic_v1_cqe **cqe) 81 { 82 struct ionic_v1_cqe *qcqe = ionic_queue_at_prod(&cq->q); 83 84 if (unlikely(cq->color != ionic_v1_cqe_color(qcqe))) 85 return false; 86 87 /* Prevent out-of-order reads of the CQE */ 88 dma_rmb(); 89 *cqe = qcqe; 90 91 return true; 92 } 93 94 static void ionic_admin_poll_locked(struct ionic_aq *aq) 95 { 96 struct ionic_cq *cq = &aq->vcq->cq[0]; 97 struct ionic_admin_wr *wr, *wr_next; 98 struct ionic_ibdev *dev = aq->dev; 99 u32 wr_strides, avlbl_strides; 100 struct ionic_v1_cqe *cqe; 101 u32 qtf, qid; 102 u16 old_prod; 103 u8 type; 104 105 lockdep_assert_held(&aq->lock); 106 107 if (atomic_read(&aq->admin_state) == IONIC_ADMIN_KILLED) { 108 list_for_each_entry_safe(wr, wr_next, &aq->wr_prod, aq_ent) { 109 INIT_LIST_HEAD(&wr->aq_ent); 110 aq->q_wr[wr->status].wr = NULL; 111 wr->status = atomic_read(&aq->admin_state); 112 complete_all(&wr->work); 113 } 114 INIT_LIST_HEAD(&aq->wr_prod); 115 116 list_for_each_entry_safe(wr, wr_next, &aq->wr_post, aq_ent) { 117 INIT_LIST_HEAD(&wr->aq_ent); 118 wr->status = atomic_read(&aq->admin_state); 119 complete_all(&wr->work); 120 } 121 INIT_LIST_HEAD(&aq->wr_post); 122 123 return; 124 } 125 126 old_prod = cq->q.prod; 127 128 while (ionic_admin_next_cqe(dev, cq, &cqe)) { 129 qtf = ionic_v1_cqe_qtf(cqe); 130 qid = ionic_v1_cqe_qtf_qid(qtf); 131 type = ionic_v1_cqe_qtf_type(qtf); 132 133 if (unlikely(type != IONIC_V1_CQE_TYPE_ADMIN)) { 134 ibdev_warn_ratelimited(&dev->ibdev, 135 "bad cqe type %u\n", type); 136 goto cq_next; 137 } 138 139 if (unlikely(qid != aq->aqid)) { 140 ibdev_warn_ratelimited(&dev->ibdev, 141 "bad cqe qid %u\n", qid); 142 goto cq_next; 143 } 144 145 if (unlikely(be16_to_cpu(cqe->admin.cmd_idx) != aq->q.cons)) { 146 ibdev_warn_ratelimited(&dev->ibdev, 147 "bad idx %u cons %u qid %u\n", 148 be16_to_cpu(cqe->admin.cmd_idx), 149 aq->q.cons, qid); 150 goto cq_next; 151 } 152 153 if (unlikely(ionic_queue_empty(&aq->q))) { 154 ibdev_warn_ratelimited(&dev->ibdev, 155 "bad cqe for empty adminq\n"); 156 goto cq_next; 157 } 158 159 wr = aq->q_wr[aq->q.cons].wr; 160 if (wr) { 161 aq->q_wr[aq->q.cons].wr = NULL; 162 list_del_init(&wr->aq_ent); 163 164 wr->cqe = *cqe; 165 wr->status = atomic_read(&aq->admin_state); 166 complete_all(&wr->work); 167 } 168 169 ionic_queue_consume_entries(&aq->q, 170 aq->q_wr[aq->q.cons].wqe_strides); 171 172 cq_next: 173 ionic_queue_produce(&cq->q); 174 cq->color = ionic_color_wrap(cq->q.prod, cq->color); 175 } 176 177 if (old_prod != cq->q.prod) { 178 ionic_admin_reset_wdog(aq); 179 cq->q.cons = cq->q.prod; 180 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, 181 ionic_queue_dbell_val(&cq->q)); 182 queue_work(ionic_evt_workq, &aq->work); 183 } else if (!aq->armed) { 184 aq->armed = true; 185 cq->arm_any_prod = ionic_queue_next(&cq->q, cq->arm_any_prod); 186 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.cq_qtype, 187 cq->q.dbell | IONIC_CQ_RING_ARM | 188 cq->arm_any_prod); 189 queue_work(ionic_evt_workq, &aq->work); 190 } 191 192 if (atomic_read(&aq->admin_state) != IONIC_ADMIN_ACTIVE) 193 return; 194 195 old_prod = aq->q.prod; 196 197 if (ionic_queue_empty(&aq->q) && !list_empty(&aq->wr_post)) 198 ionic_admin_reset_wdog(aq); 199 200 if (list_empty(&aq->wr_post)) 201 return; 202 203 do { 204 u8 *src; 205 int i, src_len; 206 size_t stride_len; 207 208 wr = list_first_entry(&aq->wr_post, struct ionic_admin_wr, 209 aq_ent); 210 wr_strides = (le16_to_cpu(wr->wqe.len) + ADMIN_WQE_HDR_LEN + 211 (ADMIN_WQE_STRIDE - 1)) >> aq->q.stride_log2; 212 avlbl_strides = ionic_queue_length_remaining(&aq->q); 213 214 if (wr_strides > avlbl_strides) 215 break; 216 217 list_move(&wr->aq_ent, &aq->wr_prod); 218 wr->status = aq->q.prod; 219 aq->q_wr[aq->q.prod].wr = wr; 220 aq->q_wr[aq->q.prod].wqe_strides = wr_strides; 221 222 src_len = le16_to_cpu(wr->wqe.len); 223 src = (uint8_t *)&wr->wqe.cmd; 224 225 /* First stride */ 226 memcpy(ionic_queue_at_prod(&aq->q), &wr->wqe, 227 ADMIN_WQE_HDR_LEN); 228 stride_len = ADMIN_WQE_STRIDE - ADMIN_WQE_HDR_LEN; 229 if (stride_len > src_len) 230 stride_len = src_len; 231 memcpy(ionic_queue_at_prod(&aq->q) + ADMIN_WQE_HDR_LEN, 232 src, stride_len); 233 ibdev_dbg(&dev->ibdev, "post admin prod %u (%u strides)\n", 234 aq->q.prod, wr_strides); 235 print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, 236 ionic_queue_at_prod(&aq->q), 237 BIT(aq->q.stride_log2), true); 238 ionic_queue_produce(&aq->q); 239 240 /* Remaining strides */ 241 for (i = stride_len; i < src_len; i += stride_len) { 242 stride_len = ADMIN_WQE_STRIDE; 243 244 if (i + stride_len > src_len) 245 stride_len = src_len - i; 246 247 memcpy(ionic_queue_at_prod(&aq->q), src + i, 248 stride_len); 249 print_hex_dump_debug("wqe ", DUMP_PREFIX_OFFSET, 16, 1, 250 ionic_queue_at_prod(&aq->q), 251 BIT(aq->q.stride_log2), true); 252 ionic_queue_produce(&aq->q); 253 } 254 } while (!list_empty(&aq->wr_post)); 255 256 if (old_prod != aq->q.prod) 257 ionic_dbell_ring(dev->lif_cfg.dbpage, dev->lif_cfg.aq_qtype, 258 ionic_queue_dbell_val(&aq->q)); 259 } 260 261 static void ionic_admin_dwork(struct work_struct *ws) 262 { 263 struct ionic_ibdev *dev = 264 container_of(ws, struct ionic_ibdev, admin_dwork.work); 265 struct ionic_aq *aq, *bad_aq = NULL; 266 bool do_reschedule = false; 267 unsigned long irqflags; 268 bool do_reset = false; 269 u16 pos; 270 int i; 271 272 for (i = 0; i < dev->lif_cfg.aq_count; i++) { 273 aq = dev->aq_vec[i]; 274 275 spin_lock_irqsave(&aq->lock, irqflags); 276 277 if (ionic_queue_empty(&aq->q)) 278 goto next_aq; 279 280 /* Reschedule if any queue has outstanding work */ 281 do_reschedule = true; 282 283 if (time_is_after_eq_jiffies(aq->stamp + IONIC_ADMIN_WARN)) 284 /* Warning threshold not met, nothing to do */ 285 goto next_aq; 286 287 /* See if polling now makes some progress */ 288 pos = aq->q.cons; 289 ionic_admin_poll_locked(aq); 290 if (pos != aq->q.cons) { 291 ibdev_dbg(&dev->ibdev, 292 "missed event for acq %d\n", aq->cqid); 293 goto next_aq; 294 } 295 296 if (time_is_after_eq_jiffies(aq->stamp + 297 IONIC_ADMIN_TIMEOUT)) { 298 /* Timeout threshold not met */ 299 ibdev_dbg(&dev->ibdev, "no progress after %ums\n", 300 (u32)jiffies_to_msecs(jiffies - aq->stamp)); 301 goto next_aq; 302 } 303 304 /* Queue timed out */ 305 bad_aq = aq; 306 do_reset = true; 307 next_aq: 308 spin_unlock_irqrestore(&aq->lock, irqflags); 309 } 310 311 if (do_reset) 312 /* Reset RDMA lif on a timeout */ 313 ionic_admin_timedout(bad_aq); 314 else if (do_reschedule) 315 /* Try to poll again later */ 316 ionic_admin_reset_dwork(dev); 317 } 318 319 static void ionic_admin_work(struct work_struct *ws) 320 { 321 struct ionic_aq *aq = container_of(ws, struct ionic_aq, work); 322 unsigned long irqflags; 323 324 spin_lock_irqsave(&aq->lock, irqflags); 325 ionic_admin_poll_locked(aq); 326 spin_unlock_irqrestore(&aq->lock, irqflags); 327 } 328 329 static void ionic_admin_post_aq(struct ionic_aq *aq, struct ionic_admin_wr *wr) 330 { 331 unsigned long irqflags; 332 bool poll; 333 334 wr->status = IONIC_ADMIN_POSTED; 335 wr->aq = aq; 336 337 spin_lock_irqsave(&aq->lock, irqflags); 338 poll = list_empty(&aq->wr_post); 339 list_add(&wr->aq_ent, &aq->wr_post); 340 if (poll) 341 ionic_admin_poll_locked(aq); 342 spin_unlock_irqrestore(&aq->lock, irqflags); 343 } 344 345 void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr) 346 { 347 int aq_idx; 348 349 /* Use cpu id for the adminq selection */ 350 aq_idx = raw_smp_processor_id() % dev->lif_cfg.aq_count; 351 ionic_admin_post_aq(dev->aq_vec[aq_idx], wr); 352 } 353 354 static void ionic_admin_cancel(struct ionic_admin_wr *wr) 355 { 356 struct ionic_aq *aq = wr->aq; 357 unsigned long irqflags; 358 359 spin_lock_irqsave(&aq->lock, irqflags); 360 361 if (!list_empty(&wr->aq_ent)) { 362 list_del(&wr->aq_ent); 363 if (wr->status != IONIC_ADMIN_POSTED) 364 aq->q_wr[wr->status].wr = NULL; 365 } 366 367 spin_unlock_irqrestore(&aq->lock, irqflags); 368 } 369 370 static int ionic_admin_busy_wait(struct ionic_admin_wr *wr) 371 { 372 struct ionic_aq *aq = wr->aq; 373 unsigned long irqflags; 374 int try_i; 375 376 for (try_i = 0; try_i < IONIC_ADMIN_BUSY_RETRY_COUNT; ++try_i) { 377 if (completion_done(&wr->work)) 378 return 0; 379 380 mdelay(IONIC_ADMIN_BUSY_RETRY_MS); 381 382 spin_lock_irqsave(&aq->lock, irqflags); 383 ionic_admin_poll_locked(aq); 384 spin_unlock_irqrestore(&aq->lock, irqflags); 385 } 386 387 /* 388 * we timed out. Initiate RDMA LIF reset and indicate 389 * error to caller. 390 */ 391 ionic_admin_timedout(aq); 392 return -ETIMEDOUT; 393 } 394 395 int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr, 396 enum ionic_admin_flags flags) 397 { 398 int rc, timo; 399 400 if (flags & IONIC_ADMIN_F_BUSYWAIT) { 401 /* Spin */ 402 rc = ionic_admin_busy_wait(wr); 403 } else if (flags & IONIC_ADMIN_F_INTERRUPT) { 404 /* 405 * Interruptible sleep, 1s timeout 406 * This is used for commands which are safe for the caller 407 * to clean up without killing and resetting the adminq. 408 */ 409 timo = wait_for_completion_interruptible_timeout(&wr->work, 410 HZ); 411 if (timo > 0) 412 rc = 0; 413 else if (timo == 0) 414 rc = -ETIMEDOUT; 415 else 416 rc = timo; 417 } else { 418 /* 419 * Uninterruptible sleep 420 * This is used for commands which are NOT safe for the 421 * caller to clean up. Cleanup must be handled by the 422 * adminq kill and reset process so that host memory is 423 * not corrupted by the device. 424 */ 425 wait_for_completion(&wr->work); 426 rc = 0; 427 } 428 429 if (rc) { 430 ibdev_warn(&dev->ibdev, "wait status %d\n", rc); 431 ionic_admin_cancel(wr); 432 } else if (wr->status == IONIC_ADMIN_KILLED) { 433 ibdev_dbg(&dev->ibdev, "admin killed\n"); 434 435 /* No error if admin already killed during teardown */ 436 rc = (flags & IONIC_ADMIN_F_TEARDOWN) ? 0 : -ENODEV; 437 } else if (ionic_v1_cqe_error(&wr->cqe)) { 438 ibdev_warn(&dev->ibdev, "opcode %u error %u\n", 439 wr->wqe.op, 440 be32_to_cpu(wr->cqe.status_length)); 441 rc = -EINVAL; 442 } 443 return rc; 444 } 445 446 static int ionic_rdma_devcmd(struct ionic_ibdev *dev, 447 struct ionic_admin_ctx *admin) 448 { 449 int rc; 450 451 rc = ionic_adminq_post_wait(dev->lif_cfg.lif, admin); 452 if (rc) 453 return rc; 454 455 return ionic_error_to_errno(admin->comp.comp.status); 456 } 457 458 int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev) 459 { 460 struct ionic_admin_ctx admin = { 461 .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), 462 .cmd.rdma_reset = { 463 .opcode = IONIC_CMD_RDMA_RESET_LIF, 464 .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), 465 }, 466 }; 467 468 return ionic_rdma_devcmd(dev, &admin); 469 } 470 471 static int ionic_rdma_queue_devcmd(struct ionic_ibdev *dev, 472 struct ionic_queue *q, 473 u32 qid, u32 cid, u16 opcode) 474 { 475 struct ionic_admin_ctx admin = { 476 .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), 477 .cmd.rdma_queue = { 478 .opcode = opcode, 479 .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), 480 .qid_ver = cpu_to_le32(qid), 481 .cid = cpu_to_le32(cid), 482 .dbid = cpu_to_le16(dev->lif_cfg.dbid), 483 .depth_log2 = q->depth_log2, 484 .stride_log2 = q->stride_log2, 485 .dma_addr = cpu_to_le64(q->dma), 486 }, 487 }; 488 489 return ionic_rdma_devcmd(dev, &admin); 490 } 491 492 static int ionic_rdma_cq_devcmd(struct ionic_vcq *vcq, 493 struct ionic_queue *q, 494 u32 *qid, u32 cid, u16 opcode, 495 u8 udma_idx) 496 { 497 struct ionic_ibdev *dev = to_ionic_ibdev(vcq->ibcq.device); 498 struct ionic_admin_ctx admin = { 499 .work = COMPLETION_INITIALIZER_ONSTACK(admin.work), 500 .cmd.rdma_queue = { 501 .opcode = opcode, 502 .lif_index = cpu_to_le16(dev->lif_cfg.lif_index), 503 .qid_ver = cpu_to_le32(*qid), 504 .cid = cpu_to_le32(cid), 505 .dbid = cpu_to_le16(dev->lif_cfg.dbid), 506 .depth_log2 = q->depth_log2, 507 .stride_log2 = q->stride_log2, 508 .dma_addr = cpu_to_le64(q->dma), 509 }, 510 }; 511 int rc; 512 513 if (ionic_fw_has_qid_alloc(dev, IONIC_LIF_RDMA_ALLOC_QID_CQ)) { 514 struct ionic_admin_create_cq_resp resp_buf; 515 516 /* Indicates use of CQ create V2 response format. */ 517 admin.cmd.rdma_queue.qid_ver = cpu_to_le32(IONIC_CREATE_CQ_CMD_V2_MAGIC); 518 admin.cmd.rdma_queue.udma_idx = udma_idx; 519 520 rc = ionic_rdma_devcmd(dev, &admin); 521 if (rc) 522 return rc; 523 524 memcpy(&resp_buf, admin.comp.comp.cmd_data, sizeof(resp_buf)); 525 *qid = le32_to_cpu(resp_buf.id); 526 } else { 527 rc = ionic_rdma_devcmd(dev, &admin); 528 if (rc) 529 return rc; 530 } 531 532 return 0; 533 } 534 535 static void ionic_rdma_admincq_comp(struct ib_cq *ibcq, void *cq_context) 536 { 537 struct ionic_aq *aq = cq_context; 538 unsigned long irqflags; 539 540 spin_lock_irqsave(&aq->lock, irqflags); 541 aq->armed = false; 542 if (atomic_read(&aq->admin_state) < IONIC_ADMIN_KILLED) 543 queue_work(ionic_evt_workq, &aq->work); 544 spin_unlock_irqrestore(&aq->lock, irqflags); 545 } 546 547 static void ionic_rdma_admincq_event(struct ib_event *event, void *cq_context) 548 { 549 struct ionic_aq *aq = cq_context; 550 551 ibdev_err(&aq->dev->ibdev, "admincq event %d\n", event->event); 552 } 553 554 static struct ionic_vcq *ionic_create_rdma_admincq(struct ionic_ibdev *dev, 555 int comp_vector) 556 { 557 struct ib_cq_init_attr attr = { 558 .cqe = IONIC_AQ_DEPTH, 559 .comp_vector = comp_vector, 560 }; 561 struct ionic_tbl_buf buf = {}; 562 struct ionic_vcq *vcq; 563 struct ionic_cq *cq; 564 int rc; 565 566 vcq = kzalloc_obj(*vcq); 567 if (!vcq) 568 return ERR_PTR(-ENOMEM); 569 570 vcq->ibcq.device = &dev->ibdev; 571 vcq->ibcq.comp_handler = ionic_rdma_admincq_comp; 572 vcq->ibcq.event_handler = ionic_rdma_admincq_event; 573 atomic_set(&vcq->ibcq.usecnt, 0); 574 575 vcq->udma_mask = 1; 576 cq = &vcq->cq[0]; 577 578 rc = ionic_create_cq_common(vcq, &buf, &attr, NULL, NULL, 579 NULL, 0); 580 if (rc) 581 goto err_init; 582 583 rc = ionic_rdma_cq_devcmd(vcq, &cq->q, &cq->cqid, cq->eqid, 584 IONIC_CMD_RDMA_CREATE_CQ, 0); 585 if (rc) 586 goto err_cmd; 587 588 rc = ionic_post_create_cq_cmd(cq, NULL, NULL); 589 if (rc) 590 goto err_cmd; 591 592 return vcq; 593 594 err_cmd: 595 ionic_destroy_cq_common(dev, cq); 596 err_init: 597 kfree(vcq); 598 599 return ERR_PTR(rc); 600 } 601 602 static void ionic_destroy_rdma_admincq(struct ionic_ibdev *dev, 603 struct ionic_vcq *vcq) 604 { 605 ionic_pre_destroy_cq_cmd(dev, &vcq->cq[0]); 606 ionic_destroy_cq_common(dev, &vcq->cq[0]); 607 kfree(vcq); 608 } 609 610 static struct ionic_aq *__ionic_create_rdma_adminq(struct ionic_ibdev *dev, 611 u32 aqid, u32 cqid) 612 { 613 struct ionic_aq *aq; 614 int rc; 615 616 aq = kzalloc_obj(*aq); 617 if (!aq) 618 return ERR_PTR(-ENOMEM); 619 620 atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED); 621 aq->dev = dev; 622 aq->aqid = aqid; 623 aq->cqid = cqid; 624 spin_lock_init(&aq->lock); 625 626 rc = ionic_queue_init(&aq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH, 627 ADMIN_WQE_STRIDE); 628 if (rc) 629 goto err_q; 630 631 ionic_queue_dbell_init(&aq->q, aq->aqid); 632 633 aq->q_wr = kzalloc_objs(*aq->q_wr, (u32)aq->q.mask + 1); 634 if (!aq->q_wr) { 635 rc = -ENOMEM; 636 goto err_wr; 637 } 638 639 INIT_LIST_HEAD(&aq->wr_prod); 640 INIT_LIST_HEAD(&aq->wr_post); 641 642 INIT_WORK(&aq->work, ionic_admin_work); 643 aq->armed = false; 644 645 return aq; 646 647 err_wr: 648 ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev); 649 err_q: 650 kfree(aq); 651 652 return ERR_PTR(rc); 653 } 654 655 static void __ionic_destroy_rdma_adminq(struct ionic_ibdev *dev, 656 struct ionic_aq *aq) 657 { 658 kfree(aq->q_wr); 659 ionic_queue_destroy(&aq->q, dev->lif_cfg.hwdev); 660 kfree(aq); 661 } 662 663 static struct ionic_aq *ionic_create_rdma_adminq(struct ionic_ibdev *dev, 664 u32 aqid, u32 cqid) 665 { 666 struct ionic_aq *aq; 667 int rc; 668 669 aq = __ionic_create_rdma_adminq(dev, aqid, cqid); 670 if (IS_ERR(aq)) 671 return aq; 672 673 rc = ionic_rdma_queue_devcmd(dev, &aq->q, aq->aqid, aq->cqid, 674 IONIC_CMD_RDMA_CREATE_ADMINQ); 675 if (rc) 676 goto err_cmd; 677 678 return aq; 679 680 err_cmd: 681 __ionic_destroy_rdma_adminq(dev, aq); 682 683 return ERR_PTR(rc); 684 } 685 686 static void ionic_flush_qs(struct ionic_ibdev *dev) 687 { 688 struct ionic_qp *qp, *qp_tmp; 689 struct ionic_cq *cq, *cq_tmp; 690 LIST_HEAD(flush_list); 691 unsigned long index; 692 693 WARN_ON(!irqs_disabled()); 694 695 /* Flush qp send and recv */ 696 xa_lock(&dev->qp_tbl); 697 xa_for_each(&dev->qp_tbl, index, qp) { 698 kref_get(&qp->qp_kref); 699 list_add_tail(&qp->ibkill_flush_ent, &flush_list); 700 } 701 xa_unlock(&dev->qp_tbl); 702 703 list_for_each_entry_safe(qp, qp_tmp, &flush_list, ibkill_flush_ent) { 704 ionic_flush_qp(dev, qp); 705 kref_put(&qp->qp_kref, ionic_qp_complete); 706 list_del(&qp->ibkill_flush_ent); 707 } 708 709 /* Notify completions */ 710 xa_lock(&dev->cq_tbl); 711 xa_for_each(&dev->cq_tbl, index, cq) { 712 kref_get(&cq->cq_kref); 713 list_add_tail(&cq->ibkill_flush_ent, &flush_list); 714 } 715 xa_unlock(&dev->cq_tbl); 716 717 list_for_each_entry_safe(cq, cq_tmp, &flush_list, ibkill_flush_ent) { 718 ionic_notify_flush_cq(cq); 719 kref_put(&cq->cq_kref, ionic_cq_complete); 720 list_del(&cq->ibkill_flush_ent); 721 } 722 } 723 724 static void ionic_kill_ibdev(struct ionic_ibdev *dev, bool fatal_path) 725 { 726 unsigned long irqflags; 727 bool do_flush = false; 728 int i; 729 730 /* Mark AQs for drain and flush the QPs while irq is disabled */ 731 local_irq_save(irqflags); 732 733 /* Mark the admin queue, flushing at most once */ 734 for (i = 0; i < dev->lif_cfg.aq_count; i++) { 735 struct ionic_aq *aq = dev->aq_vec[i]; 736 737 spin_lock(&aq->lock); 738 if (atomic_read(&aq->admin_state) != IONIC_ADMIN_KILLED) { 739 atomic_set(&aq->admin_state, IONIC_ADMIN_KILLED); 740 /* Flush incomplete admin commands */ 741 ionic_admin_poll_locked(aq); 742 do_flush = true; 743 } 744 spin_unlock(&aq->lock); 745 } 746 747 if (do_flush) 748 ionic_flush_qs(dev); 749 750 local_irq_restore(irqflags); 751 752 /* Post a fatal event if requested */ 753 if (fatal_path) { 754 struct ib_event ev; 755 756 ev.device = &dev->ibdev; 757 ev.element.port_num = 1; 758 ev.event = IB_EVENT_DEVICE_FATAL; 759 760 ib_dispatch_event(&ev); 761 } 762 763 atomic_set(&dev->admin_state, IONIC_ADMIN_KILLED); 764 } 765 766 void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path) 767 { 768 enum ionic_admin_state old_state; 769 unsigned long irqflags = 0; 770 int i, rc; 771 772 if (!dev->aq_vec) 773 return; 774 775 /* 776 * Admin queues are transitioned from active to paused to killed state. 777 * When in paused state, no new commands are issued to the device, 778 * nor are any completed locally. After resetting the lif, it will be 779 * safe to resume the rdma admin queues in the killed state. Commands 780 * will not be issued to the device, but will complete locally with status 781 * IONIC_ADMIN_KILLED. Handling completion will ensure that creating or 782 * modifying resources fails, but destroying resources succeeds. 783 * If there was a failure resetting the lif using this strategy, 784 * then the state of the device is unknown. 785 */ 786 old_state = atomic_cmpxchg(&dev->admin_state, IONIC_ADMIN_ACTIVE, 787 IONIC_ADMIN_PAUSED); 788 if (old_state != IONIC_ADMIN_ACTIVE) 789 return; 790 791 /* Pause all the AQs */ 792 local_irq_save(irqflags); 793 for (i = 0; i < dev->lif_cfg.aq_count; i++) { 794 struct ionic_aq *aq = dev->aq_vec[i]; 795 796 spin_lock(&aq->lock); 797 /* pause rdma admin queues to reset lif */ 798 if (atomic_read(&aq->admin_state) == IONIC_ADMIN_ACTIVE) 799 atomic_set(&aq->admin_state, IONIC_ADMIN_PAUSED); 800 spin_unlock(&aq->lock); 801 } 802 local_irq_restore(irqflags); 803 804 rc = ionic_rdma_reset_devcmd(dev); 805 if (unlikely(rc)) { 806 ibdev_err(&dev->ibdev, "failed to reset rdma %d\n", rc); 807 ionic_request_rdma_reset(dev->lif_cfg.lif); 808 } 809 810 ionic_kill_ibdev(dev, fatal_path); 811 } 812 813 static void ionic_reset_work(struct work_struct *ws) 814 { 815 struct ionic_ibdev *dev = 816 container_of(ws, struct ionic_ibdev, reset_work); 817 818 ionic_kill_rdma_admin(dev, true); 819 } 820 821 static bool ionic_next_eqe(struct ionic_eq *eq, struct ionic_v1_eqe *eqe) 822 { 823 struct ionic_v1_eqe *qeqe; 824 bool color; 825 826 qeqe = ionic_queue_at_prod(&eq->q); 827 color = ionic_v1_eqe_color(qeqe); 828 829 /* cons is color for eq */ 830 if (eq->q.cons != color) 831 return false; 832 833 /* Prevent out-of-order reads of the EQE */ 834 dma_rmb(); 835 836 ibdev_dbg(&eq->dev->ibdev, "poll eq prod %u\n", eq->q.prod); 837 print_hex_dump_debug("eqe ", DUMP_PREFIX_OFFSET, 16, 1, 838 qeqe, BIT(eq->q.stride_log2), true); 839 *eqe = *qeqe; 840 841 return true; 842 } 843 844 static void ionic_cq_event(struct ionic_ibdev *dev, u32 cqid, u8 code) 845 { 846 unsigned long irqflags; 847 struct ib_event ibev; 848 struct ionic_cq *cq; 849 850 xa_lock_irqsave(&dev->cq_tbl, irqflags); 851 cq = xa_load(&dev->cq_tbl, cqid); 852 if (cq) 853 kref_get(&cq->cq_kref); 854 xa_unlock_irqrestore(&dev->cq_tbl, irqflags); 855 856 if (!cq) { 857 ibdev_dbg(&dev->ibdev, 858 "missing cqid %#x code %u\n", cqid, code); 859 return; 860 } 861 862 switch (code) { 863 case IONIC_V1_EQE_CQ_NOTIFY: 864 if (cq->vcq->ibcq.comp_handler) 865 cq->vcq->ibcq.comp_handler(&cq->vcq->ibcq, 866 cq->vcq->ibcq.cq_context); 867 break; 868 869 case IONIC_V1_EQE_CQ_ERR: 870 if (cq->vcq->ibcq.event_handler) { 871 ibev.event = IB_EVENT_CQ_ERR; 872 ibev.device = &dev->ibdev; 873 ibev.element.cq = &cq->vcq->ibcq; 874 875 cq->vcq->ibcq.event_handler(&ibev, 876 cq->vcq->ibcq.cq_context); 877 } 878 break; 879 880 default: 881 ibdev_dbg(&dev->ibdev, 882 "unrecognized cqid %#x code %u\n", cqid, code); 883 break; 884 } 885 886 kref_put(&cq->cq_kref, ionic_cq_complete); 887 } 888 889 static void ionic_qp_event(struct ionic_ibdev *dev, u32 qpid, u8 code) 890 { 891 unsigned long irqflags; 892 struct ib_event ibev; 893 struct ionic_qp *qp; 894 895 xa_lock_irqsave(&dev->qp_tbl, irqflags); 896 qp = xa_load(&dev->qp_tbl, qpid); 897 if (qp) 898 kref_get(&qp->qp_kref); 899 xa_unlock_irqrestore(&dev->qp_tbl, irqflags); 900 901 if (!qp) { 902 ibdev_dbg(&dev->ibdev, 903 "missing qpid %#x code %u\n", qpid, code); 904 return; 905 } 906 907 ibev.device = &dev->ibdev; 908 ibev.element.qp = &qp->ibqp; 909 910 switch (code) { 911 case IONIC_V1_EQE_SQ_DRAIN: 912 ibev.event = IB_EVENT_SQ_DRAINED; 913 break; 914 915 case IONIC_V1_EQE_QP_COMM_EST: 916 ibev.event = IB_EVENT_COMM_EST; 917 break; 918 919 case IONIC_V1_EQE_QP_LAST_WQE: 920 ibev.event = IB_EVENT_QP_LAST_WQE_REACHED; 921 break; 922 923 case IONIC_V1_EQE_QP_ERR: 924 ibev.event = IB_EVENT_QP_FATAL; 925 break; 926 927 case IONIC_V1_EQE_QP_ERR_REQUEST: 928 ibev.event = IB_EVENT_QP_REQ_ERR; 929 break; 930 931 case IONIC_V1_EQE_QP_ERR_ACCESS: 932 ibev.event = IB_EVENT_QP_ACCESS_ERR; 933 break; 934 935 default: 936 ibdev_dbg(&dev->ibdev, 937 "unrecognized qpid %#x code %u\n", qpid, code); 938 goto out; 939 } 940 941 if (qp->ibqp.event_handler) 942 qp->ibqp.event_handler(&ibev, qp->ibqp.qp_context); 943 944 out: 945 kref_put(&qp->qp_kref, ionic_qp_complete); 946 } 947 948 static void ionic_srq_event(struct ionic_ibdev *dev, u32 srqid, u8 code) 949 { 950 unsigned long irqflags; 951 struct ionic_srq *srq; 952 struct ib_event ibev; 953 954 xa_lock_irqsave(&dev->srq_tbl, irqflags); 955 srq = xa_load(&dev->srq_tbl, srqid); 956 if (srq) 957 kref_get(&srq->kref); 958 xa_unlock_irqrestore(&dev->srq_tbl, irqflags); 959 960 if (!srq) { 961 ibdev_dbg(&dev->ibdev, 962 "missing srqid %#x code %u\n", srqid, code); 963 return; 964 } 965 966 ibev.device = &dev->ibdev; 967 ibev.element.srq = &srq->ibsrq; 968 969 switch (code) { 970 case IONIC_V1_EQE_SRQ_LIMIT_REACHED: 971 ibev.event = IB_EVENT_SRQ_LIMIT_REACHED; 972 break; 973 974 case IONIC_V1_EQE_SRQ_ERR: 975 ibev.event = IB_EVENT_SRQ_ERR; 976 break; 977 978 default: 979 ibdev_dbg(&dev->ibdev, 980 "unrecognized srqid %#x code %u\n", srqid, code); 981 goto out; 982 } 983 984 if (srq->ibsrq.event_handler) 985 srq->ibsrq.event_handler(&ibev, srq->ibsrq.srq_context); 986 987 out: 988 kref_put(&srq->kref, ionic_srq_complete); 989 } 990 991 static u16 ionic_poll_eq(struct ionic_eq *eq, u16 budget) 992 { 993 struct ionic_ibdev *dev = eq->dev; 994 struct ionic_v1_eqe eqe; 995 u16 npolled = 0; 996 u8 type, code; 997 u32 evt, qid; 998 999 while (npolled < budget) { 1000 if (!ionic_next_eqe(eq, &eqe)) 1001 break; 1002 1003 ionic_queue_produce(&eq->q); 1004 1005 /* cons is color for eq */ 1006 eq->q.cons = ionic_color_wrap(eq->q.prod, eq->q.cons); 1007 1008 ++npolled; 1009 1010 evt = ionic_v1_eqe_evt(&eqe); 1011 type = ionic_v1_eqe_evt_type(evt); 1012 code = ionic_v1_eqe_evt_code(evt); 1013 qid = ionic_v1_eqe_evt_qid(evt); 1014 1015 switch (type) { 1016 case IONIC_V1_EQE_TYPE_CQ: 1017 ionic_cq_event(dev, qid, code); 1018 break; 1019 1020 case IONIC_V1_EQE_TYPE_QP: 1021 ionic_qp_event(dev, qid, code); 1022 break; 1023 1024 case IONIC_V1_EQE_TYPE_SRQ: 1025 ionic_srq_event(dev, qid, code); 1026 break; 1027 1028 default: 1029 ibdev_dbg(&dev->ibdev, 1030 "unknown event %#x type %u\n", evt, type); 1031 } 1032 } 1033 1034 return npolled; 1035 } 1036 1037 static void ionic_poll_eq_work(struct work_struct *work) 1038 { 1039 struct ionic_eq *eq = container_of(work, struct ionic_eq, work); 1040 u32 npolled; 1041 1042 if (unlikely(!eq->enable) || WARN_ON(eq->armed)) 1043 return; 1044 1045 npolled = ionic_poll_eq(eq, IONIC_EQ_WORK_BUDGET); 1046 if (npolled == IONIC_EQ_WORK_BUDGET) { 1047 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 1048 npolled, 0); 1049 queue_work(ionic_evt_workq, &eq->work); 1050 } else { 1051 xchg(&eq->armed, 1); 1052 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 1053 0, IONIC_INTR_CRED_UNMASK); 1054 } 1055 } 1056 1057 static irqreturn_t ionic_poll_eq_isr(int irq, void *eqptr) 1058 { 1059 struct ionic_eq *eq = eqptr; 1060 int was_armed; 1061 u32 npolled; 1062 1063 was_armed = xchg(&eq->armed, 0); 1064 1065 if (unlikely(!eq->enable) || !was_armed) 1066 return IRQ_HANDLED; 1067 1068 npolled = ionic_poll_eq(eq, IONIC_EQ_ISR_BUDGET); 1069 if (npolled == IONIC_EQ_ISR_BUDGET) { 1070 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 1071 npolled, 0); 1072 queue_work(ionic_evt_workq, &eq->work); 1073 } else { 1074 xchg(&eq->armed, 1); 1075 ionic_intr_credits(eq->dev->lif_cfg.intr_ctrl, eq->intr, 1076 0, IONIC_INTR_CRED_UNMASK); 1077 } 1078 1079 return IRQ_HANDLED; 1080 } 1081 1082 static struct ionic_eq *ionic_create_eq(struct ionic_ibdev *dev, int eqid) 1083 { 1084 struct ionic_intr_info intr_obj = { }; 1085 struct ionic_eq *eq; 1086 int rc; 1087 1088 eq = kzalloc_obj(*eq); 1089 if (!eq) 1090 return ERR_PTR(-ENOMEM); 1091 1092 eq->dev = dev; 1093 1094 rc = ionic_queue_init(&eq->q, dev->lif_cfg.hwdev, IONIC_EQ_DEPTH, 1095 sizeof(struct ionic_v1_eqe)); 1096 if (rc) 1097 goto err_q; 1098 1099 eq->eqid = eqid; 1100 1101 eq->armed = true; 1102 eq->enable = false; 1103 INIT_WORK(&eq->work, ionic_poll_eq_work); 1104 1105 rc = ionic_intr_alloc(dev->lif_cfg.lif, &intr_obj); 1106 if (rc < 0) 1107 goto err_intr; 1108 1109 eq->irq = intr_obj.vector; 1110 eq->intr = intr_obj.index; 1111 1112 ionic_queue_dbell_init(&eq->q, eq->eqid); 1113 1114 /* cons is color for eq */ 1115 eq->q.cons = true; 1116 1117 snprintf(eq->name, sizeof(eq->name), "%s-%d-%d-eq", 1118 "ionr", dev->lif_cfg.lif_index, eq->eqid); 1119 1120 ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET); 1121 ionic_intr_mask_assert(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_SET); 1122 ionic_intr_coal_init(dev->lif_cfg.intr_ctrl, eq->intr, 0); 1123 ionic_intr_clean(dev->lif_cfg.intr_ctrl, eq->intr); 1124 1125 eq->enable = true; 1126 1127 rc = request_irq(eq->irq, ionic_poll_eq_isr, 0, eq->name, eq); 1128 if (rc) 1129 goto err_irq; 1130 1131 rc = ionic_rdma_queue_devcmd(dev, &eq->q, eq->eqid, eq->intr, 1132 IONIC_CMD_RDMA_CREATE_EQ); 1133 if (rc) 1134 goto err_cmd; 1135 1136 ionic_intr_mask(dev->lif_cfg.intr_ctrl, eq->intr, IONIC_INTR_MASK_CLEAR); 1137 1138 return eq; 1139 1140 err_cmd: 1141 eq->enable = false; 1142 free_irq(eq->irq, eq); 1143 flush_work(&eq->work); 1144 err_irq: 1145 ionic_intr_free(dev->lif_cfg.lif, eq->intr); 1146 err_intr: 1147 ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev); 1148 err_q: 1149 kfree(eq); 1150 1151 return ERR_PTR(rc); 1152 } 1153 1154 static void ionic_destroy_eq(struct ionic_eq *eq) 1155 { 1156 struct ionic_ibdev *dev = eq->dev; 1157 1158 eq->enable = false; 1159 free_irq(eq->irq, eq); 1160 flush_work(&eq->work); 1161 1162 ionic_intr_free(dev->lif_cfg.lif, eq->intr); 1163 ionic_queue_destroy(&eq->q, dev->lif_cfg.hwdev); 1164 kfree(eq); 1165 } 1166 1167 int ionic_create_rdma_admin(struct ionic_ibdev *dev) 1168 { 1169 int eq_i = 0, aq_i = 0, rc = 0; 1170 struct ionic_vcq *vcq; 1171 struct ionic_aq *aq; 1172 struct ionic_eq *eq; 1173 1174 dev->eq_vec = NULL; 1175 dev->aq_vec = NULL; 1176 1177 INIT_WORK(&dev->reset_work, ionic_reset_work); 1178 INIT_DELAYED_WORK(&dev->admin_dwork, ionic_admin_dwork); 1179 atomic_set(&dev->admin_state, IONIC_ADMIN_KILLED); 1180 1181 if (dev->lif_cfg.aq_count > IONIC_AQ_COUNT) { 1182 ibdev_dbg(&dev->ibdev, "limiting adminq count to %d\n", 1183 IONIC_AQ_COUNT); 1184 dev->lif_cfg.aq_count = IONIC_AQ_COUNT; 1185 } 1186 1187 if (dev->lif_cfg.eq_count > IONIC_EQ_COUNT) { 1188 dev_dbg(&dev->ibdev.dev, "limiting eventq count to %d\n", 1189 IONIC_EQ_COUNT); 1190 dev->lif_cfg.eq_count = IONIC_EQ_COUNT; 1191 } 1192 1193 /* need at least two eq and one aq */ 1194 if (dev->lif_cfg.eq_count < IONIC_EQ_COUNT_MIN || 1195 dev->lif_cfg.aq_count < IONIC_AQ_COUNT_MIN) { 1196 rc = -EINVAL; 1197 goto out; 1198 } 1199 1200 dev->eq_vec = kmalloc_objs(*dev->eq_vec, dev->lif_cfg.eq_count); 1201 if (!dev->eq_vec) { 1202 rc = -ENOMEM; 1203 goto out; 1204 } 1205 1206 for (eq_i = 0; eq_i < dev->lif_cfg.eq_count; ++eq_i) { 1207 eq = ionic_create_eq(dev, eq_i + dev->lif_cfg.eq_base); 1208 if (IS_ERR(eq)) { 1209 rc = PTR_ERR(eq); 1210 1211 if (eq_i < IONIC_EQ_COUNT_MIN) { 1212 ibdev_err(&dev->ibdev, 1213 "fail create eq %pe\n", eq); 1214 goto out; 1215 } 1216 1217 /* ok, just fewer eq than device supports */ 1218 ibdev_dbg(&dev->ibdev, "eq count %d want %d rc %pe\n", 1219 eq_i, dev->lif_cfg.eq_count, eq); 1220 1221 rc = 0; 1222 break; 1223 } 1224 1225 dev->eq_vec[eq_i] = eq; 1226 } 1227 1228 dev->lif_cfg.eq_count = eq_i; 1229 1230 dev->aq_vec = kmalloc_objs(*dev->aq_vec, dev->lif_cfg.aq_count); 1231 if (!dev->aq_vec) { 1232 rc = -ENOMEM; 1233 goto out; 1234 } 1235 1236 /* Create one CQ per AQ */ 1237 for (aq_i = 0; aq_i < dev->lif_cfg.aq_count; ++aq_i) { 1238 vcq = ionic_create_rdma_admincq(dev, aq_i % eq_i); 1239 if (IS_ERR(vcq)) { 1240 rc = PTR_ERR(vcq); 1241 1242 if (!aq_i) { 1243 ibdev_err(&dev->ibdev, 1244 "failed to create acq %pe\n", vcq); 1245 goto out; 1246 } 1247 1248 /* ok, just fewer adminq than device supports */ 1249 ibdev_dbg(&dev->ibdev, "acq count %d want %d rc %pe\n", 1250 aq_i, dev->lif_cfg.aq_count, vcq); 1251 break; 1252 } 1253 1254 aq = ionic_create_rdma_adminq(dev, aq_i + dev->lif_cfg.aq_base, 1255 vcq->cq[0].cqid); 1256 if (IS_ERR(aq)) { 1257 /* Clean up the dangling CQ */ 1258 ionic_destroy_rdma_admincq(dev, vcq); 1259 1260 rc = PTR_ERR(aq); 1261 1262 if (!aq_i) { 1263 ibdev_err(&dev->ibdev, 1264 "failed to create aq %pe\n", aq); 1265 goto out; 1266 } 1267 1268 /* ok, just fewer adminq than device supports */ 1269 ibdev_dbg(&dev->ibdev, "aq count %d want %d rc %pe\n", 1270 aq_i, dev->lif_cfg.aq_count, aq); 1271 break; 1272 } 1273 1274 vcq->ibcq.cq_context = aq; 1275 aq->vcq = vcq; 1276 1277 atomic_set(&aq->admin_state, IONIC_ADMIN_ACTIVE); 1278 dev->aq_vec[aq_i] = aq; 1279 } 1280 1281 atomic_set(&dev->admin_state, IONIC_ADMIN_ACTIVE); 1282 out: 1283 dev->lif_cfg.eq_count = eq_i; 1284 dev->lif_cfg.aq_count = aq_i; 1285 1286 return rc; 1287 } 1288 1289 void ionic_destroy_rdma_admin(struct ionic_ibdev *dev) 1290 { 1291 struct ionic_vcq *vcq; 1292 struct ionic_aq *aq; 1293 struct ionic_eq *eq; 1294 1295 /* 1296 * Killing the admin before destroy makes sure all admin and 1297 * completions are flushed. admin_state = IONIC_ADMIN_KILLED 1298 * stops queueing up further works. 1299 */ 1300 cancel_delayed_work_sync(&dev->admin_dwork); 1301 cancel_work_sync(&dev->reset_work); 1302 1303 if (dev->aq_vec) { 1304 while (dev->lif_cfg.aq_count > 0) { 1305 aq = dev->aq_vec[--dev->lif_cfg.aq_count]; 1306 vcq = aq->vcq; 1307 1308 cancel_work_sync(&aq->work); 1309 1310 __ionic_destroy_rdma_adminq(dev, aq); 1311 if (vcq) 1312 ionic_destroy_rdma_admincq(dev, vcq); 1313 } 1314 1315 kfree(dev->aq_vec); 1316 } 1317 1318 if (dev->eq_vec) { 1319 while (dev->lif_cfg.eq_count > 0) { 1320 eq = dev->eq_vec[--dev->lif_cfg.eq_count]; 1321 ionic_destroy_eq(eq); 1322 } 1323 1324 kfree(dev->eq_vec); 1325 } 1326 } 1327