1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */ 3 4 #include <linux/kernel.h> 5 #include <linux/types.h> 6 #include <linux/errno.h> 7 #include <linux/io.h> 8 #include <linux/slab.h> 9 #include <linux/etherdevice.h> 10 #include "ionic.h" 11 #include "ionic_dev.h" 12 #include "ionic_lif.h" 13 14 static void ionic_watchdog_cb(struct timer_list *t) 15 { 16 struct ionic *ionic = from_timer(ionic, t, watchdog_timer); 17 struct ionic_lif *lif = ionic->lif; 18 int hb; 19 20 mod_timer(&ionic->watchdog_timer, 21 round_jiffies(jiffies + ionic->watchdog_period)); 22 23 if (!lif) 24 return; 25 26 hb = ionic_heartbeat_check(ionic); 27 dev_dbg(ionic->dev, "%s: hb %d running %d UP %d\n", 28 __func__, hb, netif_running(lif->netdev), 29 test_bit(IONIC_LIF_F_UP, lif->state)); 30 31 if (hb >= 0 && 32 !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 33 ionic_link_status_check_request(lif, CAN_NOT_SLEEP); 34 } 35 36 void ionic_init_devinfo(struct ionic *ionic) 37 { 38 struct ionic_dev *idev = &ionic->idev; 39 40 idev->dev_info.asic_type = ioread8(&idev->dev_info_regs->asic_type); 41 idev->dev_info.asic_rev = ioread8(&idev->dev_info_regs->asic_rev); 42 43 memcpy_fromio(idev->dev_info.fw_version, 44 idev->dev_info_regs->fw_version, 45 IONIC_DEVINFO_FWVERS_BUFLEN); 46 47 memcpy_fromio(idev->dev_info.serial_num, 48 idev->dev_info_regs->serial_num, 49 IONIC_DEVINFO_SERIAL_BUFLEN); 50 51 idev->dev_info.fw_version[IONIC_DEVINFO_FWVERS_BUFLEN] = 0; 52 idev->dev_info.serial_num[IONIC_DEVINFO_SERIAL_BUFLEN] = 0; 53 54 dev_dbg(ionic->dev, "fw_version %s\n", idev->dev_info.fw_version); 55 } 56 57 int ionic_dev_setup(struct ionic *ionic) 58 { 59 struct ionic_dev_bar *bar = ionic->bars; 60 unsigned int num_bars = ionic->num_bars; 61 struct ionic_dev *idev = &ionic->idev; 62 struct device *dev = ionic->dev; 63 u32 sig; 64 65 /* BAR0: dev_cmd and interrupts */ 66 if (num_bars < 1) { 67 dev_err(dev, "No bars found, aborting\n"); 68 return -EFAULT; 69 } 70 71 if (bar->len < IONIC_BAR0_SIZE) { 72 dev_err(dev, "Resource bar size %lu too small, aborting\n", 73 bar->len); 74 return -EFAULT; 75 } 76 77 idev->dev_info_regs = bar->vaddr + IONIC_BAR0_DEV_INFO_REGS_OFFSET; 78 idev->dev_cmd_regs = bar->vaddr + IONIC_BAR0_DEV_CMD_REGS_OFFSET; 79 idev->intr_status = bar->vaddr + IONIC_BAR0_INTR_STATUS_OFFSET; 80 idev->intr_ctrl = bar->vaddr + IONIC_BAR0_INTR_CTRL_OFFSET; 81 82 idev->hwstamp_regs = &idev->dev_info_regs->hwstamp; 83 84 sig = ioread32(&idev->dev_info_regs->signature); 85 if (sig != IONIC_DEV_INFO_SIGNATURE) { 86 dev_err(dev, "Incompatible firmware signature %x", sig); 87 return -EFAULT; 88 } 89 90 ionic_init_devinfo(ionic); 91 92 /* BAR1: doorbells */ 93 bar++; 94 if (num_bars < 2) { 95 dev_err(dev, "Doorbell bar missing, aborting\n"); 96 return -EFAULT; 97 } 98 99 timer_setup(&ionic->watchdog_timer, ionic_watchdog_cb, 0); 100 ionic->watchdog_period = IONIC_WATCHDOG_SECS * HZ; 101 102 /* set times to ensure the first check will proceed */ 103 atomic_long_set(&idev->last_check_time, jiffies - 2 * HZ); 104 idev->last_hb_time = jiffies - 2 * ionic->watchdog_period; 105 /* init as ready, so no transition if the first check succeeds */ 106 idev->last_fw_hb = 0; 107 idev->fw_hb_ready = true; 108 idev->fw_status_ready = true; 109 110 mod_timer(&ionic->watchdog_timer, 111 round_jiffies(jiffies + ionic->watchdog_period)); 112 113 idev->db_pages = bar->vaddr; 114 idev->phy_db_pages = bar->bus_addr; 115 116 return 0; 117 } 118 119 /* Devcmd Interface */ 120 int ionic_heartbeat_check(struct ionic *ionic) 121 { 122 struct ionic_dev *idev = &ionic->idev; 123 unsigned long check_time, last_check_time; 124 bool fw_status_ready, fw_hb_ready; 125 u8 fw_status; 126 u32 fw_hb; 127 128 /* wait a least one second before testing again */ 129 check_time = jiffies; 130 last_check_time = atomic_long_read(&idev->last_check_time); 131 do_check_time: 132 if (time_before(check_time, last_check_time + HZ)) 133 return 0; 134 if (!atomic_long_try_cmpxchg_relaxed(&idev->last_check_time, 135 &last_check_time, check_time)) { 136 /* if called concurrently, only the first should proceed. */ 137 dev_dbg(ionic->dev, "%s: do_check_time again\n", __func__); 138 goto do_check_time; 139 } 140 141 /* firmware is useful only if the running bit is set and 142 * fw_status != 0xff (bad PCI read) 143 */ 144 fw_status = ioread8(&idev->dev_info_regs->fw_status); 145 fw_status_ready = (fw_status != 0xff) && (fw_status & IONIC_FW_STS_F_RUNNING); 146 147 /* is this a transition? */ 148 if (fw_status_ready != idev->fw_status_ready) { 149 struct ionic_lif *lif = ionic->lif; 150 bool trigger = false; 151 152 idev->fw_status_ready = fw_status_ready; 153 154 if (!fw_status_ready) { 155 dev_info(ionic->dev, "FW stopped %u\n", fw_status); 156 if (lif && !test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 157 trigger = true; 158 } else { 159 dev_info(ionic->dev, "FW running %u\n", fw_status); 160 if (lif && test_bit(IONIC_LIF_F_FW_RESET, lif->state)) 161 trigger = true; 162 } 163 164 if (trigger) { 165 struct ionic_deferred_work *work; 166 167 work = kzalloc(sizeof(*work), GFP_ATOMIC); 168 if (work) { 169 work->type = IONIC_DW_TYPE_LIF_RESET; 170 work->fw_status = fw_status_ready; 171 ionic_lif_deferred_enqueue(&lif->deferred, work); 172 } 173 } 174 } 175 176 if (!fw_status_ready) 177 return -ENXIO; 178 179 /* wait at least one watchdog period since the last heartbeat */ 180 last_check_time = idev->last_hb_time; 181 if (time_before(check_time, last_check_time + ionic->watchdog_period)) 182 return 0; 183 184 fw_hb = ioread32(&idev->dev_info_regs->fw_heartbeat); 185 fw_hb_ready = fw_hb != idev->last_fw_hb; 186 187 /* early FW version had no heartbeat, so fake it */ 188 if (!fw_hb_ready && !fw_hb) 189 fw_hb_ready = true; 190 191 dev_dbg(ionic->dev, "%s: fw_hb %u last_fw_hb %u ready %u\n", 192 __func__, fw_hb, idev->last_fw_hb, fw_hb_ready); 193 194 idev->last_fw_hb = fw_hb; 195 196 /* log a transition */ 197 if (fw_hb_ready != idev->fw_hb_ready) { 198 idev->fw_hb_ready = fw_hb_ready; 199 if (!fw_hb_ready) 200 dev_info(ionic->dev, "FW heartbeat stalled at %d\n", fw_hb); 201 else 202 dev_info(ionic->dev, "FW heartbeat restored at %d\n", fw_hb); 203 } 204 205 if (!fw_hb_ready) 206 return -ENXIO; 207 208 idev->last_hb_time = check_time; 209 210 return 0; 211 } 212 213 u8 ionic_dev_cmd_status(struct ionic_dev *idev) 214 { 215 return ioread8(&idev->dev_cmd_regs->comp.comp.status); 216 } 217 218 bool ionic_dev_cmd_done(struct ionic_dev *idev) 219 { 220 return ioread32(&idev->dev_cmd_regs->done) & IONIC_DEV_CMD_DONE; 221 } 222 223 void ionic_dev_cmd_comp(struct ionic_dev *idev, union ionic_dev_cmd_comp *comp) 224 { 225 memcpy_fromio(comp, &idev->dev_cmd_regs->comp, sizeof(*comp)); 226 } 227 228 void ionic_dev_cmd_go(struct ionic_dev *idev, union ionic_dev_cmd *cmd) 229 { 230 memcpy_toio(&idev->dev_cmd_regs->cmd, cmd, sizeof(*cmd)); 231 iowrite32(0, &idev->dev_cmd_regs->done); 232 iowrite32(1, &idev->dev_cmd_regs->doorbell); 233 } 234 235 /* Device commands */ 236 void ionic_dev_cmd_identify(struct ionic_dev *idev, u8 ver) 237 { 238 union ionic_dev_cmd cmd = { 239 .identify.opcode = IONIC_CMD_IDENTIFY, 240 .identify.ver = ver, 241 }; 242 243 ionic_dev_cmd_go(idev, &cmd); 244 } 245 246 void ionic_dev_cmd_init(struct ionic_dev *idev) 247 { 248 union ionic_dev_cmd cmd = { 249 .init.opcode = IONIC_CMD_INIT, 250 .init.type = 0, 251 }; 252 253 ionic_dev_cmd_go(idev, &cmd); 254 } 255 256 void ionic_dev_cmd_reset(struct ionic_dev *idev) 257 { 258 union ionic_dev_cmd cmd = { 259 .reset.opcode = IONIC_CMD_RESET, 260 }; 261 262 ionic_dev_cmd_go(idev, &cmd); 263 } 264 265 /* Port commands */ 266 void ionic_dev_cmd_port_identify(struct ionic_dev *idev) 267 { 268 union ionic_dev_cmd cmd = { 269 .port_init.opcode = IONIC_CMD_PORT_IDENTIFY, 270 .port_init.index = 0, 271 }; 272 273 ionic_dev_cmd_go(idev, &cmd); 274 } 275 276 void ionic_dev_cmd_port_init(struct ionic_dev *idev) 277 { 278 union ionic_dev_cmd cmd = { 279 .port_init.opcode = IONIC_CMD_PORT_INIT, 280 .port_init.index = 0, 281 .port_init.info_pa = cpu_to_le64(idev->port_info_pa), 282 }; 283 284 ionic_dev_cmd_go(idev, &cmd); 285 } 286 287 void ionic_dev_cmd_port_reset(struct ionic_dev *idev) 288 { 289 union ionic_dev_cmd cmd = { 290 .port_reset.opcode = IONIC_CMD_PORT_RESET, 291 .port_reset.index = 0, 292 }; 293 294 ionic_dev_cmd_go(idev, &cmd); 295 } 296 297 void ionic_dev_cmd_port_state(struct ionic_dev *idev, u8 state) 298 { 299 union ionic_dev_cmd cmd = { 300 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 301 .port_setattr.index = 0, 302 .port_setattr.attr = IONIC_PORT_ATTR_STATE, 303 .port_setattr.state = state, 304 }; 305 306 ionic_dev_cmd_go(idev, &cmd); 307 } 308 309 void ionic_dev_cmd_port_speed(struct ionic_dev *idev, u32 speed) 310 { 311 union ionic_dev_cmd cmd = { 312 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 313 .port_setattr.index = 0, 314 .port_setattr.attr = IONIC_PORT_ATTR_SPEED, 315 .port_setattr.speed = cpu_to_le32(speed), 316 }; 317 318 ionic_dev_cmd_go(idev, &cmd); 319 } 320 321 void ionic_dev_cmd_port_autoneg(struct ionic_dev *idev, u8 an_enable) 322 { 323 union ionic_dev_cmd cmd = { 324 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 325 .port_setattr.index = 0, 326 .port_setattr.attr = IONIC_PORT_ATTR_AUTONEG, 327 .port_setattr.an_enable = an_enable, 328 }; 329 330 ionic_dev_cmd_go(idev, &cmd); 331 } 332 333 void ionic_dev_cmd_port_fec(struct ionic_dev *idev, u8 fec_type) 334 { 335 union ionic_dev_cmd cmd = { 336 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 337 .port_setattr.index = 0, 338 .port_setattr.attr = IONIC_PORT_ATTR_FEC, 339 .port_setattr.fec_type = fec_type, 340 }; 341 342 ionic_dev_cmd_go(idev, &cmd); 343 } 344 345 void ionic_dev_cmd_port_pause(struct ionic_dev *idev, u8 pause_type) 346 { 347 union ionic_dev_cmd cmd = { 348 .port_setattr.opcode = IONIC_CMD_PORT_SETATTR, 349 .port_setattr.index = 0, 350 .port_setattr.attr = IONIC_PORT_ATTR_PAUSE, 351 .port_setattr.pause_type = pause_type, 352 }; 353 354 ionic_dev_cmd_go(idev, &cmd); 355 } 356 357 /* VF commands */ 358 int ionic_set_vf_config(struct ionic *ionic, int vf, u8 attr, u8 *data) 359 { 360 union ionic_dev_cmd cmd = { 361 .vf_setattr.opcode = IONIC_CMD_VF_SETATTR, 362 .vf_setattr.attr = attr, 363 .vf_setattr.vf_index = cpu_to_le16(vf), 364 }; 365 int err; 366 367 switch (attr) { 368 case IONIC_VF_ATTR_SPOOFCHK: 369 cmd.vf_setattr.spoofchk = *data; 370 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n", 371 __func__, vf, *data); 372 break; 373 case IONIC_VF_ATTR_TRUST: 374 cmd.vf_setattr.trust = *data; 375 dev_dbg(ionic->dev, "%s: vf %d trust %d\n", 376 __func__, vf, *data); 377 break; 378 case IONIC_VF_ATTR_LINKSTATE: 379 cmd.vf_setattr.linkstate = *data; 380 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n", 381 __func__, vf, *data); 382 break; 383 case IONIC_VF_ATTR_MAC: 384 ether_addr_copy(cmd.vf_setattr.macaddr, data); 385 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n", 386 __func__, vf, data); 387 break; 388 case IONIC_VF_ATTR_VLAN: 389 cmd.vf_setattr.vlanid = cpu_to_le16(*(u16 *)data); 390 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n", 391 __func__, vf, *(u16 *)data); 392 break; 393 case IONIC_VF_ATTR_RATE: 394 cmd.vf_setattr.maxrate = cpu_to_le32(*(u32 *)data); 395 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n", 396 __func__, vf, *(u32 *)data); 397 break; 398 case IONIC_VF_ATTR_STATSADDR: 399 cmd.vf_setattr.stats_pa = cpu_to_le64(*(u64 *)data); 400 dev_dbg(ionic->dev, "%s: vf %d stats_pa 0x%08llx\n", 401 __func__, vf, *(u64 *)data); 402 break; 403 default: 404 return -EINVAL; 405 } 406 407 mutex_lock(&ionic->dev_cmd_lock); 408 ionic_dev_cmd_go(&ionic->idev, &cmd); 409 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT); 410 mutex_unlock(&ionic->dev_cmd_lock); 411 412 return err; 413 } 414 415 /* LIF commands */ 416 void ionic_dev_cmd_queue_identify(struct ionic_dev *idev, 417 u16 lif_type, u8 qtype, u8 qver) 418 { 419 union ionic_dev_cmd cmd = { 420 .q_identify.opcode = IONIC_CMD_Q_IDENTIFY, 421 .q_identify.lif_type = cpu_to_le16(lif_type), 422 .q_identify.type = qtype, 423 .q_identify.ver = qver, 424 }; 425 426 ionic_dev_cmd_go(idev, &cmd); 427 } 428 429 void ionic_dev_cmd_lif_identify(struct ionic_dev *idev, u8 type, u8 ver) 430 { 431 union ionic_dev_cmd cmd = { 432 .lif_identify.opcode = IONIC_CMD_LIF_IDENTIFY, 433 .lif_identify.type = type, 434 .lif_identify.ver = ver, 435 }; 436 437 ionic_dev_cmd_go(idev, &cmd); 438 } 439 440 void ionic_dev_cmd_lif_init(struct ionic_dev *idev, u16 lif_index, 441 dma_addr_t info_pa) 442 { 443 union ionic_dev_cmd cmd = { 444 .lif_init.opcode = IONIC_CMD_LIF_INIT, 445 .lif_init.index = cpu_to_le16(lif_index), 446 .lif_init.info_pa = cpu_to_le64(info_pa), 447 }; 448 449 ionic_dev_cmd_go(idev, &cmd); 450 } 451 452 void ionic_dev_cmd_lif_reset(struct ionic_dev *idev, u16 lif_index) 453 { 454 union ionic_dev_cmd cmd = { 455 .lif_init.opcode = IONIC_CMD_LIF_RESET, 456 .lif_init.index = cpu_to_le16(lif_index), 457 }; 458 459 ionic_dev_cmd_go(idev, &cmd); 460 } 461 462 void ionic_dev_cmd_adminq_init(struct ionic_dev *idev, struct ionic_qcq *qcq, 463 u16 lif_index, u16 intr_index) 464 { 465 struct ionic_queue *q = &qcq->q; 466 struct ionic_cq *cq = &qcq->cq; 467 468 union ionic_dev_cmd cmd = { 469 .q_init.opcode = IONIC_CMD_Q_INIT, 470 .q_init.lif_index = cpu_to_le16(lif_index), 471 .q_init.type = q->type, 472 .q_init.ver = qcq->q.lif->qtype_info[q->type].version, 473 .q_init.index = cpu_to_le32(q->index), 474 .q_init.flags = cpu_to_le16(IONIC_QINIT_F_IRQ | 475 IONIC_QINIT_F_ENA), 476 .q_init.pid = cpu_to_le16(q->pid), 477 .q_init.intr_index = cpu_to_le16(intr_index), 478 .q_init.ring_size = ilog2(q->num_descs), 479 .q_init.ring_base = cpu_to_le64(q->base_pa), 480 .q_init.cq_ring_base = cpu_to_le64(cq->base_pa), 481 }; 482 483 ionic_dev_cmd_go(idev, &cmd); 484 } 485 486 int ionic_db_page_num(struct ionic_lif *lif, int pid) 487 { 488 return (lif->hw_index * lif->dbid_count) + pid; 489 } 490 491 int ionic_cq_init(struct ionic_lif *lif, struct ionic_cq *cq, 492 struct ionic_intr_info *intr, 493 unsigned int num_descs, size_t desc_size) 494 { 495 unsigned int ring_size; 496 497 if (desc_size == 0 || !is_power_of_2(num_descs)) 498 return -EINVAL; 499 500 ring_size = ilog2(num_descs); 501 if (ring_size < 2 || ring_size > 16) 502 return -EINVAL; 503 504 cq->lif = lif; 505 cq->bound_intr = intr; 506 cq->num_descs = num_descs; 507 cq->desc_size = desc_size; 508 cq->tail_idx = 0; 509 cq->done_color = 1; 510 511 return 0; 512 } 513 514 void ionic_cq_map(struct ionic_cq *cq, void *base, dma_addr_t base_pa) 515 { 516 struct ionic_cq_info *cur; 517 unsigned int i; 518 519 cq->base = base; 520 cq->base_pa = base_pa; 521 522 for (i = 0, cur = cq->info; i < cq->num_descs; i++, cur++) 523 cur->cq_desc = base + (i * cq->desc_size); 524 } 525 526 void ionic_cq_bind(struct ionic_cq *cq, struct ionic_queue *q) 527 { 528 cq->bound_q = q; 529 } 530 531 unsigned int ionic_cq_service(struct ionic_cq *cq, unsigned int work_to_do, 532 ionic_cq_cb cb, ionic_cq_done_cb done_cb, 533 void *done_arg) 534 { 535 struct ionic_cq_info *cq_info; 536 unsigned int work_done = 0; 537 538 if (work_to_do == 0) 539 return 0; 540 541 cq_info = &cq->info[cq->tail_idx]; 542 while (cb(cq, cq_info)) { 543 if (cq->tail_idx == cq->num_descs - 1) 544 cq->done_color = !cq->done_color; 545 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 546 cq_info = &cq->info[cq->tail_idx]; 547 DEBUG_STATS_CQE_CNT(cq); 548 549 if (++work_done >= work_to_do) 550 break; 551 } 552 553 if (work_done && done_cb) 554 done_cb(done_arg); 555 556 return work_done; 557 } 558 559 int ionic_q_init(struct ionic_lif *lif, struct ionic_dev *idev, 560 struct ionic_queue *q, unsigned int index, const char *name, 561 unsigned int num_descs, size_t desc_size, 562 size_t sg_desc_size, unsigned int pid) 563 { 564 unsigned int ring_size; 565 566 if (desc_size == 0 || !is_power_of_2(num_descs)) 567 return -EINVAL; 568 569 ring_size = ilog2(num_descs); 570 if (ring_size < 2 || ring_size > 16) 571 return -EINVAL; 572 573 q->lif = lif; 574 q->idev = idev; 575 q->index = index; 576 q->num_descs = num_descs; 577 q->desc_size = desc_size; 578 q->sg_desc_size = sg_desc_size; 579 q->tail_idx = 0; 580 q->head_idx = 0; 581 q->pid = pid; 582 583 snprintf(q->name, sizeof(q->name), "L%d-%s%u", lif->index, name, index); 584 585 return 0; 586 } 587 588 void ionic_q_map(struct ionic_queue *q, void *base, dma_addr_t base_pa) 589 { 590 struct ionic_desc_info *cur; 591 unsigned int i; 592 593 q->base = base; 594 q->base_pa = base_pa; 595 596 for (i = 0, cur = q->info; i < q->num_descs; i++, cur++) 597 cur->desc = base + (i * q->desc_size); 598 } 599 600 void ionic_q_sg_map(struct ionic_queue *q, void *base, dma_addr_t base_pa) 601 { 602 struct ionic_desc_info *cur; 603 unsigned int i; 604 605 q->sg_base = base; 606 q->sg_base_pa = base_pa; 607 608 for (i = 0, cur = q->info; i < q->num_descs; i++, cur++) 609 cur->sg_desc = base + (i * q->sg_desc_size); 610 } 611 612 void ionic_q_post(struct ionic_queue *q, bool ring_doorbell, ionic_desc_cb cb, 613 void *cb_arg) 614 { 615 struct ionic_desc_info *desc_info; 616 struct ionic_lif *lif = q->lif; 617 struct device *dev = q->dev; 618 619 desc_info = &q->info[q->head_idx]; 620 desc_info->cb = cb; 621 desc_info->cb_arg = cb_arg; 622 623 q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); 624 625 dev_dbg(dev, "lif=%d qname=%s qid=%d qtype=%d p_index=%d ringdb=%d\n", 626 q->lif->index, q->name, q->hw_type, q->hw_index, 627 q->head_idx, ring_doorbell); 628 629 if (ring_doorbell) 630 ionic_dbell_ring(lif->kern_dbpage, q->hw_type, 631 q->dbval | q->head_idx); 632 } 633 634 static bool ionic_q_is_posted(struct ionic_queue *q, unsigned int pos) 635 { 636 unsigned int mask, tail, head; 637 638 mask = q->num_descs - 1; 639 tail = q->tail_idx; 640 head = q->head_idx; 641 642 return ((pos - tail) & mask) < ((head - tail) & mask); 643 } 644 645 void ionic_q_service(struct ionic_queue *q, struct ionic_cq_info *cq_info, 646 unsigned int stop_index) 647 { 648 struct ionic_desc_info *desc_info; 649 ionic_desc_cb cb; 650 void *cb_arg; 651 u16 index; 652 653 /* check for empty queue */ 654 if (q->tail_idx == q->head_idx) 655 return; 656 657 /* stop index must be for a descriptor that is not yet completed */ 658 if (unlikely(!ionic_q_is_posted(q, stop_index))) 659 dev_err(q->dev, 660 "ionic stop is not posted %s stop %u tail %u head %u\n", 661 q->name, stop_index, q->tail_idx, q->head_idx); 662 663 do { 664 desc_info = &q->info[q->tail_idx]; 665 index = q->tail_idx; 666 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 667 668 cb = desc_info->cb; 669 cb_arg = desc_info->cb_arg; 670 671 desc_info->cb = NULL; 672 desc_info->cb_arg = NULL; 673 674 if (cb) 675 cb(q, desc_info, cq_info, cb_arg); 676 } while (index != stop_index); 677 } 678