480     register struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_init()  local
492     dqrr->ring = ptr_ADD(portal->addr.addr_ce, CL_DQRR);  in qm_dqrr_init()
493     dqrr->pi = (uint8_t)(qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1));  in qm_dqrr_init()
494     dqrr->ci = (uint8_t)(qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1));  in qm_dqrr_init()
495     dqrr->cursor = dqrr->ring + dqrr->ci;  in qm_dqrr_init()
496     dqrr->fill = cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);  in qm_dqrr_init()
497     dqrr->vbit = (uint8_t)((qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?  in qm_dqrr_init()
499     dqrr->ithresh = (uint8_t)qm_in(DQRR_ITR);  in qm_dqrr_init()
502     dqrr->dmode = dmode;  in qm_dqrr_init()
503     dqrr->pmode = pmode;  in qm_dqrr_init()
504     dqrr->cmode = cmode;  in qm_dqrr_init()
505     dqrr->flags = 0;  in qm_dqrr_init()
507         dqrr->flags |= QM_DQRR_FLAG_RE;  in qm_dqrr_init()
509         dqrr->flags |= QM_DQRR_FLAG_SE;  in qm_dqrr_init()
529     register struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_finish()  local
530     if (dqrr->ci != DQRR_PTR2IDX(dqrr->cursor))  in qm_dqrr_finish()
537     register struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_current()  local
538     if (!dqrr->fill)  in qm_dqrr_current()
540     return dqrr->cursor;  in qm_dqrr_current()
545     register struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_cursor()  local
546     return DQRR_PTR2IDX(dqrr->cursor);  in qm_dqrr_cursor()
551     register struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_next()  local
553     ASSERT_COND(dqrr->fill);  in qm_dqrr_next()
555     dqrr->cursor = DQRR_INC(dqrr->cursor);  in qm_dqrr_next()
556     return --dqrr->fill;  in qm_dqrr_next()
561     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrPciUpdate()  local
562     uint8_t diff, old_pi = dqrr->pi;  in qmPortalDqrrPciUpdate()
564     ASSERT_COND(dqrr->pmode == e_QmPortalPCI);  in qmPortalDqrrPciUpdate()
566     dqrr->pi = (uint8_t)(qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1));  in qmPortalDqrrPciUpdate()
567     diff = cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);  in qmPortalDqrrPciUpdate()
568     dqrr->fill += diff;  in qmPortalDqrrPciUpdate()
575     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrPcePrefetch()  local
576     ASSERT_COND(dqrr->pmode == e_QmPortalPCE);  in qmPortalDqrrPcePrefetch()
584     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrPceUpdate()  local
585     uint8_t diff, old_pi = dqrr->pi;  in qmPortalDqrrPceUpdate()
587     ASSERT_COND(dqrr->pmode == e_QmPortalPCE);  in qmPortalDqrrPceUpdate()
589     dqrr->pi = (uint8_t)(qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1));  in qmPortalDqrrPceUpdate()
590     diff = cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);  in qmPortalDqrrPceUpdate()
591     dqrr->fill += diff;  in qmPortalDqrrPceUpdate()
597     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrPvbPrefetch()  local
599     ASSERT_COND(dqrr->pmode == e_QmPortalPVB);  in qmPortalDqrrPvbPrefetch()
601     if (!(dqrr->flags & QM_DQRR_FLAG_RE))  in qmPortalDqrrPvbPrefetch()
603         dcbit_ro(ptr_ADD(dqrr->ring, qm_cl(dqrr->pi)));  in qmPortalDqrrPvbPrefetch()
608     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrPvbUpdate()  local
609     struct qm_dqrr_entry *res = ptr_ADD(dqrr->ring, qm_cl(dqrr->pi));  in qmPortalDqrrPvbUpdate()
611     ASSERT_COND(dqrr->pmode == e_QmPortalPVB);  in qmPortalDqrrPvbUpdate()
613     if ((res->verb & QM_DQRR_VERB_VBIT) == dqrr->vbit) {  in qmPortalDqrrPvbUpdate()
614         dqrr->pi = (uint8_t)((dqrr->pi + 1) & (QM_DQRR_SIZE - 1));  in qmPortalDqrrPvbUpdate()
615         if (!dqrr->pi)  in qmPortalDqrrPvbUpdate()
616             dqrr->vbit ^= QM_DQRR_VERB_VBIT;  in qmPortalDqrrPvbUpdate()
617         dqrr->fill++;  in qmPortalDqrrPvbUpdate()
625     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrCciConsume()  local
627     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCI);  in qmPortalDqrrCciConsume()
629     dqrr->ci = (uint8_t)((dqrr->ci + num) & (QM_DQRR_SIZE - 1));  in qmPortalDqrrCciConsume()
630     qm_out(DQRR_CI_CINH, dqrr->ci);  in qmPortalDqrrCciConsume()
635     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrCciConsumeToCurrent()  local
637     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCI);  in qmPortalDqrrCciConsumeToCurrent()
639     dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);  in qmPortalDqrrCciConsumeToCurrent()
640     qm_out(DQRR_CI_CINH, dqrr->ci);  in qmPortalDqrrCciConsumeToCurrent()
646     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrCcePrefetch()  local
647     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCE);  in qmPortalDqrrCcePrefetch()
655     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrCceConsume()  local
657     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCE);  in qmPortalDqrrCceConsume()
659     dqrr->ci = (uint8_t)((dqrr->ci + num) & (QM_DQRR_SIZE - 1));  in qmPortalDqrrCceConsume()
660     qm_cl_out(DQRR_CI, dqrr->ci);  in qmPortalDqrrCceConsume()
665     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrCceConsume_to_current()  local
667     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrCCE);  in qmPortalDqrrCceConsume_to_current()
669     dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);  in qmPortalDqrrCceConsume_to_current()
670     qm_cl_out(DQRR_CI, dqrr->ci);  in qmPortalDqrrCceConsume_to_current()
676     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrDcaConsume1()  local
677     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA);  in qmPortalDqrrDcaConsume1()
691     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrDcaConsume1ptr()  local
693     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA);  in qmPortalDqrrDcaConsume1ptr()
694     ASSERT_COND((dqrr->ring + idx) == dq);  in qmPortalDqrrDcaConsume1ptr()
705     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrDcaConsumeN()  local
706     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA);  in qmPortalDqrrDcaConsumeN()
715     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrDcaCci()  local
716     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA);  in qmPortalDqrrDcaCci()
724     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrDcaCcePrefetch()  local
725     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA);  in qmPortalDqrrDcaCcePrefetch()
734     register struct qm_dqrr *dqrr = &portal->dqrr;  in qmPortalDqrrDcaCce()  local
735     ASSERT_COND(dqrr->cmode == e_QmPortalDqrrDCA);  in qmPortalDqrrDcaCce()
742     register struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_get_ci()  local
744     ASSERT_COND(dqrr->cmode != e_QmPortalDqrrDCA);  in qm_dqrr_get_ci()
747     return dqrr->ci;  in qm_dqrr_get_ci()
753     register struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_park()  local
754     ASSERT_COND(dqrr->cmode != e_QmPortalDqrrDCA);  in qm_dqrr_park()
764     register struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_park_ci()  local
766     ASSERT_COND(dqrr->cmode != e_QmPortalDqrrDCA);  in qm_dqrr_park_ci()
770         (dqrr->ci & (QM_DQRR_SIZE - 1)));/* DCAP_CI */  in qm_dqrr_park_ci()
805     register struct qm_dqrr *dqrr = &portal->dqrr;  in qm_dqrr_get_ithresh()  local
806     return dqrr->ithresh;  in qm_dqrr_get_ithresh()