167 static __inline__ void RCR_INC(struct bm_rcr *rcr)  in RCR_INC()  argument
172     struct bm_rcr_entry *partial = rcr->cursor + 1;  in RCR_INC()
173     rcr->cursor = RCR_CARRYCLEAR(partial);  in RCR_INC()
174     if (partial != rcr->cursor)  in RCR_INC()
175         rcr->vbit ^= BM_RCR_VERB_VBIT;  in RCR_INC()
182     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_init()  local
186     rcr->ring = ptr_ADD(portal->addr.addr_ce, CL_RCR);  in bm_rcr_init()
187     rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));  in bm_rcr_init()
189     rcr->cursor = rcr->ring + pi;  in bm_rcr_init()
190     rcr->vbit = (uint8_t)((bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ?  BM_RCR_VERB_VBIT : 0);  in bm_rcr_init()
191     rcr->available = (uint8_t)(BM_RCR_SIZE - 1 - cyc_diff(BM_RCR_SIZE, rcr->ci, pi));  in bm_rcr_init()
192     rcr->ithresh = (uint8_t)bm_in(RCR_ITR);  in bm_rcr_init()
194     rcr->busy = 0;  in bm_rcr_init()
195     rcr->pmode = pmode;  in bm_rcr_init()
196     rcr->cmode = cmode;  in bm_rcr_init()
207     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_finish()  local
210     ASSERT_COND(!rcr->busy);  in bm_rcr_finish()
211     if (pi != RCR_PTR2IDX(rcr->cursor))  in bm_rcr_finish()
213     if (ci != rcr->ci)  in bm_rcr_finish()
215     if (rcr->ci != RCR_PTR2IDX(rcr->cursor))  in bm_rcr_finish()
221     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_start()  local
222     ASSERT_COND(!rcr->busy);  in bm_rcr_start()
223     if (!rcr->available)  in bm_rcr_start()
226     rcr->busy = 1;  in bm_rcr_start()
228     dcbz_64(rcr->cursor);  in bm_rcr_start()
229     return rcr->cursor;  in bm_rcr_start()
234     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_abort()  local
235     ASSERT_COND(rcr->busy);  in bm_rcr_abort()
237     rcr->busy = 0;  in bm_rcr_abort()
239     UNUSED(rcr);  in bm_rcr_abort()
245     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_pend_and_next()  local
246     ASSERT_COND(rcr->busy);  in bm_rcr_pend_and_next()
247     ASSERT_COND(rcr->pmode != e_BmPortalPVB);  in bm_rcr_pend_and_next()
248     if (rcr->available == 1)  in bm_rcr_pend_and_next()
250     rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);  in bm_rcr_pend_and_next()
251     dcbf_64(rcr->cursor);  in bm_rcr_pend_and_next()
252     RCR_INC(rcr);  in bm_rcr_pend_and_next()
253     rcr->available--;  in bm_rcr_pend_and_next()
254     dcbz_64(rcr->cursor);  in bm_rcr_pend_and_next()
255     return rcr->cursor;  in bm_rcr_pend_and_next()
260     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_pci_commit()  local
261     ASSERT_COND(rcr->busy);  in bm_rcr_pci_commit()
262     ASSERT_COND(rcr->pmode == e_BmPortalPCI);  in bm_rcr_pci_commit()
263     rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);  in bm_rcr_pci_commit()
264     RCR_INC(rcr);  in bm_rcr_pci_commit()
265     rcr->available--;  in bm_rcr_pci_commit()
267     bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));  in bm_rcr_pci_commit()
269     rcr->busy = 0;  in bm_rcr_pci_commit()
275     ASSERT_COND(((struct bm_rcr *)&portal->rcr)->pmode == e_BmPortalPCE);  in bm_rcr_pce_prefetch()
282     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_pce_commit()  local
283     ASSERT_COND(rcr->busy);  in bm_rcr_pce_commit()
284     ASSERT_COND(rcr->pmode == e_BmPortalPCE);  in bm_rcr_pce_commit()
285     rcr->cursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);  in bm_rcr_pce_commit()
286     RCR_INC(rcr);  in bm_rcr_pce_commit()
287     rcr->available--;  in bm_rcr_pce_commit()
289     bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));  in bm_rcr_pce_commit()
291     rcr->busy = 0;  in bm_rcr_pce_commit()
297     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_pvb_commit()  local
299     ASSERT_COND(rcr->busy);  in bm_rcr_pvb_commit()
300     ASSERT_COND(rcr->pmode == e_BmPortalPVB);  in bm_rcr_pvb_commit()
302     rcursor = rcr->cursor;  in bm_rcr_pvb_commit()
303     rcursor->__dont_write_directly__verb = (uint8_t)(myverb | rcr->vbit);  in bm_rcr_pvb_commit()
305     RCR_INC(rcr);  in bm_rcr_pvb_commit()
306     rcr->available--;  in bm_rcr_pvb_commit()
308     rcr->busy = 0;  in bm_rcr_pvb_commit()
315     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_cci_update()  local
316     uint8_t diff, old_ci = rcr->ci;  in bm_rcr_cci_update()
317     ASSERT_COND(rcr->cmode == e_BmPortalRcrCCI);  in bm_rcr_cci_update()
318     rcr->ci = (uint8_t)(bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1));  in bm_rcr_cci_update()
319     diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);  in bm_rcr_cci_update()
320     rcr->available += diff;  in bm_rcr_cci_update()
327     ASSERT_COND(((struct bm_rcr *)&portal->rcr)->cmode == e_BmPortalRcrCCE);  in bm_rcr_cce_prefetch()
334     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_cce_update()  local
335     uint8_t diff, old_ci = rcr->ci;  in bm_rcr_cce_update()
336     ASSERT_COND(rcr->cmode == e_BmPortalRcrCCE);  in bm_rcr_cce_update()
337     rcr->ci = (uint8_t)(bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1));  in bm_rcr_cce_update()
339     diff = cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);  in bm_rcr_cce_update()
340     rcr->available += diff;  in bm_rcr_cce_update()
347     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_get_ithresh()  local
348     return rcr->ithresh;  in bm_rcr_get_ithresh()
354     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_set_ithresh()  local
355     rcr->ithresh = ithresh;  in bm_rcr_set_ithresh()
362     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_get_avail()  local
363     return rcr->available;  in bm_rcr_get_avail()
369     register struct bm_rcr *rcr = &portal->rcr;  in bm_rcr_get_fill()  local
370     return (uint8_t)(BM_RCR_SIZE - 1 - rcr->available);  in bm_rcr_get_fill()