Lines Matching +full:bman +full:- +full:portal
1 /* Copyright 2008 - 2016 Freescale Semiconductor, Inc.
33 #define IRQNAME "BMan portal %d"
34 #define MAX_IRQNAME 16 /* big enough for "BMan portal %d" */
36 /* Portal register assists */
39 /* Cache-inhibited register offsets */
50 /* Cache-enabled register offsets */
59 /* Cache-inhibited register offsets */
70 /* Cache-enabled register offsets */
80 * Portal modes.
84 * Enum values use 3 letter codes. First letter matches the portal mode,
86 * ci == cache-inhibited portal register
87 * ce == cache-enabled portal register
88 * vb == in-band valid-bit (cache-enabled)
91 bm_rcr_pci = 0, /* PI index, cache-inhibited */
92 bm_rcr_pce = 1, /* PI index, cache-enabled */
93 bm_rcr_pvb = 2 /* valid-bit */
95 enum bm_rcr_cmode { /* s/w-only */
96 bm_rcr_cci, /* CI index, cache-inhibited */
97 bm_rcr_cce /* CI index, cache-enabled */
101 /* --- Portal structures --- */
109 u8 _ncw_verb; /* writes to this are non-coherent */
134 u8 _ncw_verb; /* writes to this are non-coherent */
179 void *ce; /* cache-enabled */
181 void __iomem *ci; /* cache-inhibited */
190 /* Cache-inhibited register access. */
193 return ioread32be(p->addr.ci + offset); in bm_in()
198 iowrite32be(val, p->addr.ci + offset); in bm_out()
201 /* Cache Enabled Portal Access */
204 dpaa_invalidate(p->addr.ce + offset); in bm_cl_invalidate()
209 dpaa_touch_ro(p->addr.ce + offset); in bm_cl_touch_ro()
214 return be32_to_cpu(*(p->addr.ce_be + (offset/4))); in bm_ce_in()
221 /* probing time config params for cpu-affine portals */
242 * more than one such object per BMan buffer pool, eg. if different users of the
246 /* index of the buffer pool to encapsulate (0-63) */
248 /* Used for hash-table admin when using depletion notifications. */
249 struct bman_portal *portal; member
258 struct bm_portal *portal = &p->p; in portal_isr() local
259 u32 clear = p->irq_sources; in portal_isr()
260 u32 is = bm_in(portal, BM_REG_ISR) & p->irq_sources; in portal_isr()
266 bm_out(portal, BM_REG_ISR, clear); in portal_isr()
270 /* --- RCR API --- */
275 /* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
286 /* Bit-wise logic to convert a ring pointer to a ring index */
289 return ((uintptr_t)e >> RCR_SHIFT) & (BM_RCR_SIZE - 1); in rcr_ptr2idx()
297 struct bm_rcr_entry *partial = rcr->cursor + 1; in rcr_inc()
299 rcr->cursor = rcr_carryclear(partial); in rcr_inc()
300 if (partial != rcr->cursor) in rcr_inc()
301 rcr->vbit ^= BM_RCR_VERB_VBIT; in rcr_inc()
304 static int bm_rcr_get_avail(struct bm_portal *portal) in bm_rcr_get_avail() argument
306 struct bm_rcr *rcr = &portal->rcr; in bm_rcr_get_avail()
308 return rcr->available; in bm_rcr_get_avail()
311 static int bm_rcr_get_fill(struct bm_portal *portal) in bm_rcr_get_fill() argument
313 struct bm_rcr *rcr = &portal->rcr; in bm_rcr_get_fill()
315 return BM_RCR_SIZE - 1 - rcr->available; in bm_rcr_get_fill()
318 static void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh) in bm_rcr_set_ithresh() argument
320 struct bm_rcr *rcr = &portal->rcr; in bm_rcr_set_ithresh()
322 rcr->ithresh = ithresh; in bm_rcr_set_ithresh()
323 bm_out(portal, BM_REG_RCR_ITR, ithresh); in bm_rcr_set_ithresh()
326 static void bm_rcr_cce_prefetch(struct bm_portal *portal) in bm_rcr_cce_prefetch() argument
328 __maybe_unused struct bm_rcr *rcr = &portal->rcr; in bm_rcr_cce_prefetch()
330 DPAA_ASSERT(rcr->cmode == bm_rcr_cce); in bm_rcr_cce_prefetch()
331 bm_cl_touch_ro(portal, BM_CL_RCR_CI_CENA); in bm_rcr_cce_prefetch()
334 static u8 bm_rcr_cce_update(struct bm_portal *portal) in bm_rcr_cce_update() argument
336 struct bm_rcr *rcr = &portal->rcr; in bm_rcr_cce_update()
337 u8 diff, old_ci = rcr->ci; in bm_rcr_cce_update()
339 DPAA_ASSERT(rcr->cmode == bm_rcr_cce); in bm_rcr_cce_update()
340 rcr->ci = bm_ce_in(portal, BM_CL_RCR_CI_CENA) & (BM_RCR_SIZE - 1); in bm_rcr_cce_update()
341 bm_cl_invalidate(portal, BM_CL_RCR_CI_CENA); in bm_rcr_cce_update()
342 diff = dpaa_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci); in bm_rcr_cce_update()
343 rcr->available += diff; in bm_rcr_cce_update()
347 static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal) in bm_rcr_start() argument
349 struct bm_rcr *rcr = &portal->rcr; in bm_rcr_start()
351 DPAA_ASSERT(!rcr->busy); in bm_rcr_start()
352 if (!rcr->available) in bm_rcr_start()
355 rcr->busy = 1; in bm_rcr_start()
357 dpaa_zero(rcr->cursor); in bm_rcr_start()
358 return rcr->cursor; in bm_rcr_start()
361 static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb) in bm_rcr_pvb_commit() argument
363 struct bm_rcr *rcr = &portal->rcr; in bm_rcr_pvb_commit()
366 DPAA_ASSERT(rcr->busy); in bm_rcr_pvb_commit()
367 DPAA_ASSERT(rcr->pmode == bm_rcr_pvb); in bm_rcr_pvb_commit()
368 DPAA_ASSERT(rcr->available >= 1); in bm_rcr_pvb_commit()
370 rcursor = rcr->cursor; in bm_rcr_pvb_commit()
371 rcursor->_ncw_verb = myverb | rcr->vbit; in bm_rcr_pvb_commit()
374 rcr->available--; in bm_rcr_pvb_commit()
376 rcr->busy = 0; in bm_rcr_pvb_commit()
380 static int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode, in bm_rcr_init() argument
383 struct bm_rcr *rcr = &portal->rcr; in bm_rcr_init()
387 rcr->ring = portal->addr.ce + BM_CL_RCR; in bm_rcr_init()
388 rcr->ci = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1); in bm_rcr_init()
389 pi = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1); in bm_rcr_init()
390 rcr->cursor = rcr->ring + pi; in bm_rcr_init()
391 rcr->vbit = (bm_in(portal, BM_REG_RCR_PI_CINH) & BM_RCR_SIZE) ? in bm_rcr_init()
393 rcr->available = BM_RCR_SIZE - 1 in bm_rcr_init()
394 - dpaa_cyc_diff(BM_RCR_SIZE, rcr->ci, pi); in bm_rcr_init()
395 rcr->ithresh = bm_in(portal, BM_REG_RCR_ITR); in bm_rcr_init()
397 rcr->busy = 0; in bm_rcr_init()
398 rcr->pmode = pmode; in bm_rcr_init()
399 rcr->cmode = cmode; in bm_rcr_init()
401 cfg = (bm_in(portal, BM_REG_CFG) & 0xffffffe0) in bm_rcr_init()
403 bm_out(portal, BM_REG_CFG, cfg); in bm_rcr_init()
407 static void bm_rcr_finish(struct bm_portal *portal) in bm_rcr_finish() argument
410 struct bm_rcr *rcr = &portal->rcr; in bm_rcr_finish()
413 DPAA_ASSERT(!rcr->busy); in bm_rcr_finish()
415 i = bm_in(portal, BM_REG_RCR_PI_CINH) & (BM_RCR_SIZE - 1); in bm_rcr_finish()
416 if (i != rcr_ptr2idx(rcr->cursor)) in bm_rcr_finish()
419 i = bm_in(portal, BM_REG_RCR_CI_CINH) & (BM_RCR_SIZE - 1); in bm_rcr_finish()
420 if (i != rcr->ci) in bm_rcr_finish()
422 if (rcr->ci != rcr_ptr2idx(rcr->cursor)) in bm_rcr_finish()
427 /* --- Management command API --- */
428 static int bm_mc_init(struct bm_portal *portal) in bm_mc_init() argument
430 struct bm_mc *mc = &portal->mc; in bm_mc_init()
432 mc->cr = portal->addr.ce + BM_CL_CR; in bm_mc_init()
433 mc->rr = portal->addr.ce + BM_CL_RR0; in bm_mc_init()
434 mc->rridx = (mc->cr->_ncw_verb & BM_MCC_VERB_VBIT) ? in bm_mc_init()
436 mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0; in bm_mc_init()
438 mc->state = mc_idle; in bm_mc_init()
443 static void bm_mc_finish(struct bm_portal *portal) in bm_mc_finish() argument
446 struct bm_mc *mc = &portal->mc; in bm_mc_finish()
448 DPAA_ASSERT(mc->state == mc_idle); in bm_mc_finish()
449 if (mc->state != mc_idle) in bm_mc_finish()
454 static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal) in bm_mc_start() argument
456 struct bm_mc *mc = &portal->mc; in bm_mc_start()
458 DPAA_ASSERT(mc->state == mc_idle); in bm_mc_start()
460 mc->state = mc_user; in bm_mc_start()
462 dpaa_zero(mc->cr); in bm_mc_start()
463 return mc->cr; in bm_mc_start()
466 static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb) in bm_mc_commit() argument
468 struct bm_mc *mc = &portal->mc; in bm_mc_commit()
469 union bm_mc_result *rr = mc->rr + mc->rridx; in bm_mc_commit()
471 DPAA_ASSERT(mc->state == mc_user); in bm_mc_commit()
473 mc->cr->_ncw_verb = myverb | mc->vbit; in bm_mc_commit()
474 dpaa_flush(mc->cr); in bm_mc_commit()
477 mc->state = mc_hw; in bm_mc_commit()
481 static inline union bm_mc_result *bm_mc_result(struct bm_portal *portal) in bm_mc_result() argument
483 struct bm_mc *mc = &portal->mc; in bm_mc_result()
484 union bm_mc_result *rr = mc->rr + mc->rridx; in bm_mc_result()
486 DPAA_ASSERT(mc->state == mc_hw); in bm_mc_result()
489 * its command is submitted and completed. This includes the valid-bit, in bm_mc_result()
492 if (!rr->verb) { in bm_mc_result()
496 mc->rridx ^= 1; in bm_mc_result()
497 mc->vbit ^= BM_MCC_VERB_VBIT; in bm_mc_result()
499 mc->state = mc_idle; in bm_mc_result()
504 static inline int bm_mc_result_timeout(struct bm_portal *portal, in bm_mc_result_timeout() argument
510 *mcr = bm_mc_result(portal); in bm_mc_result_timeout()
514 } while (--timeout); in bm_mc_result_timeout()
519 /* Disable all BSCN interrupts for the portal */
520 static void bm_isr_bscn_disable(struct bm_portal *portal) in bm_isr_bscn_disable() argument
522 bm_out(portal, BM_REG_SCN(0), 0); in bm_isr_bscn_disable()
523 bm_out(portal, BM_REG_SCN(1), 0); in bm_isr_bscn_disable()
526 static int bman_create_portal(struct bman_portal *portal, in bman_create_portal() argument
532 p = &portal->p; in bman_create_portal()
534 * prep the low-level portal struct with the mapped addresses from the in bman_create_portal()
538 p->addr.ce = c->addr_virt_ce; in bman_create_portal()
539 p->addr.ce_be = c->addr_virt_ce; in bman_create_portal()
540 p->addr.ci = c->addr_virt_ci; in bman_create_portal()
542 dev_err(c->dev, "RCR initialisation failed\n"); in bman_create_portal()
546 dev_err(c->dev, "MC initialisation failed\n"); in bman_create_portal()
551 * run-time. in bman_create_portal()
555 /* Write-to-clear any stale interrupt status bits */ in bman_create_portal()
557 portal->irq_sources = 0; in bman_create_portal()
560 snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, c->cpu); in bman_create_portal()
561 if (request_irq(c->irq, portal_isr, 0, portal->irqname, portal)) { in bman_create_portal()
562 dev_err(c->dev, "request_irq() failed\n"); in bman_create_portal()
566 if (dpaa_set_portal_irq_affinity(c->dev, c->irq, c->cpu)) in bman_create_portal()
572 dev_err(c->dev, "RCR unclean\n"); in bman_create_portal()
576 portal->config = c; in bman_create_portal()
585 free_irq(c->irq, portal); in bman_create_portal()
591 return -EIO; in bman_create_portal()
596 struct bman_portal *portal; in bman_create_affine_portal() local
599 portal = &per_cpu(bman_affine_portal, c->cpu); in bman_create_affine_portal()
600 err = bman_create_portal(portal, c); in bman_create_affine_portal()
605 cpumask_set_cpu(c->cpu, &affine_mask); in bman_create_affine_portal()
608 return portal; in bman_create_affine_portal()
616 bm_rcr_cce_update(&p->p); in poll_portal_slow()
617 bm_rcr_set_ithresh(&p->p, 0); in poll_portal_slow()
618 bm_out(&p->p, BM_REG_ISR, BM_PIRQ_RCRI); in poll_portal_slow()
632 p->irq_sources |= bits & BM_PIRQ_VISIBLE; in bman_p_irqsource_add()
633 bm_out(&p->p, BM_REG_IER, p->irq_sources); in bman_p_irqsource_add()
648 bm_cmd = bm_mc_start(&p->p); in bm_shutdown_pool()
649 bm_cmd->bpid = bpid; in bm_shutdown_pool()
650 bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | 1); in bm_shutdown_pool()
651 if (!bm_mc_result_timeout(&p->p, &bm_res)) { in bm_shutdown_pool()
652 pr_crit("BMan Acquire Command timedout\n"); in bm_shutdown_pool()
653 err = -ETIMEDOUT; in bm_shutdown_pool()
656 if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) { in bm_shutdown_pool()
674 return -ENOMEM; in bm_alloc_bpid_range()
707 pool->bpid = bpid; in bman_new_pool()
718 bm_release_bpid(pool->bpid); in bman_free_pool()
726 return pool->bpid; in bman_get_bpid()
733 bm_rcr_cce_prefetch(&p->p); in update_rcr_ci()
735 bm_rcr_cce_update(&p->p); in update_rcr_ci()
744 int i = num - 1; in bman_release()
751 avail = bm_rcr_get_avail(&p->p); in bman_release()
754 r = bm_rcr_start(&p->p); in bman_release()
761 } while (--timeout); in bman_release()
764 return -ETIMEDOUT; in bman_release()
770 * with the valid-bit in bman_release()
772 bm_buffer_set64(r->bufs, bm_buffer_get64(bufs)); in bman_release()
773 bm_buffer_set_bpid(r->bufs, pool->bpid); in bman_release()
775 memcpy(&r->bufs[1], &bufs[1], i * sizeof(bufs[0])); in bman_release()
777 bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE | in bman_release()
795 mcc = bm_mc_start(&p->p); in bman_acquire()
796 mcc->bpid = pool->bpid; in bman_acquire()
797 bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE | in bman_acquire()
799 if (!bm_mc_result_timeout(&p->p, &mcr)) { in bman_acquire()
801 pr_crit("BMan Acquire Timeout\n"); in bman_acquire()
802 return -ETIMEDOUT; in bman_acquire()
804 ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT; in bman_acquire()
806 memcpy(&bufs[0], &mcr->bufs[0], num * sizeof(bufs[0])); in bman_acquire()
810 ret = -ENOMEM; in bman_acquire()
816 bman_get_bm_portal_config(const struct bman_portal *portal) in bman_get_bm_portal_config() argument
818 return portal->config; in bman_get_bm_portal_config()