Lines Matching +full:burst +full:- +full:wrap
1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
3 * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
4 * Copyright 2016-2019 NXP
12 #include <soc/fsl/dpaa2-global.h>
14 #include "qbman-portal.h"
54 /* CENA register offsets in memory-backed mode */
178 return readl_relaxed(p->addr_cinh + offset); in qbman_read_register()
184 writel_relaxed(value, p->addr_cinh + offset); in qbman_write_register()
189 return p->addr_cena + offset; in qbman_get_cmd()
234 return last - first; in qm_cyc_diff()
236 return (2 * ringsize) - (first - last); in qm_cyc_diff()
240 * qbman_swp_init() - Create a functional object representing the given
257 spin_lock_init(&p->access_spinlock); in qbman_swp_init()
259 p->desc = d; in qbman_swp_init()
260 p->mc.valid_bit = QB_VALID_BIT; in qbman_swp_init()
261 p->sdq = 0; in qbman_swp_init()
262 p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; in qbman_swp_init()
263 p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; in qbman_swp_init()
264 p->sdq |= QMAN_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; in qbman_swp_init()
265 if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) in qbman_swp_init()
266 p->mr.valid_bit = QB_VALID_BIT; in qbman_swp_init()
268 atomic_set(&p->vdq.available, 1); in qbman_swp_init()
269 p->vdq.valid_bit = QB_VALID_BIT; in qbman_swp_init()
270 p->dqrr.next_idx = 0; in qbman_swp_init()
271 p->dqrr.valid_bit = QB_VALID_BIT; in qbman_swp_init()
273 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_4100) { in qbman_swp_init()
274 p->dqrr.dqrr_size = 4; in qbman_swp_init()
275 p->dqrr.reset_bug = 1; in qbman_swp_init()
277 p->dqrr.dqrr_size = 8; in qbman_swp_init()
278 p->dqrr.reset_bug = 0; in qbman_swp_init()
281 p->addr_cena = d->cena_bar; in qbman_swp_init()
282 p->addr_cinh = d->cinh_bar; in qbman_swp_init()
284 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { in qbman_swp_init()
286 reg = qbman_set_swp_cfg(p->dqrr.dqrr_size, in qbman_swp_init()
287 1, /* Writes Non-cacheable */ in qbman_swp_init()
299 memset(p->addr_cena, 0, 64 * 1024); in qbman_swp_init()
300 reg = qbman_set_swp_cfg(p->dqrr.dqrr_size, in qbman_swp_init()
301 1, /* Writes Non-cacheable */ in qbman_swp_init()
312 reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */ in qbman_swp_init()
325 if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { in qbman_swp_init()
337 p->eqcr.pi_ring_size = 8; in qbman_swp_init()
338 if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) { in qbman_swp_init()
339 p->eqcr.pi_ring_size = 32; in qbman_swp_init()
351 for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1) in qbman_swp_init()
352 p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1; in qbman_swp_init()
354 p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask; in qbman_swp_init()
355 p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT; in qbman_swp_init()
356 p->eqcr.ci = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_CI) in qbman_swp_init()
357 & p->eqcr.pi_ci_mask; in qbman_swp_init()
358 p->eqcr.available = p->eqcr.pi_ring_size; in qbman_swp_init()
361 qbman_swp_set_irq_coalescing(p, p->dqrr.dqrr_size - 1, 0); in qbman_swp_init()
367 * qbman_swp_finish() - Create and destroy a functional object representing
398 * qbman_swp_interrupt_get_trigger() - read interrupt enable register
409 * qbman_swp_interrupt_set_trigger() - enable interrupts for a swp
419 * qbman_swp_interrupt_get_inhibit() - read interrupt mask register
430 * qbman_swp_interrupt_set_inhibit() - write interrupt mask register
450 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) in qbman_swp_mc_start()
457 * Commits merges in the caller-supplied command verb (which should not include
458 * the valid-bit) and submits the command to hardware
464 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { in qbman_swp_mc_submit()
466 *v = cmd_verb | p->mc.valid_bit; in qbman_swp_mc_submit()
468 *v = cmd_verb | p->mc.valid_bit; in qbman_swp_mc_submit()
475 * Checks for a completed response (returns non-NULL if only if the response
482 if ((p->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) { in qbman_swp_mc_result()
483 ret = qbman_get_cmd(p, QBMAN_CENA_SWP_RR(p->mc.valid_bit)); in qbman_swp_mc_result()
484 /* Remove the valid-bit - command completed if the rest in qbman_swp_mc_result()
485 * is non-zero. in qbman_swp_mc_result()
490 p->mc.valid_bit ^= QB_VALID_BIT; in qbman_swp_mc_result()
494 if (p->mr.valid_bit != (ret[0] & QB_VALID_BIT)) in qbman_swp_mc_result()
496 /* Command completed if the rest is non-zero */ in qbman_swp_mc_result()
500 p->mr.valid_bit ^= QB_VALID_BIT; in qbman_swp_mc_result()
519 * qbman_eq_desc_clear() - Clear the contents of a descriptor to
528 * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp
535 d->verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT); in qbman_eq_desc_set_no_orp()
537 d->verb |= enqueue_response_always; in qbman_eq_desc_set_no_orp()
539 d->verb |= enqueue_rejects_to_fq; in qbman_eq_desc_set_no_orp()
545 * -enqueue to a frame queue
546 * -enqueue to a queuing destination
550 * qbman_eq_desc_set_fq() - set the FQ for the enqueue command
556 d->verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT); in qbman_eq_desc_set_fq()
557 d->tgtid = cpu_to_le32(fqid); in qbman_eq_desc_set_fq()
561 * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command
570 d->verb |= 1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT; in qbman_eq_desc_set_qd()
571 d->tgtid = cpu_to_le32(qdid); in qbman_eq_desc_set_qd()
572 d->qdbin = cpu_to_le16(qd_bin); in qbman_eq_desc_set_qd()
573 d->qpri = qd_prio; in qbman_eq_desc_set_qd()
582 * qbman_swp_enqueue_direct() - Issue an enqueue command
590 * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
603 ret = -EBUSY; in qbman_swp_enqueue_direct()
608 * qbman_swp_enqueue_mem_back() - Issue an enqueue command
616 * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
629 ret = -EBUSY; in qbman_swp_enqueue_mem_back()
634 * qbman_swp_enqueue_multiple_direct() - Issue a multi enqueue command
656 spin_lock(&s->access_spinlock); in qbman_swp_enqueue_multiple_direct()
657 half_mask = (s->eqcr.pi_ci_mask>>1); in qbman_swp_enqueue_multiple_direct()
658 full_mask = s->eqcr.pi_ci_mask; in qbman_swp_enqueue_multiple_direct()
660 if (!s->eqcr.available) { in qbman_swp_enqueue_multiple_direct()
661 eqcr_ci = s->eqcr.ci; in qbman_swp_enqueue_multiple_direct()
662 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI; in qbman_swp_enqueue_multiple_direct()
663 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); in qbman_swp_enqueue_multiple_direct()
664 s->eqcr.ci &= full_mask; in qbman_swp_enqueue_multiple_direct()
666 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, in qbman_swp_enqueue_multiple_direct()
667 eqcr_ci, s->eqcr.ci); in qbman_swp_enqueue_multiple_direct()
668 if (!s->eqcr.available) { in qbman_swp_enqueue_multiple_direct()
669 spin_unlock(&s->access_spinlock); in qbman_swp_enqueue_multiple_direct()
674 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_direct()
675 num_enqueued = (s->eqcr.available < num_frames) ? in qbman_swp_enqueue_multiple_direct()
676 s->eqcr.available : num_frames; in qbman_swp_enqueue_multiple_direct()
677 s->eqcr.available -= num_enqueued; in qbman_swp_enqueue_multiple_direct()
680 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); in qbman_swp_enqueue_multiple_direct()
682 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); in qbman_swp_enqueue_multiple_direct()
690 /* Set the verb byte, have to substitute in the valid-bit */ in qbman_swp_enqueue_multiple_direct()
691 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_direct()
693 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); in qbman_swp_enqueue_multiple_direct()
694 p[0] = cl[0] | s->eqcr.pi_vb; in qbman_swp_enqueue_multiple_direct()
698 eq_desc->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | in qbman_swp_enqueue_multiple_direct()
703 s->eqcr.pi_vb ^= QB_VALID_BIT; in qbman_swp_enqueue_multiple_direct()
707 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_direct()
710 s->eqcr.pi = eqcr_pi & full_mask; in qbman_swp_enqueue_multiple_direct()
711 spin_unlock(&s->access_spinlock); in qbman_swp_enqueue_multiple_direct()
717 * qbman_swp_enqueue_multiple_mem_back() - Issue a multi enqueue command
740 spin_lock_irqsave(&s->access_spinlock, irq_flags); in qbman_swp_enqueue_multiple_mem_back()
742 half_mask = (s->eqcr.pi_ci_mask>>1); in qbman_swp_enqueue_multiple_mem_back()
743 full_mask = s->eqcr.pi_ci_mask; in qbman_swp_enqueue_multiple_mem_back()
744 if (!s->eqcr.available) { in qbman_swp_enqueue_multiple_mem_back()
745 eqcr_ci = s->eqcr.ci; in qbman_swp_enqueue_multiple_mem_back()
746 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); in qbman_swp_enqueue_multiple_mem_back()
747 s->eqcr.ci &= full_mask; in qbman_swp_enqueue_multiple_mem_back()
748 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, in qbman_swp_enqueue_multiple_mem_back()
749 eqcr_ci, s->eqcr.ci); in qbman_swp_enqueue_multiple_mem_back()
750 if (!s->eqcr.available) { in qbman_swp_enqueue_multiple_mem_back()
751 spin_unlock_irqrestore(&s->access_spinlock, irq_flags); in qbman_swp_enqueue_multiple_mem_back()
756 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_mem_back()
757 num_enqueued = (s->eqcr.available < num_frames) ? in qbman_swp_enqueue_multiple_mem_back()
758 s->eqcr.available : num_frames; in qbman_swp_enqueue_multiple_mem_back()
759 s->eqcr.available -= num_enqueued; in qbman_swp_enqueue_multiple_mem_back()
762 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); in qbman_swp_enqueue_multiple_mem_back()
764 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); in qbman_swp_enqueue_multiple_mem_back()
770 /* Set the verb byte, have to substitute in the valid-bit */ in qbman_swp_enqueue_multiple_mem_back()
771 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_mem_back()
773 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); in qbman_swp_enqueue_multiple_mem_back()
774 p[0] = cl[0] | s->eqcr.pi_vb; in qbman_swp_enqueue_multiple_mem_back()
778 eq_desc->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) | in qbman_swp_enqueue_multiple_mem_back()
783 s->eqcr.pi_vb ^= QB_VALID_BIT; in qbman_swp_enqueue_multiple_mem_back()
785 s->eqcr.pi = eqcr_pi & full_mask; in qbman_swp_enqueue_multiple_mem_back()
789 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); in qbman_swp_enqueue_multiple_mem_back()
790 spin_unlock_irqrestore(&s->access_spinlock, irq_flags); in qbman_swp_enqueue_multiple_mem_back()
796 * qbman_swp_enqueue_multiple_desc_direct() - Issue a multi enqueue command
816 half_mask = (s->eqcr.pi_ci_mask>>1); in qbman_swp_enqueue_multiple_desc_direct()
817 full_mask = s->eqcr.pi_ci_mask; in qbman_swp_enqueue_multiple_desc_direct()
818 if (!s->eqcr.available) { in qbman_swp_enqueue_multiple_desc_direct()
819 eqcr_ci = s->eqcr.ci; in qbman_swp_enqueue_multiple_desc_direct()
820 p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI; in qbman_swp_enqueue_multiple_desc_direct()
821 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); in qbman_swp_enqueue_multiple_desc_direct()
822 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, in qbman_swp_enqueue_multiple_desc_direct()
823 eqcr_ci, s->eqcr.ci); in qbman_swp_enqueue_multiple_desc_direct()
824 if (!s->eqcr.available) in qbman_swp_enqueue_multiple_desc_direct()
828 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_desc_direct()
829 num_enqueued = (s->eqcr.available < num_frames) ? in qbman_swp_enqueue_multiple_desc_direct()
830 s->eqcr.available : num_frames; in qbman_swp_enqueue_multiple_desc_direct()
831 s->eqcr.available -= num_enqueued; in qbman_swp_enqueue_multiple_desc_direct()
834 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); in qbman_swp_enqueue_multiple_desc_direct()
837 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); in qbman_swp_enqueue_multiple_desc_direct()
845 /* Set the verb byte, have to substitute in the valid-bit */ in qbman_swp_enqueue_multiple_desc_direct()
846 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_desc_direct()
848 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); in qbman_swp_enqueue_multiple_desc_direct()
850 p[0] = cl[0] | s->eqcr.pi_vb; in qbman_swp_enqueue_multiple_desc_direct()
853 s->eqcr.pi_vb ^= QB_VALID_BIT; in qbman_swp_enqueue_multiple_desc_direct()
857 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_desc_direct()
860 s->eqcr.pi = eqcr_pi & full_mask; in qbman_swp_enqueue_multiple_desc_direct()
866 * qbman_swp_enqueue_multiple_desc_mem_back() - Issue a multi enqueue command
886 half_mask = (s->eqcr.pi_ci_mask>>1); in qbman_swp_enqueue_multiple_desc_mem_back()
887 full_mask = s->eqcr.pi_ci_mask; in qbman_swp_enqueue_multiple_desc_mem_back()
888 if (!s->eqcr.available) { in qbman_swp_enqueue_multiple_desc_mem_back()
889 eqcr_ci = s->eqcr.ci; in qbman_swp_enqueue_multiple_desc_mem_back()
890 s->eqcr.ci = qbman_read_register(s, QBMAN_CINH_SWP_EQCR_CI); in qbman_swp_enqueue_multiple_desc_mem_back()
891 s->eqcr.ci &= full_mask; in qbman_swp_enqueue_multiple_desc_mem_back()
892 s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size, in qbman_swp_enqueue_multiple_desc_mem_back()
893 eqcr_ci, s->eqcr.ci); in qbman_swp_enqueue_multiple_desc_mem_back()
894 if (!s->eqcr.available) in qbman_swp_enqueue_multiple_desc_mem_back()
898 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_desc_mem_back()
899 num_enqueued = (s->eqcr.available < num_frames) ? in qbman_swp_enqueue_multiple_desc_mem_back()
900 s->eqcr.available : num_frames; in qbman_swp_enqueue_multiple_desc_mem_back()
901 s->eqcr.available -= num_enqueued; in qbman_swp_enqueue_multiple_desc_mem_back()
904 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); in qbman_swp_enqueue_multiple_desc_mem_back()
907 memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1); in qbman_swp_enqueue_multiple_desc_mem_back()
913 /* Set the verb byte, have to substitute in the valid-bit */ in qbman_swp_enqueue_multiple_desc_mem_back()
914 eqcr_pi = s->eqcr.pi; in qbman_swp_enqueue_multiple_desc_mem_back()
916 p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask)); in qbman_swp_enqueue_multiple_desc_mem_back()
918 p[0] = cl[0] | s->eqcr.pi_vb; in qbman_swp_enqueue_multiple_desc_mem_back()
921 s->eqcr.pi_vb ^= QB_VALID_BIT; in qbman_swp_enqueue_multiple_desc_mem_back()
924 s->eqcr.pi = eqcr_pi & full_mask; in qbman_swp_enqueue_multiple_desc_mem_back()
928 (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb); in qbman_swp_enqueue_multiple_desc_mem_back()
936 * qbman_swp_push_get() - Get the push dequeue setup
944 u16 src = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; in qbman_swp_push_get()
951 * qbman_swp_push_set() - Enable or disable push dequeue
962 s->sdq |= 1 << channel_idx; in qbman_swp_push_set()
964 s->sdq &= ~(1 << channel_idx); in qbman_swp_push_set()
969 dqsrc = (s->sdq >> QB_SDQCR_SRC_SHIFT) & QB_SDQCR_SRC_MASK; in qbman_swp_push_set()
971 qbman_write_register(s, QBMAN_CINH_SWP_SDQCR, s->sdq); in qbman_swp_push_set()
988 * qbman_pull_desc_clear() - Clear the contents of a descriptor to
998 * qbman_pull_desc_set_storage()- Set the pull dequeue storage
1005 * will produce results to DQRR. If 'storage' is non-NULL, then results are
1008 * those writes to main-memory express a cache-warming attribute.
1016 d->rsp_addr_virt = (u64)(uintptr_t)storage; in qbman_pull_desc_set_storage()
1019 d->verb &= ~(1 << QB_VDQCR_VERB_RLS_SHIFT); in qbman_pull_desc_set_storage()
1022 d->verb |= 1 << QB_VDQCR_VERB_RLS_SHIFT; in qbman_pull_desc_set_storage()
1024 d->verb |= 1 << QB_VDQCR_VERB_WAE_SHIFT; in qbman_pull_desc_set_storage()
1026 d->verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT); in qbman_pull_desc_set_storage()
1028 d->rsp_addr = cpu_to_le64(storage_phys); in qbman_pull_desc_set_storage()
1032 * qbman_pull_desc_set_numframes() - Set the number of frames to be dequeued
1038 d->numf = numframes - 1; in qbman_pull_desc_set_numframes()
1044 * - pull dequeue from the given frame queue (FQ)
1045 * - pull dequeue from any FQ in the given work queue (WQ)
1046 * - pull dequeue from any FQ in any WQ in the given channel
1050 * qbman_pull_desc_set_fq() - Set fqid from which the dequeue command dequeues
1056 d->verb |= 1 << QB_VDQCR_VERB_DCT_SHIFT; in qbman_pull_desc_set_fq()
1057 d->verb |= qb_pull_dt_framequeue << QB_VDQCR_VERB_DT_SHIFT; in qbman_pull_desc_set_fq()
1058 d->dq_src = cpu_to_le32(fqid); in qbman_pull_desc_set_fq()
1062 * qbman_pull_desc_set_wq() - Set wqid from which the dequeue command dequeues
1070 d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; in qbman_pull_desc_set_wq()
1071 d->verb |= qb_pull_dt_workqueue << QB_VDQCR_VERB_DT_SHIFT; in qbman_pull_desc_set_wq()
1072 d->dq_src = cpu_to_le32(wqid); in qbman_pull_desc_set_wq()
1076 * qbman_pull_desc_set_channel() - Set channelid from which the dequeue command
1085 d->verb |= dct << QB_VDQCR_VERB_DCT_SHIFT; in qbman_pull_desc_set_channel()
1086 d->verb |= qb_pull_dt_channel << QB_VDQCR_VERB_DT_SHIFT; in qbman_pull_desc_set_channel()
1087 d->dq_src = cpu_to_le32(chid); in qbman_pull_desc_set_channel()
1091 * qbman_swp_pull_direct() - Issue the pull dequeue command
1096 * Return 0 for success, and -EBUSY if the software portal is not ready
1104 if (!atomic_dec_and_test(&s->vdq.available)) { in qbman_swp_pull_direct()
1105 atomic_inc(&s->vdq.available); in qbman_swp_pull_direct()
1106 return -EBUSY; in qbman_swp_pull_direct()
1108 s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt; in qbman_swp_pull_direct()
1109 if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) in qbman_swp_pull_direct()
1113 p->numf = d->numf; in qbman_swp_pull_direct()
1114 p->tok = QMAN_DQ_TOKEN_VALID; in qbman_swp_pull_direct()
1115 p->dq_src = d->dq_src; in qbman_swp_pull_direct()
1116 p->rsp_addr = d->rsp_addr; in qbman_swp_pull_direct()
1117 p->rsp_addr_virt = d->rsp_addr_virt; in qbman_swp_pull_direct()
1119 /* Set the verb byte, have to substitute in the valid-bit */ in qbman_swp_pull_direct()
1120 p->verb = d->verb | s->vdq.valid_bit; in qbman_swp_pull_direct()
1121 s->vdq.valid_bit ^= QB_VALID_BIT; in qbman_swp_pull_direct()
1127 * qbman_swp_pull_mem_back() - Issue the pull dequeue command
1132 * Return 0 for success, and -EBUSY if the software portal is not ready
1140 if (!atomic_dec_and_test(&s->vdq.available)) { in qbman_swp_pull_mem_back()
1141 atomic_inc(&s->vdq.available); in qbman_swp_pull_mem_back()
1142 return -EBUSY; in qbman_swp_pull_mem_back()
1144 s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt; in qbman_swp_pull_mem_back()
1145 if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) in qbman_swp_pull_mem_back()
1149 p->numf = d->numf; in qbman_swp_pull_mem_back()
1150 p->tok = QMAN_DQ_TOKEN_VALID; in qbman_swp_pull_mem_back()
1151 p->dq_src = d->dq_src; in qbman_swp_pull_mem_back()
1152 p->rsp_addr = d->rsp_addr; in qbman_swp_pull_mem_back()
1153 p->rsp_addr_virt = d->rsp_addr_virt; in qbman_swp_pull_mem_back()
1155 /* Set the verb byte, have to substitute in the valid-bit */ in qbman_swp_pull_mem_back()
1156 p->verb = d->verb | s->vdq.valid_bit; in qbman_swp_pull_mem_back()
1157 s->vdq.valid_bit ^= QB_VALID_BIT; in qbman_swp_pull_mem_back()
1167 * qbman_swp_dqrr_next_direct() - Get an valid DQRR entry
1181 /* Before using valid-bit to detect if something is there, we have to in qbman_swp_dqrr_next_direct()
1184 if (unlikely(s->dqrr.reset_bug)) { in qbman_swp_dqrr_next_direct()
1186 * We pick up new entries by cache-inhibited producer index, in qbman_swp_dqrr_next_direct()
1187 * which means that a non-coherent mapping would require us to in qbman_swp_dqrr_next_direct()
1197 if (pi == s->dqrr.next_idx) in qbman_swp_dqrr_next_direct()
1203 * entries have now been DMA'd to so valid-bit checking is in qbman_swp_dqrr_next_direct()
1206 * can burst and wrap-around between our snapshots of it). in qbman_swp_dqrr_next_direct()
1208 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) { in qbman_swp_dqrr_next_direct()
1210 s->dqrr.next_idx, pi); in qbman_swp_dqrr_next_direct()
1211 s->dqrr.reset_bug = 0; in qbman_swp_dqrr_next_direct()
1214 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); in qbman_swp_dqrr_next_direct()
1217 p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)); in qbman_swp_dqrr_next_direct()
1218 verb = p->dq.verb; in qbman_swp_dqrr_next_direct()
1221 * If the valid-bit isn't of the expected polarity, nothing there. Note, in qbman_swp_dqrr_next_direct()
1225 * valid-bit behaviour is repaired and should tell us what we already in qbman_swp_dqrr_next_direct()
1228 if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) { in qbman_swp_dqrr_next_direct()
1230 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); in qbman_swp_dqrr_next_direct()
1237 s->dqrr.next_idx++; in qbman_swp_dqrr_next_direct()
1238 s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */ in qbman_swp_dqrr_next_direct()
1239 if (!s->dqrr.next_idx) in qbman_swp_dqrr_next_direct()
1240 s->dqrr.valid_bit ^= QB_VALID_BIT; in qbman_swp_dqrr_next_direct()
1246 flags = p->dq.stat; in qbman_swp_dqrr_next_direct()
1251 atomic_inc(&s->vdq.available); in qbman_swp_dqrr_next_direct()
1253 prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); in qbman_swp_dqrr_next_direct()
1259 * qbman_swp_dqrr_next_mem_back() - Get an valid DQRR entry
1273 /* Before using valid-bit to detect if something is there, we have to in qbman_swp_dqrr_next_mem_back()
1276 if (unlikely(s->dqrr.reset_bug)) { in qbman_swp_dqrr_next_mem_back()
1278 * We pick up new entries by cache-inhibited producer index, in qbman_swp_dqrr_next_mem_back()
1279 * which means that a non-coherent mapping would require us to in qbman_swp_dqrr_next_mem_back()
1289 if (pi == s->dqrr.next_idx) in qbman_swp_dqrr_next_mem_back()
1295 * entries have now been DMA'd to so valid-bit checking is in qbman_swp_dqrr_next_mem_back()
1298 * can burst and wrap-around between our snapshots of it). in qbman_swp_dqrr_next_mem_back()
1300 if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) { in qbman_swp_dqrr_next_mem_back()
1302 s->dqrr.next_idx, pi); in qbman_swp_dqrr_next_mem_back()
1303 s->dqrr.reset_bug = 0; in qbman_swp_dqrr_next_mem_back()
1306 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); in qbman_swp_dqrr_next_mem_back()
1309 p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx)); in qbman_swp_dqrr_next_mem_back()
1310 verb = p->dq.verb; in qbman_swp_dqrr_next_mem_back()
1313 * If the valid-bit isn't of the expected polarity, nothing there. Note, in qbman_swp_dqrr_next_mem_back()
1317 * valid-bit behaviour is repaired and should tell us what we already in qbman_swp_dqrr_next_mem_back()
1320 if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) { in qbman_swp_dqrr_next_mem_back()
1322 QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); in qbman_swp_dqrr_next_mem_back()
1329 s->dqrr.next_idx++; in qbman_swp_dqrr_next_mem_back()
1330 s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */ in qbman_swp_dqrr_next_mem_back()
1331 if (!s->dqrr.next_idx) in qbman_swp_dqrr_next_mem_back()
1332 s->dqrr.valid_bit ^= QB_VALID_BIT; in qbman_swp_dqrr_next_mem_back()
1338 flags = p->dq.stat; in qbman_swp_dqrr_next_mem_back()
1343 atomic_inc(&s->vdq.available); in qbman_swp_dqrr_next_mem_back()
1345 prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx))); in qbman_swp_dqrr_next_mem_back()
1351 * qbman_swp_dqrr_consume() - Consume DQRR entries previously returned from
1362 * qbman_result_has_new_result() - Check and get the dequeue response from the
1370 * Only used for user-provided storage of dequeue results, not DQRR. For
1372 * conversion to ensure that the user's dequeue result storage is in host-endian
1380 if (dq->dq.tok != QMAN_DQ_TOKEN_VALID) in qbman_result_has_new_result()
1388 ((struct dpaa2_dq *)dq)->dq.tok = 0; in qbman_result_has_new_result()
1395 if (s->vdq.storage == dq) { in qbman_result_has_new_result()
1396 s->vdq.storage = NULL; in qbman_result_has_new_result()
1397 atomic_inc(&s->vdq.available); in qbman_result_has_new_result()
1404 * qbman_release_desc_clear() - Clear the contents of a descriptor to
1411 d->verb = 1 << 5; /* Release Command Valid */ in qbman_release_desc_clear()
1415 * qbman_release_desc_set_bpid() - Set the ID of the buffer pool to release to
1421 d->bpid = cpu_to_le16(bpid); in qbman_release_desc_set_bpid()
1425 * qbman_release_desc_set_rcdi() - Determines whether or not the portal's RCDI
1433 d->verb |= 1 << 6; in qbman_release_desc_set_rcdi()
1435 d->verb &= ~(1 << 6); in qbman_release_desc_set_rcdi()
1443 * qbman_swp_release_direct() - Issue a buffer release command
1449 * Return 0 for success, -EBUSY if the release command ring is not ready.
1460 return -EINVAL; in qbman_swp_release_direct()
1464 return -EBUSY; in qbman_swp_release_direct()
1471 p->buf[i] = cpu_to_le64(buffers[i]); in qbman_swp_release_direct()
1472 p->bpid = d->bpid; in qbman_swp_release_direct()
1475 * Set the verb byte, have to substitute in the valid-bit in qbman_swp_release_direct()
1479 p->verb = d->verb | RAR_VB(rar) | num_buffers; in qbman_swp_release_direct()
1485 * qbman_swp_release_mem_back() - Issue a buffer release command
1491 * Return 0 for success, -EBUSY if the release command ring is not ready.
1502 return -EINVAL; in qbman_swp_release_mem_back()
1506 return -EBUSY; in qbman_swp_release_mem_back()
1513 p->buf[i] = cpu_to_le64(buffers[i]); in qbman_swp_release_mem_back()
1514 p->bpid = d->bpid; in qbman_swp_release_mem_back()
1516 p->verb = d->verb | RAR_VB(rar) | num_buffers; in qbman_swp_release_mem_back()
1542 * qbman_swp_acquire() - Issue a buffer acquire command
1559 return -EINVAL; in qbman_swp_acquire()
1565 return -EBUSY; in qbman_swp_acquire()
1567 /* Encode the caller-provided attributes */ in qbman_swp_acquire()
1568 p->bpid = cpu_to_le16(bpid); in qbman_swp_acquire()
1569 p->num = num_buffers; in qbman_swp_acquire()
1576 return -EIO; in qbman_swp_acquire()
1580 WARN_ON((r->verb & 0x7f) != QBMAN_MC_ACQUIRE); in qbman_swp_acquire()
1583 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { in qbman_swp_acquire()
1585 bpid, r->rslt); in qbman_swp_acquire()
1586 return -EIO; in qbman_swp_acquire()
1589 WARN_ON(r->num > num_buffers); in qbman_swp_acquire()
1592 for (i = 0; i < r->num; i++) in qbman_swp_acquire()
1593 buffers[i] = le64_to_cpu(r->buf[i]); in qbman_swp_acquire()
1595 return (int)r->num; in qbman_swp_acquire()
1622 return -EBUSY; in qbman_swp_alt_fq_state()
1624 p->fqid = cpu_to_le32(fqid & ALT_FQ_FQID_MASK); in qbman_swp_alt_fq_state()
1631 return -EIO; in qbman_swp_alt_fq_state()
1635 WARN_ON((r->verb & QBMAN_RESULT_MASK) != alt_fq_verb); in qbman_swp_alt_fq_state()
1638 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { in qbman_swp_alt_fq_state()
1640 fqid, r->verb, r->rslt); in qbman_swp_alt_fq_state()
1641 return -EIO; in qbman_swp_alt_fq_state()
1676 return -EBUSY; in qbman_swp_CDAN_set()
1678 /* Encode the caller-provided attributes */ in qbman_swp_CDAN_set()
1679 p->ch = cpu_to_le16(channelid); in qbman_swp_CDAN_set()
1680 p->we = we_mask; in qbman_swp_CDAN_set()
1682 p->ctrl = 1; in qbman_swp_CDAN_set()
1684 p->ctrl = 0; in qbman_swp_CDAN_set()
1685 p->cdan_ctx = cpu_to_le64(ctx); in qbman_swp_CDAN_set()
1691 return -EIO; in qbman_swp_CDAN_set()
1694 WARN_ON((r->verb & 0x7f) != QBMAN_WQCHAN_CONFIGURE); in qbman_swp_CDAN_set()
1697 if (unlikely(r->rslt != QBMAN_MC_RSLT_OK)) { in qbman_swp_CDAN_set()
1699 channelid, r->rslt); in qbman_swp_CDAN_set()
1700 return -EIO; in qbman_swp_CDAN_set()
1725 return -EBUSY; in qbman_fq_query_state()
1728 p->fqid = cpu_to_le32(fqid & 0x00FFFFFF); in qbman_fq_query_state()
1733 return -EIO; in qbman_fq_query_state()
1737 WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_FQ_QUERY_NP); in qbman_fq_query_state()
1740 if (r->rslt != QBMAN_MC_RSLT_OK) { in qbman_fq_query_state()
1742 p->fqid, r->rslt); in qbman_fq_query_state()
1743 return -EIO; in qbman_fq_query_state()
1751 return (le32_to_cpu(r->frm_cnt) & 0x00FFFFFF); in qbman_fq_state_frame_count()
1756 return le32_to_cpu(r->byte_cnt); in qbman_fq_state_byte_count()
1774 return -EBUSY; in qbman_bp_query()
1776 p->bpid = cpu_to_le16(bpid); in qbman_bp_query()
1781 return -EIO; in qbman_bp_query()
1785 WARN_ON((r->verb & QBMAN_RESPONSE_VERB_MASK) != QBMAN_BP_QUERY); in qbman_bp_query()
1788 if (r->rslt != QBMAN_MC_RSLT_OK) { in qbman_bp_query()
1790 bpid, r->rslt); in qbman_bp_query()
1791 return -EIO; in qbman_bp_query()
1799 return le32_to_cpu(a->fill); in qbman_bp_info_num_free_bufs()
1803 * qbman_swp_set_irq_coalescing() - Set new IRQ coalescing values
1818 itp = (irq_holdoff * 1000) / p->desc->qman_256_cycles_per_ns; in qbman_swp_set_irq_coalescing()
1820 max_holdoff = (p->desc->qman_256_cycles_per_ns * 4096) / 1000; in qbman_swp_set_irq_coalescing()
1822 return -EINVAL; in qbman_swp_set_irq_coalescing()
1825 if (irq_threshold >= p->dqrr.dqrr_size) { in qbman_swp_set_irq_coalescing()
1826 pr_err("irq_threshold must be < %u\n", p->dqrr.dqrr_size - 1); in qbman_swp_set_irq_coalescing()
1827 return -EINVAL; in qbman_swp_set_irq_coalescing()
1830 p->irq_threshold = irq_threshold; in qbman_swp_set_irq_coalescing()
1831 p->irq_holdoff = irq_holdoff; in qbman_swp_set_irq_coalescing()
1840 * qbman_swp_get_irq_coalescing() - Get the current IRQ coalescing parameters
1850 *irq_threshold = p->irq_threshold; in qbman_swp_get_irq_coalescing()
1852 *irq_holdoff = p->irq_holdoff; in qbman_swp_get_irq_coalescing()