Lines Matching +full:qman +full:- +full:portal +full:- +full:1
1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause
4 * Copyright © 2014-2016 Freescale Semiconductor, Inc.
5 * Copyright © 2016-2019 NXP
11 * 1. Redistributions of source code must retain the above copyright notice,
35 * drivers/soc/fsl/dpio/qbman-portal.c
42 * Copyright © 2021-2022 Dmitry Salychev
47 * 1. Redistributions of source code must retain the above copyright
68 * DPAA2 QBMan software portal.
106 #define ENQ_CMD_RESPONSE_ALWAYS 1u
112 #define ENQ_FLAG_DCA (1ull << 31)
114 /* QBMan portal command codes. */
119 /* QBMan portal command result codes. */
134 #define QB_VDQCR_VERB_DCT1_SHIFT 1
150 MALLOC_DEFINE(M_DPAA2_SWP, "dpaa2_swp", "DPAA2 QBMan Software Portal");
161 qbman_sdqcr_fc_up_to_3 = 1
164 /* Routines to execute software portal commands. */
198 mtx_init(&p->lock, "swp_sleep_lock", NULL, MTX_DEF); in dpaa2_swp_init_portal()
200 p->cfg.mem_backed = false; in dpaa2_swp_init_portal()
201 p->cfg.writes_cinh = true; in dpaa2_swp_init_portal()
203 p->desc = desc; in dpaa2_swp_init_portal()
204 p->flags = flags; in dpaa2_swp_init_portal()
205 p->mc.valid_bit = DPAA2_SWP_VALID_BIT; in dpaa2_swp_init_portal()
206 p->mr.valid_bit = DPAA2_SWP_VALID_BIT; in dpaa2_swp_init_portal()
208 /* FIXME: Memory-backed mode doesn't work now. Why? */ in dpaa2_swp_init_portal()
209 p->cena_res = desc->cena_res; in dpaa2_swp_init_portal()
210 p->cena_map = desc->cena_map; in dpaa2_swp_init_portal()
211 p->cinh_res = desc->cinh_res; in dpaa2_swp_init_portal()
212 p->cinh_map = desc->cinh_map; in dpaa2_swp_init_portal()
215 p->sdq = 0; in dpaa2_swp_init_portal()
216 p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT; in dpaa2_swp_init_portal()
217 p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT; in dpaa2_swp_init_portal()
218 p->sdq |= DPAA2_SWP_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT; in dpaa2_swp_init_portal()
221 p->vdq.valid_bit = DPAA2_SWP_VALID_BIT; in dpaa2_swp_init_portal()
224 p->dqrr.next_idx = 0; in dpaa2_swp_init_portal()
225 p->dqrr.valid_bit = DPAA2_SWP_VALID_BIT; in dpaa2_swp_init_portal()
226 if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_4100) { in dpaa2_swp_init_portal()
227 p->dqrr.ring_size = 4; in dpaa2_swp_init_portal()
228 p->dqrr.reset_bug = 1; in dpaa2_swp_init_portal()
230 p->dqrr.ring_size = 8; in dpaa2_swp_init_portal()
231 p->dqrr.reset_bug = 0; in dpaa2_swp_init_portal()
234 if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_5000) { in dpaa2_swp_init_portal()
236 p->dqrr.ring_size, /* max. entries QMan writes to DQRR */ in dpaa2_swp_init_portal()
237 1, /* writes enabled in the CINH memory only */ in dpaa2_swp_init_portal()
242 1, /* mem stashing drop enable enable */ in dpaa2_swp_init_portal()
243 1, /* mem stashing priority enable */ in dpaa2_swp_init_portal()
244 1, /* mem stashing enable */ in dpaa2_swp_init_portal()
245 1, /* dequeue stashing priority enable */ in dpaa2_swp_init_portal()
249 reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */ in dpaa2_swp_init_portal()
251 bus_set_region_4(p->cena_map, 0, 0, in dpaa2_swp_init_portal()
252 rman_get_size(p->cena_res) / 4); in dpaa2_swp_init_portal()
255 p->dqrr.ring_size, /* max. entries QMan writes to DQRR */ /* DQRR_MF */ in dpaa2_swp_init_portal()
256 1, /* writes enabled in the CINH memory only */ /* WN */ in dpaa2_swp_init_portal()
261 1, /* Dequeued frame data, annotation, and FQ context stashing drop enable */ /* SD */ in dpaa2_swp_init_portal()
262 1, /* Dequeued frame data, annotation, and FQ context stashing priority */ /* SP */ in dpaa2_swp_init_portal()
263 1, /* Dequeued frame data, annotation, and FQ context stashing enable */ /* SE */ in dpaa2_swp_init_portal()
264 1, /* Dequeue response ring (DQRR) entry stashing priority */ /* DP */ in dpaa2_swp_init_portal()
265 … 0, /* Dequeue response ring (DQRR) entry, or cacheable portal area, stashing enable. */ /* DE */ in dpaa2_swp_init_portal()
268 /* TODO: Switch to memory-backed mode. */ in dpaa2_swp_init_portal()
269 reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */ in dpaa2_swp_init_portal()
280 * channels are being dequeued from or else the QMan HW will indicate an in dpaa2_swp_init_portal()
286 p->eqcr.pi_ring_size = 8; in dpaa2_swp_init_portal()
287 /* if ((desc->swp_version & DPAA2_SWP_REV_MASK) >= DPAA2_SWP_REV_5000) */ in dpaa2_swp_init_portal()
288 /* p->eqcr.pi_ring_size = 32; */ in dpaa2_swp_init_portal()
290 for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1) in dpaa2_swp_init_portal()
291 p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1; in dpaa2_swp_init_portal()
294 p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask; in dpaa2_swp_init_portal()
295 p->eqcr.pi_vb = eqcr_pi & DPAA2_SWP_VALID_BIT; in dpaa2_swp_init_portal()
296 p->eqcr.ci = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_CI) in dpaa2_swp_init_portal()
297 & p->eqcr.pi_ci_mask; in dpaa2_swp_init_portal()
298 p->eqcr.available = p->eqcr.pi_ring_size; in dpaa2_swp_init_portal()
301 /* Initialize the portal with an IRQ threshold and timeout of 120us. */ in dpaa2_swp_init_portal()
302 dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, 120); in dpaa2_swp_init_portal()
317 swp->flags |= DPAA2_SWP_DESTROYED; in dpaa2_swp_free_portal()
320 /* Let threads stop using this portal. */ in dpaa2_swp_free_portal()
323 mtx_destroy(&swp->lock); in dpaa2_swp_free_portal()
347 /* Read/write registers of a software portal. */
352 bus_write_4(swp->cinh_map, o, v); in dpaa2_swp_write_reg()
358 return (bus_read_4(swp->cinh_map, o)); in dpaa2_swp_read_reg()
367 * resp_always: Enqueue with response always (1); FD from a rejected enqueue
373 ed->verb &= ~(1 << ENQ_CMD_ORP_ENABLE_SHIFT); in dpaa2_swp_set_ed_norp()
375 ed->verb |= ENQ_CMD_RESPONSE_ALWAYS; in dpaa2_swp_set_ed_norp()
377 ed->verb |= ENQ_CMD_REJECTS_TO_FQ; in dpaa2_swp_set_ed_norp()
386 ed->verb &= ~(1 << ENQ_CMD_TARGET_TYPE_SHIFT); in dpaa2_swp_set_ed_fq()
387 ed->tgtid = fqid; in dpaa2_swp_set_ed_fq()
391 * @brief Enable interrupts for a software portal.
435 * swp: the software portal object
446 device_printf(swp->desc->dpio_dev, "channel index " in dpaa2_swp_set_push_dequeue()
452 swp->sdq |= 1 << chan_idx; in dpaa2_swp_set_push_dequeue()
454 swp->sdq &= ~(1 << chan_idx); in dpaa2_swp_set_push_dequeue()
457 * the SDQCR must be 0 or else QMan will assert errors. in dpaa2_swp_set_push_dequeue()
459 dqsrc = (swp->sdq >> DPAA2_SDQCR_SRC_SHIFT) & in dpaa2_swp_set_push_dequeue()
462 ? swp->sdq : 0); in dpaa2_swp_set_push_dequeue()
469 * swp: The software portal object.
486 itp = (holdoff * 1000u) / swp->desc->swp_cycles_ratio; in dpaa2_swp_set_irq_coalescing()
489 if (threshold >= swp->dqrr.ring_size) in dpaa2_swp_set_irq_coalescing()
490 threshold = swp->dqrr.ring_size - 1; in dpaa2_swp_set_irq_coalescing()
492 swp->dqrr.irq_threshold = threshold; in dpaa2_swp_set_irq_coalescing()
493 swp->dqrr.irq_itp = itp; in dpaa2_swp_set_irq_coalescing()
502 * Software portal commands.
537 cmd.ctrl = cdan_en ? 1u : 0u; in dpaa2_swp_conf_wq_channel()
546 device_printf(swp->desc->dpio_dev, "WQ channel configuration " in dpaa2_swp_conf_wq_channel()
592 device_printf(swp->desc->dpio_dev, "BP query error: bpid=%d, " in dpaa2_swp_query_bp()
597 conf->bdi = rsp.bdi; in dpaa2_swp_query_bp()
598 conf->state = rsp.state; in dpaa2_swp_query_bp()
599 conf->free_bufn = rsp.fill; in dpaa2_swp_query_bp()
625 cmd.verb |= 1 << 5; /* Switch release buffer command to valid. */ in dpaa2_swp_release_bufs()
630 device_printf(swp->desc->dpio_dev, "buffers release command " in dpaa2_swp_release_bufs()
642 struct resource_map *map = swp->cinh_map; in dpaa2_swp_dqrr_next_locked()
645 uint32_t offset = swp->cfg.mem_backed in dpaa2_swp_dqrr_next_locked()
646 ? DPAA2_SWP_CENA_DQRR_MEM(swp->dqrr.next_idx) in dpaa2_swp_dqrr_next_locked()
647 : DPAA2_SWP_CENA_DQRR(swp->dqrr.next_idx); in dpaa2_swp_dqrr_next_locked()
653 * Before using valid-bit to detect if something is there, we have to in dpaa2_swp_dqrr_next_locked()
656 if (swp->dqrr.reset_bug) { in dpaa2_swp_dqrr_next_locked()
658 * We pick up new entries by cache-inhibited producer index, in dpaa2_swp_dqrr_next_locked()
659 * which means that a non-coherent mapping would require us to in dpaa2_swp_dqrr_next_locked()
668 if (pi == swp->dqrr.next_idx) in dpaa2_swp_dqrr_next_locked()
674 * entries have now been DMA'd to so valid-bit checking is in dpaa2_swp_dqrr_next_locked()
679 * can burst and wrap-around between our snapshots of it). in dpaa2_swp_dqrr_next_locked()
681 if (swp->dqrr.next_idx == (swp->dqrr.ring_size - 1)) in dpaa2_swp_dqrr_next_locked()
682 swp->dqrr.reset_bug = 0; in dpaa2_swp_dqrr_next_locked()
686 if ((verb & DPAA2_SWP_VALID_BIT) != swp->dqrr.valid_bit) in dpaa2_swp_dqrr_next_locked()
691 rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t)); in dpaa2_swp_dqrr_next_locked()
695 *idx = swp->dqrr.next_idx; in dpaa2_swp_dqrr_next_locked()
701 swp->dqrr.next_idx++; in dpaa2_swp_dqrr_next_locked()
702 swp->dqrr.next_idx &= swp->dqrr.ring_size - 1; /* wrap around */ in dpaa2_swp_dqrr_next_locked()
703 if (swp->dqrr.next_idx == 0u) in dpaa2_swp_dqrr_next_locked()
704 swp->dqrr.valid_bit ^= DPAA2_SWP_VALID_BIT; in dpaa2_swp_dqrr_next_locked()
730 cmd.numf = frames_n - 1; in dpaa2_swp_pull()
733 cmd.rsp_addr = (uint64_t)buf->paddr; in dpaa2_swp_pull()
736 cmd.verb &= ~(1 << QB_VDQCR_VERB_DCT0_SHIFT); in dpaa2_swp_pull()
737 cmd.verb |= (1 << QB_VDQCR_VERB_DCT1_SHIFT); in dpaa2_swp_pull()
738 /* Dequeue from a specific software portal channel (ID's in DQ_SRC). */ in dpaa2_swp_pull()
739 cmd.verb &= ~(1 << QB_VDQCR_VERB_DT0_SHIFT); in dpaa2_swp_pull()
740 cmd.verb &= ~(1 << QB_VDQCR_VERB_DT1_SHIFT); in dpaa2_swp_pull()
742 cmd.verb |= (1 << QB_VDQCR_VERB_RLS_SHIFT); in dpaa2_swp_pull()
744 cmd.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT); in dpaa2_swp_pull()
745 /* Allow the FQ to remain active in the portal after dequeue. */ in dpaa2_swp_pull()
746 cmd.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT); in dpaa2_swp_pull()
750 /* Terminate operation if portal is destroyed. */ in dpaa2_swp_pull()
763 bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD); in dpaa2_swp_pull()
766 msg = (struct dpaa2_dq *)buf->vaddr; in dpaa2_swp_pull()
767 for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) { in dpaa2_swp_pull()
768 if ((msg->fdr.desc.stat & DPAA2_DQ_STAT_VOLATILE) && in dpaa2_swp_pull()
769 (msg->fdr.desc.tok == DPAA2_SWP_VDQCR_TOKEN)) { in dpaa2_swp_pull()
771 msg->fdr.desc.tok = 0; in dpaa2_swp_pull()
785 * swp: Software portal used to send this command to.
794 int rc = dpaa2_swp_enq_mult(swp, ed, fd, &flags, 1); in dpaa2_swp_enq()
802 * swp: Software portal used to send this command to.
830 /* Terminate operation if portal is destroyed. */ in dpaa2_swp_enq_mult()
835 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map; in dpaa2_swp_enq_mult()
836 ci_offset = swp->cfg.mem_backed in dpaa2_swp_enq_mult()
840 half_mask = swp->eqcr.pi_ci_mask >> 1; in dpaa2_swp_enq_mult()
841 full_mask = swp->eqcr.pi_ci_mask; in dpaa2_swp_enq_mult()
843 if (swp->eqcr.available == 0) { in dpaa2_swp_enq_mult()
845 eqcr_ci = swp->eqcr.ci; in dpaa2_swp_enq_mult()
846 swp->eqcr.ci = val & full_mask; in dpaa2_swp_enq_mult()
848 swp->eqcr.available = dpaa2_swp_cyc_diff(swp->eqcr.pi_ring_size, in dpaa2_swp_enq_mult()
849 eqcr_ci, swp->eqcr.ci); in dpaa2_swp_enq_mult()
851 if (swp->eqcr.available == 0) { in dpaa2_swp_enq_mult()
857 eqcr_pi = swp->eqcr.pi; in dpaa2_swp_enq_mult()
858 num_enq = swp->eqcr.available < frames_n in dpaa2_swp_enq_mult()
859 ? swp->eqcr.available : frames_n; in dpaa2_swp_enq_mult()
860 swp->eqcr.available -= num_enq; in dpaa2_swp_enq_mult()
862 KASSERT(num_enq >= 0 && num_enq <= swp->eqcr.pi_ring_size, in dpaa2_swp_enq_mult()
864 KASSERT(swp->eqcr.available >= 0 && in dpaa2_swp_enq_mult()
865 swp->eqcr.available <= swp->eqcr.pi_ring_size, in dpaa2_swp_enq_mult()
866 ("%s: unexpected eqcr.available=%d", __func__, swp->eqcr.available)); in dpaa2_swp_enq_mult()
871 for (int j = 1; j <= 3; j++) in dpaa2_swp_enq_mult()
878 ed_pdat32[1]); in dpaa2_swp_enq_mult()
880 for (int j = 1; j <= 3; j++) in dpaa2_swp_enq_mult()
897 eqcr_pi = swp->eqcr.pi; in dpaa2_swp_enq_mult()
901 ed_pdat8[0] | swp->eqcr.pi_vb); in dpaa2_swp_enq_mult()
906 DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + 1, in dpaa2_swp_enq_mult()
907 (1 << ENQ_CMD_DCA_EN_SHIFT) | in dpaa2_swp_enq_mult()
912 swp->eqcr.pi_vb ^= DPAA2_SWP_VALID_BIT; in dpaa2_swp_enq_mult()
914 swp->eqcr.pi = eqcr_pi & full_mask; in dpaa2_swp_enq_mult()
926 ? (last - first) : ((2 * ringsize) - (first - last))); in dpaa2_swp_cyc_diff()
940 const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params; in dpaa2_swp_exec_br_command()
941 const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params; in dpaa2_swp_exec_br_command()
951 /* Terminate operation if portal is destroyed. */ in dpaa2_swp_exec_br_command()
962 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map; in dpaa2_swp_exec_br_command()
963 offset = swp->cfg.mem_backed in dpaa2_swp_exec_br_command()
969 for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++) in dpaa2_swp_exec_br_command()
970 bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]); in dpaa2_swp_exec_br_command()
971 bus_write_4(map, offset + 4, cmd_pdat32[1]); in dpaa2_swp_exec_br_command()
972 for (uint32_t i = 1; i <= 3; i++) in dpaa2_swp_exec_br_command()
976 if (swp->cfg.mem_backed) { in dpaa2_swp_exec_br_command()
977 bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num); in dpaa2_swp_exec_br_command()
983 bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num); in dpaa2_swp_exec_br_command()
995 * number of frames (1-16 or 1-32 depending on QBMan version) to the driver via
996 * DQRR or arbitrary DMA-mapped memory.
1010 const uint8_t *p8 = (const uint8_t *) cmd->params; in dpaa2_swp_exec_vdc_command_locked()
1011 const uint32_t *p32 = (const uint32_t *) cmd->params; in dpaa2_swp_exec_vdc_command_locked()
1015 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map; in dpaa2_swp_exec_vdc_command_locked()
1016 offset = swp->cfg.mem_backed in dpaa2_swp_exec_vdc_command_locked()
1021 for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++) in dpaa2_swp_exec_vdc_command_locked()
1022 bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]); in dpaa2_swp_exec_vdc_command_locked()
1023 bus_write_4(map, offset + 4, p32[1]); in dpaa2_swp_exec_vdc_command_locked()
1024 for (uint32_t i = 1; i <= 3; i++) in dpaa2_swp_exec_vdc_command_locked()
1028 if (swp->cfg.mem_backed) { in dpaa2_swp_exec_vdc_command_locked()
1029 bus_write_1(map, offset, c->verb | swp->vdq.valid_bit); in dpaa2_swp_exec_vdc_command_locked()
1030 swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT; in dpaa2_swp_exec_vdc_command_locked()
1036 bus_write_1(map, offset, c->verb | swp->vdq.valid_bit); in dpaa2_swp_exec_vdc_command_locked()
1037 swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT; in dpaa2_swp_exec_vdc_command_locked()
1064 /* Terminate operation if portal is destroyed. */ in dpaa2_swp_exec_mgmt_command()
1083 KASSERT((r->verb & CMD_VERB_MASK) == cmdid, in dpaa2_swp_exec_mgmt_command()
1085 r->verb, cmdid)); in dpaa2_swp_exec_mgmt_command()
1095 const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params; in dpaa2_swp_send_mgmt_command()
1096 const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params; in dpaa2_swp_send_mgmt_command()
1100 map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map; in dpaa2_swp_send_mgmt_command()
1101 offset = swp->cfg.mem_backed ? DPAA2_SWP_CENA_CR_MEM : DPAA2_SWP_CENA_CR; in dpaa2_swp_send_mgmt_command()
1104 for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++) in dpaa2_swp_send_mgmt_command()
1105 bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]); in dpaa2_swp_send_mgmt_command()
1106 bus_write_4(map, offset + 4, cmd_pdat32[1]); in dpaa2_swp_send_mgmt_command()
1107 for (uint32_t i = 1; i <= 3; i++) in dpaa2_swp_send_mgmt_command()
1111 if (swp->cfg.mem_backed) { in dpaa2_swp_send_mgmt_command()
1112 bus_write_1(map, offset, cmdid | swp->mr.valid_bit); in dpaa2_swp_send_mgmt_command()
1118 bus_write_1(map, offset, cmdid | swp->mc.valid_bit); in dpaa2_swp_send_mgmt_command()
1127 struct resource_map *map = swp->cfg.mem_backed in dpaa2_swp_wait_for_mgmt_response()
1128 ? swp->cena_map : swp->cinh_map; in dpaa2_swp_wait_for_mgmt_response()
1130 const uint32_t offset = swp->cfg.mem_backed in dpaa2_swp_wait_for_mgmt_response()
1132 : DPAA2_SWP_CENA_RR(swp->mc.valid_bit); in dpaa2_swp_wait_for_mgmt_response()
1137 for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) { in dpaa2_swp_wait_for_mgmt_response()
1138 if (swp->cfg.mem_backed) { in dpaa2_swp_wait_for_mgmt_response()
1140 if (swp->mr.valid_bit != (verb & DPAA2_SWP_VALID_BIT)) in dpaa2_swp_wait_for_mgmt_response()
1144 swp->mr.valid_bit ^= DPAA2_SWP_VALID_BIT; in dpaa2_swp_wait_for_mgmt_response()
1150 swp->mc.valid_bit ^= DPAA2_SWP_VALID_BIT; in dpaa2_swp_wait_for_mgmt_response()
1161 rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t)); in dpaa2_swp_wait_for_mgmt_response()