xref: /freebsd/sys/dev/dpaa2/dpaa2_swp.c (revision a259b98fa211ed87bfee58c575de4e2de94ee0fa)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause
3  *
4  * Copyright © 2014-2016 Freescale Semiconductor, Inc.
5  * Copyright © 2016-2019 NXP
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions are met:
10  *
11  * 1. Redistributions of source code must retain the above copyright notice,
12  *    this list of conditions and the following disclaimer.
13  *
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * 3. Neither the name of the copyright holder nor the names of its
19  *    contributors may be used to endorse or promote products derived from this
20  *    software without specific prior written permission.
21  *
22  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
26  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32  * POSSIBILITY OF SUCH DAMAGE.
33  *
34  * Original source file obtained from:
35  * drivers/soc/fsl/dpio/qbman-portal.c
36  *
37  * Commit: 4c86114194e644b6da9107d75910635c9e87179e
38  * Repository: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
39  */
40 
41 /*
42  * Copyright © 2021-2022 Dmitry Salychev
43  *
44  * Redistribution and use in source and binary forms, with or without
45  * modification, are permitted provided that the following conditions
46  * are met:
47  * 1. Redistributions of source code must retain the above copyright
48  *    notice, this list of conditions and the following disclaimer.
49  * 2. Redistributions in binary form must reproduce the above copyright
50  *    notice, this list of conditions and the following disclaimer in the
51  *    documentation and/or other materials provided with the distribution.
52  *
53  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
54  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
57  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63  * SUCH DAMAGE.
64  */
65 
66 #include <sys/cdefs.h>
67 /*
68  * DPAA2 QBMan software portal.
69  */
70 
71 #include <sys/param.h>
72 #include <sys/kernel.h>
73 #include <sys/bus.h>
74 #include <sys/rman.h>
75 #include <sys/module.h>
76 #include <sys/malloc.h>
77 #include <sys/mutex.h>
78 #include <sys/time.h>
79 #include <sys/types.h>
80 #include <sys/systm.h>
81 #include <sys/lock.h>
82 
83 #include <machine/bus.h>
84 #include <machine/resource.h>
85 #include <machine/atomic.h>
86 
87 #include "pcib_if.h"
88 #include "pci_if.h"
89 
90 #include "dpaa2_swp.h"
91 #include "dpaa2_mc.h"
92 #include "dpaa2_bp.h"
93 
94 #define CMD_SPIN_TIMEOUT		100u	/* us */
95 #define CMD_SPIN_ATTEMPTS		2000u	/* 200 ms max. */
96 
97 #define CMD_VERB_MASK			0x7Fu
98 
99 /* Shifts in the VERB byte of the enqueue command descriptor. */
100 #define ENQ_CMD_ORP_ENABLE_SHIFT	2
101 #define ENQ_CMD_IRQ_ON_DISPATCH_SHIFT	3
102 #define ENQ_CMD_TARGET_TYPE_SHIFT	4
103 #define ENQ_CMD_DCA_EN_SHIFT		7
104 /* VERB byte options of the enqueue command descriptor. */
105 #define ENQ_CMD_EMPTY			0u
106 #define ENQ_CMD_RESPONSE_ALWAYS		1u
107 #define ENQ_CMD_REJECTS_TO_FQ		2u
108 
109 #define ENQ_DESC_FD_OFFSET		32u
110 
111 #define ENQ_DCA_IDXMASK			0x0Fu
112 #define ENQ_FLAG_DCA			(1ull << 31)
113 
114 /* QBMan portal command codes. */
115 #define CMDID_SWP_MC_ACQUIRE		0x30
116 #define CMDID_SWP_BP_QUERY		0x32
117 #define CMDID_SWP_WQCHAN_CONFIGURE	0x46
118 
119 /* QBMan portal command result codes. */
120 #define QBMAN_CMD_RC_OK			0xF0
121 
122 /* SDQCR attribute codes */
123 #define QB_SDQCR_FC_SHIFT 		29u
124 #define QB_SDQCR_FC_MASK		0x1u
125 #define QB_SDQCR_DCT_SHIFT		24u
126 #define QB_SDQCR_DCT_MASK		0x3u
127 #define QB_SDQCR_TOK_SHIFT		16u
128 #define QB_SDQCR_TOK_MASK		0xFFu
129 #define QB_SDQCR_SRC_SHIFT		0u
130 #define QB_SDQCR_SRC_MASK		0xFFFFu
131 
132 /* Shifts in the VERB byte of the volatile dequeue command. */
133 #define QB_VDQCR_VERB_DCT0_SHIFT	0
134 #define QB_VDQCR_VERB_DCT1_SHIFT	1
135 #define QB_VDQCR_VERB_DT0_SHIFT		2
136 #define QB_VDQCR_VERB_DT1_SHIFT		3
137 #define QB_VDQCR_VERB_RLS_SHIFT		4
138 #define QB_VDQCR_VERB_WAE_SHIFT		5
139 #define QB_VDQCR_VERB_RAD_SHIFT		6
140 
141 /* Maximum timeout period for the DQRR interrupt. */
142 #define DQRR_MAX_ITP			4096u
143 #define DQRR_PI_MASK			0x0Fu
144 
145 /* Release Array Allocation register helpers. */
146 #define RAR_IDX(rar)			((rar) & 0x7u)
147 #define RAR_VB(rar)			((rar) & 0x80u)
148 #define RAR_SUCCESS(rar)		((rar) & 0x100u)
149 
150 MALLOC_DEFINE(M_DPAA2_SWP, "dpaa2_swp", "DPAA2 QBMan Software Portal");
151 
152 enum qbman_sdqcr_dct {
153 	qbman_sdqcr_dct_null = 0,
154 	qbman_sdqcr_dct_prio_ics,
155 	qbman_sdqcr_dct_active_ics,
156 	qbman_sdqcr_dct_active
157 };
158 
159 enum qbman_sdqcr_fc {
160 	qbman_sdqcr_fc_one = 0,
161 	qbman_sdqcr_fc_up_to_3 = 1
162 };
163 
164 /* Routines to execute software portal commands. */
165 static int dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *,
166     struct dpaa2_swp_cmd *, struct dpaa2_swp_rsp *, uint8_t);
167 static int dpaa2_swp_exec_br_command(struct dpaa2_swp *, struct dpaa2_swp_cmd *,
168     uint32_t);
169 static int dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *,
170     struct dpaa2_swp_cmd *);
171 
172 /* Management Commands helpers. */
173 static int dpaa2_swp_send_mgmt_command(struct dpaa2_swp *,
174     struct dpaa2_swp_cmd *, uint8_t);
175 static int dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *,
176     struct dpaa2_swp_rsp *);
177 
178 /* Helper subroutines. */
179 static int dpaa2_swp_cyc_diff(uint8_t, uint8_t, uint8_t);
180 
181 int
182 dpaa2_swp_init_portal(struct dpaa2_io_softc *sc, uint16_t flags)
183 {
184 	struct dpaa2_swp *p;
185 	struct dpaa2_swp_desc *desc;
186 	uint32_t reg, mask_size, eqcr_pi; /* EQCR producer index */
187 
188 	if (!sc)
189 		return (DPAA2_SWP_STAT_EINVAL);
190 	desc = &sc->swp_desc;
191 	p = malloc(sizeof(struct dpaa2_swp), M_DPAA2_SWP,
192 	    flags & DPAA2_SWP_NOWAIT_ALLOC
193 	    ? (M_NOWAIT | M_ZERO)
194 	    : (M_WAITOK | M_ZERO));
195 	if (!p)
196 		return (DPAA2_SWP_STAT_NO_MEMORY);
197 
198 	mtx_init(&p->lock, "swp_sleep_lock", NULL, MTX_DEF);
199 
200 	p->cfg.mem_backed = false;
201 	p->cfg.writes_cinh = true;
202 
203 	p->desc = desc;
204 	p->flags = flags;
205 	p->mc.valid_bit = DPAA2_SWP_VALID_BIT;
206 	p->mr.valid_bit = DPAA2_SWP_VALID_BIT;
207 
208 	/* FIXME: Memory-backed mode doesn't work now. Why? */
209 	p->cena_res = desc->cena_res;
210 	p->cena_map = desc->cena_map;
211 	p->cinh_res = desc->cinh_res;
212 	p->cinh_map = desc->cinh_map;
213 
214 	/* Static Dequeue Command Register configuration. */
215 	p->sdq = 0;
216 	p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
217 	p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
218 	p->sdq |= DPAA2_SWP_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;
219 
220 	/* Volatile Dequeue Command configuration. */
221 	p->vdq.valid_bit = DPAA2_SWP_VALID_BIT;
222 
223 	/* Dequeue Response Ring configuration */
224 	p->dqrr.next_idx = 0;
225 	p->dqrr.valid_bit = DPAA2_SWP_VALID_BIT;
226 	if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_4100) {
227 		p->dqrr.ring_size = 4;
228 		p->dqrr.reset_bug = 1;
229 	} else {
230 		p->dqrr.ring_size = 8;
231 		p->dqrr.reset_bug = 0;
232 	}
233 
234 	if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_5000) {
235 		reg = dpaa2_swp_set_cfg(
236 		    p->dqrr.ring_size, /* max. entries QMan writes to DQRR */
237 		    1, /* writes enabled in the CINH memory only */
238 		    0, /* EQCR_CI stashing threshold */
239 		    3, /* RPM: RCR in array mode */
240 		    2, /* DCM: Discrete consumption ack */
241 		    2, /* EPM: EQCR in ring mode (FIFO) */
242 		    1, /* mem stashing drop enable enable */
243 		    1, /* mem stashing priority enable */
244 		    1, /* mem stashing enable */
245 		    1, /* dequeue stashing priority enable */
246 		    0, /* dequeue stashing enable enable */
247 		    0  /* EQCR_CI stashing priority enable */
248 		);
249 		reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */
250 	} else {
251 		bus_set_region_4(p->cena_map, 0, 0,
252 		    rman_get_size(p->cena_res) / 4);
253 
254 		reg = dpaa2_swp_set_cfg(
255 		    p->dqrr.ring_size, /* max. entries QMan writes to DQRR */					/* DQRR_MF */
256 		    1, /* writes enabled in the CINH memory only */						/* WN */
257 		    0, /* EQCR_CI stashing is disabled */							/* EST */
258 		    3, /* RPM: RCR in array mode */								/* RPM */
259 		    2, /* DCM: Discrete consumption ack */							/* DCM */
260 		    2, /* EPM: EQCR in ring mode (FIFO) */							/* EPM */
261 		    1, /* Dequeued frame data, annotation, and FQ context stashing drop enable */		/* SD */
262 		    1, /* Dequeued frame data, annotation, and FQ context stashing priority */			/* SP */
263 		    1, /* Dequeued frame data, annotation, and FQ context stashing enable */			/* SE */
264 		    1, /* Dequeue response ring (DQRR) entry stashing priority */				/* DP */
265 		    0, /* Dequeue response ring (DQRR) entry, or cacheable portal area, stashing enable. */	/* DE */
266 		    0  /* EQCR_CI stashing priority */								/* EP */
267 		);
268 		/* TODO: Switch to memory-backed mode. */
269 		reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */
270 	}
271 	dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_CFG, reg);
272 	reg = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_CFG);
273 	if (!reg) {
274 		free(p, M_DPAA2_SWP);
275 		return (DPAA2_SWP_STAT_PORTAL_DISABLED);
276 	}
277 
278 	/*
279 	 * Static Dequeue Command Register needs to be initialized to 0 when no
280 	 * channels are being dequeued from or else the QMan HW will indicate an
281 	 * error. The values that were calculated above will be applied when
282 	 * dequeues from a specific channel are enabled.
283 	 */
284 	dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_SDQCR, 0);
285 
286 	p->eqcr.pi_ring_size = 8;
287 	/* if ((desc->swp_version & DPAA2_SWP_REV_MASK) >= DPAA2_SWP_REV_5000) */
288 	/* 	p->eqcr.pi_ring_size = 32; */
289 
290 	for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
291 		p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1;
292 
293 	eqcr_pi = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_PI);
294 	p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
295 	p->eqcr.pi_vb = eqcr_pi & DPAA2_SWP_VALID_BIT;
296 	p->eqcr.ci = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_CI)
297 	    & p->eqcr.pi_ci_mask;
298 	p->eqcr.available = p->eqcr.pi_ring_size;
299 
300 	/*
301 	 * Initialize the portal with an IRQ threshold and timeout from the
302 	 * device context.
303 	 */
304 	dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, sc->irq_holdoff);
305 
306 	sc->swp = p;
307 
308 	return (0);
309 }
310 
311 void
312 dpaa2_swp_free_portal(struct dpaa2_swp *swp)
313 {
314 	uint16_t flags;
315 
316 	KASSERT(swp != NULL, ("%s: swp is NULL", __func__));
317 
318 	DPAA2_SWP_LOCK(swp, &flags);
319 	swp->flags |= DPAA2_SWP_DESTROYED;
320 	DPAA2_SWP_UNLOCK(swp);
321 
322 	/* Let threads stop using this portal. */
323 	DELAY(DPAA2_SWP_TIMEOUT);
324 
325 	mtx_destroy(&swp->lock);
326 	free(swp, M_DPAA2_SWP);
327 }
328 
329 uint32_t
330 dpaa2_swp_set_cfg(uint8_t max_fill, uint8_t wn, uint8_t est, uint8_t rpm,
331     uint8_t dcm, uint8_t epm, int sd, int sp, int se, int dp, int de, int ep)
332 {
333 	return (
334 	    max_fill	<< DPAA2_SWP_CFG_DQRR_MF_SHIFT |
335 	    est		<< DPAA2_SWP_CFG_EST_SHIFT |
336 	    wn		<< DPAA2_SWP_CFG_WN_SHIFT |
337 	    rpm		<< DPAA2_SWP_CFG_RPM_SHIFT |
338 	    dcm		<< DPAA2_SWP_CFG_DCM_SHIFT |
339 	    epm		<< DPAA2_SWP_CFG_EPM_SHIFT |
340 	    sd		<< DPAA2_SWP_CFG_SD_SHIFT |
341 	    sp		<< DPAA2_SWP_CFG_SP_SHIFT |
342 	    se		<< DPAA2_SWP_CFG_SE_SHIFT |
343 	    dp		<< DPAA2_SWP_CFG_DP_SHIFT |
344 	    de		<< DPAA2_SWP_CFG_DE_SHIFT |
345 	    ep		<< DPAA2_SWP_CFG_EP_SHIFT
346 	);
347 }
348 
349 /* Read/write registers of a software portal. */
350 
351 void
352 dpaa2_swp_write_reg(struct dpaa2_swp *swp, uint32_t o, uint32_t v)
353 {
354 	bus_write_4(swp->cinh_map, o, v);
355 }
356 
357 uint32_t
358 dpaa2_swp_read_reg(struct dpaa2_swp *swp, uint32_t o)
359 {
360 	return (bus_read_4(swp->cinh_map, o));
361 }
362 
363 /* Helper routines. */
364 
365 /**
366  * @brief Set enqueue descriptor without Order Point Record ID.
367  *
368  * ed:		Enqueue descriptor.
369  * resp_always:	Enqueue with response always (1); FD from a rejected enqueue
370  *		will be returned on a FQ (0).
371  */
372 void
373 dpaa2_swp_set_ed_norp(struct dpaa2_eq_desc *ed, bool resp_always)
374 {
375 	ed->verb &= ~(1 << ENQ_CMD_ORP_ENABLE_SHIFT);
376 	if (resp_always)
377 		ed->verb |= ENQ_CMD_RESPONSE_ALWAYS;
378 	else
379 		ed->verb |= ENQ_CMD_REJECTS_TO_FQ;
380 }
381 
382 /**
383  * @brief Set FQ of the enqueue descriptor.
384  */
385 void
386 dpaa2_swp_set_ed_fq(struct dpaa2_eq_desc *ed, uint32_t fqid)
387 {
388 	ed->verb &= ~(1 << ENQ_CMD_TARGET_TYPE_SHIFT);
389 	ed->tgtid = fqid;
390 }
391 
392 /**
393  * @brief Enable interrupts for a software portal.
394  */
395 void
396 dpaa2_swp_set_intr_trigger(struct dpaa2_swp *swp, uint32_t mask)
397 {
398 	if (swp != NULL)
399 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_IER, mask);
400 }
401 
402 /**
403  * @brief Return the value in the SWP_IER register.
404  */
405 uint32_t
406 dpaa2_swp_get_intr_trigger(struct dpaa2_swp *swp)
407 {
408 	if (swp != NULL)
409 		return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_IER);
410 	return (0);
411 }
412 
413 /**
414  * @brief Return the value in the SWP_ISR register.
415  */
416 uint32_t
417 dpaa2_swp_read_intr_status(struct dpaa2_swp *swp)
418 {
419 	if (swp != NULL)
420 		return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_ISR);
421 	return (0);
422 }
423 
424 /**
425  * @brief Clear SWP_ISR register according to the given mask.
426  */
427 void
428 dpaa2_swp_clear_intr_status(struct dpaa2_swp *swp, uint32_t mask)
429 {
430 	if (swp != NULL)
431 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ISR, mask);
432 }
433 
434 /**
435  * @brief Enable or disable push dequeue.
436  *
437  * swp:		the software portal object
438  * chan_idx:	the channel index (0 to 15)
439  * en:		enable or disable push dequeue
440  */
441 void
442 dpaa2_swp_set_push_dequeue(struct dpaa2_swp *swp, uint8_t chan_idx, bool en)
443 {
444 	uint16_t dqsrc;
445 
446 	if (swp != NULL) {
447 		if (chan_idx > 15u) {
448 			device_printf(swp->desc->dpio_dev, "channel index "
449 			    "should be <= 15: chan_idx=%d\n", chan_idx);
450 			return;
451 		}
452 
453 		if (en)
454 			swp->sdq |= 1 << chan_idx;
455 		else
456 			swp->sdq &= ~(1 << chan_idx);
457 		/*
458 		 * Read make the complete src map. If no channels are enabled
459 		 * the SDQCR must be 0 or else QMan will assert errors.
460 		 */
461 		dqsrc = (swp->sdq >> DPAA2_SDQCR_SRC_SHIFT) &
462 		    DPAA2_SDQCR_SRC_MASK;
463 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_SDQCR, dqsrc != 0
464 		    ? swp->sdq : 0);
465 	}
466 }
467 
468 /**
469  * @brief Set new IRQ coalescing values.
470  *
471  * swp:		The software portal object.
472  * threshold:	Threshold for DQRR interrupt generation. The DQRR interrupt
473  *		asserts when the ring contains greater than "threshold" entries.
474  * holdoff:	DQRR interrupt holdoff (timeout) period in us.
475  */
476 int dpaa2_swp_set_irq_coalescing(struct dpaa2_swp *swp, uint32_t threshold,
477     uint32_t holdoff)
478 {
479 	uint32_t itp; /* Interrupt Timeout Period */
480 
481 	if (swp == NULL)
482 		return (EINVAL);
483 
484 	/*
485 	 * Convert "holdoff" value from us to 256 QBMAN clock cycles
486 	 * increments. This depends on the QBMAN internal frequency.
487 	 */
488 	itp = (holdoff * 1000u) / swp->desc->swp_cycles_ratio;
489 	if (itp > DQRR_MAX_ITP)
490 		itp = DQRR_MAX_ITP;
491 	if (threshold >= swp->dqrr.ring_size)
492 		threshold = swp->dqrr.ring_size - 1;
493 
494 	swp->dqrr.irq_threshold = threshold;
495 	swp->dqrr.irq_itp = itp;
496 
497 	dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_DQRR_ITR, threshold);
498 	dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ITPR, itp);
499 
500 	return (0);
501 }
502 
503 /*
504  * Software portal commands.
505  */
506 
507 /**
508  * @brief Configure the channel data availability notification (CDAN)
509  * in a particular WQ channel.
510  */
511 int
512 dpaa2_swp_conf_wq_channel(struct dpaa2_swp *swp, uint16_t chan_id,
513     uint8_t we_mask, bool cdan_en, uint64_t ctx)
514 {
515 	/* NOTE: 64 bytes command. */
516 	struct __packed {
517 		uint8_t		verb;
518 		uint8_t		result; /* in response only! */
519 		uint16_t	chan_id;
520 		uint8_t		we;
521 		uint8_t		ctrl;
522 		uint16_t	_reserved2;
523 		uint64_t	ctx;
524 		uint8_t		_reserved3[48];
525 	} cmd = {0};
526 	struct __packed {
527 		uint8_t		verb;
528 		uint8_t		result;
529 		uint16_t	chan_id;
530 		uint8_t		_reserved[60];
531 	} rsp;
532 	int error;
533 
534 	if (swp == NULL)
535 		return (EINVAL);
536 
537 	cmd.chan_id = chan_id;
538 	cmd.we = we_mask;
539 	cmd.ctrl = cdan_en ? 1u : 0u;
540 	cmd.ctx = ctx;
541 
542 	error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,
543 	    (struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_WQCHAN_CONFIGURE);
544 	if (error)
545 		return (error);
546 
547 	if (rsp.result != QBMAN_CMD_RC_OK) {
548 		device_printf(swp->desc->dpio_dev, "WQ channel configuration "
549 		    "error: channel_id=%d, result=0x%02x\n", chan_id,
550 		    rsp.result);
551 		return (EIO);
552 	}
553 
554 	return (0);
555 }
556 
557 /**
558  * @brief Query current configuration/state of the buffer pool.
559  */
560 int
561 dpaa2_swp_query_bp(struct dpaa2_swp *swp, uint16_t bpid,
562     struct dpaa2_bp_conf *conf)
563 {
564 	/* NOTE: 64 bytes command. */
565 	struct __packed {
566 		uint8_t		verb;
567 		uint8_t		_reserved1;
568 		uint16_t	bpid;
569 		uint8_t		_reserved2[60];
570 	} cmd = {0};
571 	struct __packed {
572 		uint8_t		verb;
573 		uint8_t		result;
574 		uint32_t	_reserved1;
575 		uint8_t		bdi;
576 		uint8_t		state;
577 		uint32_t	fill;
578 		/* TODO: Support the other fields as well. */
579 		uint8_t		_reserved2[52];
580 	} rsp;
581 	int error;
582 
583 	if (swp == NULL || conf == NULL)
584 		return (EINVAL);
585 
586 	cmd.bpid = bpid;
587 
588 	error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,
589 	    (struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_BP_QUERY);
590 	if (error)
591 		return (error);
592 
593 	if (rsp.result != QBMAN_CMD_RC_OK) {
594 		device_printf(swp->desc->dpio_dev, "BP query error: bpid=%d, "
595 		    "result=0x%02x\n", bpid, rsp.result);
596 		return (EIO);
597 	}
598 
599 	conf->bdi = rsp.bdi;
600 	conf->state = rsp.state;
601 	conf->free_bufn = rsp.fill;
602 
603 	return (0);
604 }
605 
606 int
607 dpaa2_swp_release_bufs(struct dpaa2_swp *swp, uint16_t bpid, bus_addr_t *buf,
608     uint32_t buf_num)
609 {
610 	/* NOTE: 64 bytes command. */
611 	struct __packed {
612 		uint8_t		verb;
613 		uint8_t		_reserved1;
614 		uint16_t	bpid;
615 		uint32_t	_reserved2;
616 		uint64_t	buf[DPAA2_SWP_BUFS_PER_CMD];
617 	} cmd = {0};
618 	int error;
619 
620 	if (swp == NULL || buf == NULL || buf_num == 0u ||
621 	    buf_num > DPAA2_SWP_BUFS_PER_CMD)
622 		return (EINVAL);
623 
624 	for (uint32_t i = 0; i < buf_num; i++)
625 		cmd.buf[i] = buf[i];
626 	cmd.bpid = bpid;
627 	cmd.verb |= 1 << 5; /* Switch release buffer command to valid. */
628 
629 	error = dpaa2_swp_exec_br_command(swp, (struct dpaa2_swp_cmd *) &cmd,
630 	    buf_num);
631 	if (error) {
632 		device_printf(swp->desc->dpio_dev, "buffers release command "
633 		    "failed\n");
634 		return (error);
635 	}
636 
637 	return (0);
638 }
639 
640 int
641 dpaa2_swp_dqrr_next_locked(struct dpaa2_swp *swp, struct dpaa2_dq *dq,
642     uint32_t *idx)
643 {
644 	struct resource_map *map = swp->cinh_map;
645 	struct dpaa2_swp_rsp *rsp = (struct dpaa2_swp_rsp *) dq;
646 	uint32_t verb, pi; /* producer index */
647 	uint32_t offset = swp->cfg.mem_backed
648 	    ? DPAA2_SWP_CENA_DQRR_MEM(swp->dqrr.next_idx)
649 	    : DPAA2_SWP_CENA_DQRR(swp->dqrr.next_idx);
650 
651 	if (swp == NULL || dq == NULL)
652 		return (EINVAL);
653 
654 	/*
655 	 * Before using valid-bit to detect if something is there, we have to
656 	 * handle the case of the DQRR reset bug...
657 	 */
658 	if (swp->dqrr.reset_bug) {
659 		/*
660 		 * We pick up new entries by cache-inhibited producer index,
661 		 * which means that a non-coherent mapping would require us to
662 		 * invalidate and read *only* once that PI has indicated that
663 		 * there's an entry here. The first trip around the DQRR ring
664 		 * will be much less efficient than all subsequent trips around
665 		 * it...
666 		 */
667 		pi = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_DQPI) & DQRR_PI_MASK;
668 
669 		/* There are new entries if pi != next_idx */
670 		if (pi == swp->dqrr.next_idx)
671 			return (ENOENT);
672 
673 		/*
674 		 * If next_idx is/was the last ring index, and 'pi' is
675 		 * different, we can disable the workaround as all the ring
676 		 * entries have now been DMA'd to so valid-bit checking is
677 		 * repaired.
678 		 *
679 		 * NOTE: This logic needs to be based on next_idx (which
680 		 *	 increments one at a time), rather than on pi (which
681 		 *	 can burst and wrap-around between our snapshots of it).
682 		 */
683 		if (swp->dqrr.next_idx == (swp->dqrr.ring_size - 1))
684 			swp->dqrr.reset_bug = 0;
685 	}
686 
687 	verb = bus_read_4(map, offset);
688 	if ((verb & DPAA2_SWP_VALID_BIT) != swp->dqrr.valid_bit)
689 		return (ENOENT);
690 
691 	/* Read dequeue response message. */
692 	for (int i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++)
693 		rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t));
694 
695 	/* Return index of the current entry (if requested). */
696 	if (idx != NULL)
697 		*idx = swp->dqrr.next_idx;
698 
699 	/*
700 	 * There's something there. Move "next_idx" attention to the next ring
701 	 * entry before returning what we found.
702 	 */
703 	swp->dqrr.next_idx++;
704 	swp->dqrr.next_idx &= swp->dqrr.ring_size - 1; /* wrap around */
705 	if (swp->dqrr.next_idx == 0u)
706 		swp->dqrr.valid_bit ^= DPAA2_SWP_VALID_BIT;
707 
708 	return (0);
709 }
710 
711 int
712 dpaa2_swp_pull(struct dpaa2_swp *swp, uint16_t chan_id, struct dpaa2_buf *buf,
713     uint32_t frames_n)
714 {
715 	/* NOTE: 64 bytes command. */
716 	struct __packed {
717 		uint8_t		verb;
718 		uint8_t		numf;
719 		uint8_t		tok;
720 		uint8_t		_reserved;
721 		uint32_t	dq_src;
722 		uint64_t	rsp_addr;
723 		uint64_t	_reserved1[6];
724 	} cmd = {0};
725 	struct dpaa2_dq *msg;
726 	uint16_t flags;
727 	int i, error;
728 
729 	KASSERT(frames_n != 0u, ("%s: cannot pull zero frames", __func__));
730 	KASSERT(frames_n <= 16u, ("%s: too much frames to pull", __func__));
731 
732 	cmd.numf = frames_n - 1;
733 	cmd.tok = DPAA2_SWP_VDQCR_TOKEN;
734 	cmd.dq_src = chan_id;
735 	cmd.rsp_addr = (uint64_t)buf->paddr;
736 
737 	/* Dequeue command type */
738 	cmd.verb &= ~(1 << QB_VDQCR_VERB_DCT0_SHIFT);
739 	cmd.verb |=  (1 << QB_VDQCR_VERB_DCT1_SHIFT);
740 	/* Dequeue from a specific software portal channel (ID's in DQ_SRC). */
741 	cmd.verb &= ~(1 << QB_VDQCR_VERB_DT0_SHIFT);
742 	cmd.verb &= ~(1 << QB_VDQCR_VERB_DT1_SHIFT);
743 	/* Write the response to this command into memory (at the RSP_ADDR). */
744 	cmd.verb |=  (1 << QB_VDQCR_VERB_RLS_SHIFT);
745 	/* Response writes won't attempt to allocate into a cache. */
746 	cmd.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
747 	/* Allow the FQ to remain active in the portal after dequeue. */
748 	cmd.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT);
749 
750 	DPAA2_SWP_LOCK(swp, &flags);
751 	if (flags & DPAA2_SWP_DESTROYED) {
752 		/* Terminate operation if portal is destroyed. */
753 		DPAA2_SWP_UNLOCK(swp);
754 		return (ENOENT);
755 	}
756 
757 	error = dpaa2_swp_exec_vdc_command_locked(swp,
758 	    (struct dpaa2_swp_cmd *) &cmd);
759 	if (error != 0) {
760 		DPAA2_SWP_UNLOCK(swp);
761 		return (error);
762 	}
763 
764 	/* Let's sync before reading VDQ response from QBMan. */
765 	bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD);
766 
767 	/* Read VDQ response from QBMan. */
768 	msg = (struct dpaa2_dq *)buf->vaddr;
769 	for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) {
770 		if ((msg->fdr.desc.stat & DPAA2_DQ_STAT_VOLATILE) &&
771 		    (msg->fdr.desc.tok == DPAA2_SWP_VDQCR_TOKEN)) {
772 			/* Reset token. */
773 			msg->fdr.desc.tok = 0;
774 			break;
775 		}
776 		DELAY(CMD_SPIN_TIMEOUT);
777 	}
778 	DPAA2_SWP_UNLOCK(swp);
779 
780 	/* Return an error on expired timeout. */
781 	return (i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0);
782 }
783 
784 /**
785  * @brief Issue a command to enqueue a frame using one enqueue descriptor.
786  *
787  * swp:		Software portal used to send this command to.
788  * ed:		Enqueue command descriptor.
789  * fd:		Frame descriptor to enqueue.
790  */
791 int
792 dpaa2_swp_enq(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,
793     struct dpaa2_fd *fd)
794 {
795 	uint32_t flags = 0;
796 	int rc = dpaa2_swp_enq_mult(swp, ed, fd, &flags, 1);
797 
798 	return (rc >= 0 ? 0 : EBUSY);
799 }
800 
801 /**
802  * @brief Issue a command to enqueue frames using one enqueue descriptor.
803  *
804  * swp:		Software portal used to send this command to.
805  * ed:		Enqueue command descriptor.
806  * fd:		Frame descriptor to enqueue.
807  * flags:	Table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL.
808  * frames_n:	Number of FDs to enqueue.
809  *
810  * NOTE: Enqueue command (64 bytes): 32 (eq. descriptor) + 32 (frame descriptor).
811  */
812 int
813 dpaa2_swp_enq_mult(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,
814     struct dpaa2_fd *fd, uint32_t *flags, int frames_n)
815 {
816 	const uint8_t  *ed_pdat8 =  (const uint8_t *) ed;
817 	const uint32_t *ed_pdat32 = (const uint32_t *) ed;
818 	const uint64_t *ed_pdat64 = (const uint64_t *) ed;
819 	const uint64_t *fd_pdat64 = (const uint64_t *) fd;
820 	struct resource_map *map;
821 	uint32_t eqcr_ci, eqcr_pi; /* EQCR consumer/producer index */
822 	uint32_t half_mask, full_mask, val, ci_offset;
823 	uint16_t swp_flags;
824 	int num_enq = 0;
825 
826 	if (swp == NULL || ed == NULL || fd == NULL || flags == NULL ||
827 	    frames_n == 0)
828 		return (EINVAL);
829 
830 	DPAA2_SWP_LOCK(swp, &swp_flags);
831 	if (swp_flags & DPAA2_SWP_DESTROYED) {
832 		/* Terminate operation if portal is destroyed. */
833 		DPAA2_SWP_UNLOCK(swp);
834 		return (ENOENT);
835 	}
836 
837 	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
838 	ci_offset = swp->cfg.mem_backed
839 	    ? DPAA2_SWP_CENA_EQCR_CI_MEMBACK
840 	    : DPAA2_SWP_CENA_EQCR_CI;
841 
842 	half_mask = swp->eqcr.pi_ci_mask >> 1;
843 	full_mask = swp->eqcr.pi_ci_mask;
844 
845 	if (swp->eqcr.available == 0) {
846 		val = dpaa2_swp_read_reg(swp, ci_offset);
847 		eqcr_ci = swp->eqcr.ci;
848 		swp->eqcr.ci = val & full_mask;
849 
850 		swp->eqcr.available = dpaa2_swp_cyc_diff(swp->eqcr.pi_ring_size,
851 		    eqcr_ci, swp->eqcr.ci);
852 
853 		if (swp->eqcr.available == 0) {
854 			DPAA2_SWP_UNLOCK(swp);
855 			return (0);
856 		}
857 	}
858 
859 	eqcr_pi = swp->eqcr.pi;
860 	num_enq = swp->eqcr.available < frames_n
861 	    ? swp->eqcr.available : frames_n;
862 	swp->eqcr.available -= num_enq;
863 
864 	KASSERT(num_enq >= 0 && num_enq <= swp->eqcr.pi_ring_size,
865 	    ("%s: unexpected num_enq=%d", __func__, num_enq));
866 	KASSERT(swp->eqcr.available >= 0 &&
867 	    swp->eqcr.available <= swp->eqcr.pi_ring_size,
868 	    ("%s: unexpected eqcr.available=%d", __func__, swp->eqcr.available));
869 
870 	/* Fill in the EQCR ring. */
871 	for (int i = 0; i < num_enq; i++) {
872 		/* Write enq. desc. without the VERB, DCA, SEQNUM and OPRID. */
873 		for (int j = 1; j <= 3; j++)
874 			bus_write_8(map,
875 			    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
876 			    sizeof(uint64_t) * j, ed_pdat64[j]);
877 		/* Write OPRID. */
878 		bus_write_4(map,
879 		    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + sizeof(uint32_t),
880 		    ed_pdat32[1]);
881 		/* Write DCA and SEQNUM without VERB byte. */
882 		for (int j = 1; j <= 3; j++)
883 			bus_write_1(map,
884 			    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
885 			    sizeof(uint8_t) * j, ed_pdat8[j]);
886 
887 		/* Write frame descriptor. */
888 		for (int j = 0; j <= 3; j++)
889 			bus_write_8(map,
890 			    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
891 			    ENQ_DESC_FD_OFFSET +
892 			    sizeof(uint64_t) * j, fd_pdat64[j]);
893 		eqcr_pi++;
894 	}
895 
896 	wmb();
897 
898 	/* Write the VERB byte of enqueue descriptor. */
899 	eqcr_pi = swp->eqcr.pi;
900 	for (int i = 0; i < num_enq; i++) {
901 		bus_write_1(map,
902 		    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask),
903 		    ed_pdat8[0] | swp->eqcr.pi_vb);
904 
905 		if (flags && (flags[i] & ENQ_FLAG_DCA)) {
906 			/* Update DCA byte. */
907 			bus_write_1(map,
908 			    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + 1,
909 			    (1 << ENQ_CMD_DCA_EN_SHIFT) |
910 			    (flags[i] & ENQ_DCA_IDXMASK));
911 		}
912 		eqcr_pi++;
913 		if (!(eqcr_pi & half_mask))
914 			swp->eqcr.pi_vb ^= DPAA2_SWP_VALID_BIT;
915 	}
916 	swp->eqcr.pi = eqcr_pi & full_mask;
917 
918 	DPAA2_SWP_UNLOCK(swp);
919 
920 	return (num_enq);
921 }
922 
923 static int
924 dpaa2_swp_cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last)
925 {
926 	/* 'first' is included, 'last' is excluded */
927 	return ((first <= last)
928 	    ? (last - first) : ((2 * ringsize) - (first - last)));
929 }
930 
931 /**
932  * @brief Execute Buffer Release Command (BRC).
933  */
934 static int
935 dpaa2_swp_exec_br_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
936     uint32_t buf_num)
937 {
938 	struct __packed with_verb {
939 		uint8_t	verb;
940 		uint8_t	_reserved[63];
941 	} *c;
942 	const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params;
943 	const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params;
944 	struct resource_map *map;
945 	uint32_t offset, rar; /* Release Array Allocation register */
946 	uint16_t flags;
947 
948 	if (!swp || !cmd)
949 		return (EINVAL);
950 
951 	DPAA2_SWP_LOCK(swp, &flags);
952 	if (flags & DPAA2_SWP_DESTROYED) {
953 		/* Terminate operation if portal is destroyed. */
954 		DPAA2_SWP_UNLOCK(swp);
955 		return (ENOENT);
956 	}
957 
958 	rar = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_RAR);
959 	if (!RAR_SUCCESS(rar)) {
960 		DPAA2_SWP_UNLOCK(swp);
961 		return (EBUSY);
962 	}
963 
964 	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
965 	offset = swp->cfg.mem_backed
966 	    ? DPAA2_SWP_CENA_RCR_MEM(RAR_IDX(rar))
967 	    : DPAA2_SWP_CENA_RCR(RAR_IDX(rar));
968 	c = (struct with_verb *) cmd;
969 
970 	/* Write command bytes (without VERB byte). */
971 	for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
972 		bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
973 	bus_write_4(map, offset + 4, cmd_pdat32[1]);
974 	for (uint32_t i = 1; i <= 3; i++)
975 		bus_write_1(map, offset + i, cmd_pdat8[i]);
976 
977 	/* Write VERB byte and trigger command execution. */
978 	if (swp->cfg.mem_backed) {
979 		bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num);
980 		wmb();
981 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_RCR_AM_RT +
982 		    RAR_IDX(rar) * 4, DPAA2_SWP_RT_MODE);
983 	} else {
984 		wmb();
985 		bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num);
986 	}
987 
988 	DPAA2_SWP_UNLOCK(swp);
989 
990 	return (0);
991 }
992 
993 /**
994  * @brief Execute Volatile Dequeue Command (VDC).
995  *
996  * This command will be executed by QBMan only once in order to deliver requested
997  * number of frames (1-16 or 1-32 depending on QBMan version) to the driver via
998  * DQRR or arbitrary DMA-mapped memory.
999  *
1000  * NOTE: There is a counterpart to the volatile dequeue command called static
1001  *	 dequeue command (SDQC) which is executed periodically all the time the
1002  *	 command is present in the SDQCR register.
1003  */
1004 static int
1005 dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *swp,
1006     struct dpaa2_swp_cmd *cmd)
1007 {
1008 	struct __packed with_verb {
1009 		uint8_t	verb;
1010 		uint8_t	_reserved[63];
1011 	} *c;
1012 	const uint8_t *p8 = (const uint8_t *) cmd->params;
1013 	const uint32_t *p32 = (const uint32_t *) cmd->params;
1014 	struct resource_map *map;
1015 	uint32_t offset;
1016 
1017 	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
1018 	offset = swp->cfg.mem_backed
1019 	    ? DPAA2_SWP_CENA_VDQCR_MEM : DPAA2_SWP_CENA_VDQCR;
1020 	c = (struct with_verb *) cmd;
1021 
1022 	/* Write command bytes (without VERB byte). */
1023 	for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
1024 		bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
1025 	bus_write_4(map, offset + 4, p32[1]);
1026 	for (uint32_t i = 1; i <= 3; i++)
1027 		bus_write_1(map, offset + i, p8[i]);
1028 
1029 	/* Write VERB byte and trigger command execution. */
1030 	if (swp->cfg.mem_backed) {
1031 		bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);
1032 		swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;
1033 		wmb();
1034 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_VDQCR_RT,
1035 		    DPAA2_SWP_RT_MODE);
1036 	} else {
1037 		wmb();
1038 		bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);
1039 		swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;
1040 	}
1041 
1042 	return (0);
1043 }
1044 
1045 /**
1046  * @brief Execute a QBMan management command.
1047  */
1048 static int
1049 dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
1050     struct dpaa2_swp_rsp *rsp, uint8_t cmdid)
1051 {
1052 #if (defined(_KERNEL) && defined(INVARIANTS))
1053 	struct __packed with_verb {
1054 		uint8_t	verb;
1055 		uint8_t	_reserved[63];
1056 	} *r;
1057 #endif
1058 	uint16_t flags;
1059 	int error;
1060 
1061 	if (swp == NULL || cmd == NULL || rsp == NULL)
1062 		return (EINVAL);
1063 
1064 	DPAA2_SWP_LOCK(swp, &flags);
1065 	if (flags & DPAA2_SWP_DESTROYED) {
1066 		/* Terminate operation if portal is destroyed. */
1067 		DPAA2_SWP_UNLOCK(swp);
1068 		return (ENOENT);
1069 	}
1070 
1071 	/*
1072 	 * Send a command to QBMan using Management Command register and wait
1073 	 * for response from the Management Response registers.
1074 	 */
1075 	dpaa2_swp_send_mgmt_command(swp, cmd, cmdid);
1076 	error = dpaa2_swp_wait_for_mgmt_response(swp, rsp);
1077 	if (error) {
1078 		DPAA2_SWP_UNLOCK(swp);
1079 		return (error);
1080 	}
1081 	DPAA2_SWP_UNLOCK(swp);
1082 
1083 #if (defined(_KERNEL) && defined(INVARIANTS))
1084 	r = (struct with_verb *) rsp;
1085 	KASSERT((r->verb & CMD_VERB_MASK) == cmdid,
1086 	    ("wrong VERB byte in response: resp=0x%02x, expected=0x%02x",
1087 	    r->verb, cmdid));
1088 #endif
1089 
1090 	return (0);
1091 }
1092 
1093 static int
1094 dpaa2_swp_send_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
1095     uint8_t cmdid)
1096 {
1097 	const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params;
1098 	const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params;
1099 	struct resource_map *map;
1100 	uint32_t offset;
1101 
1102 	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
1103 	offset = swp->cfg.mem_backed ? DPAA2_SWP_CENA_CR_MEM : DPAA2_SWP_CENA_CR;
1104 
1105 	/* Write command bytes (without VERB byte). */
1106 	for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
1107 		bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
1108 	bus_write_4(map, offset + 4, cmd_pdat32[1]);
1109 	for (uint32_t i = 1; i <= 3; i++)
1110 		bus_write_1(map, offset + i, cmd_pdat8[i]);
1111 
1112 	/* Write VERB byte and trigger command execution. */
1113 	if (swp->cfg.mem_backed) {
1114 		bus_write_1(map, offset, cmdid | swp->mr.valid_bit);
1115 		wmb();
1116 		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_CR_RT,
1117 		    DPAA2_SWP_RT_MODE);
1118 	} else {
1119 		wmb();
1120 		bus_write_1(map, offset, cmdid | swp->mc.valid_bit);
1121 	}
1122 
1123 	return (0);
1124 }
1125 
1126 static int
1127 dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *swp, struct dpaa2_swp_rsp *rsp)
1128 {
1129 	struct resource_map *map = swp->cfg.mem_backed
1130 	    ? swp->cena_map : swp->cinh_map;
1131 	/* Management command response to be read from the only RR or RR0/RR1. */
1132 	const uint32_t offset = swp->cfg.mem_backed
1133 	    ? DPAA2_SWP_CENA_RR_MEM
1134 	    : DPAA2_SWP_CENA_RR(swp->mc.valid_bit);
1135 	uint32_t i, verb, ret;
1136 	int rc;
1137 
1138 	/* Wait for a command response from QBMan. */
1139 	for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) {
1140 		if (swp->cfg.mem_backed) {
1141 			verb = (uint32_t) (bus_read_4(map, offset) & 0xFFu);
1142 			if (swp->mr.valid_bit != (verb & DPAA2_SWP_VALID_BIT))
1143 				goto wait;
1144 			if (!(verb & ~DPAA2_SWP_VALID_BIT))
1145 				goto wait;
1146 			swp->mr.valid_bit ^= DPAA2_SWP_VALID_BIT;
1147 		} else {
1148 			ret = bus_read_4(map, offset);
1149 			verb = ret & ~DPAA2_SWP_VALID_BIT; /* remove valid bit */
1150 			if (verb == 0u)
1151 				goto wait;
1152 			swp->mc.valid_bit ^= DPAA2_SWP_VALID_BIT;
1153 		}
1154 		break;
1155  wait:
1156 		DELAY(CMD_SPIN_TIMEOUT);
1157 	}
1158 	/* Return an error on expired timeout. */
1159 	rc = i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0;
1160 
1161 	/* Read command response. */
1162 	for (i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++)
1163 		rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t));
1164 
1165 	return (rc);
1166 }
1167