xref: /freebsd/sys/dev/liquidio/base/lio_droq.c (revision 71625ec9ad2a9bc8c09784fbd23b759830e0ee5f)
1f173c2b7SSean Bruno /*
2f173c2b7SSean Bruno  *   BSD LICENSE
3f173c2b7SSean Bruno  *
4f173c2b7SSean Bruno  *   Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5f173c2b7SSean Bruno  *   All rights reserved.
6f173c2b7SSean Bruno  *
7f173c2b7SSean Bruno  *   Redistribution and use in source and binary forms, with or without
8f173c2b7SSean Bruno  *   modification, are permitted provided that the following conditions
9f173c2b7SSean Bruno  *   are met:
10f173c2b7SSean Bruno  *
11f173c2b7SSean Bruno  *     * Redistributions of source code must retain the above copyright
12f173c2b7SSean Bruno  *       notice, this list of conditions and the following disclaimer.
13f173c2b7SSean Bruno  *     * Redistributions in binary form must reproduce the above copyright
14f173c2b7SSean Bruno  *       notice, this list of conditions and the following disclaimer in
15f173c2b7SSean Bruno  *       the documentation and/or other materials provided with the
16f173c2b7SSean Bruno  *       distribution.
17f173c2b7SSean Bruno  *     * Neither the name of Cavium, Inc. nor the names of its
18f173c2b7SSean Bruno  *       contributors may be used to endorse or promote products derived
19f173c2b7SSean Bruno  *       from this software without specific prior written permission.
20f173c2b7SSean Bruno  *
21f173c2b7SSean Bruno  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22f173c2b7SSean Bruno  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23f173c2b7SSean Bruno  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24f173c2b7SSean Bruno  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25f173c2b7SSean Bruno  *   OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26f173c2b7SSean Bruno  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27f173c2b7SSean Bruno  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28f173c2b7SSean Bruno  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29f173c2b7SSean Bruno  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30f173c2b7SSean Bruno  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31f173c2b7SSean Bruno  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32f173c2b7SSean Bruno  */
33f173c2b7SSean Bruno 
34f173c2b7SSean Bruno #include "lio_bsd.h"
35f173c2b7SSean Bruno #include "lio_common.h"
36f173c2b7SSean Bruno #include "lio_droq.h"
37f173c2b7SSean Bruno #include "lio_iq.h"
38f173c2b7SSean Bruno #include "lio_response_manager.h"
39f173c2b7SSean Bruno #include "lio_device.h"
40f173c2b7SSean Bruno #include "lio_main.h"
41f173c2b7SSean Bruno #include "cn23xx_pf_device.h"
42f173c2b7SSean Bruno #include "lio_network.h"
43f173c2b7SSean Bruno 
44f173c2b7SSean Bruno struct __dispatch {
45f173c2b7SSean Bruno 	struct lio_stailq_node	node;
46f173c2b7SSean Bruno 	struct lio_recv_info	*rinfo;
47f173c2b7SSean Bruno 	lio_dispatch_fn_t	disp_fn;
48f173c2b7SSean Bruno };
49f173c2b7SSean Bruno 
50f173c2b7SSean Bruno void	*lio_get_dispatch_arg(struct octeon_device *oct,
51f173c2b7SSean Bruno 			      uint16_t opcode, uint16_t subcode);
52f173c2b7SSean Bruno 
53f173c2b7SSean Bruno /*
54f173c2b7SSean Bruno  *  Get the argument that the user set when registering dispatch
55f173c2b7SSean Bruno  *  function for a given opcode/subcode.
56f173c2b7SSean Bruno  *  @param  octeon_dev - the octeon device pointer.
57f173c2b7SSean Bruno  *  @param  opcode     - the opcode for which the dispatch argument
58f173c2b7SSean Bruno  *                       is to be checked.
59f173c2b7SSean Bruno  *  @param  subcode    - the subcode for which the dispatch argument
60f173c2b7SSean Bruno  *                       is to be checked.
61f173c2b7SSean Bruno  *  @return  Success: void * (argument to the dispatch function)
62f173c2b7SSean Bruno  *  @return  Failure: NULL
63f173c2b7SSean Bruno  *
64f173c2b7SSean Bruno  */
65f173c2b7SSean Bruno void   *
lio_get_dispatch_arg(struct octeon_device * octeon_dev,uint16_t opcode,uint16_t subcode)66f173c2b7SSean Bruno lio_get_dispatch_arg(struct octeon_device *octeon_dev,
67f173c2b7SSean Bruno 		     uint16_t opcode, uint16_t subcode)
68f173c2b7SSean Bruno {
69f173c2b7SSean Bruno 	struct lio_stailq_node	*dispatch;
70f173c2b7SSean Bruno 	void			*fn_arg = NULL;
71f173c2b7SSean Bruno 	int			idx;
72f173c2b7SSean Bruno 	uint16_t		combined_opcode;
73f173c2b7SSean Bruno 
74f173c2b7SSean Bruno 	combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode);
75f173c2b7SSean Bruno 
76f173c2b7SSean Bruno 	idx = combined_opcode & LIO_OPCODE_MASK;
77f173c2b7SSean Bruno 
78f173c2b7SSean Bruno 	mtx_lock(&octeon_dev->dispatch.lock);
79f173c2b7SSean Bruno 
80f173c2b7SSean Bruno 	if (octeon_dev->dispatch.count == 0) {
81f173c2b7SSean Bruno 		mtx_unlock(&octeon_dev->dispatch.lock);
82f173c2b7SSean Bruno 		return (NULL);
83f173c2b7SSean Bruno 	}
84f173c2b7SSean Bruno 
85f173c2b7SSean Bruno 	if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
86f173c2b7SSean Bruno 		fn_arg = octeon_dev->dispatch.dlist[idx].arg;
87f173c2b7SSean Bruno 	} else {
88f173c2b7SSean Bruno 		STAILQ_FOREACH(dispatch,
89f173c2b7SSean Bruno 			       &octeon_dev->dispatch.dlist[idx].head, entries) {
90f173c2b7SSean Bruno 			if (((struct lio_dispatch *)dispatch)->opcode ==
91f173c2b7SSean Bruno 			    combined_opcode) {
92f173c2b7SSean Bruno 				fn_arg = ((struct lio_dispatch *)dispatch)->arg;
93f173c2b7SSean Bruno 				break;
94f173c2b7SSean Bruno 			}
95f173c2b7SSean Bruno 		}
96f173c2b7SSean Bruno 	}
97f173c2b7SSean Bruno 
98f173c2b7SSean Bruno 	mtx_unlock(&octeon_dev->dispatch.lock);
99f173c2b7SSean Bruno 	return (fn_arg);
100f173c2b7SSean Bruno }
101f173c2b7SSean Bruno 
102f173c2b7SSean Bruno /*
103f173c2b7SSean Bruno  *  Check for packets on Droq. This function should be called with lock held.
104f173c2b7SSean Bruno  *  @param  droq - Droq on which count is checked.
105f173c2b7SSean Bruno  *  @return Returns packet count.
106f173c2b7SSean Bruno  */
107f173c2b7SSean Bruno uint32_t
lio_droq_check_hw_for_pkts(struct lio_droq * droq)108f173c2b7SSean Bruno lio_droq_check_hw_for_pkts(struct lio_droq *droq)
109f173c2b7SSean Bruno {
110f173c2b7SSean Bruno 	struct octeon_device	*oct = droq->oct_dev;
111f173c2b7SSean Bruno 	uint32_t		last_count;
112f173c2b7SSean Bruno 	uint32_t		pkt_count = 0;
113f173c2b7SSean Bruno 
114f173c2b7SSean Bruno 	pkt_count = lio_read_csr32(oct, droq->pkts_sent_reg);
115f173c2b7SSean Bruno 
116f173c2b7SSean Bruno 	last_count = pkt_count - droq->pkt_count;
117f173c2b7SSean Bruno 	droq->pkt_count = pkt_count;
118f173c2b7SSean Bruno 
119f173c2b7SSean Bruno 	/* we shall write to cnts at the end of processing */
120f173c2b7SSean Bruno 	if (last_count)
121f173c2b7SSean Bruno 		atomic_add_int(&droq->pkts_pending, last_count);
122f173c2b7SSean Bruno 
123f173c2b7SSean Bruno 	return (last_count);
124f173c2b7SSean Bruno }
125f173c2b7SSean Bruno 
126f173c2b7SSean Bruno static void
lio_droq_compute_max_packet_bufs(struct lio_droq * droq)127f173c2b7SSean Bruno lio_droq_compute_max_packet_bufs(struct lio_droq *droq)
128f173c2b7SSean Bruno {
129f173c2b7SSean Bruno 	uint32_t	count = 0;
130f173c2b7SSean Bruno 
131f173c2b7SSean Bruno 	/*
132f173c2b7SSean Bruno 	 * max_empty_descs is the max. no. of descs that can have no buffers.
133f173c2b7SSean Bruno 	 * If the empty desc count goes beyond this value, we cannot safely
134f173c2b7SSean Bruno 	 * read in a 64K packet sent by Octeon
135f173c2b7SSean Bruno 	 * (64K is max pkt size from Octeon)
136f173c2b7SSean Bruno 	 */
137f173c2b7SSean Bruno 	droq->max_empty_descs = 0;
138f173c2b7SSean Bruno 
139f173c2b7SSean Bruno 	do {
140f173c2b7SSean Bruno 		droq->max_empty_descs++;
141f173c2b7SSean Bruno 		count += droq->buffer_size;
142f173c2b7SSean Bruno 	} while (count < (64 * 1024));
143f173c2b7SSean Bruno 
144f173c2b7SSean Bruno 	droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
145f173c2b7SSean Bruno }
146f173c2b7SSean Bruno 
147f173c2b7SSean Bruno static void
lio_droq_reset_indices(struct lio_droq * droq)148f173c2b7SSean Bruno lio_droq_reset_indices(struct lio_droq *droq)
149f173c2b7SSean Bruno {
150f173c2b7SSean Bruno 
151f173c2b7SSean Bruno 	droq->read_idx = 0;
152f173c2b7SSean Bruno 	droq->refill_idx = 0;
153f173c2b7SSean Bruno 	droq->refill_count = 0;
154f173c2b7SSean Bruno 	atomic_store_rel_int(&droq->pkts_pending, 0);
155f173c2b7SSean Bruno }
156f173c2b7SSean Bruno 
157f173c2b7SSean Bruno static void
lio_droq_destroy_ring_buffers(struct octeon_device * oct,struct lio_droq * droq)158f173c2b7SSean Bruno lio_droq_destroy_ring_buffers(struct octeon_device *oct,
159f173c2b7SSean Bruno 			      struct lio_droq *droq)
160f173c2b7SSean Bruno {
161f173c2b7SSean Bruno 	uint32_t	i;
162f173c2b7SSean Bruno 
163f173c2b7SSean Bruno 	for (i = 0; i < droq->max_count; i++) {
164f173c2b7SSean Bruno 		if (droq->recv_buf_list[i].buffer != NULL) {
165f173c2b7SSean Bruno 			lio_recv_buffer_free(droq->recv_buf_list[i].buffer);
166f173c2b7SSean Bruno 			droq->recv_buf_list[i].buffer = NULL;
167f173c2b7SSean Bruno 		}
168f173c2b7SSean Bruno 	}
169f173c2b7SSean Bruno 
170f173c2b7SSean Bruno 	lio_droq_reset_indices(droq);
171f173c2b7SSean Bruno }
172f173c2b7SSean Bruno 
173f173c2b7SSean Bruno static int
lio_droq_setup_ring_buffers(struct octeon_device * oct,struct lio_droq * droq)174f173c2b7SSean Bruno lio_droq_setup_ring_buffers(struct octeon_device *oct,
175f173c2b7SSean Bruno 			    struct lio_droq *droq)
176f173c2b7SSean Bruno {
177f173c2b7SSean Bruno 	struct lio_droq_desc	*desc_ring = droq->desc_ring;
178f173c2b7SSean Bruno 	void			*buf;
179f173c2b7SSean Bruno 	uint32_t		i;
180f173c2b7SSean Bruno 
181f173c2b7SSean Bruno 	for (i = 0; i < droq->max_count; i++) {
182f173c2b7SSean Bruno 		buf = lio_recv_buffer_alloc(droq->buffer_size);
183f173c2b7SSean Bruno 
184f173c2b7SSean Bruno 		if (buf == NULL) {
185f173c2b7SSean Bruno 			lio_dev_err(oct, "%s buffer alloc failed\n",
186f173c2b7SSean Bruno 				    __func__);
187f173c2b7SSean Bruno 			droq->stats.rx_alloc_failure++;
188f173c2b7SSean Bruno 			return (-ENOMEM);
189f173c2b7SSean Bruno 		}
190f173c2b7SSean Bruno 
191f173c2b7SSean Bruno 		droq->recv_buf_list[i].buffer = buf;
192f173c2b7SSean Bruno 		droq->recv_buf_list[i].data = ((struct mbuf *)buf)->m_data;
193f173c2b7SSean Bruno 		desc_ring[i].info_ptr = 0;
194f173c2b7SSean Bruno 		desc_ring[i].buffer_ptr =
195f173c2b7SSean Bruno 			lio_map_ring(oct->device, droq->recv_buf_list[i].buffer,
196f173c2b7SSean Bruno 				     droq->buffer_size);
197f173c2b7SSean Bruno 	}
198f173c2b7SSean Bruno 
199f173c2b7SSean Bruno 	lio_droq_reset_indices(droq);
200f173c2b7SSean Bruno 
201f173c2b7SSean Bruno 	lio_droq_compute_max_packet_bufs(droq);
202f173c2b7SSean Bruno 
203f173c2b7SSean Bruno 	return (0);
204f173c2b7SSean Bruno }
205f173c2b7SSean Bruno 
206f173c2b7SSean Bruno int
lio_delete_droq(struct octeon_device * oct,uint32_t q_no)207f173c2b7SSean Bruno lio_delete_droq(struct octeon_device *oct, uint32_t q_no)
208f173c2b7SSean Bruno {
209f173c2b7SSean Bruno 	struct lio_droq	*droq = oct->droq[q_no];
210f173c2b7SSean Bruno 
211f173c2b7SSean Bruno 	lio_dev_dbg(oct, "%s[%d]\n", __func__, q_no);
212f173c2b7SSean Bruno 
213f173c2b7SSean Bruno 	while (taskqueue_cancel(droq->droq_taskqueue, &droq->droq_task, NULL))
214f173c2b7SSean Bruno 		taskqueue_drain(droq->droq_taskqueue, &droq->droq_task);
215f173c2b7SSean Bruno 
216f173c2b7SSean Bruno 	taskqueue_free(droq->droq_taskqueue);
217f173c2b7SSean Bruno 	droq->droq_taskqueue = NULL;
218f173c2b7SSean Bruno 
219f173c2b7SSean Bruno 	lio_droq_destroy_ring_buffers(oct, droq);
220f173c2b7SSean Bruno 	free(droq->recv_buf_list, M_DEVBUF);
221f173c2b7SSean Bruno 
222f173c2b7SSean Bruno 	if (droq->desc_ring != NULL)
223f173c2b7SSean Bruno 		lio_dma_free((droq->max_count * LIO_DROQ_DESC_SIZE),
224f173c2b7SSean Bruno 			     droq->desc_ring);
225f173c2b7SSean Bruno 
226f173c2b7SSean Bruno 	oct->io_qmask.oq &= ~(1ULL << q_no);
227f173c2b7SSean Bruno 	bzero(oct->droq[q_no], sizeof(struct lio_droq));
228f173c2b7SSean Bruno 	oct->num_oqs--;
229f173c2b7SSean Bruno 
230f173c2b7SSean Bruno 	return (0);
231f173c2b7SSean Bruno }
232f173c2b7SSean Bruno 
233f173c2b7SSean Bruno void
lio_droq_bh(void * ptr,int pending __unused)234f173c2b7SSean Bruno lio_droq_bh(void *ptr, int pending __unused)
235f173c2b7SSean Bruno {
236f173c2b7SSean Bruno 	struct lio_droq		*droq = ptr;
237f173c2b7SSean Bruno 	struct octeon_device	*oct = droq->oct_dev;
238f173c2b7SSean Bruno 	struct lio_instr_queue	*iq = oct->instr_queue[droq->q_no];
239f173c2b7SSean Bruno 	int	reschedule, tx_done = 1;
240f173c2b7SSean Bruno 
241f173c2b7SSean Bruno 	reschedule = lio_droq_process_packets(oct, droq, oct->rx_budget);
242f173c2b7SSean Bruno 
243f173c2b7SSean Bruno 	if (atomic_load_acq_int(&iq->instr_pending))
244f173c2b7SSean Bruno 		tx_done = lio_flush_iq(oct, iq, oct->tx_budget);
245f173c2b7SSean Bruno 
246f173c2b7SSean Bruno 	if (reschedule || !tx_done)
247f173c2b7SSean Bruno 		taskqueue_enqueue(droq->droq_taskqueue, &droq->droq_task);
248f173c2b7SSean Bruno 	else
249f173c2b7SSean Bruno 		lio_enable_irq(droq, iq);
250f173c2b7SSean Bruno }
251f173c2b7SSean Bruno 
252f173c2b7SSean Bruno int
lio_init_droq(struct octeon_device * oct,uint32_t q_no,uint32_t num_descs,uint32_t desc_size,void * app_ctx)253f173c2b7SSean Bruno lio_init_droq(struct octeon_device *oct, uint32_t q_no,
254f173c2b7SSean Bruno 	      uint32_t num_descs, uint32_t desc_size, void *app_ctx)
255f173c2b7SSean Bruno {
256f173c2b7SSean Bruno 	struct lio_droq	*droq;
257f173c2b7SSean Bruno 	unsigned long	size;
258f173c2b7SSean Bruno 	uint32_t	c_buf_size = 0, c_num_descs = 0, c_pkts_per_intr = 0;
259f173c2b7SSean Bruno 	uint32_t	c_refill_threshold = 0, desc_ring_size = 0;
260f173c2b7SSean Bruno 
261f173c2b7SSean Bruno 	lio_dev_dbg(oct, "%s[%d]\n", __func__, q_no);
262f173c2b7SSean Bruno 
263f173c2b7SSean Bruno 	droq = oct->droq[q_no];
264f173c2b7SSean Bruno 	bzero(droq, LIO_DROQ_SIZE);
265f173c2b7SSean Bruno 
266f173c2b7SSean Bruno 	droq->oct_dev = oct;
267f173c2b7SSean Bruno 	droq->q_no = q_no;
268f173c2b7SSean Bruno 	if (app_ctx != NULL)
269f173c2b7SSean Bruno 		droq->app_ctx = app_ctx;
270f173c2b7SSean Bruno 	else
271f173c2b7SSean Bruno 		droq->app_ctx = (void *)(size_t)q_no;
272f173c2b7SSean Bruno 
273f173c2b7SSean Bruno 	c_num_descs = num_descs;
274f173c2b7SSean Bruno 	c_buf_size = desc_size;
275f173c2b7SSean Bruno 	if (LIO_CN23XX_PF(oct)) {
276f173c2b7SSean Bruno 		struct lio_config *conf23 = LIO_CHIP_CONF(oct, cn23xx_pf);
277f173c2b7SSean Bruno 
278f173c2b7SSean Bruno 		c_pkts_per_intr =
279f173c2b7SSean Bruno 			(uint32_t)LIO_GET_OQ_PKTS_PER_INTR_CFG(conf23);
280f173c2b7SSean Bruno 		c_refill_threshold =
281f173c2b7SSean Bruno 			(uint32_t)LIO_GET_OQ_REFILL_THRESHOLD_CFG(conf23);
282f173c2b7SSean Bruno 	} else {
283f173c2b7SSean Bruno 		return (1);
284f173c2b7SSean Bruno 	}
285f173c2b7SSean Bruno 
286f173c2b7SSean Bruno 	droq->max_count = c_num_descs;
287f173c2b7SSean Bruno 	droq->buffer_size = c_buf_size;
288f173c2b7SSean Bruno 
289f173c2b7SSean Bruno 	desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE;
290f173c2b7SSean Bruno 	droq->desc_ring = lio_dma_alloc(desc_ring_size, &droq->desc_ring_dma);
291f173c2b7SSean Bruno 	if (droq->desc_ring == NULL) {
292f173c2b7SSean Bruno 		lio_dev_err(oct, "Output queue %d ring alloc failed\n", q_no);
293f173c2b7SSean Bruno 		return (1);
294f173c2b7SSean Bruno 	}
295f173c2b7SSean Bruno 
2963de0952fSSean Bruno 	lio_dev_dbg(oct, "droq[%d]: desc_ring: virt: 0x%p, dma: %llx\n", q_no,
2973de0952fSSean Bruno 		    droq->desc_ring, LIO_CAST64(droq->desc_ring_dma));
298f173c2b7SSean Bruno 	lio_dev_dbg(oct, "droq[%d]: num_desc: %d\n", q_no, droq->max_count);
299f173c2b7SSean Bruno 
300f173c2b7SSean Bruno 	size = droq->max_count * LIO_DROQ_RECVBUF_SIZE;
301f173c2b7SSean Bruno 	droq->recv_buf_list =
302f173c2b7SSean Bruno 		(struct lio_recv_buffer *)malloc(size, M_DEVBUF,
303f173c2b7SSean Bruno 						 M_NOWAIT | M_ZERO);
304f173c2b7SSean Bruno 	if (droq->recv_buf_list == NULL) {
305f173c2b7SSean Bruno 		lio_dev_err(oct, "Output queue recv buf list alloc failed\n");
306f173c2b7SSean Bruno 		goto init_droq_fail;
307f173c2b7SSean Bruno 	}
308f173c2b7SSean Bruno 
309f173c2b7SSean Bruno 	if (lio_droq_setup_ring_buffers(oct, droq))
310f173c2b7SSean Bruno 		goto init_droq_fail;
311f173c2b7SSean Bruno 
312f173c2b7SSean Bruno 	droq->pkts_per_intr = c_pkts_per_intr;
313f173c2b7SSean Bruno 	droq->refill_threshold = c_refill_threshold;
314f173c2b7SSean Bruno 
315f173c2b7SSean Bruno 	lio_dev_dbg(oct, "DROQ INIT: max_empty_descs: %d\n",
316f173c2b7SSean Bruno 		    droq->max_empty_descs);
317f173c2b7SSean Bruno 
318f173c2b7SSean Bruno 	mtx_init(&droq->lock, "droq_lock", NULL, MTX_DEF);
319f173c2b7SSean Bruno 
320f173c2b7SSean Bruno 	STAILQ_INIT(&droq->dispatch_stq_head);
321f173c2b7SSean Bruno 
322f173c2b7SSean Bruno 	oct->fn_list.setup_oq_regs(oct, q_no);
323f173c2b7SSean Bruno 
324f173c2b7SSean Bruno 	oct->io_qmask.oq |= BIT_ULL(q_no);
325f173c2b7SSean Bruno 
326f173c2b7SSean Bruno 	/*
327f173c2b7SSean Bruno 	 * Initialize the taskqueue that handles
328f173c2b7SSean Bruno 	 * output queue packet processing.
329f173c2b7SSean Bruno 	 */
330f173c2b7SSean Bruno 	lio_dev_dbg(oct, "Initializing droq%d taskqueue\n", q_no);
331*301a87acSGleb Smirnoff 	NET_TASK_INIT(&droq->droq_task, 0, lio_droq_bh, (void *)droq);
332f173c2b7SSean Bruno 
333f173c2b7SSean Bruno 	droq->droq_taskqueue = taskqueue_create_fast("lio_droq_task", M_NOWAIT,
334f173c2b7SSean Bruno 						     taskqueue_thread_enqueue,
335f173c2b7SSean Bruno 						     &droq->droq_taskqueue);
336f173c2b7SSean Bruno 	taskqueue_start_threads_cpuset(&droq->droq_taskqueue, 1, PI_NET,
337f173c2b7SSean Bruno 				       &oct->ioq_vector[q_no].affinity_mask,
338f173c2b7SSean Bruno 				       "lio%d_droq%d_task", oct->octeon_id,
339f173c2b7SSean Bruno 				       q_no);
340f173c2b7SSean Bruno 
341f173c2b7SSean Bruno 	return (0);
342f173c2b7SSean Bruno 
343f173c2b7SSean Bruno init_droq_fail:
344f173c2b7SSean Bruno 	lio_delete_droq(oct, q_no);
345f173c2b7SSean Bruno 	return (1);
346f173c2b7SSean Bruno }
347f173c2b7SSean Bruno 
348f173c2b7SSean Bruno /*
349f173c2b7SSean Bruno  * lio_create_recv_info
350f173c2b7SSean Bruno  * Parameters:
351f173c2b7SSean Bruno  *  octeon_dev - pointer to the octeon device structure
352f173c2b7SSean Bruno  *  droq       - droq in which the packet arrived.
353f173c2b7SSean Bruno  *  buf_cnt    - no. of buffers used by the packet.
354f173c2b7SSean Bruno  *  idx        - index in the descriptor for the first buffer in the packet.
355f173c2b7SSean Bruno  * Description:
356f173c2b7SSean Bruno  *  Allocates a recv_info_t and copies the buffer addresses for packet data
357f173c2b7SSean Bruno  *  into the recv_pkt space which starts at an 8B offset from recv_info_t.
358f173c2b7SSean Bruno  *  Flags the descriptors for refill later. If available descriptors go
359f173c2b7SSean Bruno  *  below the threshold to receive a 64K pkt, new buffers are first allocated
360f173c2b7SSean Bruno  *  before the recv_pkt_t is created.
361f173c2b7SSean Bruno  *  This routine will be called in interrupt context.
362f173c2b7SSean Bruno  * Returns:
363f173c2b7SSean Bruno  *  Success: Pointer to recv_info_t
364f173c2b7SSean Bruno  *  Failure: NULL.
365f173c2b7SSean Bruno  * Locks:
366f173c2b7SSean Bruno  *  The droq->lock is held when this routine is called.
367f173c2b7SSean Bruno  */
368f173c2b7SSean Bruno static inline struct lio_recv_info *
lio_create_recv_info(struct octeon_device * octeon_dev,struct lio_droq * droq,uint32_t buf_cnt,uint32_t idx)369f173c2b7SSean Bruno lio_create_recv_info(struct octeon_device *octeon_dev, struct lio_droq *droq,
370f173c2b7SSean Bruno 		     uint32_t buf_cnt, uint32_t idx)
371f173c2b7SSean Bruno {
372f173c2b7SSean Bruno 	struct lio_droq_info	*info;
373f173c2b7SSean Bruno 	struct lio_recv_pkt	*recv_pkt;
374f173c2b7SSean Bruno 	struct lio_recv_info	*recv_info;
375f173c2b7SSean Bruno 	uint32_t		bytes_left, i;
376f173c2b7SSean Bruno 
377f173c2b7SSean Bruno 	info = (struct lio_droq_info *)droq->recv_buf_list[idx].data;
378f173c2b7SSean Bruno 
379f173c2b7SSean Bruno 	recv_info = lio_alloc_recv_info(sizeof(struct __dispatch));
380f173c2b7SSean Bruno 	if (recv_info == NULL)
381f173c2b7SSean Bruno 		return (NULL);
382f173c2b7SSean Bruno 
383f173c2b7SSean Bruno 	recv_pkt = recv_info->recv_pkt;
384f173c2b7SSean Bruno 	recv_pkt->rh = info->rh;
385f173c2b7SSean Bruno 	recv_pkt->length = (uint32_t)info->length;
386f173c2b7SSean Bruno 	recv_pkt->buffer_count = (uint16_t)buf_cnt;
387f173c2b7SSean Bruno 	recv_pkt->octeon_id = (uint16_t)octeon_dev->octeon_id;
388f173c2b7SSean Bruno 
389f173c2b7SSean Bruno 	i = 0;
390f173c2b7SSean Bruno 	bytes_left = (uint32_t)info->length;
391f173c2b7SSean Bruno 
392f173c2b7SSean Bruno 	while (buf_cnt) {
393f173c2b7SSean Bruno 		recv_pkt->buffer_size[i] = (bytes_left >= droq->buffer_size) ?
394f173c2b7SSean Bruno 			droq->buffer_size : bytes_left;
395f173c2b7SSean Bruno 
396f173c2b7SSean Bruno 		recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
397f173c2b7SSean Bruno 		droq->recv_buf_list[idx].buffer = NULL;
398f173c2b7SSean Bruno 
399f173c2b7SSean Bruno 		idx = lio_incr_index(idx, 1, droq->max_count);
400f173c2b7SSean Bruno 		bytes_left -= droq->buffer_size;
401f173c2b7SSean Bruno 		i++;
402f173c2b7SSean Bruno 		buf_cnt--;
403f173c2b7SSean Bruno 	}
404f173c2b7SSean Bruno 
405f173c2b7SSean Bruno 	return (recv_info);
406f173c2b7SSean Bruno }
407f173c2b7SSean Bruno 
408f173c2b7SSean Bruno /*
409f173c2b7SSean Bruno  * If we were not able to refill all buffers, try to move around
410f173c2b7SSean Bruno  * the buffers that were not dispatched.
411f173c2b7SSean Bruno  */
412f173c2b7SSean Bruno static inline uint32_t
lio_droq_refill_pullup_descs(struct lio_droq * droq,struct lio_droq_desc * desc_ring)413f173c2b7SSean Bruno lio_droq_refill_pullup_descs(struct lio_droq *droq,
414f173c2b7SSean Bruno 			     struct lio_droq_desc *desc_ring)
415f173c2b7SSean Bruno {
416f173c2b7SSean Bruno 	uint32_t	desc_refilled = 0;
417f173c2b7SSean Bruno 	uint32_t	refill_index = droq->refill_idx;
418f173c2b7SSean Bruno 
419f173c2b7SSean Bruno 	while (refill_index != droq->read_idx) {
420f173c2b7SSean Bruno 		if (droq->recv_buf_list[refill_index].buffer != NULL) {
421f173c2b7SSean Bruno 			droq->recv_buf_list[droq->refill_idx].buffer =
422f173c2b7SSean Bruno 				droq->recv_buf_list[refill_index].buffer;
423f173c2b7SSean Bruno 			droq->recv_buf_list[droq->refill_idx].data =
424f173c2b7SSean Bruno 				droq->recv_buf_list[refill_index].data;
425f173c2b7SSean Bruno 			desc_ring[droq->refill_idx].buffer_ptr =
426f173c2b7SSean Bruno 				desc_ring[refill_index].buffer_ptr;
427f173c2b7SSean Bruno 			droq->recv_buf_list[refill_index].buffer = NULL;
428f173c2b7SSean Bruno 			desc_ring[refill_index].buffer_ptr = 0;
429f173c2b7SSean Bruno 			do {
430f173c2b7SSean Bruno 				droq->refill_idx =
431f173c2b7SSean Bruno 					lio_incr_index(droq->refill_idx, 1,
432f173c2b7SSean Bruno 						       droq->max_count);
433f173c2b7SSean Bruno 				desc_refilled++;
434f173c2b7SSean Bruno 				droq->refill_count--;
435f173c2b7SSean Bruno 			} while (droq->recv_buf_list[droq->refill_idx].buffer !=
436f173c2b7SSean Bruno 				 NULL);
437f173c2b7SSean Bruno 		}
438f173c2b7SSean Bruno 		refill_index = lio_incr_index(refill_index, 1, droq->max_count);
439f173c2b7SSean Bruno 	}	/* while */
440f173c2b7SSean Bruno 	return (desc_refilled);
441f173c2b7SSean Bruno }
442f173c2b7SSean Bruno 
443f173c2b7SSean Bruno /*
444f173c2b7SSean Bruno  * lio_droq_refill
445f173c2b7SSean Bruno  * Parameters:
446f173c2b7SSean Bruno  *  droq       - droq in which descriptors require new buffers.
447f173c2b7SSean Bruno  * Description:
448f173c2b7SSean Bruno  *  Called during normal DROQ processing in interrupt mode or by the poll
449f173c2b7SSean Bruno  *  thread to refill the descriptors from which buffers were dispatched
450f173c2b7SSean Bruno  *  to upper layers. Attempts to allocate new buffers. If that fails, moves
451f173c2b7SSean Bruno  *  up buffers (that were not dispatched) to form a contiguous ring.
452f173c2b7SSean Bruno  * Returns:
453f173c2b7SSean Bruno  *  No of descriptors refilled.
454f173c2b7SSean Bruno  * Locks:
455f173c2b7SSean Bruno  *  This routine is called with droq->lock held.
456f173c2b7SSean Bruno  */
457f173c2b7SSean Bruno uint32_t
lio_droq_refill(struct octeon_device * octeon_dev,struct lio_droq * droq)458f173c2b7SSean Bruno lio_droq_refill(struct octeon_device *octeon_dev, struct lio_droq *droq)
459f173c2b7SSean Bruno {
460f173c2b7SSean Bruno 	struct lio_droq_desc	*desc_ring;
461f173c2b7SSean Bruno 	void			*buf = NULL;
462f173c2b7SSean Bruno 	uint32_t		desc_refilled = 0;
463f173c2b7SSean Bruno 	uint8_t			*data;
464f173c2b7SSean Bruno 
465f173c2b7SSean Bruno 	desc_ring = droq->desc_ring;
466f173c2b7SSean Bruno 
467f173c2b7SSean Bruno 	while (droq->refill_count && (desc_refilled < droq->max_count)) {
468f173c2b7SSean Bruno 		/*
469f173c2b7SSean Bruno 		 * If a valid buffer exists (happens if there is no dispatch),
470f173c2b7SSean Bruno 		 * reuse
471f173c2b7SSean Bruno 		 * the buffer, else allocate.
472f173c2b7SSean Bruno 		 */
473f173c2b7SSean Bruno 		if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) {
474f173c2b7SSean Bruno 			buf = lio_recv_buffer_alloc(droq->buffer_size);
475f173c2b7SSean Bruno 			/*
476f173c2b7SSean Bruno 			 * If a buffer could not be allocated, no point in
477f173c2b7SSean Bruno 			 * continuing
478f173c2b7SSean Bruno 			 */
479f173c2b7SSean Bruno 			if (buf == NULL) {
480f173c2b7SSean Bruno 				droq->stats.rx_alloc_failure++;
481f173c2b7SSean Bruno 				break;
482f173c2b7SSean Bruno 			}
483f173c2b7SSean Bruno 
484f173c2b7SSean Bruno 			droq->recv_buf_list[droq->refill_idx].buffer = buf;
485f173c2b7SSean Bruno 			data = ((struct mbuf *)buf)->m_data;
486f173c2b7SSean Bruno 		} else {
487f173c2b7SSean Bruno 			data = ((struct mbuf *)droq->recv_buf_list
488f173c2b7SSean Bruno 				[droq->refill_idx].buffer)->m_data;
489f173c2b7SSean Bruno 		}
490f173c2b7SSean Bruno 
491f173c2b7SSean Bruno 		droq->recv_buf_list[droq->refill_idx].data = data;
492f173c2b7SSean Bruno 
493f173c2b7SSean Bruno 		desc_ring[droq->refill_idx].buffer_ptr =
494f173c2b7SSean Bruno 		    lio_map_ring(octeon_dev->device,
495f173c2b7SSean Bruno 				 droq->recv_buf_list[droq->refill_idx].buffer,
496f173c2b7SSean Bruno 				 droq->buffer_size);
497f173c2b7SSean Bruno 
498f173c2b7SSean Bruno 		droq->refill_idx = lio_incr_index(droq->refill_idx, 1,
499f173c2b7SSean Bruno 						  droq->max_count);
500f173c2b7SSean Bruno 		desc_refilled++;
501f173c2b7SSean Bruno 		droq->refill_count--;
502f173c2b7SSean Bruno 	}
503f173c2b7SSean Bruno 
504f173c2b7SSean Bruno 	if (droq->refill_count)
505f173c2b7SSean Bruno 		desc_refilled += lio_droq_refill_pullup_descs(droq, desc_ring);
506f173c2b7SSean Bruno 
507f173c2b7SSean Bruno 	/*
508f173c2b7SSean Bruno 	 * if droq->refill_count
509f173c2b7SSean Bruno 	 * The refill count would not change in pass two. We only moved buffers
510f173c2b7SSean Bruno 	 * to close the gap in the ring, but we would still have the same no. of
511f173c2b7SSean Bruno 	 * buffers to refill.
512f173c2b7SSean Bruno 	 */
513f173c2b7SSean Bruno 	return (desc_refilled);
514f173c2b7SSean Bruno }
515f173c2b7SSean Bruno 
516f173c2b7SSean Bruno static inline uint32_t
lio_droq_get_bufcount(uint32_t buf_size,uint32_t total_len)517f173c2b7SSean Bruno lio_droq_get_bufcount(uint32_t buf_size, uint32_t total_len)
518f173c2b7SSean Bruno {
519f173c2b7SSean Bruno 
520f173c2b7SSean Bruno 	return ((total_len + buf_size - 1) / buf_size);
521f173c2b7SSean Bruno }
522f173c2b7SSean Bruno 
523f173c2b7SSean Bruno static int
lio_droq_dispatch_pkt(struct octeon_device * oct,struct lio_droq * droq,union octeon_rh * rh,struct lio_droq_info * info)524f173c2b7SSean Bruno lio_droq_dispatch_pkt(struct octeon_device *oct, struct lio_droq *droq,
525f173c2b7SSean Bruno 		      union octeon_rh *rh, struct lio_droq_info *info)
526f173c2b7SSean Bruno {
527f173c2b7SSean Bruno 	struct lio_recv_info	*rinfo;
528f173c2b7SSean Bruno 	lio_dispatch_fn_t	disp_fn;
529f173c2b7SSean Bruno 	uint32_t		cnt;
530f173c2b7SSean Bruno 
531f173c2b7SSean Bruno 	cnt = lio_droq_get_bufcount(droq->buffer_size, (uint32_t)info->length);
532f173c2b7SSean Bruno 
533f173c2b7SSean Bruno 	disp_fn = lio_get_dispatch(oct, (uint16_t)rh->r.opcode,
534f173c2b7SSean Bruno 				   (uint16_t)rh->r.subcode);
535f173c2b7SSean Bruno 	if (disp_fn) {
536f173c2b7SSean Bruno 		rinfo = lio_create_recv_info(oct, droq, cnt, droq->read_idx);
537f173c2b7SSean Bruno 		if (rinfo != NULL) {
538f173c2b7SSean Bruno 			struct __dispatch *rdisp = rinfo->rsvd;
539f173c2b7SSean Bruno 
540f173c2b7SSean Bruno 			rdisp->rinfo = rinfo;
541f173c2b7SSean Bruno 			rdisp->disp_fn = disp_fn;
542f173c2b7SSean Bruno 			rinfo->recv_pkt->rh = *rh;
543f173c2b7SSean Bruno 			STAILQ_INSERT_TAIL(&droq->dispatch_stq_head,
544f173c2b7SSean Bruno 					   &rdisp->node, entries);
545f173c2b7SSean Bruno 		} else {
546f173c2b7SSean Bruno 			droq->stats.dropped_nomem++;
547f173c2b7SSean Bruno 		}
548f173c2b7SSean Bruno 	} else {
549f173c2b7SSean Bruno 		lio_dev_err(oct, "DROQ: No dispatch function (opcode %u/%u)\n",
550f173c2b7SSean Bruno 			    (unsigned int)rh->r.opcode,
551f173c2b7SSean Bruno 			    (unsigned int)rh->r.subcode);
552f173c2b7SSean Bruno 		droq->stats.dropped_nodispatch++;
553f173c2b7SSean Bruno 	}
554f173c2b7SSean Bruno 
555f173c2b7SSean Bruno 	return (cnt);
556f173c2b7SSean Bruno }
557f173c2b7SSean Bruno 
558f173c2b7SSean Bruno static inline void
lio_droq_drop_packets(struct octeon_device * oct,struct lio_droq * droq,uint32_t cnt)559f173c2b7SSean Bruno lio_droq_drop_packets(struct octeon_device *oct, struct lio_droq *droq,
560f173c2b7SSean Bruno 		      uint32_t cnt)
561f173c2b7SSean Bruno {
562f173c2b7SSean Bruno 	struct lio_droq_info	*info;
563f173c2b7SSean Bruno 	uint32_t		i = 0, buf_cnt;
564f173c2b7SSean Bruno 
565f173c2b7SSean Bruno 	for (i = 0; i < cnt; i++) {
566f173c2b7SSean Bruno 		info = (struct lio_droq_info *)
567f173c2b7SSean Bruno 			droq->recv_buf_list[droq->read_idx].data;
568f173c2b7SSean Bruno 
569f173c2b7SSean Bruno 		lio_swap_8B_data((uint64_t *)info, 2);
570f173c2b7SSean Bruno 
571f173c2b7SSean Bruno 		if (info->length) {
572f173c2b7SSean Bruno 			info->length += 8;
573f173c2b7SSean Bruno 			droq->stats.bytes_received += info->length;
574f173c2b7SSean Bruno 			buf_cnt = lio_droq_get_bufcount(droq->buffer_size,
575f173c2b7SSean Bruno 							(uint32_t)info->length);
576f173c2b7SSean Bruno 		} else {
577f173c2b7SSean Bruno 			lio_dev_err(oct, "DROQ: In drop: pkt with len 0\n");
578f173c2b7SSean Bruno 			buf_cnt = 1;
579f173c2b7SSean Bruno 		}
580f173c2b7SSean Bruno 
581f173c2b7SSean Bruno 		droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt,
582f173c2b7SSean Bruno 						droq->max_count);
583f173c2b7SSean Bruno 		droq->refill_count += buf_cnt;
584f173c2b7SSean Bruno 	}
585f173c2b7SSean Bruno }
586f173c2b7SSean Bruno 
587f173c2b7SSean Bruno static uint32_t
lio_droq_fast_process_packets(struct octeon_device * oct,struct lio_droq * droq,uint32_t pkts_to_process)588f173c2b7SSean Bruno lio_droq_fast_process_packets(struct octeon_device *oct, struct lio_droq *droq,
589f173c2b7SSean Bruno 			      uint32_t pkts_to_process)
590f173c2b7SSean Bruno {
591f173c2b7SSean Bruno 	struct lio_droq_info	*info;
592f173c2b7SSean Bruno 	union			octeon_rh *rh;
593f173c2b7SSean Bruno 	uint32_t		pkt, pkt_count, total_len = 0;
594f173c2b7SSean Bruno 
595f173c2b7SSean Bruno 	pkt_count = pkts_to_process;
596f173c2b7SSean Bruno 
597f173c2b7SSean Bruno 	for (pkt = 0; pkt < pkt_count; pkt++) {
598f173c2b7SSean Bruno 		struct mbuf	*nicbuf = NULL;
599f173c2b7SSean Bruno 		uint32_t	pkt_len = 0;
600f173c2b7SSean Bruno 
601f173c2b7SSean Bruno 		info = (struct lio_droq_info *)
602f173c2b7SSean Bruno 		    droq->recv_buf_list[droq->read_idx].data;
603f173c2b7SSean Bruno 
604f173c2b7SSean Bruno 		lio_swap_8B_data((uint64_t *)info, 2);
605f173c2b7SSean Bruno 
606f173c2b7SSean Bruno 		if (!info->length) {
607f173c2b7SSean Bruno 			lio_dev_err(oct,
608f173c2b7SSean Bruno 				    "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
609f173c2b7SSean Bruno 				    droq->q_no, droq->read_idx, pkt_count);
610f173c2b7SSean Bruno 			hexdump((uint8_t *)info, LIO_DROQ_INFO_SIZE, NULL,
611f173c2b7SSean Bruno 				HD_OMIT_CHARS);
612f173c2b7SSean Bruno 			pkt++;
613f173c2b7SSean Bruno 			lio_incr_index(droq->read_idx, 1, droq->max_count);
614f173c2b7SSean Bruno 			droq->refill_count++;
615f173c2b7SSean Bruno 			break;
616f173c2b7SSean Bruno 		}
617f173c2b7SSean Bruno 
618f173c2b7SSean Bruno 		rh = &info->rh;
619f173c2b7SSean Bruno 
620f173c2b7SSean Bruno 		info->length += 8;
621f173c2b7SSean Bruno 		rh->r_dh.len += (LIO_DROQ_INFO_SIZE + 7) / 8;
622f173c2b7SSean Bruno 
623f173c2b7SSean Bruno 		total_len += (uint32_t)info->length;
624f173c2b7SSean Bruno 		if (lio_opcode_slow_path(rh)) {
625f173c2b7SSean Bruno 			uint32_t	buf_cnt;
626f173c2b7SSean Bruno 
627f173c2b7SSean Bruno 			buf_cnt = lio_droq_dispatch_pkt(oct, droq, rh, info);
628f173c2b7SSean Bruno 			droq->read_idx = lio_incr_index(droq->read_idx,	buf_cnt,
629f173c2b7SSean Bruno 							droq->max_count);
630f173c2b7SSean Bruno 			droq->refill_count += buf_cnt;
631f173c2b7SSean Bruno 		} else {
632f173c2b7SSean Bruno 			if (info->length <= droq->buffer_size) {
633f173c2b7SSean Bruno 				pkt_len = (uint32_t)info->length;
634f173c2b7SSean Bruno 				nicbuf = droq->recv_buf_list[
635f173c2b7SSean Bruno 						       droq->read_idx].buffer;
636f173c2b7SSean Bruno 				nicbuf->m_len = pkt_len;
637f173c2b7SSean Bruno 				droq->recv_buf_list[droq->read_idx].buffer =
638f173c2b7SSean Bruno 					NULL;
639f173c2b7SSean Bruno 
640f173c2b7SSean Bruno 				droq->read_idx =
641f173c2b7SSean Bruno 					lio_incr_index(droq->read_idx,
642f173c2b7SSean Bruno 						       1, droq->max_count);
643f173c2b7SSean Bruno 				droq->refill_count++;
644f173c2b7SSean Bruno 			} else {
645f173c2b7SSean Bruno 				bool	secondary_frag = false;
646f173c2b7SSean Bruno 
647f173c2b7SSean Bruno 				pkt_len = 0;
648f173c2b7SSean Bruno 
649f173c2b7SSean Bruno 				while (pkt_len < info->length) {
650f173c2b7SSean Bruno 					int	frag_len, idx = droq->read_idx;
651f173c2b7SSean Bruno 					struct mbuf	*buffer;
652f173c2b7SSean Bruno 
653f173c2b7SSean Bruno 					frag_len =
654f173c2b7SSean Bruno 						((pkt_len + droq->buffer_size) >
655f173c2b7SSean Bruno 						 info->length) ?
656f173c2b7SSean Bruno 						((uint32_t)info->length -
657f173c2b7SSean Bruno 						 pkt_len) : droq->buffer_size;
658f173c2b7SSean Bruno 
659f173c2b7SSean Bruno 					buffer = ((struct mbuf *)
660f173c2b7SSean Bruno 						  droq->recv_buf_list[idx].
661f173c2b7SSean Bruno 						  buffer);
662f173c2b7SSean Bruno 					buffer->m_len = frag_len;
663f173c2b7SSean Bruno 					if (__predict_true(secondary_frag)) {
664f173c2b7SSean Bruno 						m_cat(nicbuf, buffer);
665f173c2b7SSean Bruno 					} else {
666f173c2b7SSean Bruno 						nicbuf = buffer;
667f173c2b7SSean Bruno 						secondary_frag = true;
668f173c2b7SSean Bruno 					}
669f173c2b7SSean Bruno 
670f173c2b7SSean Bruno 					droq->recv_buf_list[droq->read_idx].
671f173c2b7SSean Bruno 						buffer = NULL;
672f173c2b7SSean Bruno 
673f173c2b7SSean Bruno 					pkt_len += frag_len;
674f173c2b7SSean Bruno 					droq->read_idx =
675f173c2b7SSean Bruno 						lio_incr_index(droq->read_idx,
676f173c2b7SSean Bruno 							       1,
677f173c2b7SSean Bruno 							       droq->max_count);
678f173c2b7SSean Bruno 					droq->refill_count++;
679f173c2b7SSean Bruno 				}
680f173c2b7SSean Bruno 			}
681f173c2b7SSean Bruno 
682f173c2b7SSean Bruno 			if (nicbuf != NULL) {
683f173c2b7SSean Bruno 				if (droq->ops.fptr != NULL) {
684f173c2b7SSean Bruno 					droq->ops.fptr(nicbuf, pkt_len, rh,
685f173c2b7SSean Bruno 						       droq, droq->ops.farg);
686f173c2b7SSean Bruno 				} else {
687f173c2b7SSean Bruno 					lio_recv_buffer_free(nicbuf);
688f173c2b7SSean Bruno 				}
689f173c2b7SSean Bruno 			}
690f173c2b7SSean Bruno 		}
691f173c2b7SSean Bruno 
692f173c2b7SSean Bruno 		if (droq->refill_count >= droq->refill_threshold) {
693f173c2b7SSean Bruno 			int desc_refilled = lio_droq_refill(oct, droq);
694f173c2b7SSean Bruno 
695f173c2b7SSean Bruno 			/*
696f173c2b7SSean Bruno 			 * Flush the droq descriptor data to memory to be sure
697f173c2b7SSean Bruno 			 * that when we update the credits the data in memory
698f173c2b7SSean Bruno 			 * is accurate.
699f173c2b7SSean Bruno 			 */
700f173c2b7SSean Bruno 			wmb();
701f173c2b7SSean Bruno 			lio_write_csr32(oct, droq->pkts_credit_reg,
702f173c2b7SSean Bruno 					desc_refilled);
703f173c2b7SSean Bruno 			/* make sure mmio write completes */
704f173c2b7SSean Bruno 			__compiler_membar();
705f173c2b7SSean Bruno 		}
706f173c2b7SSean Bruno 	}	/* for (each packet)... */
707f173c2b7SSean Bruno 
708f173c2b7SSean Bruno 	/* Increment refill_count by the number of buffers processed. */
709f173c2b7SSean Bruno 	droq->stats.pkts_received += pkt;
710f173c2b7SSean Bruno 	droq->stats.bytes_received += total_len;
711f173c2b7SSean Bruno 
712f173c2b7SSean Bruno 	tcp_lro_flush_all(&droq->lro);
713f173c2b7SSean Bruno 
714f173c2b7SSean Bruno 	if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
715f173c2b7SSean Bruno 		lio_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
716f173c2b7SSean Bruno 
717f173c2b7SSean Bruno 		droq->stats.dropped_toomany += (pkts_to_process - pkt);
718f173c2b7SSean Bruno 		return (pkts_to_process);
719f173c2b7SSean Bruno 	}
720f173c2b7SSean Bruno 
721f173c2b7SSean Bruno 	return (pkt);
722f173c2b7SSean Bruno }
723f173c2b7SSean Bruno 
724f173c2b7SSean Bruno int
lio_droq_process_packets(struct octeon_device * oct,struct lio_droq * droq,uint32_t budget)725f173c2b7SSean Bruno lio_droq_process_packets(struct octeon_device *oct, struct lio_droq *droq,
726f173c2b7SSean Bruno 			 uint32_t budget)
727f173c2b7SSean Bruno {
728f173c2b7SSean Bruno 	struct lio_stailq_node	*tmp, *tmp2;
729f173c2b7SSean Bruno 	uint32_t		pkt_count = 0, pkts_processed = 0;
730f173c2b7SSean Bruno 
731f173c2b7SSean Bruno 	/* Grab the droq lock */
732f173c2b7SSean Bruno 	mtx_lock(&droq->lock);
733f173c2b7SSean Bruno 
734f173c2b7SSean Bruno 	lio_droq_check_hw_for_pkts(droq);
735f173c2b7SSean Bruno 	pkt_count = atomic_load_acq_int(&droq->pkts_pending);
736f173c2b7SSean Bruno 
737f173c2b7SSean Bruno 	if (!pkt_count) {
738f173c2b7SSean Bruno 		mtx_unlock(&droq->lock);
739f173c2b7SSean Bruno 		return (0);
740f173c2b7SSean Bruno 	}
741f173c2b7SSean Bruno 	if (pkt_count > budget)
742f173c2b7SSean Bruno 		pkt_count = budget;
743f173c2b7SSean Bruno 
744f173c2b7SSean Bruno 	pkts_processed = lio_droq_fast_process_packets(oct, droq, pkt_count);
745f173c2b7SSean Bruno 
746f173c2b7SSean Bruno 	atomic_subtract_int(&droq->pkts_pending, pkts_processed);
747f173c2b7SSean Bruno 
748f173c2b7SSean Bruno 	/* Release the lock */
749f173c2b7SSean Bruno 	mtx_unlock(&droq->lock);
750f173c2b7SSean Bruno 
751f173c2b7SSean Bruno 	STAILQ_FOREACH_SAFE(tmp, &droq->dispatch_stq_head, entries, tmp2) {
752f173c2b7SSean Bruno 		struct __dispatch *rdisp = (struct __dispatch *)tmp;
753f173c2b7SSean Bruno 
754f173c2b7SSean Bruno 		STAILQ_REMOVE_HEAD(&droq->dispatch_stq_head, entries);
755f173c2b7SSean Bruno 		rdisp->disp_fn(rdisp->rinfo, lio_get_dispatch_arg(oct,
756f173c2b7SSean Bruno 			(uint16_t)rdisp->rinfo->recv_pkt->rh.r.opcode,
757f173c2b7SSean Bruno 			(uint16_t)rdisp->rinfo->recv_pkt->rh.r.subcode));
758f173c2b7SSean Bruno 	}
759f173c2b7SSean Bruno 
760f173c2b7SSean Bruno 	/* If there are packets pending. schedule tasklet again */
761f173c2b7SSean Bruno 	if (atomic_load_acq_int(&droq->pkts_pending))
762f173c2b7SSean Bruno 		return (1);
763f173c2b7SSean Bruno 
764f173c2b7SSean Bruno 	return (0);
765f173c2b7SSean Bruno }
766f173c2b7SSean Bruno 
767f173c2b7SSean Bruno int
lio_register_droq_ops(struct octeon_device * oct,uint32_t q_no,struct lio_droq_ops * ops)768f173c2b7SSean Bruno lio_register_droq_ops(struct octeon_device *oct, uint32_t q_no,
769f173c2b7SSean Bruno 		      struct lio_droq_ops *ops)
770f173c2b7SSean Bruno {
771f173c2b7SSean Bruno 	struct lio_droq		*droq;
772f173c2b7SSean Bruno 	struct lio_config	*lio_cfg = NULL;
773f173c2b7SSean Bruno 
774f173c2b7SSean Bruno 	lio_cfg = lio_get_conf(oct);
775f173c2b7SSean Bruno 
776f173c2b7SSean Bruno 	if (lio_cfg == NULL)
777f173c2b7SSean Bruno 		return (-EINVAL);
778f173c2b7SSean Bruno 
779f173c2b7SSean Bruno 	if (ops == NULL) {
780f173c2b7SSean Bruno 		lio_dev_err(oct, "%s: droq_ops pointer is NULL\n", __func__);
781f173c2b7SSean Bruno 		return (-EINVAL);
782f173c2b7SSean Bruno 	}
783f173c2b7SSean Bruno 
784f173c2b7SSean Bruno 	if (q_no >= LIO_GET_OQ_MAX_Q_CFG(lio_cfg)) {
785f173c2b7SSean Bruno 		lio_dev_err(oct, "%s: droq id (%d) exceeds MAX (%d)\n",
786f173c2b7SSean Bruno 			    __func__, q_no, (oct->num_oqs - 1));
787f173c2b7SSean Bruno 		return (-EINVAL);
788f173c2b7SSean Bruno 	}
789f173c2b7SSean Bruno 	droq = oct->droq[q_no];
790f173c2b7SSean Bruno 
791f173c2b7SSean Bruno 	mtx_lock(&droq->lock);
792f173c2b7SSean Bruno 
793f173c2b7SSean Bruno 	memcpy(&droq->ops, ops, sizeof(struct lio_droq_ops));
794f173c2b7SSean Bruno 
795f173c2b7SSean Bruno 	mtx_unlock(&droq->lock);
796f173c2b7SSean Bruno 
797f173c2b7SSean Bruno 	return (0);
798f173c2b7SSean Bruno }
799f173c2b7SSean Bruno 
800f173c2b7SSean Bruno int
lio_unregister_droq_ops(struct octeon_device * oct,uint32_t q_no)801f173c2b7SSean Bruno lio_unregister_droq_ops(struct octeon_device *oct, uint32_t q_no)
802f173c2b7SSean Bruno {
803f173c2b7SSean Bruno 	struct lio_droq		*droq;
804f173c2b7SSean Bruno 	struct lio_config	*lio_cfg = NULL;
805f173c2b7SSean Bruno 
806f173c2b7SSean Bruno 	lio_cfg = lio_get_conf(oct);
807f173c2b7SSean Bruno 
808f173c2b7SSean Bruno 	if (lio_cfg == NULL)
809f173c2b7SSean Bruno 		return (-EINVAL);
810f173c2b7SSean Bruno 
811f173c2b7SSean Bruno 	if (q_no >= LIO_GET_OQ_MAX_Q_CFG(lio_cfg)) {
812f173c2b7SSean Bruno 		lio_dev_err(oct, "%s: droq id (%d) exceeds MAX (%d)\n",
813f173c2b7SSean Bruno 			    __func__, q_no, oct->num_oqs - 1);
814f173c2b7SSean Bruno 		return (-EINVAL);
815f173c2b7SSean Bruno 	}
816f173c2b7SSean Bruno 
817f173c2b7SSean Bruno 	droq = oct->droq[q_no];
818f173c2b7SSean Bruno 
819f173c2b7SSean Bruno 	if (droq == NULL) {
820f173c2b7SSean Bruno 		lio_dev_info(oct, "Droq id (%d) not available.\n", q_no);
821f173c2b7SSean Bruno 		return (0);
822f173c2b7SSean Bruno 	}
823f173c2b7SSean Bruno 
824f173c2b7SSean Bruno 	mtx_lock(&droq->lock);
825f173c2b7SSean Bruno 
826f173c2b7SSean Bruno 	droq->ops.fptr = NULL;
827f173c2b7SSean Bruno 	droq->ops.farg = NULL;
828f173c2b7SSean Bruno 	droq->ops.drop_on_max = 0;
829f173c2b7SSean Bruno 
830f173c2b7SSean Bruno 	mtx_unlock(&droq->lock);
831f173c2b7SSean Bruno 
832f173c2b7SSean Bruno 	return (0);
833f173c2b7SSean Bruno }
834f173c2b7SSean Bruno 
835f173c2b7SSean Bruno int
lio_create_droq(struct octeon_device * oct,uint32_t q_no,uint32_t num_descs,uint32_t desc_size,void * app_ctx)836f173c2b7SSean Bruno lio_create_droq(struct octeon_device *oct, uint32_t q_no, uint32_t num_descs,
837f173c2b7SSean Bruno 		uint32_t desc_size, void *app_ctx)
838f173c2b7SSean Bruno {
839f173c2b7SSean Bruno 
840f173c2b7SSean Bruno 	if (oct->droq[q_no]->oct_dev != NULL) {
841f173c2b7SSean Bruno 		lio_dev_dbg(oct, "Droq already in use. Cannot create droq %d again\n",
842f173c2b7SSean Bruno 			    q_no);
843f173c2b7SSean Bruno 		return (1);
844f173c2b7SSean Bruno 	}
845f173c2b7SSean Bruno 
846f173c2b7SSean Bruno 	/* Initialize the Droq */
847f173c2b7SSean Bruno 	if (lio_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) {
848f173c2b7SSean Bruno 		bzero(oct->droq[q_no], sizeof(struct lio_droq));
849f173c2b7SSean Bruno 		goto create_droq_fail;
850f173c2b7SSean Bruno 	}
851f173c2b7SSean Bruno 
852f173c2b7SSean Bruno 	oct->num_oqs++;
853f173c2b7SSean Bruno 
854f173c2b7SSean Bruno 	lio_dev_dbg(oct, "%s: Total number of OQ: %d\n", __func__,
855f173c2b7SSean Bruno 		    oct->num_oqs);
856f173c2b7SSean Bruno 
857f173c2b7SSean Bruno 	/* Global Droq register settings */
858f173c2b7SSean Bruno 
859f173c2b7SSean Bruno 	/*
860f173c2b7SSean Bruno 	 * As of now not required, as setting are done for all 32 Droqs at
861f173c2b7SSean Bruno 	 * the same time.
862f173c2b7SSean Bruno 	 */
863f173c2b7SSean Bruno 	return (0);
864f173c2b7SSean Bruno 
865f173c2b7SSean Bruno create_droq_fail:
866f173c2b7SSean Bruno 	return (-ENOMEM);
867f173c2b7SSean Bruno }
868