1f173c2b7SSean Bruno /*
2f173c2b7SSean Bruno * BSD LICENSE
3f173c2b7SSean Bruno *
4f173c2b7SSean Bruno * Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5f173c2b7SSean Bruno * All rights reserved.
6f173c2b7SSean Bruno *
7f173c2b7SSean Bruno * Redistribution and use in source and binary forms, with or without
8f173c2b7SSean Bruno * modification, are permitted provided that the following conditions
9f173c2b7SSean Bruno * are met:
10f173c2b7SSean Bruno *
11f173c2b7SSean Bruno * * Redistributions of source code must retain the above copyright
12f173c2b7SSean Bruno * notice, this list of conditions and the following disclaimer.
13f173c2b7SSean Bruno * * Redistributions in binary form must reproduce the above copyright
14f173c2b7SSean Bruno * notice, this list of conditions and the following disclaimer in
15f173c2b7SSean Bruno * the documentation and/or other materials provided with the
16f173c2b7SSean Bruno * distribution.
17f173c2b7SSean Bruno * * Neither the name of Cavium, Inc. nor the names of its
18f173c2b7SSean Bruno * contributors may be used to endorse or promote products derived
19f173c2b7SSean Bruno * from this software without specific prior written permission.
20f173c2b7SSean Bruno *
21f173c2b7SSean Bruno * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22f173c2b7SSean Bruno * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23f173c2b7SSean Bruno * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24f173c2b7SSean Bruno * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25f173c2b7SSean Bruno * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26f173c2b7SSean Bruno * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27f173c2b7SSean Bruno * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28f173c2b7SSean Bruno * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29f173c2b7SSean Bruno * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30f173c2b7SSean Bruno * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31f173c2b7SSean Bruno * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32f173c2b7SSean Bruno */
33f173c2b7SSean Bruno
34f173c2b7SSean Bruno #include "lio_bsd.h"
35f173c2b7SSean Bruno #include "lio_common.h"
36f173c2b7SSean Bruno #include "lio_droq.h"
37f173c2b7SSean Bruno #include "lio_iq.h"
38f173c2b7SSean Bruno #include "lio_response_manager.h"
39f173c2b7SSean Bruno #include "lio_device.h"
40f173c2b7SSean Bruno #include "lio_main.h"
41f173c2b7SSean Bruno #include "cn23xx_pf_device.h"
42f173c2b7SSean Bruno #include "lio_network.h"
43f173c2b7SSean Bruno
44f173c2b7SSean Bruno struct __dispatch {
45f173c2b7SSean Bruno struct lio_stailq_node node;
46f173c2b7SSean Bruno struct lio_recv_info *rinfo;
47f173c2b7SSean Bruno lio_dispatch_fn_t disp_fn;
48f173c2b7SSean Bruno };
49f173c2b7SSean Bruno
50f173c2b7SSean Bruno void *lio_get_dispatch_arg(struct octeon_device *oct,
51f173c2b7SSean Bruno uint16_t opcode, uint16_t subcode);
52f173c2b7SSean Bruno
53f173c2b7SSean Bruno /*
54f173c2b7SSean Bruno * Get the argument that the user set when registering dispatch
55f173c2b7SSean Bruno * function for a given opcode/subcode.
56f173c2b7SSean Bruno * @param octeon_dev - the octeon device pointer.
57f173c2b7SSean Bruno * @param opcode - the opcode for which the dispatch argument
58f173c2b7SSean Bruno * is to be checked.
59f173c2b7SSean Bruno * @param subcode - the subcode for which the dispatch argument
60f173c2b7SSean Bruno * is to be checked.
61f173c2b7SSean Bruno * @return Success: void * (argument to the dispatch function)
62f173c2b7SSean Bruno * @return Failure: NULL
63f173c2b7SSean Bruno *
64f173c2b7SSean Bruno */
65f173c2b7SSean Bruno void *
lio_get_dispatch_arg(struct octeon_device * octeon_dev,uint16_t opcode,uint16_t subcode)66f173c2b7SSean Bruno lio_get_dispatch_arg(struct octeon_device *octeon_dev,
67f173c2b7SSean Bruno uint16_t opcode, uint16_t subcode)
68f173c2b7SSean Bruno {
69f173c2b7SSean Bruno struct lio_stailq_node *dispatch;
70f173c2b7SSean Bruno void *fn_arg = NULL;
71f173c2b7SSean Bruno int idx;
72f173c2b7SSean Bruno uint16_t combined_opcode;
73f173c2b7SSean Bruno
74f173c2b7SSean Bruno combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode);
75f173c2b7SSean Bruno
76f173c2b7SSean Bruno idx = combined_opcode & LIO_OPCODE_MASK;
77f173c2b7SSean Bruno
78f173c2b7SSean Bruno mtx_lock(&octeon_dev->dispatch.lock);
79f173c2b7SSean Bruno
80f173c2b7SSean Bruno if (octeon_dev->dispatch.count == 0) {
81f173c2b7SSean Bruno mtx_unlock(&octeon_dev->dispatch.lock);
82f173c2b7SSean Bruno return (NULL);
83f173c2b7SSean Bruno }
84f173c2b7SSean Bruno
85f173c2b7SSean Bruno if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
86f173c2b7SSean Bruno fn_arg = octeon_dev->dispatch.dlist[idx].arg;
87f173c2b7SSean Bruno } else {
88f173c2b7SSean Bruno STAILQ_FOREACH(dispatch,
89f173c2b7SSean Bruno &octeon_dev->dispatch.dlist[idx].head, entries) {
90f173c2b7SSean Bruno if (((struct lio_dispatch *)dispatch)->opcode ==
91f173c2b7SSean Bruno combined_opcode) {
92f173c2b7SSean Bruno fn_arg = ((struct lio_dispatch *)dispatch)->arg;
93f173c2b7SSean Bruno break;
94f173c2b7SSean Bruno }
95f173c2b7SSean Bruno }
96f173c2b7SSean Bruno }
97f173c2b7SSean Bruno
98f173c2b7SSean Bruno mtx_unlock(&octeon_dev->dispatch.lock);
99f173c2b7SSean Bruno return (fn_arg);
100f173c2b7SSean Bruno }
101f173c2b7SSean Bruno
102f173c2b7SSean Bruno /*
103f173c2b7SSean Bruno * Check for packets on Droq. This function should be called with lock held.
104f173c2b7SSean Bruno * @param droq - Droq on which count is checked.
105f173c2b7SSean Bruno * @return Returns packet count.
106f173c2b7SSean Bruno */
107f173c2b7SSean Bruno uint32_t
lio_droq_check_hw_for_pkts(struct lio_droq * droq)108f173c2b7SSean Bruno lio_droq_check_hw_for_pkts(struct lio_droq *droq)
109f173c2b7SSean Bruno {
110f173c2b7SSean Bruno struct octeon_device *oct = droq->oct_dev;
111f173c2b7SSean Bruno uint32_t last_count;
112f173c2b7SSean Bruno uint32_t pkt_count = 0;
113f173c2b7SSean Bruno
114f173c2b7SSean Bruno pkt_count = lio_read_csr32(oct, droq->pkts_sent_reg);
115f173c2b7SSean Bruno
116f173c2b7SSean Bruno last_count = pkt_count - droq->pkt_count;
117f173c2b7SSean Bruno droq->pkt_count = pkt_count;
118f173c2b7SSean Bruno
119f173c2b7SSean Bruno /* we shall write to cnts at the end of processing */
120f173c2b7SSean Bruno if (last_count)
121f173c2b7SSean Bruno atomic_add_int(&droq->pkts_pending, last_count);
122f173c2b7SSean Bruno
123f173c2b7SSean Bruno return (last_count);
124f173c2b7SSean Bruno }
125f173c2b7SSean Bruno
126f173c2b7SSean Bruno static void
lio_droq_compute_max_packet_bufs(struct lio_droq * droq)127f173c2b7SSean Bruno lio_droq_compute_max_packet_bufs(struct lio_droq *droq)
128f173c2b7SSean Bruno {
129f173c2b7SSean Bruno uint32_t count = 0;
130f173c2b7SSean Bruno
131f173c2b7SSean Bruno /*
132f173c2b7SSean Bruno * max_empty_descs is the max. no. of descs that can have no buffers.
133f173c2b7SSean Bruno * If the empty desc count goes beyond this value, we cannot safely
134f173c2b7SSean Bruno * read in a 64K packet sent by Octeon
135f173c2b7SSean Bruno * (64K is max pkt size from Octeon)
136f173c2b7SSean Bruno */
137f173c2b7SSean Bruno droq->max_empty_descs = 0;
138f173c2b7SSean Bruno
139f173c2b7SSean Bruno do {
140f173c2b7SSean Bruno droq->max_empty_descs++;
141f173c2b7SSean Bruno count += droq->buffer_size;
142f173c2b7SSean Bruno } while (count < (64 * 1024));
143f173c2b7SSean Bruno
144f173c2b7SSean Bruno droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
145f173c2b7SSean Bruno }
146f173c2b7SSean Bruno
147f173c2b7SSean Bruno static void
lio_droq_reset_indices(struct lio_droq * droq)148f173c2b7SSean Bruno lio_droq_reset_indices(struct lio_droq *droq)
149f173c2b7SSean Bruno {
150f173c2b7SSean Bruno
151f173c2b7SSean Bruno droq->read_idx = 0;
152f173c2b7SSean Bruno droq->refill_idx = 0;
153f173c2b7SSean Bruno droq->refill_count = 0;
154f173c2b7SSean Bruno atomic_store_rel_int(&droq->pkts_pending, 0);
155f173c2b7SSean Bruno }
156f173c2b7SSean Bruno
157f173c2b7SSean Bruno static void
lio_droq_destroy_ring_buffers(struct octeon_device * oct,struct lio_droq * droq)158f173c2b7SSean Bruno lio_droq_destroy_ring_buffers(struct octeon_device *oct,
159f173c2b7SSean Bruno struct lio_droq *droq)
160f173c2b7SSean Bruno {
161f173c2b7SSean Bruno uint32_t i;
162f173c2b7SSean Bruno
163f173c2b7SSean Bruno for (i = 0; i < droq->max_count; i++) {
164f173c2b7SSean Bruno if (droq->recv_buf_list[i].buffer != NULL) {
165f173c2b7SSean Bruno lio_recv_buffer_free(droq->recv_buf_list[i].buffer);
166f173c2b7SSean Bruno droq->recv_buf_list[i].buffer = NULL;
167f173c2b7SSean Bruno }
168f173c2b7SSean Bruno }
169f173c2b7SSean Bruno
170f173c2b7SSean Bruno lio_droq_reset_indices(droq);
171f173c2b7SSean Bruno }
172f173c2b7SSean Bruno
173f173c2b7SSean Bruno static int
lio_droq_setup_ring_buffers(struct octeon_device * oct,struct lio_droq * droq)174f173c2b7SSean Bruno lio_droq_setup_ring_buffers(struct octeon_device *oct,
175f173c2b7SSean Bruno struct lio_droq *droq)
176f173c2b7SSean Bruno {
177f173c2b7SSean Bruno struct lio_droq_desc *desc_ring = droq->desc_ring;
178f173c2b7SSean Bruno void *buf;
179f173c2b7SSean Bruno uint32_t i;
180f173c2b7SSean Bruno
181f173c2b7SSean Bruno for (i = 0; i < droq->max_count; i++) {
182f173c2b7SSean Bruno buf = lio_recv_buffer_alloc(droq->buffer_size);
183f173c2b7SSean Bruno
184f173c2b7SSean Bruno if (buf == NULL) {
185f173c2b7SSean Bruno lio_dev_err(oct, "%s buffer alloc failed\n",
186f173c2b7SSean Bruno __func__);
187f173c2b7SSean Bruno droq->stats.rx_alloc_failure++;
188f173c2b7SSean Bruno return (-ENOMEM);
189f173c2b7SSean Bruno }
190f173c2b7SSean Bruno
191f173c2b7SSean Bruno droq->recv_buf_list[i].buffer = buf;
192f173c2b7SSean Bruno droq->recv_buf_list[i].data = ((struct mbuf *)buf)->m_data;
193f173c2b7SSean Bruno desc_ring[i].info_ptr = 0;
194f173c2b7SSean Bruno desc_ring[i].buffer_ptr =
195f173c2b7SSean Bruno lio_map_ring(oct->device, droq->recv_buf_list[i].buffer,
196f173c2b7SSean Bruno droq->buffer_size);
197f173c2b7SSean Bruno }
198f173c2b7SSean Bruno
199f173c2b7SSean Bruno lio_droq_reset_indices(droq);
200f173c2b7SSean Bruno
201f173c2b7SSean Bruno lio_droq_compute_max_packet_bufs(droq);
202f173c2b7SSean Bruno
203f173c2b7SSean Bruno return (0);
204f173c2b7SSean Bruno }
205f173c2b7SSean Bruno
206f173c2b7SSean Bruno int
lio_delete_droq(struct octeon_device * oct,uint32_t q_no)207f173c2b7SSean Bruno lio_delete_droq(struct octeon_device *oct, uint32_t q_no)
208f173c2b7SSean Bruno {
209f173c2b7SSean Bruno struct lio_droq *droq = oct->droq[q_no];
210f173c2b7SSean Bruno
211f173c2b7SSean Bruno lio_dev_dbg(oct, "%s[%d]\n", __func__, q_no);
212f173c2b7SSean Bruno
213f173c2b7SSean Bruno while (taskqueue_cancel(droq->droq_taskqueue, &droq->droq_task, NULL))
214f173c2b7SSean Bruno taskqueue_drain(droq->droq_taskqueue, &droq->droq_task);
215f173c2b7SSean Bruno
216f173c2b7SSean Bruno taskqueue_free(droq->droq_taskqueue);
217f173c2b7SSean Bruno droq->droq_taskqueue = NULL;
218f173c2b7SSean Bruno
219f173c2b7SSean Bruno lio_droq_destroy_ring_buffers(oct, droq);
220f173c2b7SSean Bruno free(droq->recv_buf_list, M_DEVBUF);
221f173c2b7SSean Bruno
222f173c2b7SSean Bruno if (droq->desc_ring != NULL)
223f173c2b7SSean Bruno lio_dma_free((droq->max_count * LIO_DROQ_DESC_SIZE),
224f173c2b7SSean Bruno droq->desc_ring);
225f173c2b7SSean Bruno
226f173c2b7SSean Bruno oct->io_qmask.oq &= ~(1ULL << q_no);
227f173c2b7SSean Bruno bzero(oct->droq[q_no], sizeof(struct lio_droq));
228f173c2b7SSean Bruno oct->num_oqs--;
229f173c2b7SSean Bruno
230f173c2b7SSean Bruno return (0);
231f173c2b7SSean Bruno }
232f173c2b7SSean Bruno
233f173c2b7SSean Bruno void
lio_droq_bh(void * ptr,int pending __unused)234f173c2b7SSean Bruno lio_droq_bh(void *ptr, int pending __unused)
235f173c2b7SSean Bruno {
236f173c2b7SSean Bruno struct lio_droq *droq = ptr;
237f173c2b7SSean Bruno struct octeon_device *oct = droq->oct_dev;
238f173c2b7SSean Bruno struct lio_instr_queue *iq = oct->instr_queue[droq->q_no];
239f173c2b7SSean Bruno int reschedule, tx_done = 1;
240f173c2b7SSean Bruno
241f173c2b7SSean Bruno reschedule = lio_droq_process_packets(oct, droq, oct->rx_budget);
242f173c2b7SSean Bruno
243f173c2b7SSean Bruno if (atomic_load_acq_int(&iq->instr_pending))
244f173c2b7SSean Bruno tx_done = lio_flush_iq(oct, iq, oct->tx_budget);
245f173c2b7SSean Bruno
246f173c2b7SSean Bruno if (reschedule || !tx_done)
247f173c2b7SSean Bruno taskqueue_enqueue(droq->droq_taskqueue, &droq->droq_task);
248f173c2b7SSean Bruno else
249f173c2b7SSean Bruno lio_enable_irq(droq, iq);
250f173c2b7SSean Bruno }
251f173c2b7SSean Bruno
252f173c2b7SSean Bruno int
lio_init_droq(struct octeon_device * oct,uint32_t q_no,uint32_t num_descs,uint32_t desc_size,void * app_ctx)253f173c2b7SSean Bruno lio_init_droq(struct octeon_device *oct, uint32_t q_no,
254f173c2b7SSean Bruno uint32_t num_descs, uint32_t desc_size, void *app_ctx)
255f173c2b7SSean Bruno {
256f173c2b7SSean Bruno struct lio_droq *droq;
257f173c2b7SSean Bruno unsigned long size;
258f173c2b7SSean Bruno uint32_t c_buf_size = 0, c_num_descs = 0, c_pkts_per_intr = 0;
259f173c2b7SSean Bruno uint32_t c_refill_threshold = 0, desc_ring_size = 0;
260f173c2b7SSean Bruno
261f173c2b7SSean Bruno lio_dev_dbg(oct, "%s[%d]\n", __func__, q_no);
262f173c2b7SSean Bruno
263f173c2b7SSean Bruno droq = oct->droq[q_no];
264f173c2b7SSean Bruno bzero(droq, LIO_DROQ_SIZE);
265f173c2b7SSean Bruno
266f173c2b7SSean Bruno droq->oct_dev = oct;
267f173c2b7SSean Bruno droq->q_no = q_no;
268f173c2b7SSean Bruno if (app_ctx != NULL)
269f173c2b7SSean Bruno droq->app_ctx = app_ctx;
270f173c2b7SSean Bruno else
271f173c2b7SSean Bruno droq->app_ctx = (void *)(size_t)q_no;
272f173c2b7SSean Bruno
273f173c2b7SSean Bruno c_num_descs = num_descs;
274f173c2b7SSean Bruno c_buf_size = desc_size;
275f173c2b7SSean Bruno if (LIO_CN23XX_PF(oct)) {
276f173c2b7SSean Bruno struct lio_config *conf23 = LIO_CHIP_CONF(oct, cn23xx_pf);
277f173c2b7SSean Bruno
278f173c2b7SSean Bruno c_pkts_per_intr =
279f173c2b7SSean Bruno (uint32_t)LIO_GET_OQ_PKTS_PER_INTR_CFG(conf23);
280f173c2b7SSean Bruno c_refill_threshold =
281f173c2b7SSean Bruno (uint32_t)LIO_GET_OQ_REFILL_THRESHOLD_CFG(conf23);
282f173c2b7SSean Bruno } else {
283f173c2b7SSean Bruno return (1);
284f173c2b7SSean Bruno }
285f173c2b7SSean Bruno
286f173c2b7SSean Bruno droq->max_count = c_num_descs;
287f173c2b7SSean Bruno droq->buffer_size = c_buf_size;
288f173c2b7SSean Bruno
289f173c2b7SSean Bruno desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE;
290f173c2b7SSean Bruno droq->desc_ring = lio_dma_alloc(desc_ring_size, &droq->desc_ring_dma);
291f173c2b7SSean Bruno if (droq->desc_ring == NULL) {
292f173c2b7SSean Bruno lio_dev_err(oct, "Output queue %d ring alloc failed\n", q_no);
293f173c2b7SSean Bruno return (1);
294f173c2b7SSean Bruno }
295f173c2b7SSean Bruno
2963de0952fSSean Bruno lio_dev_dbg(oct, "droq[%d]: desc_ring: virt: 0x%p, dma: %llx\n", q_no,
2973de0952fSSean Bruno droq->desc_ring, LIO_CAST64(droq->desc_ring_dma));
298f173c2b7SSean Bruno lio_dev_dbg(oct, "droq[%d]: num_desc: %d\n", q_no, droq->max_count);
299f173c2b7SSean Bruno
300f173c2b7SSean Bruno size = droq->max_count * LIO_DROQ_RECVBUF_SIZE;
301f173c2b7SSean Bruno droq->recv_buf_list =
302f173c2b7SSean Bruno (struct lio_recv_buffer *)malloc(size, M_DEVBUF,
303f173c2b7SSean Bruno M_NOWAIT | M_ZERO);
304f173c2b7SSean Bruno if (droq->recv_buf_list == NULL) {
305f173c2b7SSean Bruno lio_dev_err(oct, "Output queue recv buf list alloc failed\n");
306f173c2b7SSean Bruno goto init_droq_fail;
307f173c2b7SSean Bruno }
308f173c2b7SSean Bruno
309f173c2b7SSean Bruno if (lio_droq_setup_ring_buffers(oct, droq))
310f173c2b7SSean Bruno goto init_droq_fail;
311f173c2b7SSean Bruno
312f173c2b7SSean Bruno droq->pkts_per_intr = c_pkts_per_intr;
313f173c2b7SSean Bruno droq->refill_threshold = c_refill_threshold;
314f173c2b7SSean Bruno
315f173c2b7SSean Bruno lio_dev_dbg(oct, "DROQ INIT: max_empty_descs: %d\n",
316f173c2b7SSean Bruno droq->max_empty_descs);
317f173c2b7SSean Bruno
318f173c2b7SSean Bruno mtx_init(&droq->lock, "droq_lock", NULL, MTX_DEF);
319f173c2b7SSean Bruno
320f173c2b7SSean Bruno STAILQ_INIT(&droq->dispatch_stq_head);
321f173c2b7SSean Bruno
322f173c2b7SSean Bruno oct->fn_list.setup_oq_regs(oct, q_no);
323f173c2b7SSean Bruno
324f173c2b7SSean Bruno oct->io_qmask.oq |= BIT_ULL(q_no);
325f173c2b7SSean Bruno
326f173c2b7SSean Bruno /*
327f173c2b7SSean Bruno * Initialize the taskqueue that handles
328f173c2b7SSean Bruno * output queue packet processing.
329f173c2b7SSean Bruno */
330f173c2b7SSean Bruno lio_dev_dbg(oct, "Initializing droq%d taskqueue\n", q_no);
331*301a87acSGleb Smirnoff NET_TASK_INIT(&droq->droq_task, 0, lio_droq_bh, (void *)droq);
332f173c2b7SSean Bruno
333f173c2b7SSean Bruno droq->droq_taskqueue = taskqueue_create_fast("lio_droq_task", M_NOWAIT,
334f173c2b7SSean Bruno taskqueue_thread_enqueue,
335f173c2b7SSean Bruno &droq->droq_taskqueue);
336f173c2b7SSean Bruno taskqueue_start_threads_cpuset(&droq->droq_taskqueue, 1, PI_NET,
337f173c2b7SSean Bruno &oct->ioq_vector[q_no].affinity_mask,
338f173c2b7SSean Bruno "lio%d_droq%d_task", oct->octeon_id,
339f173c2b7SSean Bruno q_no);
340f173c2b7SSean Bruno
341f173c2b7SSean Bruno return (0);
342f173c2b7SSean Bruno
343f173c2b7SSean Bruno init_droq_fail:
344f173c2b7SSean Bruno lio_delete_droq(oct, q_no);
345f173c2b7SSean Bruno return (1);
346f173c2b7SSean Bruno }
347f173c2b7SSean Bruno
348f173c2b7SSean Bruno /*
349f173c2b7SSean Bruno * lio_create_recv_info
350f173c2b7SSean Bruno * Parameters:
351f173c2b7SSean Bruno * octeon_dev - pointer to the octeon device structure
352f173c2b7SSean Bruno * droq - droq in which the packet arrived.
353f173c2b7SSean Bruno * buf_cnt - no. of buffers used by the packet.
354f173c2b7SSean Bruno * idx - index in the descriptor for the first buffer in the packet.
355f173c2b7SSean Bruno * Description:
356f173c2b7SSean Bruno * Allocates a recv_info_t and copies the buffer addresses for packet data
357f173c2b7SSean Bruno * into the recv_pkt space which starts at an 8B offset from recv_info_t.
358f173c2b7SSean Bruno * Flags the descriptors for refill later. If available descriptors go
359f173c2b7SSean Bruno * below the threshold to receive a 64K pkt, new buffers are first allocated
360f173c2b7SSean Bruno * before the recv_pkt_t is created.
361f173c2b7SSean Bruno * This routine will be called in interrupt context.
362f173c2b7SSean Bruno * Returns:
363f173c2b7SSean Bruno * Success: Pointer to recv_info_t
364f173c2b7SSean Bruno * Failure: NULL.
365f173c2b7SSean Bruno * Locks:
366f173c2b7SSean Bruno * The droq->lock is held when this routine is called.
367f173c2b7SSean Bruno */
368f173c2b7SSean Bruno static inline struct lio_recv_info *
lio_create_recv_info(struct octeon_device * octeon_dev,struct lio_droq * droq,uint32_t buf_cnt,uint32_t idx)369f173c2b7SSean Bruno lio_create_recv_info(struct octeon_device *octeon_dev, struct lio_droq *droq,
370f173c2b7SSean Bruno uint32_t buf_cnt, uint32_t idx)
371f173c2b7SSean Bruno {
372f173c2b7SSean Bruno struct lio_droq_info *info;
373f173c2b7SSean Bruno struct lio_recv_pkt *recv_pkt;
374f173c2b7SSean Bruno struct lio_recv_info *recv_info;
375f173c2b7SSean Bruno uint32_t bytes_left, i;
376f173c2b7SSean Bruno
377f173c2b7SSean Bruno info = (struct lio_droq_info *)droq->recv_buf_list[idx].data;
378f173c2b7SSean Bruno
379f173c2b7SSean Bruno recv_info = lio_alloc_recv_info(sizeof(struct __dispatch));
380f173c2b7SSean Bruno if (recv_info == NULL)
381f173c2b7SSean Bruno return (NULL);
382f173c2b7SSean Bruno
383f173c2b7SSean Bruno recv_pkt = recv_info->recv_pkt;
384f173c2b7SSean Bruno recv_pkt->rh = info->rh;
385f173c2b7SSean Bruno recv_pkt->length = (uint32_t)info->length;
386f173c2b7SSean Bruno recv_pkt->buffer_count = (uint16_t)buf_cnt;
387f173c2b7SSean Bruno recv_pkt->octeon_id = (uint16_t)octeon_dev->octeon_id;
388f173c2b7SSean Bruno
389f173c2b7SSean Bruno i = 0;
390f173c2b7SSean Bruno bytes_left = (uint32_t)info->length;
391f173c2b7SSean Bruno
392f173c2b7SSean Bruno while (buf_cnt) {
393f173c2b7SSean Bruno recv_pkt->buffer_size[i] = (bytes_left >= droq->buffer_size) ?
394f173c2b7SSean Bruno droq->buffer_size : bytes_left;
395f173c2b7SSean Bruno
396f173c2b7SSean Bruno recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
397f173c2b7SSean Bruno droq->recv_buf_list[idx].buffer = NULL;
398f173c2b7SSean Bruno
399f173c2b7SSean Bruno idx = lio_incr_index(idx, 1, droq->max_count);
400f173c2b7SSean Bruno bytes_left -= droq->buffer_size;
401f173c2b7SSean Bruno i++;
402f173c2b7SSean Bruno buf_cnt--;
403f173c2b7SSean Bruno }
404f173c2b7SSean Bruno
405f173c2b7SSean Bruno return (recv_info);
406f173c2b7SSean Bruno }
407f173c2b7SSean Bruno
408f173c2b7SSean Bruno /*
409f173c2b7SSean Bruno * If we were not able to refill all buffers, try to move around
410f173c2b7SSean Bruno * the buffers that were not dispatched.
411f173c2b7SSean Bruno */
412f173c2b7SSean Bruno static inline uint32_t
lio_droq_refill_pullup_descs(struct lio_droq * droq,struct lio_droq_desc * desc_ring)413f173c2b7SSean Bruno lio_droq_refill_pullup_descs(struct lio_droq *droq,
414f173c2b7SSean Bruno struct lio_droq_desc *desc_ring)
415f173c2b7SSean Bruno {
416f173c2b7SSean Bruno uint32_t desc_refilled = 0;
417f173c2b7SSean Bruno uint32_t refill_index = droq->refill_idx;
418f173c2b7SSean Bruno
419f173c2b7SSean Bruno while (refill_index != droq->read_idx) {
420f173c2b7SSean Bruno if (droq->recv_buf_list[refill_index].buffer != NULL) {
421f173c2b7SSean Bruno droq->recv_buf_list[droq->refill_idx].buffer =
422f173c2b7SSean Bruno droq->recv_buf_list[refill_index].buffer;
423f173c2b7SSean Bruno droq->recv_buf_list[droq->refill_idx].data =
424f173c2b7SSean Bruno droq->recv_buf_list[refill_index].data;
425f173c2b7SSean Bruno desc_ring[droq->refill_idx].buffer_ptr =
426f173c2b7SSean Bruno desc_ring[refill_index].buffer_ptr;
427f173c2b7SSean Bruno droq->recv_buf_list[refill_index].buffer = NULL;
428f173c2b7SSean Bruno desc_ring[refill_index].buffer_ptr = 0;
429f173c2b7SSean Bruno do {
430f173c2b7SSean Bruno droq->refill_idx =
431f173c2b7SSean Bruno lio_incr_index(droq->refill_idx, 1,
432f173c2b7SSean Bruno droq->max_count);
433f173c2b7SSean Bruno desc_refilled++;
434f173c2b7SSean Bruno droq->refill_count--;
435f173c2b7SSean Bruno } while (droq->recv_buf_list[droq->refill_idx].buffer !=
436f173c2b7SSean Bruno NULL);
437f173c2b7SSean Bruno }
438f173c2b7SSean Bruno refill_index = lio_incr_index(refill_index, 1, droq->max_count);
439f173c2b7SSean Bruno } /* while */
440f173c2b7SSean Bruno return (desc_refilled);
441f173c2b7SSean Bruno }
442f173c2b7SSean Bruno
443f173c2b7SSean Bruno /*
444f173c2b7SSean Bruno * lio_droq_refill
445f173c2b7SSean Bruno * Parameters:
446f173c2b7SSean Bruno * droq - droq in which descriptors require new buffers.
447f173c2b7SSean Bruno * Description:
448f173c2b7SSean Bruno * Called during normal DROQ processing in interrupt mode or by the poll
449f173c2b7SSean Bruno * thread to refill the descriptors from which buffers were dispatched
450f173c2b7SSean Bruno * to upper layers. Attempts to allocate new buffers. If that fails, moves
451f173c2b7SSean Bruno * up buffers (that were not dispatched) to form a contiguous ring.
452f173c2b7SSean Bruno * Returns:
453f173c2b7SSean Bruno * No of descriptors refilled.
454f173c2b7SSean Bruno * Locks:
455f173c2b7SSean Bruno * This routine is called with droq->lock held.
456f173c2b7SSean Bruno */
457f173c2b7SSean Bruno uint32_t
lio_droq_refill(struct octeon_device * octeon_dev,struct lio_droq * droq)458f173c2b7SSean Bruno lio_droq_refill(struct octeon_device *octeon_dev, struct lio_droq *droq)
459f173c2b7SSean Bruno {
460f173c2b7SSean Bruno struct lio_droq_desc *desc_ring;
461f173c2b7SSean Bruno void *buf = NULL;
462f173c2b7SSean Bruno uint32_t desc_refilled = 0;
463f173c2b7SSean Bruno uint8_t *data;
464f173c2b7SSean Bruno
465f173c2b7SSean Bruno desc_ring = droq->desc_ring;
466f173c2b7SSean Bruno
467f173c2b7SSean Bruno while (droq->refill_count && (desc_refilled < droq->max_count)) {
468f173c2b7SSean Bruno /*
469f173c2b7SSean Bruno * If a valid buffer exists (happens if there is no dispatch),
470f173c2b7SSean Bruno * reuse
471f173c2b7SSean Bruno * the buffer, else allocate.
472f173c2b7SSean Bruno */
473f173c2b7SSean Bruno if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) {
474f173c2b7SSean Bruno buf = lio_recv_buffer_alloc(droq->buffer_size);
475f173c2b7SSean Bruno /*
476f173c2b7SSean Bruno * If a buffer could not be allocated, no point in
477f173c2b7SSean Bruno * continuing
478f173c2b7SSean Bruno */
479f173c2b7SSean Bruno if (buf == NULL) {
480f173c2b7SSean Bruno droq->stats.rx_alloc_failure++;
481f173c2b7SSean Bruno break;
482f173c2b7SSean Bruno }
483f173c2b7SSean Bruno
484f173c2b7SSean Bruno droq->recv_buf_list[droq->refill_idx].buffer = buf;
485f173c2b7SSean Bruno data = ((struct mbuf *)buf)->m_data;
486f173c2b7SSean Bruno } else {
487f173c2b7SSean Bruno data = ((struct mbuf *)droq->recv_buf_list
488f173c2b7SSean Bruno [droq->refill_idx].buffer)->m_data;
489f173c2b7SSean Bruno }
490f173c2b7SSean Bruno
491f173c2b7SSean Bruno droq->recv_buf_list[droq->refill_idx].data = data;
492f173c2b7SSean Bruno
493f173c2b7SSean Bruno desc_ring[droq->refill_idx].buffer_ptr =
494f173c2b7SSean Bruno lio_map_ring(octeon_dev->device,
495f173c2b7SSean Bruno droq->recv_buf_list[droq->refill_idx].buffer,
496f173c2b7SSean Bruno droq->buffer_size);
497f173c2b7SSean Bruno
498f173c2b7SSean Bruno droq->refill_idx = lio_incr_index(droq->refill_idx, 1,
499f173c2b7SSean Bruno droq->max_count);
500f173c2b7SSean Bruno desc_refilled++;
501f173c2b7SSean Bruno droq->refill_count--;
502f173c2b7SSean Bruno }
503f173c2b7SSean Bruno
504f173c2b7SSean Bruno if (droq->refill_count)
505f173c2b7SSean Bruno desc_refilled += lio_droq_refill_pullup_descs(droq, desc_ring);
506f173c2b7SSean Bruno
507f173c2b7SSean Bruno /*
508f173c2b7SSean Bruno * if droq->refill_count
509f173c2b7SSean Bruno * The refill count would not change in pass two. We only moved buffers
510f173c2b7SSean Bruno * to close the gap in the ring, but we would still have the same no. of
511f173c2b7SSean Bruno * buffers to refill.
512f173c2b7SSean Bruno */
513f173c2b7SSean Bruno return (desc_refilled);
514f173c2b7SSean Bruno }
515f173c2b7SSean Bruno
516f173c2b7SSean Bruno static inline uint32_t
lio_droq_get_bufcount(uint32_t buf_size,uint32_t total_len)517f173c2b7SSean Bruno lio_droq_get_bufcount(uint32_t buf_size, uint32_t total_len)
518f173c2b7SSean Bruno {
519f173c2b7SSean Bruno
520f173c2b7SSean Bruno return ((total_len + buf_size - 1) / buf_size);
521f173c2b7SSean Bruno }
522f173c2b7SSean Bruno
523f173c2b7SSean Bruno static int
lio_droq_dispatch_pkt(struct octeon_device * oct,struct lio_droq * droq,union octeon_rh * rh,struct lio_droq_info * info)524f173c2b7SSean Bruno lio_droq_dispatch_pkt(struct octeon_device *oct, struct lio_droq *droq,
525f173c2b7SSean Bruno union octeon_rh *rh, struct lio_droq_info *info)
526f173c2b7SSean Bruno {
527f173c2b7SSean Bruno struct lio_recv_info *rinfo;
528f173c2b7SSean Bruno lio_dispatch_fn_t disp_fn;
529f173c2b7SSean Bruno uint32_t cnt;
530f173c2b7SSean Bruno
531f173c2b7SSean Bruno cnt = lio_droq_get_bufcount(droq->buffer_size, (uint32_t)info->length);
532f173c2b7SSean Bruno
533f173c2b7SSean Bruno disp_fn = lio_get_dispatch(oct, (uint16_t)rh->r.opcode,
534f173c2b7SSean Bruno (uint16_t)rh->r.subcode);
535f173c2b7SSean Bruno if (disp_fn) {
536f173c2b7SSean Bruno rinfo = lio_create_recv_info(oct, droq, cnt, droq->read_idx);
537f173c2b7SSean Bruno if (rinfo != NULL) {
538f173c2b7SSean Bruno struct __dispatch *rdisp = rinfo->rsvd;
539f173c2b7SSean Bruno
540f173c2b7SSean Bruno rdisp->rinfo = rinfo;
541f173c2b7SSean Bruno rdisp->disp_fn = disp_fn;
542f173c2b7SSean Bruno rinfo->recv_pkt->rh = *rh;
543f173c2b7SSean Bruno STAILQ_INSERT_TAIL(&droq->dispatch_stq_head,
544f173c2b7SSean Bruno &rdisp->node, entries);
545f173c2b7SSean Bruno } else {
546f173c2b7SSean Bruno droq->stats.dropped_nomem++;
547f173c2b7SSean Bruno }
548f173c2b7SSean Bruno } else {
549f173c2b7SSean Bruno lio_dev_err(oct, "DROQ: No dispatch function (opcode %u/%u)\n",
550f173c2b7SSean Bruno (unsigned int)rh->r.opcode,
551f173c2b7SSean Bruno (unsigned int)rh->r.subcode);
552f173c2b7SSean Bruno droq->stats.dropped_nodispatch++;
553f173c2b7SSean Bruno }
554f173c2b7SSean Bruno
555f173c2b7SSean Bruno return (cnt);
556f173c2b7SSean Bruno }
557f173c2b7SSean Bruno
558f173c2b7SSean Bruno static inline void
lio_droq_drop_packets(struct octeon_device * oct,struct lio_droq * droq,uint32_t cnt)559f173c2b7SSean Bruno lio_droq_drop_packets(struct octeon_device *oct, struct lio_droq *droq,
560f173c2b7SSean Bruno uint32_t cnt)
561f173c2b7SSean Bruno {
562f173c2b7SSean Bruno struct lio_droq_info *info;
563f173c2b7SSean Bruno uint32_t i = 0, buf_cnt;
564f173c2b7SSean Bruno
565f173c2b7SSean Bruno for (i = 0; i < cnt; i++) {
566f173c2b7SSean Bruno info = (struct lio_droq_info *)
567f173c2b7SSean Bruno droq->recv_buf_list[droq->read_idx].data;
568f173c2b7SSean Bruno
569f173c2b7SSean Bruno lio_swap_8B_data((uint64_t *)info, 2);
570f173c2b7SSean Bruno
571f173c2b7SSean Bruno if (info->length) {
572f173c2b7SSean Bruno info->length += 8;
573f173c2b7SSean Bruno droq->stats.bytes_received += info->length;
574f173c2b7SSean Bruno buf_cnt = lio_droq_get_bufcount(droq->buffer_size,
575f173c2b7SSean Bruno (uint32_t)info->length);
576f173c2b7SSean Bruno } else {
577f173c2b7SSean Bruno lio_dev_err(oct, "DROQ: In drop: pkt with len 0\n");
578f173c2b7SSean Bruno buf_cnt = 1;
579f173c2b7SSean Bruno }
580f173c2b7SSean Bruno
581f173c2b7SSean Bruno droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt,
582f173c2b7SSean Bruno droq->max_count);
583f173c2b7SSean Bruno droq->refill_count += buf_cnt;
584f173c2b7SSean Bruno }
585f173c2b7SSean Bruno }
586f173c2b7SSean Bruno
587f173c2b7SSean Bruno static uint32_t
lio_droq_fast_process_packets(struct octeon_device * oct,struct lio_droq * droq,uint32_t pkts_to_process)588f173c2b7SSean Bruno lio_droq_fast_process_packets(struct octeon_device *oct, struct lio_droq *droq,
589f173c2b7SSean Bruno uint32_t pkts_to_process)
590f173c2b7SSean Bruno {
591f173c2b7SSean Bruno struct lio_droq_info *info;
592f173c2b7SSean Bruno union octeon_rh *rh;
593f173c2b7SSean Bruno uint32_t pkt, pkt_count, total_len = 0;
594f173c2b7SSean Bruno
595f173c2b7SSean Bruno pkt_count = pkts_to_process;
596f173c2b7SSean Bruno
597f173c2b7SSean Bruno for (pkt = 0; pkt < pkt_count; pkt++) {
598f173c2b7SSean Bruno struct mbuf *nicbuf = NULL;
599f173c2b7SSean Bruno uint32_t pkt_len = 0;
600f173c2b7SSean Bruno
601f173c2b7SSean Bruno info = (struct lio_droq_info *)
602f173c2b7SSean Bruno droq->recv_buf_list[droq->read_idx].data;
603f173c2b7SSean Bruno
604f173c2b7SSean Bruno lio_swap_8B_data((uint64_t *)info, 2);
605f173c2b7SSean Bruno
606f173c2b7SSean Bruno if (!info->length) {
607f173c2b7SSean Bruno lio_dev_err(oct,
608f173c2b7SSean Bruno "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
609f173c2b7SSean Bruno droq->q_no, droq->read_idx, pkt_count);
610f173c2b7SSean Bruno hexdump((uint8_t *)info, LIO_DROQ_INFO_SIZE, NULL,
611f173c2b7SSean Bruno HD_OMIT_CHARS);
612f173c2b7SSean Bruno pkt++;
613f173c2b7SSean Bruno lio_incr_index(droq->read_idx, 1, droq->max_count);
614f173c2b7SSean Bruno droq->refill_count++;
615f173c2b7SSean Bruno break;
616f173c2b7SSean Bruno }
617f173c2b7SSean Bruno
618f173c2b7SSean Bruno rh = &info->rh;
619f173c2b7SSean Bruno
620f173c2b7SSean Bruno info->length += 8;
621f173c2b7SSean Bruno rh->r_dh.len += (LIO_DROQ_INFO_SIZE + 7) / 8;
622f173c2b7SSean Bruno
623f173c2b7SSean Bruno total_len += (uint32_t)info->length;
624f173c2b7SSean Bruno if (lio_opcode_slow_path(rh)) {
625f173c2b7SSean Bruno uint32_t buf_cnt;
626f173c2b7SSean Bruno
627f173c2b7SSean Bruno buf_cnt = lio_droq_dispatch_pkt(oct, droq, rh, info);
628f173c2b7SSean Bruno droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt,
629f173c2b7SSean Bruno droq->max_count);
630f173c2b7SSean Bruno droq->refill_count += buf_cnt;
631f173c2b7SSean Bruno } else {
632f173c2b7SSean Bruno if (info->length <= droq->buffer_size) {
633f173c2b7SSean Bruno pkt_len = (uint32_t)info->length;
634f173c2b7SSean Bruno nicbuf = droq->recv_buf_list[
635f173c2b7SSean Bruno droq->read_idx].buffer;
636f173c2b7SSean Bruno nicbuf->m_len = pkt_len;
637f173c2b7SSean Bruno droq->recv_buf_list[droq->read_idx].buffer =
638f173c2b7SSean Bruno NULL;
639f173c2b7SSean Bruno
640f173c2b7SSean Bruno droq->read_idx =
641f173c2b7SSean Bruno lio_incr_index(droq->read_idx,
642f173c2b7SSean Bruno 1, droq->max_count);
643f173c2b7SSean Bruno droq->refill_count++;
644f173c2b7SSean Bruno } else {
645f173c2b7SSean Bruno bool secondary_frag = false;
646f173c2b7SSean Bruno
647f173c2b7SSean Bruno pkt_len = 0;
648f173c2b7SSean Bruno
649f173c2b7SSean Bruno while (pkt_len < info->length) {
650f173c2b7SSean Bruno int frag_len, idx = droq->read_idx;
651f173c2b7SSean Bruno struct mbuf *buffer;
652f173c2b7SSean Bruno
653f173c2b7SSean Bruno frag_len =
654f173c2b7SSean Bruno ((pkt_len + droq->buffer_size) >
655f173c2b7SSean Bruno info->length) ?
656f173c2b7SSean Bruno ((uint32_t)info->length -
657f173c2b7SSean Bruno pkt_len) : droq->buffer_size;
658f173c2b7SSean Bruno
659f173c2b7SSean Bruno buffer = ((struct mbuf *)
660f173c2b7SSean Bruno droq->recv_buf_list[idx].
661f173c2b7SSean Bruno buffer);
662f173c2b7SSean Bruno buffer->m_len = frag_len;
663f173c2b7SSean Bruno if (__predict_true(secondary_frag)) {
664f173c2b7SSean Bruno m_cat(nicbuf, buffer);
665f173c2b7SSean Bruno } else {
666f173c2b7SSean Bruno nicbuf = buffer;
667f173c2b7SSean Bruno secondary_frag = true;
668f173c2b7SSean Bruno }
669f173c2b7SSean Bruno
670f173c2b7SSean Bruno droq->recv_buf_list[droq->read_idx].
671f173c2b7SSean Bruno buffer = NULL;
672f173c2b7SSean Bruno
673f173c2b7SSean Bruno pkt_len += frag_len;
674f173c2b7SSean Bruno droq->read_idx =
675f173c2b7SSean Bruno lio_incr_index(droq->read_idx,
676f173c2b7SSean Bruno 1,
677f173c2b7SSean Bruno droq->max_count);
678f173c2b7SSean Bruno droq->refill_count++;
679f173c2b7SSean Bruno }
680f173c2b7SSean Bruno }
681f173c2b7SSean Bruno
682f173c2b7SSean Bruno if (nicbuf != NULL) {
683f173c2b7SSean Bruno if (droq->ops.fptr != NULL) {
684f173c2b7SSean Bruno droq->ops.fptr(nicbuf, pkt_len, rh,
685f173c2b7SSean Bruno droq, droq->ops.farg);
686f173c2b7SSean Bruno } else {
687f173c2b7SSean Bruno lio_recv_buffer_free(nicbuf);
688f173c2b7SSean Bruno }
689f173c2b7SSean Bruno }
690f173c2b7SSean Bruno }
691f173c2b7SSean Bruno
692f173c2b7SSean Bruno if (droq->refill_count >= droq->refill_threshold) {
693f173c2b7SSean Bruno int desc_refilled = lio_droq_refill(oct, droq);
694f173c2b7SSean Bruno
695f173c2b7SSean Bruno /*
696f173c2b7SSean Bruno * Flush the droq descriptor data to memory to be sure
697f173c2b7SSean Bruno * that when we update the credits the data in memory
698f173c2b7SSean Bruno * is accurate.
699f173c2b7SSean Bruno */
700f173c2b7SSean Bruno wmb();
701f173c2b7SSean Bruno lio_write_csr32(oct, droq->pkts_credit_reg,
702f173c2b7SSean Bruno desc_refilled);
703f173c2b7SSean Bruno /* make sure mmio write completes */
704f173c2b7SSean Bruno __compiler_membar();
705f173c2b7SSean Bruno }
706f173c2b7SSean Bruno } /* for (each packet)... */
707f173c2b7SSean Bruno
708f173c2b7SSean Bruno /* Increment refill_count by the number of buffers processed. */
709f173c2b7SSean Bruno droq->stats.pkts_received += pkt;
710f173c2b7SSean Bruno droq->stats.bytes_received += total_len;
711f173c2b7SSean Bruno
712f173c2b7SSean Bruno tcp_lro_flush_all(&droq->lro);
713f173c2b7SSean Bruno
714f173c2b7SSean Bruno if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
715f173c2b7SSean Bruno lio_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
716f173c2b7SSean Bruno
717f173c2b7SSean Bruno droq->stats.dropped_toomany += (pkts_to_process - pkt);
718f173c2b7SSean Bruno return (pkts_to_process);
719f173c2b7SSean Bruno }
720f173c2b7SSean Bruno
721f173c2b7SSean Bruno return (pkt);
722f173c2b7SSean Bruno }
723f173c2b7SSean Bruno
724f173c2b7SSean Bruno int
lio_droq_process_packets(struct octeon_device * oct,struct lio_droq * droq,uint32_t budget)725f173c2b7SSean Bruno lio_droq_process_packets(struct octeon_device *oct, struct lio_droq *droq,
726f173c2b7SSean Bruno uint32_t budget)
727f173c2b7SSean Bruno {
728f173c2b7SSean Bruno struct lio_stailq_node *tmp, *tmp2;
729f173c2b7SSean Bruno uint32_t pkt_count = 0, pkts_processed = 0;
730f173c2b7SSean Bruno
731f173c2b7SSean Bruno /* Grab the droq lock */
732f173c2b7SSean Bruno mtx_lock(&droq->lock);
733f173c2b7SSean Bruno
734f173c2b7SSean Bruno lio_droq_check_hw_for_pkts(droq);
735f173c2b7SSean Bruno pkt_count = atomic_load_acq_int(&droq->pkts_pending);
736f173c2b7SSean Bruno
737f173c2b7SSean Bruno if (!pkt_count) {
738f173c2b7SSean Bruno mtx_unlock(&droq->lock);
739f173c2b7SSean Bruno return (0);
740f173c2b7SSean Bruno }
741f173c2b7SSean Bruno if (pkt_count > budget)
742f173c2b7SSean Bruno pkt_count = budget;
743f173c2b7SSean Bruno
744f173c2b7SSean Bruno pkts_processed = lio_droq_fast_process_packets(oct, droq, pkt_count);
745f173c2b7SSean Bruno
746f173c2b7SSean Bruno atomic_subtract_int(&droq->pkts_pending, pkts_processed);
747f173c2b7SSean Bruno
748f173c2b7SSean Bruno /* Release the lock */
749f173c2b7SSean Bruno mtx_unlock(&droq->lock);
750f173c2b7SSean Bruno
751f173c2b7SSean Bruno STAILQ_FOREACH_SAFE(tmp, &droq->dispatch_stq_head, entries, tmp2) {
752f173c2b7SSean Bruno struct __dispatch *rdisp = (struct __dispatch *)tmp;
753f173c2b7SSean Bruno
754f173c2b7SSean Bruno STAILQ_REMOVE_HEAD(&droq->dispatch_stq_head, entries);
755f173c2b7SSean Bruno rdisp->disp_fn(rdisp->rinfo, lio_get_dispatch_arg(oct,
756f173c2b7SSean Bruno (uint16_t)rdisp->rinfo->recv_pkt->rh.r.opcode,
757f173c2b7SSean Bruno (uint16_t)rdisp->rinfo->recv_pkt->rh.r.subcode));
758f173c2b7SSean Bruno }
759f173c2b7SSean Bruno
760f173c2b7SSean Bruno /* If there are packets pending. schedule tasklet again */
761f173c2b7SSean Bruno if (atomic_load_acq_int(&droq->pkts_pending))
762f173c2b7SSean Bruno return (1);
763f173c2b7SSean Bruno
764f173c2b7SSean Bruno return (0);
765f173c2b7SSean Bruno }
766f173c2b7SSean Bruno
767f173c2b7SSean Bruno int
lio_register_droq_ops(struct octeon_device * oct,uint32_t q_no,struct lio_droq_ops * ops)768f173c2b7SSean Bruno lio_register_droq_ops(struct octeon_device *oct, uint32_t q_no,
769f173c2b7SSean Bruno struct lio_droq_ops *ops)
770f173c2b7SSean Bruno {
771f173c2b7SSean Bruno struct lio_droq *droq;
772f173c2b7SSean Bruno struct lio_config *lio_cfg = NULL;
773f173c2b7SSean Bruno
774f173c2b7SSean Bruno lio_cfg = lio_get_conf(oct);
775f173c2b7SSean Bruno
776f173c2b7SSean Bruno if (lio_cfg == NULL)
777f173c2b7SSean Bruno return (-EINVAL);
778f173c2b7SSean Bruno
779f173c2b7SSean Bruno if (ops == NULL) {
780f173c2b7SSean Bruno lio_dev_err(oct, "%s: droq_ops pointer is NULL\n", __func__);
781f173c2b7SSean Bruno return (-EINVAL);
782f173c2b7SSean Bruno }
783f173c2b7SSean Bruno
784f173c2b7SSean Bruno if (q_no >= LIO_GET_OQ_MAX_Q_CFG(lio_cfg)) {
785f173c2b7SSean Bruno lio_dev_err(oct, "%s: droq id (%d) exceeds MAX (%d)\n",
786f173c2b7SSean Bruno __func__, q_no, (oct->num_oqs - 1));
787f173c2b7SSean Bruno return (-EINVAL);
788f173c2b7SSean Bruno }
789f173c2b7SSean Bruno droq = oct->droq[q_no];
790f173c2b7SSean Bruno
791f173c2b7SSean Bruno mtx_lock(&droq->lock);
792f173c2b7SSean Bruno
793f173c2b7SSean Bruno memcpy(&droq->ops, ops, sizeof(struct lio_droq_ops));
794f173c2b7SSean Bruno
795f173c2b7SSean Bruno mtx_unlock(&droq->lock);
796f173c2b7SSean Bruno
797f173c2b7SSean Bruno return (0);
798f173c2b7SSean Bruno }
799f173c2b7SSean Bruno
800f173c2b7SSean Bruno int
lio_unregister_droq_ops(struct octeon_device * oct,uint32_t q_no)801f173c2b7SSean Bruno lio_unregister_droq_ops(struct octeon_device *oct, uint32_t q_no)
802f173c2b7SSean Bruno {
803f173c2b7SSean Bruno struct lio_droq *droq;
804f173c2b7SSean Bruno struct lio_config *lio_cfg = NULL;
805f173c2b7SSean Bruno
806f173c2b7SSean Bruno lio_cfg = lio_get_conf(oct);
807f173c2b7SSean Bruno
808f173c2b7SSean Bruno if (lio_cfg == NULL)
809f173c2b7SSean Bruno return (-EINVAL);
810f173c2b7SSean Bruno
811f173c2b7SSean Bruno if (q_no >= LIO_GET_OQ_MAX_Q_CFG(lio_cfg)) {
812f173c2b7SSean Bruno lio_dev_err(oct, "%s: droq id (%d) exceeds MAX (%d)\n",
813f173c2b7SSean Bruno __func__, q_no, oct->num_oqs - 1);
814f173c2b7SSean Bruno return (-EINVAL);
815f173c2b7SSean Bruno }
816f173c2b7SSean Bruno
817f173c2b7SSean Bruno droq = oct->droq[q_no];
818f173c2b7SSean Bruno
819f173c2b7SSean Bruno if (droq == NULL) {
820f173c2b7SSean Bruno lio_dev_info(oct, "Droq id (%d) not available.\n", q_no);
821f173c2b7SSean Bruno return (0);
822f173c2b7SSean Bruno }
823f173c2b7SSean Bruno
824f173c2b7SSean Bruno mtx_lock(&droq->lock);
825f173c2b7SSean Bruno
826f173c2b7SSean Bruno droq->ops.fptr = NULL;
827f173c2b7SSean Bruno droq->ops.farg = NULL;
828f173c2b7SSean Bruno droq->ops.drop_on_max = 0;
829f173c2b7SSean Bruno
830f173c2b7SSean Bruno mtx_unlock(&droq->lock);
831f173c2b7SSean Bruno
832f173c2b7SSean Bruno return (0);
833f173c2b7SSean Bruno }
834f173c2b7SSean Bruno
835f173c2b7SSean Bruno int
lio_create_droq(struct octeon_device * oct,uint32_t q_no,uint32_t num_descs,uint32_t desc_size,void * app_ctx)836f173c2b7SSean Bruno lio_create_droq(struct octeon_device *oct, uint32_t q_no, uint32_t num_descs,
837f173c2b7SSean Bruno uint32_t desc_size, void *app_ctx)
838f173c2b7SSean Bruno {
839f173c2b7SSean Bruno
840f173c2b7SSean Bruno if (oct->droq[q_no]->oct_dev != NULL) {
841f173c2b7SSean Bruno lio_dev_dbg(oct, "Droq already in use. Cannot create droq %d again\n",
842f173c2b7SSean Bruno q_no);
843f173c2b7SSean Bruno return (1);
844f173c2b7SSean Bruno }
845f173c2b7SSean Bruno
846f173c2b7SSean Bruno /* Initialize the Droq */
847f173c2b7SSean Bruno if (lio_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) {
848f173c2b7SSean Bruno bzero(oct->droq[q_no], sizeof(struct lio_droq));
849f173c2b7SSean Bruno goto create_droq_fail;
850f173c2b7SSean Bruno }
851f173c2b7SSean Bruno
852f173c2b7SSean Bruno oct->num_oqs++;
853f173c2b7SSean Bruno
854f173c2b7SSean Bruno lio_dev_dbg(oct, "%s: Total number of OQ: %d\n", __func__,
855f173c2b7SSean Bruno oct->num_oqs);
856f173c2b7SSean Bruno
857f173c2b7SSean Bruno /* Global Droq register settings */
858f173c2b7SSean Bruno
859f173c2b7SSean Bruno /*
860f173c2b7SSean Bruno * As of now not required, as setting are done for all 32 Droqs at
861f173c2b7SSean Bruno * the same time.
862f173c2b7SSean Bruno */
863f173c2b7SSean Bruno return (0);
864f173c2b7SSean Bruno
865f173c2b7SSean Bruno create_droq_fail:
866f173c2b7SSean Bruno return (-ENOMEM);
867f173c2b7SSean Bruno }
868