1 /**********************************************************************
2 * Author: Cavium, Inc.
3 *
4 * Contact: support@cavium.com
5 * Please include "LiquidIO" in the subject.
6 *
7 * Copyright (c) 2003-2016 Cavium, Inc.
8 *
9 * This file is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License, Version 2, as
11 * published by the Free Software Foundation.
12 *
13 * This file is distributed in the hope that it will be useful, but
14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16 * NONINFRINGEMENT. See the GNU General Public License for more details.
17 ***********************************************************************/
18 #include <linux/pci.h>
19 #include <linux/netdevice.h>
20 #include <linux/vmalloc.h>
21 #include "liquidio_common.h"
22 #include "octeon_droq.h"
23 #include "octeon_iq.h"
24 #include "response_manager.h"
25 #include "octeon_device.h"
26 #include "octeon_main.h"
27 #include "octeon_network.h"
28 #include "cn66xx_regs.h"
29 #include "cn66xx_device.h"
30 #include "cn23xx_pf_device.h"
31 #include "cn23xx_vf_device.h"
32
33 struct __dispatch {
34 struct list_head list;
35 struct octeon_recv_info *rinfo;
36 octeon_dispatch_fn_t disp_fn;
37 };
38
39 /** Get the argument that the user set when registering dispatch
40 * function for a given opcode/subcode.
41 * @param octeon_dev - the octeon device pointer.
42 * @param opcode - the opcode for which the dispatch argument
43 * is to be checked.
44 * @param subcode - the subcode for which the dispatch argument
45 * is to be checked.
46 * @return Success: void * (argument to the dispatch function)
47 * @return Failure: NULL
48 *
49 */
octeon_get_dispatch_arg(struct octeon_device * octeon_dev,u16 opcode,u16 subcode)50 void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
51 u16 opcode, u16 subcode)
52 {
53 int idx;
54 struct list_head *dispatch;
55 void *fn_arg = NULL;
56 u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
57
58 idx = combined_opcode & OCTEON_OPCODE_MASK;
59
60 spin_lock_bh(&octeon_dev->dispatch.lock);
61
62 if (octeon_dev->dispatch.count == 0) {
63 spin_unlock_bh(&octeon_dev->dispatch.lock);
64 return NULL;
65 }
66
67 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
68 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
69 } else {
70 list_for_each(dispatch,
71 &octeon_dev->dispatch.dlist[idx].list) {
72 if (((struct octeon_dispatch *)dispatch)->opcode ==
73 combined_opcode) {
74 fn_arg = ((struct octeon_dispatch *)
75 dispatch)->arg;
76 break;
77 }
78 }
79 }
80
81 spin_unlock_bh(&octeon_dev->dispatch.lock);
82 return fn_arg;
83 }
84
85 /** Check for packets on Droq. This function should be called with lock held.
86 * @param droq - Droq on which count is checked.
87 * @return Returns packet count.
88 */
octeon_droq_check_hw_for_pkts(struct octeon_droq * droq)89 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
90 {
91 u32 pkt_count = 0;
92 u32 last_count;
93
94 pkt_count = readl(droq->pkts_sent_reg);
95
96 last_count = pkt_count - droq->pkt_count;
97 droq->pkt_count = pkt_count;
98
99 /* we shall write to cnts at napi irq enable or end of droq tasklet */
100 if (last_count)
101 atomic_add(last_count, &droq->pkts_pending);
102
103 return last_count;
104 }
105 EXPORT_SYMBOL_GPL(octeon_droq_check_hw_for_pkts);
106
octeon_droq_compute_max_packet_bufs(struct octeon_droq * droq)107 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
108 {
109 u32 count = 0;
110
111 /* max_empty_descs is the max. no. of descs that can have no buffers.
112 * If the empty desc count goes beyond this value, we cannot safely
113 * read in a 64K packet sent by Octeon
114 * (64K is max pkt size from Octeon)
115 */
116 droq->max_empty_descs = 0;
117
118 do {
119 droq->max_empty_descs++;
120 count += droq->buffer_size;
121 } while (count < (64 * 1024));
122
123 droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
124 }
125
octeon_droq_reset_indices(struct octeon_droq * droq)126 static void octeon_droq_reset_indices(struct octeon_droq *droq)
127 {
128 droq->read_idx = 0;
129 droq->write_idx = 0;
130 droq->refill_idx = 0;
131 droq->refill_count = 0;
132 atomic_set(&droq->pkts_pending, 0);
133 }
134
135 static void
octeon_droq_destroy_ring_buffers(struct octeon_device * oct,struct octeon_droq * droq)136 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
137 struct octeon_droq *droq)
138 {
139 u32 i;
140 struct octeon_skb_page_info *pg_info;
141
142 for (i = 0; i < droq->max_count; i++) {
143 pg_info = &droq->recv_buf_list[i].pg_info;
144 if (!pg_info)
145 continue;
146
147 if (pg_info->dma)
148 lio_unmap_ring(oct->pci_dev,
149 (u64)pg_info->dma);
150 pg_info->dma = 0;
151
152 if (pg_info->page)
153 recv_buffer_destroy(droq->recv_buf_list[i].buffer,
154 pg_info);
155
156 droq->recv_buf_list[i].buffer = NULL;
157 }
158
159 octeon_droq_reset_indices(droq);
160 }
161
162 static int
octeon_droq_setup_ring_buffers(struct octeon_device * oct,struct octeon_droq * droq)163 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
164 struct octeon_droq *droq)
165 {
166 u32 i;
167 void *buf;
168 struct octeon_droq_desc *desc_ring = droq->desc_ring;
169
170 for (i = 0; i < droq->max_count; i++) {
171 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
172
173 if (!buf) {
174 dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
175 __func__);
176 droq->stats.rx_alloc_failure++;
177 return -ENOMEM;
178 }
179
180 droq->recv_buf_list[i].buffer = buf;
181 droq->recv_buf_list[i].data = get_rbd(buf);
182 desc_ring[i].info_ptr = 0;
183 desc_ring[i].buffer_ptr =
184 lio_map_ring(droq->recv_buf_list[i].buffer);
185 }
186
187 octeon_droq_reset_indices(droq);
188
189 octeon_droq_compute_max_packet_bufs(droq);
190
191 return 0;
192 }
193
octeon_delete_droq(struct octeon_device * oct,u32 q_no)194 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
195 {
196 struct octeon_droq *droq = oct->droq[q_no];
197
198 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
199
200 octeon_droq_destroy_ring_buffers(oct, droq);
201 vfree(droq->recv_buf_list);
202
203 if (droq->desc_ring)
204 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
205 droq->desc_ring, droq->desc_ring_dma);
206
207 memset(droq, 0, OCT_DROQ_SIZE);
208 oct->io_qmask.oq &= ~(1ULL << q_no);
209 vfree(oct->droq[q_no]);
210 oct->droq[q_no] = NULL;
211 oct->num_oqs--;
212
213 return 0;
214 }
215 EXPORT_SYMBOL_GPL(octeon_delete_droq);
216
octeon_init_droq(struct octeon_device * oct,u32 q_no,u32 num_descs,u32 desc_size,void * app_ctx)217 int octeon_init_droq(struct octeon_device *oct,
218 u32 q_no,
219 u32 num_descs,
220 u32 desc_size,
221 void *app_ctx)
222 {
223 struct octeon_droq *droq;
224 u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
225 u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
226 int numa_node = dev_to_node(&oct->pci_dev->dev);
227
228 dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
229
230 droq = oct->droq[q_no];
231 memset(droq, 0, OCT_DROQ_SIZE);
232
233 droq->oct_dev = oct;
234 droq->q_no = q_no;
235 if (app_ctx)
236 droq->app_ctx = app_ctx;
237 else
238 droq->app_ctx = (void *)(size_t)q_no;
239
240 c_num_descs = num_descs;
241 c_buf_size = desc_size;
242 if (OCTEON_CN6XXX(oct)) {
243 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx);
244
245 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
246 c_refill_threshold =
247 (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
248 } else if (OCTEON_CN23XX_PF(oct)) {
249 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf);
250
251 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
252 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
253 } else if (OCTEON_CN23XX_VF(oct)) {
254 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf);
255
256 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
257 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
258 } else {
259 return 1;
260 }
261
262 droq->max_count = c_num_descs;
263 droq->buffer_size = c_buf_size;
264
265 desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
266 droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
267 (dma_addr_t *)&droq->desc_ring_dma);
268
269 if (!droq->desc_ring) {
270 dev_err(&oct->pci_dev->dev,
271 "Output queue %d ring alloc failed\n", q_no);
272 return 1;
273 }
274
275 dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
276 q_no, droq->desc_ring, droq->desc_ring_dma);
277 dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
278 droq->max_count);
279
280 droq->recv_buf_list = vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE),
281 numa_node);
282 if (!droq->recv_buf_list)
283 droq->recv_buf_list = vzalloc(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE));
284 if (!droq->recv_buf_list) {
285 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
286 goto init_droq_fail;
287 }
288
289 if (octeon_droq_setup_ring_buffers(oct, droq))
290 goto init_droq_fail;
291
292 droq->pkts_per_intr = c_pkts_per_intr;
293 droq->refill_threshold = c_refill_threshold;
294
295 dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
296 droq->max_empty_descs);
297
298 INIT_LIST_HEAD(&droq->dispatch_list);
299
300 /* For 56xx Pass1, this function won't be called, so no checks. */
301 oct->fn_list.setup_oq_regs(oct, q_no);
302
303 oct->io_qmask.oq |= BIT_ULL(q_no);
304
305 return 0;
306
307 init_droq_fail:
308 octeon_delete_droq(oct, q_no);
309 return 1;
310 }
311
312 /* octeon_create_recv_info
313 * Parameters:
314 * octeon_dev - pointer to the octeon device structure
315 * droq - droq in which the packet arrived.
316 * buf_cnt - no. of buffers used by the packet.
317 * idx - index in the descriptor for the first buffer in the packet.
318 * Description:
319 * Allocates a recv_info_t and copies the buffer addresses for packet data
320 * into the recv_pkt space which starts at an 8B offset from recv_info_t.
321 * Flags the descriptors for refill later. If available descriptors go
322 * below the threshold to receive a 64K pkt, new buffers are first allocated
323 * before the recv_pkt_t is created.
324 * This routine will be called in interrupt context.
325 * Returns:
326 * Success: Pointer to recv_info_t
327 * Failure: NULL.
328 */
octeon_create_recv_info(struct octeon_device * octeon_dev,struct octeon_droq * droq,u32 buf_cnt,u32 idx)329 static inline struct octeon_recv_info *octeon_create_recv_info(
330 struct octeon_device *octeon_dev,
331 struct octeon_droq *droq,
332 u32 buf_cnt,
333 u32 idx)
334 {
335 struct octeon_droq_info *info;
336 struct octeon_recv_pkt *recv_pkt;
337 struct octeon_recv_info *recv_info;
338 u32 i, bytes_left;
339 struct octeon_skb_page_info *pg_info;
340
341 info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data;
342
343 recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
344 if (!recv_info)
345 return NULL;
346
347 recv_pkt = recv_info->recv_pkt;
348 recv_pkt->rh = info->rh;
349 recv_pkt->length = (u32)info->length;
350 recv_pkt->buffer_count = (u16)buf_cnt;
351 recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
352
353 i = 0;
354 bytes_left = (u32)info->length;
355
356 while (buf_cnt) {
357 {
358 pg_info = &droq->recv_buf_list[idx].pg_info;
359
360 lio_unmap_ring(octeon_dev->pci_dev,
361 (u64)pg_info->dma);
362 pg_info->page = NULL;
363 pg_info->dma = 0;
364 }
365
366 recv_pkt->buffer_size[i] =
367 (bytes_left >=
368 droq->buffer_size) ? droq->buffer_size : bytes_left;
369
370 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
371 droq->recv_buf_list[idx].buffer = NULL;
372
373 idx = incr_index(idx, 1, droq->max_count);
374 bytes_left -= droq->buffer_size;
375 i++;
376 buf_cnt--;
377 }
378
379 return recv_info;
380 }
381
382 /* If we were not able to refill all buffers, try to move around
383 * the buffers that were not dispatched.
384 */
385 static inline u32
octeon_droq_refill_pullup_descs(struct octeon_droq * droq,struct octeon_droq_desc * desc_ring)386 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
387 struct octeon_droq_desc *desc_ring)
388 {
389 u32 desc_refilled = 0;
390
391 u32 refill_index = droq->refill_idx;
392
393 while (refill_index != droq->read_idx) {
394 if (droq->recv_buf_list[refill_index].buffer) {
395 droq->recv_buf_list[droq->refill_idx].buffer =
396 droq->recv_buf_list[refill_index].buffer;
397 droq->recv_buf_list[droq->refill_idx].data =
398 droq->recv_buf_list[refill_index].data;
399 desc_ring[droq->refill_idx].buffer_ptr =
400 desc_ring[refill_index].buffer_ptr;
401 droq->recv_buf_list[refill_index].buffer = NULL;
402 desc_ring[refill_index].buffer_ptr = 0;
403 do {
404 droq->refill_idx = incr_index(droq->refill_idx,
405 1,
406 droq->max_count);
407 desc_refilled++;
408 droq->refill_count--;
409 } while (droq->recv_buf_list[droq->refill_idx].buffer);
410 }
411 refill_index = incr_index(refill_index, 1, droq->max_count);
412 } /* while */
413 return desc_refilled;
414 }
415
416 /* octeon_droq_refill
417 * Parameters:
418 * droq - droq in which descriptors require new buffers.
419 * Description:
420 * Called during normal DROQ processing in interrupt mode or by the poll
421 * thread to refill the descriptors from which buffers were dispatched
422 * to upper layers. Attempts to allocate new buffers. If that fails, moves
423 * up buffers (that were not dispatched) to form a contiguous ring.
424 * Returns:
425 * No of descriptors refilled.
426 */
427 static u32
octeon_droq_refill(struct octeon_device * octeon_dev,struct octeon_droq * droq)428 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
429 {
430 struct octeon_droq_desc *desc_ring;
431 void *buf = NULL;
432 u8 *data;
433 u32 desc_refilled = 0;
434 struct octeon_skb_page_info *pg_info;
435
436 desc_ring = droq->desc_ring;
437
438 while (droq->refill_count && (desc_refilled < droq->max_count)) {
439 /* If a valid buffer exists (happens if there is no dispatch),
440 * reuse the buffer, else allocate.
441 */
442 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
443 pg_info =
444 &droq->recv_buf_list[droq->refill_idx].pg_info;
445 /* Either recycle the existing pages or go for
446 * new page alloc
447 */
448 if (pg_info->page)
449 buf = recv_buffer_reuse(octeon_dev, pg_info);
450 else
451 buf = recv_buffer_alloc(octeon_dev, pg_info);
452 /* If a buffer could not be allocated, no point in
453 * continuing
454 */
455 if (!buf) {
456 droq->stats.rx_alloc_failure++;
457 break;
458 }
459 droq->recv_buf_list[droq->refill_idx].buffer =
460 buf;
461 data = get_rbd(buf);
462 } else {
463 data = get_rbd(droq->recv_buf_list
464 [droq->refill_idx].buffer);
465 }
466
467 droq->recv_buf_list[droq->refill_idx].data = data;
468
469 desc_ring[droq->refill_idx].buffer_ptr =
470 lio_map_ring(droq->recv_buf_list[
471 droq->refill_idx].buffer);
472
473 droq->refill_idx = incr_index(droq->refill_idx, 1,
474 droq->max_count);
475 desc_refilled++;
476 droq->refill_count--;
477 }
478
479 if (droq->refill_count)
480 desc_refilled +=
481 octeon_droq_refill_pullup_descs(droq, desc_ring);
482
483 /* if droq->refill_count
484 * The refill count would not change in pass two. We only moved buffers
485 * to close the gap in the ring, but we would still have the same no. of
486 * buffers to refill.
487 */
488 return desc_refilled;
489 }
490
491 /** check if we can allocate packets to get out of oom.
492 * @param droq - Droq being checked.
493 * @return 1 if fails to refill minimum
494 */
octeon_retry_droq_refill(struct octeon_droq * droq)495 int octeon_retry_droq_refill(struct octeon_droq *droq)
496 {
497 struct octeon_device *oct = droq->oct_dev;
498 int desc_refilled, reschedule = 1;
499 u32 pkts_credit;
500
501 pkts_credit = readl(droq->pkts_credit_reg);
502 desc_refilled = octeon_droq_refill(oct, droq);
503 if (desc_refilled) {
504 /* Flush the droq descriptor data to memory to be sure
505 * that when we update the credits the data in memory
506 * is accurate.
507 */
508 wmb();
509 writel(desc_refilled, droq->pkts_credit_reg);
510
511 if (pkts_credit + desc_refilled >= CN23XX_SLI_DEF_BP)
512 reschedule = 0;
513 }
514
515 return reschedule;
516 }
517
518 static inline u32
octeon_droq_get_bufcount(u32 buf_size,u32 total_len)519 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
520 {
521 return DIV_ROUND_UP(total_len, buf_size);
522 }
523
524 static int
octeon_droq_dispatch_pkt(struct octeon_device * oct,struct octeon_droq * droq,union octeon_rh * rh,struct octeon_droq_info * info)525 octeon_droq_dispatch_pkt(struct octeon_device *oct,
526 struct octeon_droq *droq,
527 union octeon_rh *rh,
528 struct octeon_droq_info *info)
529 {
530 u32 cnt;
531 octeon_dispatch_fn_t disp_fn;
532 struct octeon_recv_info *rinfo;
533
534 cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
535
536 disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
537 (u16)rh->r.subcode);
538 if (disp_fn) {
539 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
540 if (rinfo) {
541 struct __dispatch *rdisp = rinfo->rsvd;
542
543 rdisp->rinfo = rinfo;
544 rdisp->disp_fn = disp_fn;
545 rinfo->recv_pkt->rh = *rh;
546 list_add_tail(&rdisp->list,
547 &droq->dispatch_list);
548 } else {
549 droq->stats.dropped_nomem++;
550 }
551 } else {
552 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",
553 (unsigned int)rh->r.opcode,
554 (unsigned int)rh->r.subcode);
555 droq->stats.dropped_nodispatch++;
556 }
557
558 return cnt;
559 }
560
octeon_droq_drop_packets(struct octeon_device * oct,struct octeon_droq * droq,u32 cnt)561 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
562 struct octeon_droq *droq,
563 u32 cnt)
564 {
565 u32 i = 0, buf_cnt;
566 struct octeon_droq_info *info;
567
568 for (i = 0; i < cnt; i++) {
569 info = (struct octeon_droq_info *)
570 droq->recv_buf_list[droq->read_idx].data;
571 octeon_swap_8B_data((u64 *)info, 2);
572
573 if (info->length) {
574 info->length += OCTNET_FRM_LENGTH_SIZE;
575 droq->stats.bytes_received += info->length;
576 buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
577 (u32)info->length);
578 } else {
579 dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
580 buf_cnt = 1;
581 }
582
583 droq->read_idx = incr_index(droq->read_idx, buf_cnt,
584 droq->max_count);
585 droq->refill_count += buf_cnt;
586 }
587 }
588
589 static u32
octeon_droq_fast_process_packets(struct octeon_device * oct,struct octeon_droq * droq,u32 pkts_to_process)590 octeon_droq_fast_process_packets(struct octeon_device *oct,
591 struct octeon_droq *droq,
592 u32 pkts_to_process)
593 {
594 u32 pkt, total_len = 0, pkt_count, retval;
595 struct octeon_droq_info *info;
596 union octeon_rh *rh;
597
598 pkt_count = pkts_to_process;
599
600 for (pkt = 0; pkt < pkt_count; pkt++) {
601 u32 pkt_len = 0;
602 struct sk_buff *nicbuf = NULL;
603 struct octeon_skb_page_info *pg_info;
604 void *buf;
605
606 info = (struct octeon_droq_info *)
607 droq->recv_buf_list[droq->read_idx].data;
608 octeon_swap_8B_data((u64 *)info, 2);
609
610 if (!info->length) {
611 dev_err(&oct->pci_dev->dev,
612 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
613 droq->q_no, droq->read_idx, pkt_count);
614 print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
615 (u8 *)info,
616 OCT_DROQ_INFO_SIZE);
617 break;
618 }
619
620 /* Len of resp hdr in included in the received data len. */
621 rh = &info->rh;
622
623 info->length += OCTNET_FRM_LENGTH_SIZE;
624 rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64));
625 total_len += (u32)info->length;
626 if (opcode_slow_path(rh)) {
627 u32 buf_cnt;
628
629 buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
630 droq->read_idx = incr_index(droq->read_idx,
631 buf_cnt, droq->max_count);
632 droq->refill_count += buf_cnt;
633 } else {
634 if (info->length <= droq->buffer_size) {
635 pkt_len = (u32)info->length;
636 nicbuf = droq->recv_buf_list[
637 droq->read_idx].buffer;
638 pg_info = &droq->recv_buf_list[
639 droq->read_idx].pg_info;
640 if (recv_buffer_recycle(oct, pg_info))
641 pg_info->page = NULL;
642 droq->recv_buf_list[droq->read_idx].buffer =
643 NULL;
644
645 droq->read_idx = incr_index(droq->read_idx, 1,
646 droq->max_count);
647 droq->refill_count++;
648 } else {
649 nicbuf = octeon_fast_packet_alloc((u32)
650 info->length);
651 pkt_len = 0;
652 /* nicbuf allocation can fail. We'll handle it
653 * inside the loop.
654 */
655 while (pkt_len < info->length) {
656 int cpy_len, idx = droq->read_idx;
657
658 cpy_len = ((pkt_len + droq->buffer_size)
659 > info->length) ?
660 ((u32)info->length - pkt_len) :
661 droq->buffer_size;
662
663 if (nicbuf) {
664 octeon_fast_packet_next(droq,
665 nicbuf,
666 cpy_len,
667 idx);
668 buf = droq->recv_buf_list[
669 idx].buffer;
670 recv_buffer_fast_free(buf);
671 droq->recv_buf_list[idx].buffer
672 = NULL;
673 } else {
674 droq->stats.rx_alloc_failure++;
675 }
676
677 pkt_len += cpy_len;
678 droq->read_idx =
679 incr_index(droq->read_idx, 1,
680 droq->max_count);
681 droq->refill_count++;
682 }
683 }
684
685 if (nicbuf) {
686 if (droq->ops.fptr) {
687 droq->ops.fptr(oct->octeon_id,
688 nicbuf, pkt_len,
689 rh, &droq->napi,
690 droq->ops.farg);
691 } else {
692 recv_buffer_free(nicbuf);
693 }
694 }
695 }
696
697 if (droq->refill_count >= droq->refill_threshold) {
698 int desc_refilled = octeon_droq_refill(oct, droq);
699
700 if (desc_refilled) {
701 /* Flush the droq descriptor data to memory to
702 * be sure that when we update the credits the
703 * data in memory is accurate.
704 */
705 wmb();
706 writel(desc_refilled, droq->pkts_credit_reg);
707 }
708 }
709 } /* for (each packet)... */
710
711 /* Increment refill_count by the number of buffers processed. */
712 droq->stats.pkts_received += pkt;
713 droq->stats.bytes_received += total_len;
714
715 retval = pkt;
716 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
717 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
718
719 droq->stats.dropped_toomany += (pkts_to_process - pkt);
720 retval = pkts_to_process;
721 }
722
723 atomic_sub(retval, &droq->pkts_pending);
724
725 if (droq->refill_count >= droq->refill_threshold &&
726 readl(droq->pkts_credit_reg) < CN23XX_SLI_DEF_BP) {
727 octeon_droq_check_hw_for_pkts(droq);
728
729 /* Make sure there are no pkts_pending */
730 if (!atomic_read(&droq->pkts_pending))
731 octeon_schedule_rxq_oom_work(oct, droq);
732 }
733
734 return retval;
735 }
736
737 int
octeon_droq_process_packets(struct octeon_device * oct,struct octeon_droq * droq,u32 budget)738 octeon_droq_process_packets(struct octeon_device *oct,
739 struct octeon_droq *droq,
740 u32 budget)
741 {
742 u32 pkt_count = 0;
743 struct list_head *tmp, *tmp2;
744
745 octeon_droq_check_hw_for_pkts(droq);
746 pkt_count = atomic_read(&droq->pkts_pending);
747
748 if (!pkt_count)
749 return 0;
750
751 if (pkt_count > budget)
752 pkt_count = budget;
753
754 octeon_droq_fast_process_packets(oct, droq, pkt_count);
755
756 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
757 struct __dispatch *rdisp = (struct __dispatch *)tmp;
758
759 list_del(tmp);
760 rdisp->disp_fn(rdisp->rinfo,
761 octeon_get_dispatch_arg
762 (oct,
763 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
764 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
765 }
766
767 /* If there are packets pending. schedule tasklet again */
768 if (atomic_read(&droq->pkts_pending))
769 return 1;
770
771 return 0;
772 }
773 EXPORT_SYMBOL_GPL(octeon_droq_process_packets);
774
775 /*
776 * Utility function to poll for packets. check_hw_for_packets must be
777 * called before calling this routine.
778 */
779
780 int
octeon_droq_process_poll_pkts(struct octeon_device * oct,struct octeon_droq * droq,u32 budget)781 octeon_droq_process_poll_pkts(struct octeon_device *oct,
782 struct octeon_droq *droq, u32 budget)
783 {
784 struct list_head *tmp, *tmp2;
785 u32 pkts_available = 0, pkts_processed = 0;
786 u32 total_pkts_processed = 0;
787
788 if (budget > droq->max_count)
789 budget = droq->max_count;
790
791 while (total_pkts_processed < budget) {
792 octeon_droq_check_hw_for_pkts(droq);
793
794 pkts_available = min((budget - total_pkts_processed),
795 (u32)(atomic_read(&droq->pkts_pending)));
796
797 if (pkts_available == 0)
798 break;
799
800 pkts_processed =
801 octeon_droq_fast_process_packets(oct, droq,
802 pkts_available);
803
804 total_pkts_processed += pkts_processed;
805 }
806
807 list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
808 struct __dispatch *rdisp = (struct __dispatch *)tmp;
809
810 list_del(tmp);
811 rdisp->disp_fn(rdisp->rinfo,
812 octeon_get_dispatch_arg
813 (oct,
814 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
815 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
816 }
817
818 return total_pkts_processed;
819 }
820
821 /* Enable Pkt Interrupt */
822 int
octeon_enable_irq(struct octeon_device * oct,u32 q_no)823 octeon_enable_irq(struct octeon_device *oct, u32 q_no)
824 {
825 switch (oct->chip_id) {
826 case OCTEON_CN66XX:
827 case OCTEON_CN68XX: {
828 struct octeon_cn6xxx *cn6xxx =
829 (struct octeon_cn6xxx *)oct->chip;
830 unsigned long flags;
831 u32 value;
832
833 spin_lock_irqsave
834 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
835 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB);
836 value |= (1 << q_no);
837 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value);
838 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);
839 value |= (1 << q_no);
840 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value);
841
842 /* don't bother flushing the enables */
843
844 spin_unlock_irqrestore
845 (&cn6xxx->lock_for_droq_int_enb_reg, flags);
846 }
847 break;
848 case OCTEON_CN23XX_PF_VID:
849 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
850 break;
851
852 case OCTEON_CN23XX_VF_VID:
853 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
854 break;
855 default:
856 dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n", __func__);
857 return 1;
858 }
859
860 return 0;
861 }
862
octeon_register_droq_ops(struct octeon_device * oct,u32 q_no,struct octeon_droq_ops * ops)863 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
864 struct octeon_droq_ops *ops)
865 {
866 struct octeon_config *oct_cfg = NULL;
867 struct octeon_droq *droq;
868
869 oct_cfg = octeon_get_conf(oct);
870
871 if (!oct_cfg)
872 return -EINVAL;
873
874 if (!(ops)) {
875 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
876 __func__);
877 return -EINVAL;
878 }
879
880 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
881 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
882 __func__, q_no, (oct->num_oqs - 1));
883 return -EINVAL;
884 }
885
886 droq = oct->droq[q_no];
887 memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
888
889 return 0;
890 }
891
octeon_unregister_droq_ops(struct octeon_device * oct,u32 q_no)892 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
893 {
894 struct octeon_config *oct_cfg = NULL;
895 struct octeon_droq *droq;
896
897 oct_cfg = octeon_get_conf(oct);
898
899 if (!oct_cfg)
900 return -EINVAL;
901
902 if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
903 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
904 __func__, q_no, oct->num_oqs - 1);
905 return -EINVAL;
906 }
907
908 droq = oct->droq[q_no];
909
910 if (!droq) {
911 dev_info(&oct->pci_dev->dev,
912 "Droq id (%d) not available.\n", q_no);
913 return 0;
914 }
915
916 droq->ops.fptr = NULL;
917 droq->ops.farg = NULL;
918 droq->ops.drop_on_max = 0;
919
920 return 0;
921 }
922 EXPORT_SYMBOL_GPL(octeon_unregister_droq_ops);
923
octeon_create_droq(struct octeon_device * oct,u32 q_no,u32 num_descs,u32 desc_size,void * app_ctx)924 int octeon_create_droq(struct octeon_device *oct,
925 u32 q_no, u32 num_descs,
926 u32 desc_size, void *app_ctx)
927 {
928 struct octeon_droq *droq;
929 int numa_node = dev_to_node(&oct->pci_dev->dev);
930
931 if (oct->droq[q_no]) {
932 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
933 q_no);
934 return 1;
935 }
936
937 /* Allocate the DS for the new droq. */
938 droq = vmalloc_node(sizeof(*droq), numa_node);
939 if (!droq)
940 droq = vmalloc(sizeof(*droq));
941 if (!droq)
942 return -1;
943
944 memset(droq, 0, sizeof(struct octeon_droq));
945
946 /*Disable the pkt o/p for this Q */
947 octeon_set_droq_pkt_op(oct, q_no, 0);
948 oct->droq[q_no] = droq;
949
950 /* Initialize the Droq */
951 if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) {
952 vfree(oct->droq[q_no]);
953 oct->droq[q_no] = NULL;
954 return -1;
955 }
956
957 oct->num_oqs++;
958
959 dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
960 oct->num_oqs);
961
962 /* Global Droq register settings */
963
964 /* As of now not required, as setting are done for all 32 Droqs at
965 * the same time.
966 */
967 return 0;
968 }
969