1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell Octeon EP (EndPoint) Ethernet Driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/pci.h>
9 #include <linux/etherdevice.h>
10 #include <linux/vmalloc.h>
11
12 #include "octep_config.h"
13 #include "octep_main.h"
14
15 static void octep_oq_free_ring_buffers(struct octep_oq *oq);
16
octep_oq_reset_indices(struct octep_oq * oq)17 static void octep_oq_reset_indices(struct octep_oq *oq)
18 {
19 oq->host_read_idx = 0;
20 oq->host_refill_idx = 0;
21 oq->refill_count = 0;
22 oq->last_pkt_count = 0;
23 oq->pkts_pending = 0;
24 }
25
26 /**
27 * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.
28 *
29 * @oq: Octeon Rx queue data structure.
30 *
31 * Return: 0, if successfully filled receive buffers for all descriptors.
32 * -1, if failed to allocate a buffer or failed to map for DMA.
33 */
octep_oq_fill_ring_buffers(struct octep_oq * oq)34 static int octep_oq_fill_ring_buffers(struct octep_oq *oq)
35 {
36 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
37 struct page *page;
38 u32 i;
39
40 for (i = 0; i < oq->max_count; i++) {
41 page = dev_alloc_page();
42 if (unlikely(!page)) {
43 dev_err(oq->dev, "Rx buffer alloc failed\n");
44 goto rx_buf_alloc_err;
45 }
46 desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,
47 PAGE_SIZE,
48 DMA_FROM_DEVICE);
49 if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {
50 dev_err(oq->dev,
51 "OQ-%d buffer alloc: DMA mapping error!\n",
52 oq->q_no);
53 put_page(page);
54 goto dma_map_err;
55 }
56 oq->buff_info[i].page = page;
57 }
58
59 return 0;
60
61 dma_map_err:
62 rx_buf_alloc_err:
63 while (i) {
64 i--;
65 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);
66 put_page(oq->buff_info[i].page);
67 oq->buff_info[i].page = NULL;
68 }
69
70 return -1;
71 }
72
73 /**
74 * octep_oq_refill() - refill buffers for used Rx ring descriptors.
75 *
76 * @oct: Octeon device private data structure.
77 * @oq: Octeon Rx queue data structure.
78 *
79 * Return: number of descriptors successfully refilled with receive buffers.
80 */
octep_oq_refill(struct octep_device * oct,struct octep_oq * oq)81 static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq)
82 {
83 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
84 struct page *page;
85 u32 refill_idx, i;
86
87 refill_idx = oq->host_refill_idx;
88 for (i = 0; i < oq->refill_count; i++) {
89 page = dev_alloc_page();
90 if (unlikely(!page)) {
91 dev_err(oq->dev, "refill: rx buffer alloc failed\n");
92 oq->stats->alloc_failures++;
93 break;
94 }
95
96 desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,
97 PAGE_SIZE, DMA_FROM_DEVICE);
98 if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {
99 dev_err(oq->dev,
100 "OQ-%d buffer refill: DMA mapping error!\n",
101 oq->q_no);
102 put_page(page);
103 oq->stats->alloc_failures++;
104 break;
105 }
106 oq->buff_info[refill_idx].page = page;
107 refill_idx++;
108 if (refill_idx == oq->max_count)
109 refill_idx = 0;
110 }
111 oq->host_refill_idx = refill_idx;
112 oq->refill_count -= i;
113
114 return i;
115 }
116
117 /**
118 * octep_setup_oq() - Setup a Rx queue.
119 *
120 * @oct: Octeon device private data structure.
121 * @q_no: Rx queue number to be setup.
122 *
123 * Allocate resources for a Rx queue.
124 */
octep_setup_oq(struct octep_device * oct,int q_no)125 static int octep_setup_oq(struct octep_device *oct, int q_no)
126 {
127 struct octep_oq *oq;
128 u32 desc_ring_size;
129
130 oq = vzalloc(sizeof(*oq));
131 if (!oq)
132 goto create_oq_fail;
133 oct->oq[q_no] = oq;
134
135 oq->octep_dev = oct;
136 oq->netdev = oct->netdev;
137 oq->dev = &oct->pdev->dev;
138 oq->q_no = q_no;
139 oq->stats = &oct->stats_oq[q_no];
140 oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);
141 oq->ring_size_mask = oq->max_count - 1;
142 oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);
143 oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE;
144
145 /* When the hardware/firmware supports additional capabilities,
146 * additional header is filled-in by Octeon after length field in
147 * Rx packets. this header contains additional packet information.
148 */
149 if (oct->conf->fw_info.rx_ol_flags)
150 oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE;
151
152 oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);
153
154 desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE;
155 oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,
156 &oq->desc_ring_dma, GFP_KERNEL);
157
158 if (unlikely(!oq->desc_ring)) {
159 dev_err(oq->dev,
160 "Failed to allocate DMA memory for OQ-%d !!\n", q_no);
161 goto desc_dma_alloc_err;
162 }
163
164 oq->buff_info = vcalloc(oq->max_count, OCTEP_OQ_RECVBUF_SIZE);
165 if (unlikely(!oq->buff_info)) {
166 dev_err(&oct->pdev->dev,
167 "Failed to allocate buffer info for OQ-%d\n", q_no);
168 goto buf_list_err;
169 }
170
171 if (octep_oq_fill_ring_buffers(oq))
172 goto oq_fill_buff_err;
173
174 octep_oq_reset_indices(oq);
175 if (oct->hw_ops.setup_oq_regs(oct, q_no))
176 goto oq_setup_err;
177
178 oct->num_oqs++;
179
180 return 0;
181
182 oq_setup_err:
183 octep_oq_free_ring_buffers(oq);
184 oq_fill_buff_err:
185 vfree(oq->buff_info);
186 oq->buff_info = NULL;
187 buf_list_err:
188 dma_free_coherent(oq->dev, desc_ring_size,
189 oq->desc_ring, oq->desc_ring_dma);
190 oq->desc_ring = NULL;
191 desc_dma_alloc_err:
192 vfree(oq);
193 oct->oq[q_no] = NULL;
194 create_oq_fail:
195 return -1;
196 }
197
198 /**
199 * octep_oq_free_ring_buffers() - Free ring buffers.
200 *
201 * @oq: Octeon Rx queue data structure.
202 *
203 * Free receive buffers in unused Rx queue descriptors.
204 */
octep_oq_free_ring_buffers(struct octep_oq * oq)205 static void octep_oq_free_ring_buffers(struct octep_oq *oq)
206 {
207 struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
208 int i;
209
210 if (!oq->desc_ring || !oq->buff_info)
211 return;
212
213 for (i = 0; i < oq->max_count; i++) {
214 if (oq->buff_info[i].page) {
215 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,
216 PAGE_SIZE, DMA_FROM_DEVICE);
217 put_page(oq->buff_info[i].page);
218 oq->buff_info[i].page = NULL;
219 desc_ring[i].buffer_ptr = 0;
220 }
221 }
222 octep_oq_reset_indices(oq);
223 }
224
225 /**
226 * octep_free_oq() - Free Rx queue resources.
227 *
228 * @oq: Octeon Rx queue data structure.
229 *
230 * Free all resources of a Rx queue.
231 */
octep_free_oq(struct octep_oq * oq)232 static int octep_free_oq(struct octep_oq *oq)
233 {
234 struct octep_device *oct = oq->octep_dev;
235 int q_no = oq->q_no;
236
237 octep_oq_free_ring_buffers(oq);
238
239 vfree(oq->buff_info);
240
241 if (oq->desc_ring)
242 dma_free_coherent(oq->dev,
243 oq->max_count * OCTEP_OQ_DESC_SIZE,
244 oq->desc_ring, oq->desc_ring_dma);
245
246 vfree(oq);
247 oct->oq[q_no] = NULL;
248 oct->num_oqs--;
249 return 0;
250 }
251
252 /**
253 * octep_setup_oqs() - setup resources for all Rx queues.
254 *
255 * @oct: Octeon device private data structure.
256 */
octep_setup_oqs(struct octep_device * oct)257 int octep_setup_oqs(struct octep_device *oct)
258 {
259 int i, retval = 0;
260
261 oct->num_oqs = 0;
262 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
263 retval = octep_setup_oq(oct, i);
264 if (retval) {
265 dev_err(&oct->pdev->dev,
266 "Failed to setup OQ(RxQ)-%d.\n", i);
267 goto oq_setup_err;
268 }
269 dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);
270 }
271
272 return 0;
273
274 oq_setup_err:
275 while (i) {
276 i--;
277 octep_free_oq(oct->oq[i]);
278 }
279 return -1;
280 }
281
282 /**
283 * octep_oq_dbell_init() - Initialize Rx queue doorbell.
284 *
285 * @oct: Octeon device private data structure.
286 *
287 * Write number of descriptors to Rx queue doorbell register.
288 */
octep_oq_dbell_init(struct octep_device * oct)289 void octep_oq_dbell_init(struct octep_device *oct)
290 {
291 int i;
292
293 for (i = 0; i < oct->num_oqs; i++)
294 writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
295 }
296
297 /**
298 * octep_free_oqs() - Free resources of all Rx queues.
299 *
300 * @oct: Octeon device private data structure.
301 */
octep_free_oqs(struct octep_device * oct)302 void octep_free_oqs(struct octep_device *oct)
303 {
304 int i;
305
306 for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
307 if (!oct->oq[i])
308 continue;
309 octep_free_oq(oct->oq[i]);
310 dev_dbg(&oct->pdev->dev,
311 "Successfully freed OQ(RxQ)-%d.\n", i);
312 }
313 }
314
315 /**
316 * octep_oq_check_hw_for_pkts() - Check for new Rx packets.
317 *
318 * @oct: Octeon device private data structure.
319 * @oq: Octeon Rx queue data structure.
320 *
321 * Return: packets received after previous check.
322 */
octep_oq_check_hw_for_pkts(struct octep_device * oct,struct octep_oq * oq)323 static int octep_oq_check_hw_for_pkts(struct octep_device *oct,
324 struct octep_oq *oq)
325 {
326 u32 pkt_count, new_pkts;
327 u32 last_pkt_count, pkts_pending;
328
329 pkt_count = readl(oq->pkts_sent_reg);
330 last_pkt_count = READ_ONCE(oq->last_pkt_count);
331 new_pkts = pkt_count - last_pkt_count;
332
333 if (pkt_count < last_pkt_count) {
334 dev_err(oq->dev, "OQ-%u pkt_count(%u) < oq->last_pkt_count(%u)\n",
335 oq->q_no, pkt_count, last_pkt_count);
336 }
337 /* Clear the hardware packets counter register if the rx queue is
338 * being processed continuously with-in a single interrupt and
339 * reached half its max value.
340 * this counter is not cleared every time read, to save write cycles.
341 */
342 if (unlikely(pkt_count > 0xF0000000U)) {
343 writel(pkt_count, oq->pkts_sent_reg);
344 pkt_count = readl(oq->pkts_sent_reg);
345 new_pkts += pkt_count;
346 }
347 WRITE_ONCE(oq->last_pkt_count, pkt_count);
348 pkts_pending = READ_ONCE(oq->pkts_pending);
349 WRITE_ONCE(oq->pkts_pending, (pkts_pending + new_pkts));
350 return new_pkts;
351 }
352
353 /**
354 * octep_oq_next_pkt() - Move to the next packet in Rx queue.
355 *
356 * @oq: Octeon Rx queue data structure.
357 * @buff_info: Current packet buffer info.
358 * @read_idx: Current packet index in the ring.
359 * @desc_used: Current packet descriptor number.
360 *
361 * Free the resources associated with a packet.
362 * Increment packet index in the ring and packet descriptor number.
363 */
octep_oq_next_pkt(struct octep_oq * oq,struct octep_rx_buffer * buff_info,u32 * read_idx,u32 * desc_used)364 static void octep_oq_next_pkt(struct octep_oq *oq,
365 struct octep_rx_buffer *buff_info,
366 u32 *read_idx, u32 *desc_used)
367 {
368 dma_unmap_page(oq->dev, oq->desc_ring[*read_idx].buffer_ptr,
369 PAGE_SIZE, DMA_FROM_DEVICE);
370 buff_info->page = NULL;
371 (*read_idx)++;
372 (*desc_used)++;
373 if (*read_idx == oq->max_count)
374 *read_idx = 0;
375 }
376
377 /**
378 * octep_oq_drop_rx() - Free the resources associated with a packet.
379 *
380 * @oq: Octeon Rx queue data structure.
381 * @buff_info: Current packet buffer info.
382 * @read_idx: Current packet index in the ring.
383 * @desc_used: Current packet descriptor number.
384 *
385 */
octep_oq_drop_rx(struct octep_oq * oq,struct octep_rx_buffer * buff_info,u32 * read_idx,u32 * desc_used)386 static void octep_oq_drop_rx(struct octep_oq *oq,
387 struct octep_rx_buffer *buff_info,
388 u32 *read_idx, u32 *desc_used)
389 {
390 int data_len = buff_info->len - oq->max_single_buffer_size;
391
392 while (data_len > 0) {
393 octep_oq_next_pkt(oq, buff_info, read_idx, desc_used);
394 data_len -= oq->buffer_size;
395 };
396 }
397
398 /**
399 * __octep_oq_process_rx() - Process hardware Rx queue and push to stack.
400 *
401 * @oct: Octeon device private data structure.
402 * @oq: Octeon Rx queue data structure.
403 * @pkts_to_process: number of packets to be processed.
404 *
405 * Process the new packets in Rx queue.
406 * Packets larger than single Rx buffer arrive in consecutive descriptors.
407 * But, count returned by the API only accounts full packets, not fragments.
408 *
409 * Return: number of packets processed and pushed to stack.
410 */
__octep_oq_process_rx(struct octep_device * oct,struct octep_oq * oq,u16 pkts_to_process)411 static int __octep_oq_process_rx(struct octep_device *oct,
412 struct octep_oq *oq, u16 pkts_to_process)
413 {
414 struct octep_oq_resp_hw_ext *resp_hw_ext = NULL;
415 netdev_features_t feat = oq->netdev->features;
416 struct octep_rx_buffer *buff_info;
417 struct octep_oq_resp_hw *resp_hw;
418 u32 pkt, rx_bytes, desc_used;
419 struct sk_buff *skb;
420 u16 data_offset;
421 u16 rx_ol_flags;
422 u32 read_idx;
423
424 read_idx = READ_ONCE(oq->host_read_idx);
425 rx_bytes = 0;
426 desc_used = 0;
427 for (pkt = 0; pkt < pkts_to_process; pkt++) {
428 buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx];
429 resp_hw = page_address(buff_info->page);
430
431 /* Swap the length field that is in Big-Endian to CPU */
432 buff_info->len = be64_to_cpu(resp_hw->length);
433 if (oct->conf->fw_info.rx_ol_flags) {
434 /* Extended response header is immediately after
435 * response header (resp_hw)
436 */
437 resp_hw_ext = (struct octep_oq_resp_hw_ext *)
438 (resp_hw + 1);
439 buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE;
440 /* Packet Data is immediately after
441 * extended response header.
442 */
443 data_offset = OCTEP_OQ_RESP_HW_SIZE +
444 OCTEP_OQ_RESP_HW_EXT_SIZE;
445 rx_ol_flags = resp_hw_ext->rx_ol_flags;
446 } else {
447 /* Data is immediately after
448 * Hardware Rx response header.
449 */
450 data_offset = OCTEP_OQ_RESP_HW_SIZE;
451 rx_ol_flags = 0;
452 }
453
454 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);
455
456 skb = build_skb((void *)resp_hw, PAGE_SIZE);
457 if (!skb) {
458 octep_oq_drop_rx(oq, buff_info,
459 &read_idx, &desc_used);
460 oq->stats->alloc_failures++;
461 continue;
462 }
463 skb_reserve(skb, data_offset);
464
465 rx_bytes += buff_info->len;
466
467 if (buff_info->len <= oq->max_single_buffer_size) {
468 skb_put(skb, buff_info->len);
469 } else {
470 struct skb_shared_info *shinfo;
471 u16 data_len;
472
473 /* Head fragment includes response header(s);
474 * subsequent fragments contains only data.
475 */
476 skb_put(skb, oq->max_single_buffer_size);
477 shinfo = skb_shinfo(skb);
478 data_len = buff_info->len - oq->max_single_buffer_size;
479 while (data_len) {
480 buff_info = (struct octep_rx_buffer *)
481 &oq->buff_info[read_idx];
482 if (data_len < oq->buffer_size) {
483 buff_info->len = data_len;
484 data_len = 0;
485 } else {
486 buff_info->len = oq->buffer_size;
487 data_len -= oq->buffer_size;
488 }
489
490 skb_add_rx_frag(skb, shinfo->nr_frags,
491 buff_info->page, 0,
492 buff_info->len,
493 buff_info->len);
494
495 octep_oq_next_pkt(oq, buff_info, &read_idx, &desc_used);
496 }
497 }
498
499 skb->dev = oq->netdev;
500 skb->protocol = eth_type_trans(skb, skb->dev);
501 if (feat & NETIF_F_RXCSUM &&
502 OCTEP_RX_CSUM_VERIFIED(rx_ol_flags))
503 skb->ip_summed = CHECKSUM_UNNECESSARY;
504 else
505 skb->ip_summed = CHECKSUM_NONE;
506 napi_gro_receive(oq->napi, skb);
507 }
508
509 WRITE_ONCE(oq->host_read_idx, read_idx);
510 oq->refill_count += desc_used;
511 oq->stats->packets += pkt;
512 oq->stats->bytes += rx_bytes;
513
514 return pkt;
515 }
516
517 /**
518 * octep_oq_process_rx() - Process Rx queue.
519 *
520 * @oq: Octeon Rx queue data structure.
521 * @budget: max number of packets can be processed in one invocation.
522 *
523 * Check for newly received packets and process them.
524 * Keeps checking for new packets until budget is used or no new packets seen.
525 *
526 * Return: number of packets processed.
527 */
octep_oq_process_rx(struct octep_oq * oq,int budget)528 int octep_oq_process_rx(struct octep_oq *oq, int budget)
529 {
530 u32 pkts_available, pkts_processed, total_pkts_processed;
531 struct octep_device *oct = oq->octep_dev;
532 u32 pkts_pending;
533
534 pkts_available = 0;
535 pkts_processed = 0;
536 total_pkts_processed = 0;
537 while (total_pkts_processed < budget) {
538 /* update pending count only when current one exhausted */
539 pkts_pending = READ_ONCE(oq->pkts_pending);
540 if (pkts_pending == 0)
541 octep_oq_check_hw_for_pkts(oct, oq);
542 pkts_pending = READ_ONCE(oq->pkts_pending);
543 pkts_available = min(budget - total_pkts_processed,
544 pkts_pending);
545 if (!pkts_available)
546 break;
547
548 pkts_processed = __octep_oq_process_rx(oct, oq,
549 pkts_available);
550 pkts_pending = READ_ONCE(oq->pkts_pending);
551 WRITE_ONCE(oq->pkts_pending, (pkts_pending - pkts_processed));
552 total_pkts_processed += pkts_processed;
553 }
554
555 if (oq->refill_count >= oq->refill_threshold) {
556 u32 desc_refilled = octep_oq_refill(oct, oq);
557
558 /* flush pending writes before updating credits */
559 wmb();
560 writel(desc_refilled, oq->pkts_credit_reg);
561 }
562
563 return total_pkts_processed;
564 }
565