1 // SPDX-License-Identifier: GPL-2.0+
2 // Copyright (c) 2024 Hisilicon Limited.
3
4 #include <net/netdev_queues.h>
5 #include "hbg_common.h"
6 #include "hbg_irq.h"
7 #include "hbg_reg.h"
8 #include "hbg_txrx.h"
9
10 #define CREATE_TRACE_POINTS
11 #include "hbg_trace.h"
12
13 #define netdev_get_tx_ring(netdev) \
14 (&(((struct hbg_priv *)netdev_priv(netdev))->tx_ring))
15
16 #define buffer_to_dma_dir(buffer) (((buffer)->dir == HBG_DIR_RX) ? \
17 DMA_FROM_DEVICE : DMA_TO_DEVICE)
18
19 #define hbg_queue_used_num(head, tail, ring) ({ \
20 typeof(ring) _ring = (ring); \
21 ((tail) + _ring->len - (head)) % _ring->len; })
22 #define hbg_queue_left_num(head, tail, ring) ({ \
23 typeof(ring) _r = (ring); \
24 _r->len - hbg_queue_used_num((head), (tail), _r) - 1; })
25 #define hbg_queue_is_empty(head, tail, ring) \
26 (hbg_queue_used_num((head), (tail), (ring)) == 0)
27 #define hbg_queue_is_full(head, tail, ring) \
28 (hbg_queue_left_num((head), (tail), (ring)) == 0)
29 #define hbg_queue_next_prt(p, ring) (((p) + 1) % (ring)->len)
30 #define hbg_queue_move_next(p, ring) ({ \
31 typeof(ring) _ring = (ring); \
32 _ring->p = hbg_queue_next_prt(_ring->p, _ring); })
33
34 #define hbg_get_page_order(ring) ({ \
35 typeof(ring) _ring = (ring); \
36 get_order(hbg_spec_max_frame_len(_ring->priv, _ring->dir)); })
37 #define hbg_get_page_size(ring) (PAGE_SIZE << hbg_get_page_order((ring)))
38
39 #define HBG_TX_STOP_THRS 2
40 #define HBG_TX_START_THRS (2 * HBG_TX_STOP_THRS)
41
hbg_dma_map(struct hbg_buffer * buffer)42 static int hbg_dma_map(struct hbg_buffer *buffer)
43 {
44 struct hbg_priv *priv = buffer->priv;
45
46 buffer->skb_dma = dma_map_single(&priv->pdev->dev,
47 buffer->skb->data, buffer->skb_len,
48 buffer_to_dma_dir(buffer));
49 if (unlikely(dma_mapping_error(&priv->pdev->dev, buffer->skb_dma))) {
50 if (buffer->dir == HBG_DIR_RX)
51 priv->stats.rx_dma_err_cnt++;
52 else
53 priv->stats.tx_dma_err_cnt++;
54
55 return -ENOMEM;
56 }
57
58 return 0;
59 }
60
hbg_dma_unmap(struct hbg_buffer * buffer)61 static void hbg_dma_unmap(struct hbg_buffer *buffer)
62 {
63 struct hbg_priv *priv = buffer->priv;
64
65 if (unlikely(!buffer->skb_dma))
66 return;
67
68 dma_unmap_single(&priv->pdev->dev, buffer->skb_dma, buffer->skb_len,
69 buffer_to_dma_dir(buffer));
70 buffer->skb_dma = 0;
71 }
72
hbg_buffer_free_page(struct hbg_buffer * buffer)73 static void hbg_buffer_free_page(struct hbg_buffer *buffer)
74 {
75 struct hbg_ring *ring = buffer->ring;
76
77 if (unlikely(!buffer->page))
78 return;
79
80 page_pool_put_full_page(ring->page_pool, buffer->page, false);
81
82 buffer->page = NULL;
83 buffer->page_dma = 0;
84 buffer->page_addr = NULL;
85 buffer->page_size = 0;
86 buffer->page_offset = 0;
87 }
88
hbg_buffer_alloc_page(struct hbg_buffer * buffer)89 static int hbg_buffer_alloc_page(struct hbg_buffer *buffer)
90 {
91 struct hbg_ring *ring = buffer->ring;
92 u32 len = hbg_get_page_size(ring);
93 u32 offset;
94
95 if (unlikely(!ring->page_pool))
96 return 0;
97
98 buffer->page = page_pool_dev_alloc_frag(ring->page_pool, &offset, len);
99 if (unlikely(!buffer->page))
100 return -ENOMEM;
101
102 buffer->page_dma = page_pool_get_dma_addr(buffer->page) + offset;
103 buffer->page_addr = page_address(buffer->page) + offset;
104 buffer->page_size = len;
105 buffer->page_offset = offset;
106
107 return 0;
108 }
109
hbg_init_tx_desc(struct hbg_buffer * buffer,struct hbg_tx_desc * tx_desc)110 static void hbg_init_tx_desc(struct hbg_buffer *buffer,
111 struct hbg_tx_desc *tx_desc)
112 {
113 u32 ip_offset = buffer->skb->network_header - buffer->skb->mac_header;
114 u32 word0 = 0;
115
116 word0 |= FIELD_PREP(HBG_TX_DESC_W0_WB_B, HBG_STATUS_ENABLE);
117 word0 |= FIELD_PREP(HBG_TX_DESC_W0_IP_OFF_M, ip_offset);
118 if (likely(buffer->skb->ip_summed == CHECKSUM_PARTIAL)) {
119 word0 |= FIELD_PREP(HBG_TX_DESC_W0_l3_CS_B, HBG_STATUS_ENABLE);
120 word0 |= FIELD_PREP(HBG_TX_DESC_W0_l4_CS_B, HBG_STATUS_ENABLE);
121 }
122
123 tx_desc->word0 = word0;
124 tx_desc->word1 = FIELD_PREP(HBG_TX_DESC_W1_SEND_LEN_M,
125 buffer->skb->len);
126 tx_desc->word2 = buffer->skb_dma;
127 tx_desc->word3 = buffer->state_dma;
128 }
129
hbg_net_start_xmit(struct sk_buff * skb,struct net_device * netdev)130 netdev_tx_t hbg_net_start_xmit(struct sk_buff *skb, struct net_device *netdev)
131 {
132 struct hbg_ring *ring = netdev_get_tx_ring(netdev);
133 struct hbg_priv *priv = netdev_priv(netdev);
134 /* This smp_load_acquire() pairs with smp_store_release() in
135 * hbg_napi_tx_recycle() called in tx interrupt handle process.
136 */
137 u32 ntc = smp_load_acquire(&ring->ntc);
138 struct hbg_buffer *buffer;
139 struct hbg_tx_desc tx_desc;
140 u32 ntu = ring->ntu;
141
142 if (unlikely(!skb->len ||
143 skb->len > hbg_spec_max_frame_len(priv, HBG_DIR_TX))) {
144 dev_kfree_skb_any(skb);
145 netdev->stats.tx_errors++;
146 return NETDEV_TX_OK;
147 }
148
149 if (!netif_subqueue_maybe_stop(netdev, 0,
150 hbg_queue_left_num(ntc, ntu, ring),
151 HBG_TX_STOP_THRS, HBG_TX_START_THRS))
152 return NETDEV_TX_BUSY;
153
154 buffer = &ring->queue[ntu];
155 buffer->skb = skb;
156 buffer->skb_len = skb->len;
157 if (unlikely(hbg_dma_map(buffer))) {
158 dev_kfree_skb_any(skb);
159 return NETDEV_TX_OK;
160 }
161
162 buffer->state = HBG_TX_STATE_START;
163 hbg_init_tx_desc(buffer, &tx_desc);
164 hbg_hw_set_tx_desc(priv, &tx_desc);
165
166 /* This smp_store_release() pairs with smp_load_acquire() in
167 * hbg_napi_tx_recycle() called in tx interrupt handle process.
168 */
169 smp_store_release(&ring->ntu, hbg_queue_next_prt(ntu, ring));
170 dev_sw_netstats_tx_add(netdev, 1, skb->len);
171 return NETDEV_TX_OK;
172 }
173
hbg_buffer_free_skb(struct hbg_buffer * buffer)174 static void hbg_buffer_free_skb(struct hbg_buffer *buffer)
175 {
176 if (unlikely(!buffer->skb))
177 return;
178
179 dev_kfree_skb_any(buffer->skb);
180 buffer->skb = NULL;
181 }
182
hbg_buffer_free(struct hbg_buffer * buffer)183 static void hbg_buffer_free(struct hbg_buffer *buffer)
184 {
185 if (buffer->skb) {
186 hbg_dma_unmap(buffer);
187 return hbg_buffer_free_skb(buffer);
188 }
189
190 hbg_buffer_free_page(buffer);
191 }
192
hbg_napi_tx_recycle(struct napi_struct * napi,int budget)193 static int hbg_napi_tx_recycle(struct napi_struct *napi, int budget)
194 {
195 struct hbg_ring *ring = container_of(napi, struct hbg_ring, napi);
196 /* This smp_load_acquire() pairs with smp_store_release() in
197 * hbg_net_start_xmit() called in xmit process.
198 */
199 u32 ntu = smp_load_acquire(&ring->ntu);
200 struct hbg_priv *priv = ring->priv;
201 struct hbg_buffer *buffer;
202 u32 ntc = ring->ntc;
203 int packet_done = 0;
204
205 /* We need do cleanup even if budget is 0.
206 * Per NAPI documentation budget is for Rx.
207 * So We hardcode the amount of work Tx NAPI does to 128.
208 */
209 budget = 128;
210 while (packet_done < budget) {
211 if (unlikely(hbg_queue_is_empty(ntc, ntu, ring)))
212 break;
213
214 /* make sure HW write desc complete */
215 dma_rmb();
216
217 buffer = &ring->queue[ntc];
218 if (buffer->state != HBG_TX_STATE_COMPLETE)
219 break;
220
221 hbg_buffer_free(buffer);
222 ntc = hbg_queue_next_prt(ntc, ring);
223 packet_done++;
224 }
225
226 /* This smp_store_release() pairs with smp_load_acquire() in
227 * hbg_net_start_xmit() called in xmit process.
228 */
229 smp_store_release(&ring->ntc, ntc);
230 netif_wake_queue(priv->netdev);
231
232 if (likely(packet_done < budget &&
233 napi_complete_done(napi, packet_done)))
234 hbg_hw_irq_enable(priv, HBG_INT_MSK_TX_B, true);
235
236 return packet_done;
237 }
238
hbg_rx_check_l3l4_error(struct hbg_priv * priv,struct hbg_rx_desc * desc,struct sk_buff * skb)239 static bool hbg_rx_check_l3l4_error(struct hbg_priv *priv,
240 struct hbg_rx_desc *desc,
241 struct sk_buff *skb)
242 {
243 bool rx_checksum_offload = !!(priv->netdev->features & NETIF_F_RXCSUM);
244
245 skb->ip_summed = rx_checksum_offload ?
246 CHECKSUM_UNNECESSARY : CHECKSUM_NONE;
247
248 if (likely(!FIELD_GET(HBG_RX_DESC_W4_L3_ERR_CODE_M, desc->word4) &&
249 !FIELD_GET(HBG_RX_DESC_W4_L4_ERR_CODE_M, desc->word4)))
250 return true;
251
252 switch (FIELD_GET(HBG_RX_DESC_W4_L3_ERR_CODE_M, desc->word4)) {
253 case HBG_L3_OK:
254 break;
255 case HBG_L3_WRONG_HEAD:
256 priv->stats.rx_desc_l3_wrong_head_cnt++;
257 return false;
258 case HBG_L3_CSUM_ERR:
259 skb->ip_summed = CHECKSUM_NONE;
260 priv->stats.rx_desc_l3_csum_err_cnt++;
261
262 /* Don't drop packets on csum validation failure,
263 * suggest by Jakub
264 */
265 break;
266 case HBG_L3_LEN_ERR:
267 priv->stats.rx_desc_l3_len_err_cnt++;
268 return false;
269 case HBG_L3_ZERO_TTL:
270 priv->stats.rx_desc_l3_zero_ttl_cnt++;
271 return false;
272 default:
273 priv->stats.rx_desc_l3_other_cnt++;
274 return false;
275 }
276
277 switch (FIELD_GET(HBG_RX_DESC_W4_L4_ERR_CODE_M, desc->word4)) {
278 case HBG_L4_OK:
279 break;
280 case HBG_L4_WRONG_HEAD:
281 priv->stats.rx_desc_l4_wrong_head_cnt++;
282 return false;
283 case HBG_L4_LEN_ERR:
284 priv->stats.rx_desc_l4_len_err_cnt++;
285 return false;
286 case HBG_L4_CSUM_ERR:
287 skb->ip_summed = CHECKSUM_NONE;
288 priv->stats.rx_desc_l4_csum_err_cnt++;
289
290 /* Don't drop packets on csum validation failure,
291 * suggest by Jakub
292 */
293 break;
294 case HBG_L4_ZERO_PORT_NUM:
295 priv->stats.rx_desc_l4_zero_port_num_cnt++;
296 return false;
297 default:
298 priv->stats.rx_desc_l4_other_cnt++;
299 return false;
300 }
301
302 return true;
303 }
304
hbg_update_rx_ip_protocol_stats(struct hbg_priv * priv,struct hbg_rx_desc * desc)305 static void hbg_update_rx_ip_protocol_stats(struct hbg_priv *priv,
306 struct hbg_rx_desc *desc)
307 {
308 if (unlikely(!FIELD_GET(HBG_RX_DESC_W4_IP_TCP_UDP_M, desc->word4))) {
309 priv->stats.rx_desc_no_ip_pkt_cnt++;
310 return;
311 }
312
313 if (unlikely(FIELD_GET(HBG_RX_DESC_W4_IP_VERSION_ERR_B, desc->word4))) {
314 priv->stats.rx_desc_ip_ver_err_cnt++;
315 return;
316 }
317
318 /* 0:ipv4, 1:ipv6 */
319 if (FIELD_GET(HBG_RX_DESC_W4_IP_VERSION_B, desc->word4))
320 priv->stats.rx_desc_ipv6_pkt_cnt++;
321 else
322 priv->stats.rx_desc_ipv4_pkt_cnt++;
323
324 switch (FIELD_GET(HBG_RX_DESC_W4_IP_TCP_UDP_M, desc->word4)) {
325 case HBG_IP_PKT:
326 priv->stats.rx_desc_ip_pkt_cnt++;
327 if (FIELD_GET(HBG_RX_DESC_W4_OPT_B, desc->word4))
328 priv->stats.rx_desc_ip_opt_pkt_cnt++;
329 if (FIELD_GET(HBG_RX_DESC_W4_FRAG_B, desc->word4))
330 priv->stats.rx_desc_frag_cnt++;
331
332 if (FIELD_GET(HBG_RX_DESC_W4_ICMP_B, desc->word4))
333 priv->stats.rx_desc_icmp_pkt_cnt++;
334 else if (FIELD_GET(HBG_RX_DESC_W4_IPSEC_B, desc->word4))
335 priv->stats.rx_desc_ipsec_pkt_cnt++;
336 break;
337 case HBG_TCP_PKT:
338 priv->stats.rx_desc_tcp_pkt_cnt++;
339 break;
340 case HBG_UDP_PKT:
341 priv->stats.rx_desc_udp_pkt_cnt++;
342 break;
343 default:
344 priv->stats.rx_desc_no_ip_pkt_cnt++;
345 break;
346 }
347 }
348
hbg_update_rx_protocol_stats(struct hbg_priv * priv,struct hbg_rx_desc * desc)349 static void hbg_update_rx_protocol_stats(struct hbg_priv *priv,
350 struct hbg_rx_desc *desc)
351 {
352 if (unlikely(!FIELD_GET(HBG_RX_DESC_W4_IDX_MATCH_B, desc->word4))) {
353 priv->stats.rx_desc_key_not_match_cnt++;
354 return;
355 }
356
357 if (FIELD_GET(HBG_RX_DESC_W4_BRD_CST_B, desc->word4))
358 priv->stats.rx_desc_broadcast_pkt_cnt++;
359 else if (FIELD_GET(HBG_RX_DESC_W4_MUL_CST_B, desc->word4))
360 priv->stats.rx_desc_multicast_pkt_cnt++;
361
362 if (FIELD_GET(HBG_RX_DESC_W4_VLAN_FLAG_B, desc->word4))
363 priv->stats.rx_desc_vlan_pkt_cnt++;
364
365 if (FIELD_GET(HBG_RX_DESC_W4_ARP_B, desc->word4)) {
366 priv->stats.rx_desc_arp_pkt_cnt++;
367 return;
368 } else if (FIELD_GET(HBG_RX_DESC_W4_RARP_B, desc->word4)) {
369 priv->stats.rx_desc_rarp_pkt_cnt++;
370 return;
371 }
372
373 hbg_update_rx_ip_protocol_stats(priv, desc);
374 }
375
hbg_rx_pkt_check(struct hbg_priv * priv,struct hbg_rx_desc * desc,struct sk_buff * skb)376 static bool hbg_rx_pkt_check(struct hbg_priv *priv, struct hbg_rx_desc *desc,
377 struct sk_buff *skb)
378 {
379 if (unlikely(FIELD_GET(HBG_RX_DESC_W2_PKT_LEN_M, desc->word2) >
380 priv->dev_specs.max_frame_len)) {
381 priv->stats.rx_desc_pkt_len_err_cnt++;
382 return false;
383 }
384
385 if (unlikely(FIELD_GET(HBG_RX_DESC_W2_PORT_NUM_M, desc->word2) !=
386 priv->dev_specs.mac_id ||
387 FIELD_GET(HBG_RX_DESC_W4_DROP_B, desc->word4))) {
388 priv->stats.rx_desc_drop++;
389 return false;
390 }
391
392 if (unlikely(FIELD_GET(HBG_RX_DESC_W4_L2_ERR_B, desc->word4))) {
393 priv->stats.rx_desc_l2_err_cnt++;
394 return false;
395 }
396
397 if (unlikely(!hbg_rx_check_l3l4_error(priv, desc, skb))) {
398 priv->stats.rx_desc_l3l4_err_cnt++;
399 return false;
400 }
401
402 hbg_update_rx_protocol_stats(priv, desc);
403 return true;
404 }
405
hbg_rx_fill_one_buffer(struct hbg_priv * priv)406 static int hbg_rx_fill_one_buffer(struct hbg_priv *priv)
407 {
408 struct hbg_ring *ring = &priv->rx_ring;
409 struct hbg_buffer *buffer;
410 int ret;
411
412 if (hbg_queue_is_full(ring->ntc, ring->ntu, ring) ||
413 hbg_fifo_is_full(priv, ring->dir))
414 return 0;
415
416 buffer = &ring->queue[ring->ntu];
417 ret = hbg_buffer_alloc_page(buffer);
418 if (unlikely(ret))
419 return ret;
420
421 memset(buffer->page_addr, 0, HBG_PACKET_HEAD_SIZE);
422 dma_sync_single_for_device(&priv->pdev->dev, buffer->page_dma,
423 HBG_PACKET_HEAD_SIZE, DMA_TO_DEVICE);
424
425 hbg_hw_fill_buffer(priv, buffer->page_dma);
426 hbg_queue_move_next(ntu, ring);
427 return 0;
428 }
429
hbg_rx_fill_buffers(struct hbg_priv * priv)430 static int hbg_rx_fill_buffers(struct hbg_priv *priv)
431 {
432 u32 remained = hbg_hw_get_fifo_used_num(priv, HBG_DIR_RX);
433 u32 max_count = priv->dev_specs.rx_fifo_num;
434 u32 refill_count;
435 int ret;
436
437 if (unlikely(remained >= max_count))
438 return 0;
439
440 refill_count = max_count - remained;
441 while (refill_count--) {
442 ret = hbg_rx_fill_one_buffer(priv);
443 if (unlikely(ret))
444 break;
445 }
446
447 return ret;
448 }
449
hbg_sync_data_from_hw(struct hbg_priv * priv,struct hbg_buffer * buffer)450 static bool hbg_sync_data_from_hw(struct hbg_priv *priv,
451 struct hbg_buffer *buffer)
452 {
453 struct hbg_rx_desc *rx_desc;
454
455 dma_sync_single_for_cpu(&priv->pdev->dev, buffer->page_dma,
456 buffer->page_size, DMA_FROM_DEVICE);
457
458 /* make sure HW write desc complete */
459 dma_rmb();
460
461 rx_desc = (struct hbg_rx_desc *)buffer->page_addr;
462 return FIELD_GET(HBG_RX_DESC_W2_PKT_LEN_M, rx_desc->word2) != 0;
463 }
464
hbg_build_skb(struct hbg_priv * priv,struct hbg_buffer * buffer,u32 pkt_len)465 static int hbg_build_skb(struct hbg_priv *priv,
466 struct hbg_buffer *buffer, u32 pkt_len)
467 {
468 net_prefetch(buffer->page_addr);
469
470 buffer->skb = napi_build_skb(buffer->page_addr, buffer->page_size);
471 if (unlikely(!buffer->skb))
472 return -ENOMEM;
473 skb_mark_for_recycle(buffer->skb);
474
475 /* page will be freed together with the skb */
476 buffer->page = NULL;
477
478 return 0;
479 }
480
hbg_napi_rx_poll(struct napi_struct * napi,int budget)481 static int hbg_napi_rx_poll(struct napi_struct *napi, int budget)
482 {
483 struct hbg_ring *ring = container_of(napi, struct hbg_ring, napi);
484 struct hbg_priv *priv = ring->priv;
485 struct hbg_rx_desc *rx_desc;
486 struct hbg_buffer *buffer;
487 u32 packet_done = 0;
488 u32 pkt_len;
489
490 hbg_rx_fill_buffers(priv);
491 while (packet_done < budget) {
492 if (unlikely(hbg_queue_is_empty(ring->ntc, ring->ntu, ring)))
493 break;
494
495 buffer = &ring->queue[ring->ntc];
496 if (unlikely(!buffer->page))
497 goto next_buffer;
498
499 if (unlikely(!hbg_sync_data_from_hw(priv, buffer)))
500 break;
501 rx_desc = (struct hbg_rx_desc *)buffer->page_addr;
502 pkt_len = FIELD_GET(HBG_RX_DESC_W2_PKT_LEN_M, rx_desc->word2);
503 trace_hbg_rx_desc(priv, ring->ntc, rx_desc);
504
505 if (unlikely(hbg_build_skb(priv, buffer, pkt_len))) {
506 hbg_buffer_free_page(buffer);
507 goto next_buffer;
508 }
509
510 if (unlikely(!hbg_rx_pkt_check(priv, rx_desc, buffer->skb))) {
511 hbg_buffer_free_skb(buffer);
512 goto next_buffer;
513 }
514
515 skb_reserve(buffer->skb, HBG_PACKET_HEAD_SIZE + NET_IP_ALIGN);
516 skb_put(buffer->skb, pkt_len);
517 buffer->skb->protocol = eth_type_trans(buffer->skb,
518 priv->netdev);
519 dev_sw_netstats_rx_add(priv->netdev, pkt_len);
520 napi_gro_receive(napi, buffer->skb);
521 buffer->skb = NULL;
522 buffer->page = NULL;
523
524 next_buffer:
525 hbg_rx_fill_one_buffer(priv);
526 hbg_queue_move_next(ntc, ring);
527 packet_done++;
528 }
529
530 if (likely(packet_done < budget &&
531 napi_complete_done(napi, packet_done)))
532 hbg_hw_irq_enable(priv, HBG_INT_MSK_RX_B, true);
533
534 return packet_done;
535 }
536
hbg_ring_page_pool_destory(struct hbg_ring * ring)537 static void hbg_ring_page_pool_destory(struct hbg_ring *ring)
538 {
539 if (!ring->page_pool)
540 return;
541
542 page_pool_destroy(ring->page_pool);
543 ring->page_pool = NULL;
544 }
545
hbg_ring_page_pool_init(struct hbg_priv * priv,struct hbg_ring * ring)546 static int hbg_ring_page_pool_init(struct hbg_priv *priv, struct hbg_ring *ring)
547 {
548 u32 buf_size = hbg_spec_max_frame_len(priv, ring->dir);
549 struct page_pool_params pp_params = {
550 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
551 .order = hbg_get_page_order(ring),
552 .pool_size = ring->len * buf_size / hbg_get_page_size(ring),
553 .nid = dev_to_node(&priv->pdev->dev),
554 .dev = &priv->pdev->dev,
555 .napi = &ring->napi,
556 .dma_dir = DMA_FROM_DEVICE,
557 .offset = 0,
558 .max_len = hbg_get_page_size(ring),
559 };
560 int ret = 0;
561
562 ring->page_pool = page_pool_create(&pp_params);
563 if (IS_ERR(ring->page_pool)) {
564 ret = PTR_ERR(ring->page_pool);
565 dev_err(&priv->pdev->dev,
566 "failed to create page pool, ret = %d\n", ret);
567 ring->page_pool = NULL;
568 }
569
570 return ret;
571 }
572
hbg_ring_uninit(struct hbg_ring * ring)573 static void hbg_ring_uninit(struct hbg_ring *ring)
574 {
575 struct hbg_buffer *buffer;
576 u32 i;
577
578 if (!ring->queue)
579 return;
580
581 napi_disable(&ring->napi);
582 netif_napi_del(&ring->napi);
583
584 for (i = 0; i < ring->len; i++) {
585 buffer = &ring->queue[i];
586 hbg_buffer_free(buffer);
587 buffer->ring = NULL;
588 buffer->priv = NULL;
589 }
590
591 hbg_ring_page_pool_destory(ring);
592 dma_free_coherent(&ring->priv->pdev->dev,
593 ring->len * sizeof(*ring->queue),
594 ring->queue, ring->queue_dma);
595 ring->queue = NULL;
596 ring->queue_dma = 0;
597 ring->len = 0;
598 ring->priv = NULL;
599 }
600
hbg_ring_init(struct hbg_priv * priv,struct hbg_ring * ring,int (* napi_poll)(struct napi_struct *,int),enum hbg_dir dir)601 static int hbg_ring_init(struct hbg_priv *priv, struct hbg_ring *ring,
602 int (*napi_poll)(struct napi_struct *, int),
603 enum hbg_dir dir)
604 {
605 struct hbg_buffer *buffer;
606 u32 i, len;
607 int ret;
608
609 len = hbg_get_spec_fifo_max_num(priv, dir) + 1;
610 /* To improve receiving performance under high-stress scenarios,
611 * in the `hbg_napi_rx_poll()`, we first use the other half of
612 * the buffer to receive packets from the hardware via the
613 * `hbg_rx_fill_buffers()`, and then process the packets in the
614 * original half of the buffer to avoid packet loss caused by
615 * hardware overflow as much as possible.
616 */
617 if (dir == HBG_DIR_RX)
618 len += hbg_get_spec_fifo_max_num(priv, dir);
619
620 ring->queue = dma_alloc_coherent(&priv->pdev->dev,
621 len * sizeof(*ring->queue),
622 &ring->queue_dma, GFP_KERNEL);
623 if (!ring->queue)
624 return -ENOMEM;
625
626 for (i = 0; i < len; i++) {
627 buffer = &ring->queue[i];
628 buffer->skb_len = 0;
629 buffer->dir = dir;
630 buffer->ring = ring;
631 buffer->priv = priv;
632 buffer->state_dma = ring->queue_dma + (i * sizeof(*buffer));
633 }
634
635 ring->dir = dir;
636 ring->priv = priv;
637 ring->ntc = 0;
638 ring->ntu = 0;
639 ring->len = len;
640
641 if (dir == HBG_DIR_TX) {
642 netif_napi_add_tx(priv->netdev, &ring->napi, napi_poll);
643 } else {
644 netif_napi_add(priv->netdev, &ring->napi, napi_poll);
645
646 ret = hbg_ring_page_pool_init(priv, ring);
647 if (ret) {
648 netif_napi_del(&ring->napi);
649 dma_free_coherent(&ring->priv->pdev->dev,
650 ring->len * sizeof(*ring->queue),
651 ring->queue, ring->queue_dma);
652 ring->queue = NULL;
653 ring->len = 0;
654 return ret;
655 }
656 }
657
658 napi_enable(&ring->napi);
659 return 0;
660 }
661
hbg_tx_ring_init(struct hbg_priv * priv)662 static int hbg_tx_ring_init(struct hbg_priv *priv)
663 {
664 struct hbg_ring *tx_ring = &priv->tx_ring;
665
666 if (!tx_ring->tout_log_buf)
667 tx_ring->tout_log_buf = devm_kmalloc(&priv->pdev->dev,
668 HBG_TX_TIMEOUT_BUF_LEN,
669 GFP_KERNEL);
670
671 if (!tx_ring->tout_log_buf)
672 return -ENOMEM;
673
674 return hbg_ring_init(priv, tx_ring, hbg_napi_tx_recycle, HBG_DIR_TX);
675 }
676
hbg_rx_ring_init(struct hbg_priv * priv)677 static int hbg_rx_ring_init(struct hbg_priv *priv)
678 {
679 int ret;
680
681 ret = hbg_ring_init(priv, &priv->rx_ring, hbg_napi_rx_poll, HBG_DIR_RX);
682 if (ret)
683 return ret;
684
685 ret = hbg_rx_fill_buffers(priv);
686 if (ret)
687 hbg_ring_uninit(&priv->rx_ring);
688
689 return ret;
690 }
691
hbg_txrx_init(struct hbg_priv * priv)692 int hbg_txrx_init(struct hbg_priv *priv)
693 {
694 int ret;
695
696 ret = hbg_tx_ring_init(priv);
697 if (ret) {
698 dev_err(&priv->pdev->dev,
699 "failed to init tx ring, ret = %d\n", ret);
700 return ret;
701 }
702
703 ret = hbg_rx_ring_init(priv);
704 if (ret) {
705 dev_err(&priv->pdev->dev,
706 "failed to init rx ring, ret = %d\n", ret);
707 hbg_ring_uninit(&priv->tx_ring);
708 }
709
710 return ret;
711 }
712
hbg_txrx_uninit(struct hbg_priv * priv)713 void hbg_txrx_uninit(struct hbg_priv *priv)
714 {
715 hbg_ring_uninit(&priv->tx_ring);
716 hbg_ring_uninit(&priv->rx_ring);
717 }
718