1 // SPDX-License-Identifier: GPL-2.0
2
3 /* Texas Instruments ICSSG Ethernet Driver
4 *
5 * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/
6 * Copyright (C) Siemens AG, 2024
7 *
8 */
9
10 #include <linux/dma-mapping.h>
11 #include <linux/dma/ti-cppi5.h>
12 #include <linux/etherdevice.h>
13 #include <linux/interrupt.h>
14 #include <linux/kernel.h>
15 #include <linux/of.h>
16 #include <linux/of_mdio.h>
17 #include <linux/phy.h>
18 #include <linux/remoteproc/pruss.h>
19 #include <linux/regmap.h>
20 #include <linux/remoteproc.h>
21
22 #include "icssg_prueth.h"
23 #include "../k3-cppi-desc-pool.h"
24
25 /* Netif debug messages possible */
26 #define PRUETH_EMAC_DEBUG (NETIF_MSG_DRV | \
27 NETIF_MSG_PROBE | \
28 NETIF_MSG_LINK | \
29 NETIF_MSG_TIMER | \
30 NETIF_MSG_IFDOWN | \
31 NETIF_MSG_IFUP | \
32 NETIF_MSG_RX_ERR | \
33 NETIF_MSG_TX_ERR | \
34 NETIF_MSG_TX_QUEUED | \
35 NETIF_MSG_INTR | \
36 NETIF_MSG_TX_DONE | \
37 NETIF_MSG_RX_STATUS | \
38 NETIF_MSG_PKTDATA | \
39 NETIF_MSG_HW | \
40 NETIF_MSG_WOL)
41
42 #define prueth_napi_to_emac(napi) container_of(napi, struct prueth_emac, napi_rx)
43
prueth_cleanup_rx_chns(struct prueth_emac * emac,struct prueth_rx_chn * rx_chn,int max_rflows)44 void prueth_cleanup_rx_chns(struct prueth_emac *emac,
45 struct prueth_rx_chn *rx_chn,
46 int max_rflows)
47 {
48 if (rx_chn->pg_pool) {
49 page_pool_destroy(rx_chn->pg_pool);
50 rx_chn->pg_pool = NULL;
51 }
52
53 if (rx_chn->desc_pool)
54 k3_cppi_desc_pool_destroy(rx_chn->desc_pool);
55
56 if (rx_chn->rx_chn)
57 k3_udma_glue_release_rx_chn(rx_chn->rx_chn);
58 }
59 EXPORT_SYMBOL_GPL(prueth_cleanup_rx_chns);
60
prueth_cleanup_tx_chns(struct prueth_emac * emac)61 void prueth_cleanup_tx_chns(struct prueth_emac *emac)
62 {
63 int i;
64
65 for (i = 0; i < emac->tx_ch_num; i++) {
66 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];
67
68 if (tx_chn->desc_pool)
69 k3_cppi_desc_pool_destroy(tx_chn->desc_pool);
70
71 if (tx_chn->tx_chn)
72 k3_udma_glue_release_tx_chn(tx_chn->tx_chn);
73
74 /* Assume prueth_cleanup_tx_chns() is called at the
75 * end after all channel resources are freed
76 */
77 memset(tx_chn, 0, sizeof(*tx_chn));
78 }
79 }
80 EXPORT_SYMBOL_GPL(prueth_cleanup_tx_chns);
81
prueth_ndev_del_tx_napi(struct prueth_emac * emac,int num)82 void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num)
83 {
84 int i;
85
86 for (i = 0; i < num; i++) {
87 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];
88
89 if (tx_chn->irq)
90 free_irq(tx_chn->irq, tx_chn);
91 netif_napi_del(&tx_chn->napi_tx);
92 }
93 }
94 EXPORT_SYMBOL_GPL(prueth_ndev_del_tx_napi);
95
emac_xsk_xmit_zc(struct prueth_emac * emac,unsigned int q_idx)96 static void emac_xsk_xmit_zc(struct prueth_emac *emac,
97 unsigned int q_idx)
98 {
99 struct prueth_tx_chn *tx_chn = &emac->tx_chns[q_idx];
100 struct xsk_buff_pool *pool = tx_chn->xsk_pool;
101 struct net_device *ndev = emac->ndev;
102 struct cppi5_host_desc_t *host_desc;
103 dma_addr_t dma_desc, dma_buf;
104 struct prueth_swdata *swdata;
105 struct xdp_desc xdp_desc;
106 int num_tx = 0, pkt_len;
107 int descs_avail, ret;
108 u32 dst_tag_id;
109 u32 *epib;
110 int i;
111
112 descs_avail = k3_cppi_desc_pool_avail(tx_chn->desc_pool);
113 /* ensure that TX ring is not filled up by XDP, always MAX_SKB_FRAGS
114 * will be available for normal TX path and queue is stopped there if
115 * necessary
116 */
117 if (descs_avail <= MAX_SKB_FRAGS)
118 return;
119
120 descs_avail -= MAX_SKB_FRAGS;
121
122 for (i = 0; i < descs_avail; i++) {
123 if (!xsk_tx_peek_desc(pool, &xdp_desc))
124 break;
125
126 dma_buf = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
127 pkt_len = xdp_desc.len;
128 xsk_buff_raw_dma_sync_for_device(pool, dma_buf, pkt_len);
129
130 host_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
131 if (unlikely(!host_desc))
132 break;
133
134 cppi5_hdesc_init(host_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
135 PRUETH_NAV_PS_DATA_SIZE);
136 cppi5_hdesc_set_pkttype(host_desc, 0);
137 epib = host_desc->epib;
138 epib[0] = 0;
139 epib[1] = 0;
140 cppi5_hdesc_set_pktlen(host_desc, pkt_len);
141 dst_tag_id = emac->port_id | (q_idx << 8);
142
143 if (emac->prueth->is_hsr_offload_mode &&
144 (ndev->features & NETIF_F_HW_HSR_DUP))
145 dst_tag_id = PRUETH_UNDIRECTED_PKT_DST_TAG;
146
147 if (emac->prueth->is_hsr_offload_mode &&
148 (ndev->features & NETIF_F_HW_HSR_TAG_INS))
149 epib[1] |= PRUETH_UNDIRECTED_PKT_TAG_INS;
150
151 cppi5_desc_set_tags_ids(&host_desc->hdr, 0, dst_tag_id);
152 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &dma_buf);
153 cppi5_hdesc_attach_buf(host_desc, dma_buf, pkt_len, dma_buf,
154 pkt_len);
155
156 swdata = cppi5_hdesc_get_swdata(host_desc);
157 swdata->type = PRUETH_SWDATA_XSK;
158
159 dma_desc = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,
160 host_desc);
161 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn,
162 host_desc, dma_desc);
163
164 if (ret) {
165 ndev->stats.tx_errors++;
166 k3_cppi_desc_pool_free(tx_chn->desc_pool, host_desc);
167 break;
168 }
169
170 num_tx++;
171 }
172
173 if (num_tx)
174 xsk_tx_release(tx_chn->xsk_pool);
175 }
176
prueth_xmit_free(struct prueth_tx_chn * tx_chn,struct cppi5_host_desc_t * desc)177 void prueth_xmit_free(struct prueth_tx_chn *tx_chn,
178 struct cppi5_host_desc_t *desc)
179 {
180 struct cppi5_host_desc_t *first_desc, *next_desc;
181 dma_addr_t buf_dma, next_desc_dma;
182 struct prueth_swdata *swdata;
183 u32 buf_dma_len;
184
185 first_desc = desc;
186 next_desc = first_desc;
187 swdata = cppi5_hdesc_get_swdata(first_desc);
188 if (swdata->type == PRUETH_SWDATA_XSK)
189 goto free_pool;
190
191 cppi5_hdesc_get_obuf(first_desc, &buf_dma, &buf_dma_len);
192 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
193
194 dma_unmap_single(tx_chn->dma_dev, buf_dma, buf_dma_len,
195 DMA_TO_DEVICE);
196
197 next_desc_dma = cppi5_hdesc_get_next_hbdesc(first_desc);
198 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
199 while (next_desc_dma) {
200 next_desc = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
201 next_desc_dma);
202 cppi5_hdesc_get_obuf(next_desc, &buf_dma, &buf_dma_len);
203 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &buf_dma);
204
205 dma_unmap_page(tx_chn->dma_dev, buf_dma, buf_dma_len,
206 DMA_TO_DEVICE);
207
208 next_desc_dma = cppi5_hdesc_get_next_hbdesc(next_desc);
209 k3_udma_glue_tx_cppi5_to_dma_addr(tx_chn->tx_chn, &next_desc_dma);
210
211 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
212 }
213
214 free_pool:
215 k3_cppi_desc_pool_free(tx_chn->desc_pool, first_desc);
216 }
217 EXPORT_SYMBOL_GPL(prueth_xmit_free);
218
emac_tx_complete_packets(struct prueth_emac * emac,int chn,int budget,bool * tdown)219 int emac_tx_complete_packets(struct prueth_emac *emac, int chn,
220 int budget, bool *tdown)
221 {
222 struct net_device *ndev = emac->ndev;
223 struct cppi5_host_desc_t *desc_tx;
224 struct netdev_queue *netif_txq;
225 struct prueth_swdata *swdata;
226 struct prueth_tx_chn *tx_chn;
227 unsigned int total_bytes = 0;
228 int xsk_frames_done = 0;
229 struct xdp_frame *xdpf;
230 unsigned int pkt_len;
231 struct sk_buff *skb;
232 dma_addr_t desc_dma;
233 int res, num_tx = 0;
234
235 tx_chn = &emac->tx_chns[chn];
236
237 while (true) {
238 res = k3_udma_glue_pop_tx_chn(tx_chn->tx_chn, &desc_dma);
239 if (res == -ENODATA)
240 break;
241
242 /* teardown completion */
243 if (cppi5_desc_is_tdcm(desc_dma)) {
244 if (atomic_dec_and_test(&emac->tdown_cnt))
245 complete(&emac->tdown_complete);
246 *tdown = true;
247 break;
248 }
249
250 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool,
251 desc_dma);
252 swdata = cppi5_hdesc_get_swdata(desc_tx);
253
254 switch (swdata->type) {
255 case PRUETH_SWDATA_SKB:
256 skb = swdata->data.skb;
257 dev_sw_netstats_tx_add(skb->dev, 1, skb->len);
258 total_bytes += skb->len;
259 napi_consume_skb(skb, budget);
260 break;
261 case PRUETH_SWDATA_XDPF:
262 xdpf = swdata->data.xdpf;
263 dev_sw_netstats_tx_add(ndev, 1, xdpf->len);
264 total_bytes += xdpf->len;
265 xdp_return_frame(xdpf);
266 break;
267 case PRUETH_SWDATA_XSK:
268 pkt_len = cppi5_hdesc_get_pktlen(desc_tx);
269 dev_sw_netstats_tx_add(ndev, 1, pkt_len);
270 xsk_frames_done++;
271 break;
272 default:
273 prueth_xmit_free(tx_chn, desc_tx);
274 ndev->stats.tx_dropped++;
275 continue;
276 }
277
278 prueth_xmit_free(tx_chn, desc_tx);
279 num_tx++;
280 }
281
282 netif_txq = netdev_get_tx_queue(ndev, chn);
283 netdev_tx_completed_queue(netif_txq, num_tx, total_bytes);
284
285 if (netif_tx_queue_stopped(netif_txq)) {
286 /* If the TX queue was stopped, wake it now
287 * if we have enough room.
288 */
289 __netif_tx_lock(netif_txq, smp_processor_id());
290 if (netif_running(ndev) &&
291 (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=
292 MAX_SKB_FRAGS))
293 netif_tx_wake_queue(netif_txq);
294 __netif_tx_unlock(netif_txq);
295 }
296
297 if (budget && tx_chn->xsk_pool) {
298 if (xsk_frames_done) {
299 xsk_tx_completed(tx_chn->xsk_pool, xsk_frames_done);
300 txq_trans_cond_update(netif_txq);
301 }
302
303 if (xsk_uses_need_wakeup(tx_chn->xsk_pool))
304 xsk_set_tx_need_wakeup(tx_chn->xsk_pool);
305
306 __netif_tx_lock(netif_txq, smp_processor_id());
307 emac_xsk_xmit_zc(emac, chn);
308 __netif_tx_unlock(netif_txq);
309 }
310
311 return num_tx;
312 }
313
emac_tx_timer_callback(struct hrtimer * timer)314 static enum hrtimer_restart emac_tx_timer_callback(struct hrtimer *timer)
315 {
316 struct prueth_tx_chn *tx_chns =
317 container_of(timer, struct prueth_tx_chn, tx_hrtimer);
318
319 if (tx_chns->irq_disabled) {
320 tx_chns->irq_disabled = false;
321 enable_irq(tx_chns->irq);
322 }
323 return HRTIMER_NORESTART;
324 }
325
emac_napi_tx_poll(struct napi_struct * napi_tx,int budget)326 static int emac_napi_tx_poll(struct napi_struct *napi_tx, int budget)
327 {
328 struct prueth_tx_chn *tx_chn = prueth_napi_to_tx_chn(napi_tx);
329 struct prueth_emac *emac = tx_chn->emac;
330 bool tdown = false;
331 int num_tx_packets;
332
333 num_tx_packets = emac_tx_complete_packets(emac, tx_chn->id, budget,
334 &tdown);
335
336 if (num_tx_packets >= budget)
337 return budget;
338
339 if (napi_complete_done(napi_tx, num_tx_packets)) {
340 if (unlikely(tx_chn->tx_pace_timeout_ns && !tdown)) {
341 hrtimer_start(&tx_chn->tx_hrtimer,
342 ns_to_ktime(tx_chn->tx_pace_timeout_ns),
343 HRTIMER_MODE_REL_PINNED);
344 } else {
345 if (tx_chn->irq_disabled) {
346 tx_chn->irq_disabled = false;
347 enable_irq(tx_chn->irq);
348 }
349 }
350 }
351
352 return num_tx_packets;
353 }
354
prueth_tx_irq(int irq,void * dev_id)355 static irqreturn_t prueth_tx_irq(int irq, void *dev_id)
356 {
357 struct prueth_tx_chn *tx_chn = dev_id;
358
359 tx_chn->irq_disabled = true;
360 disable_irq_nosync(irq);
361 napi_schedule(&tx_chn->napi_tx);
362
363 return IRQ_HANDLED;
364 }
365
prueth_ndev_add_tx_napi(struct prueth_emac * emac)366 int prueth_ndev_add_tx_napi(struct prueth_emac *emac)
367 {
368 struct prueth *prueth = emac->prueth;
369 int i, ret;
370
371 for (i = 0; i < emac->tx_ch_num; i++) {
372 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];
373
374 netif_napi_add_tx(emac->ndev, &tx_chn->napi_tx, emac_napi_tx_poll);
375 hrtimer_setup(&tx_chn->tx_hrtimer, &emac_tx_timer_callback, CLOCK_MONOTONIC,
376 HRTIMER_MODE_REL_PINNED);
377 ret = request_irq(tx_chn->irq, prueth_tx_irq,
378 IRQF_TRIGGER_HIGH, tx_chn->name,
379 tx_chn);
380 if (ret) {
381 netif_napi_del(&tx_chn->napi_tx);
382 dev_err(prueth->dev, "unable to request TX IRQ %d\n",
383 tx_chn->irq);
384 goto fail;
385 }
386 }
387
388 return 0;
389 fail:
390 prueth_ndev_del_tx_napi(emac, i);
391 return ret;
392 }
393 EXPORT_SYMBOL_GPL(prueth_ndev_add_tx_napi);
394
prueth_init_tx_chns(struct prueth_emac * emac)395 int prueth_init_tx_chns(struct prueth_emac *emac)
396 {
397 static const struct k3_ring_cfg ring_cfg = {
398 .elm_size = K3_RINGACC_RING_ELSIZE_8,
399 .mode = K3_RINGACC_RING_MODE_RING,
400 .flags = 0,
401 .size = PRUETH_MAX_TX_DESC,
402 };
403 struct k3_udma_glue_tx_channel_cfg tx_cfg;
404 struct device *dev = emac->prueth->dev;
405 struct net_device *ndev = emac->ndev;
406 int ret, slice, i;
407 u32 hdesc_size;
408
409 slice = prueth_emac_slice(emac);
410 if (slice < 0)
411 return slice;
412
413 init_completion(&emac->tdown_complete);
414
415 hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE,
416 PRUETH_NAV_SW_DATA_SIZE);
417 memset(&tx_cfg, 0, sizeof(tx_cfg));
418 tx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE;
419 tx_cfg.tx_cfg = ring_cfg;
420 tx_cfg.txcq_cfg = ring_cfg;
421
422 for (i = 0; i < emac->tx_ch_num; i++) {
423 struct prueth_tx_chn *tx_chn = &emac->tx_chns[i];
424
425 /* To differentiate channels for SLICE0 vs SLICE1 */
426 snprintf(tx_chn->name, sizeof(tx_chn->name),
427 "tx%d-%d", slice, i);
428
429 tx_chn->emac = emac;
430 tx_chn->id = i;
431 tx_chn->descs_num = PRUETH_MAX_TX_DESC;
432
433 tx_chn->tx_chn =
434 k3_udma_glue_request_tx_chn(dev, tx_chn->name,
435 &tx_cfg);
436 if (IS_ERR(tx_chn->tx_chn)) {
437 ret = PTR_ERR(tx_chn->tx_chn);
438 tx_chn->tx_chn = NULL;
439 netdev_err(ndev,
440 "Failed to request tx dma ch: %d\n", ret);
441 goto fail;
442 }
443
444 tx_chn->dma_dev = k3_udma_glue_tx_get_dma_device(tx_chn->tx_chn);
445 tx_chn->desc_pool =
446 k3_cppi_desc_pool_create_name(tx_chn->dma_dev,
447 tx_chn->descs_num,
448 hdesc_size,
449 tx_chn->name);
450 if (IS_ERR(tx_chn->desc_pool)) {
451 ret = PTR_ERR(tx_chn->desc_pool);
452 tx_chn->desc_pool = NULL;
453 netdev_err(ndev, "Failed to create tx pool: %d\n", ret);
454 goto fail;
455 }
456
457 ret = k3_udma_glue_tx_get_irq(tx_chn->tx_chn);
458 if (ret < 0) {
459 netdev_err(ndev, "failed to get tx irq\n");
460 goto fail;
461 }
462 tx_chn->irq = ret;
463
464 snprintf(tx_chn->name, sizeof(tx_chn->name), "%s-tx%d",
465 dev_name(dev), tx_chn->id);
466 }
467
468 return 0;
469
470 fail:
471 prueth_cleanup_tx_chns(emac);
472 return ret;
473 }
474 EXPORT_SYMBOL_GPL(prueth_init_tx_chns);
475
prueth_create_page_pool(struct prueth_emac * emac,struct device * dma_dev,int size)476 static struct page_pool *prueth_create_page_pool(struct prueth_emac *emac,
477 struct device *dma_dev,
478 int size)
479 {
480 struct page_pool_params pp_params = { 0 };
481 struct page_pool *pool;
482
483 pp_params.order = 0;
484 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
485 pp_params.pool_size = size;
486 pp_params.nid = dev_to_node(emac->prueth->dev);
487 pp_params.dma_dir = DMA_BIDIRECTIONAL;
488 pp_params.dev = dma_dev;
489 pp_params.napi = &emac->napi_rx;
490 pp_params.max_len = PAGE_SIZE;
491
492 pool = page_pool_create(&pp_params);
493 if (IS_ERR(pool))
494 netdev_err(emac->ndev, "cannot create rx page pool\n");
495
496 return pool;
497 }
498
prueth_init_rx_chns(struct prueth_emac * emac,struct prueth_rx_chn * rx_chn,char * name,u32 max_rflows,u32 max_desc_num)499 int prueth_init_rx_chns(struct prueth_emac *emac,
500 struct prueth_rx_chn *rx_chn,
501 char *name, u32 max_rflows,
502 u32 max_desc_num)
503 {
504 struct k3_udma_glue_rx_channel_cfg rx_cfg;
505 struct device *dev = emac->prueth->dev;
506 struct net_device *ndev = emac->ndev;
507 u32 fdqring_id, hdesc_size;
508 struct page_pool *pool;
509 int i, ret = 0, slice;
510 int flow_id_base;
511
512 slice = prueth_emac_slice(emac);
513 if (slice < 0)
514 return slice;
515
516 /* To differentiate channels for SLICE0 vs SLICE1 */
517 snprintf(rx_chn->name, sizeof(rx_chn->name), "%s%d", name, slice);
518
519 hdesc_size = cppi5_hdesc_calc_size(true, PRUETH_NAV_PS_DATA_SIZE,
520 PRUETH_NAV_SW_DATA_SIZE);
521 memset(&rx_cfg, 0, sizeof(rx_cfg));
522 rx_cfg.swdata_size = PRUETH_NAV_SW_DATA_SIZE;
523 rx_cfg.flow_id_num = max_rflows;
524 rx_cfg.flow_id_base = -1; /* udmax will auto select flow id base */
525
526 /* init all flows */
527 rx_chn->dev = dev;
528 rx_chn->descs_num = max_desc_num;
529
530 rx_chn->rx_chn = k3_udma_glue_request_rx_chn(dev, rx_chn->name,
531 &rx_cfg);
532 if (IS_ERR(rx_chn->rx_chn)) {
533 ret = PTR_ERR(rx_chn->rx_chn);
534 rx_chn->rx_chn = NULL;
535 netdev_err(ndev, "Failed to request rx dma ch: %d\n", ret);
536 goto fail;
537 }
538
539 rx_chn->dma_dev = k3_udma_glue_rx_get_dma_device(rx_chn->rx_chn);
540 rx_chn->desc_pool = k3_cppi_desc_pool_create_name(rx_chn->dma_dev,
541 rx_chn->descs_num,
542 hdesc_size,
543 rx_chn->name);
544 if (IS_ERR(rx_chn->desc_pool)) {
545 ret = PTR_ERR(rx_chn->desc_pool);
546 rx_chn->desc_pool = NULL;
547 netdev_err(ndev, "Failed to create rx pool: %d\n", ret);
548 goto fail;
549 }
550
551 pool = prueth_create_page_pool(emac, rx_chn->dma_dev, rx_chn->descs_num);
552 if (IS_ERR(pool)) {
553 ret = PTR_ERR(pool);
554 goto fail;
555 }
556
557 rx_chn->pg_pool = pool;
558
559 flow_id_base = k3_udma_glue_rx_get_flow_id_base(rx_chn->rx_chn);
560 if (emac->is_sr1 && !strcmp(name, "rxmgm")) {
561 emac->rx_mgm_flow_id_base = flow_id_base;
562 netdev_dbg(ndev, "mgm flow id base = %d\n", flow_id_base);
563 } else {
564 emac->rx_flow_id_base = flow_id_base;
565 netdev_dbg(ndev, "flow id base = %d\n", flow_id_base);
566 }
567
568 fdqring_id = K3_RINGACC_RING_ID_ANY;
569 for (i = 0; i < rx_cfg.flow_id_num; i++) {
570 struct k3_ring_cfg rxring_cfg = {
571 .elm_size = K3_RINGACC_RING_ELSIZE_8,
572 .mode = K3_RINGACC_RING_MODE_RING,
573 .flags = 0,
574 };
575 struct k3_ring_cfg fdqring_cfg = {
576 .elm_size = K3_RINGACC_RING_ELSIZE_8,
577 .flags = K3_RINGACC_RING_SHARED,
578 };
579 struct k3_udma_glue_rx_flow_cfg rx_flow_cfg = {
580 .rx_cfg = rxring_cfg,
581 .rxfdq_cfg = fdqring_cfg,
582 .ring_rxq_id = K3_RINGACC_RING_ID_ANY,
583 .src_tag_lo_sel =
584 K3_UDMA_GLUE_SRC_TAG_LO_USE_REMOTE_SRC_TAG,
585 };
586
587 rx_flow_cfg.ring_rxfdq0_id = fdqring_id;
588 rx_flow_cfg.rx_cfg.size = max_desc_num;
589 rx_flow_cfg.rxfdq_cfg.size = max_desc_num;
590 rx_flow_cfg.rxfdq_cfg.mode = emac->prueth->pdata.fdqring_mode;
591
592 ret = k3_udma_glue_rx_flow_init(rx_chn->rx_chn,
593 i, &rx_flow_cfg);
594 if (ret) {
595 netdev_err(ndev, "Failed to init rx flow%d %d\n",
596 i, ret);
597 goto fail;
598 }
599 if (!i)
600 fdqring_id = k3_udma_glue_rx_flow_get_fdq_id(rx_chn->rx_chn,
601 i);
602 ret = k3_udma_glue_rx_get_irq(rx_chn->rx_chn, i);
603 if (ret < 0) {
604 netdev_err(ndev, "Failed to get rx dma irq");
605 goto fail;
606 }
607 rx_chn->irq[i] = ret;
608 }
609
610 return 0;
611
612 fail:
613 prueth_cleanup_rx_chns(emac, rx_chn, max_rflows);
614 return ret;
615 }
616 EXPORT_SYMBOL_GPL(prueth_init_rx_chns);
617
prueth_dma_rx_push_mapped(struct prueth_emac * emac,struct prueth_rx_chn * rx_chn,struct page * page,u32 buf_len)618 int prueth_dma_rx_push_mapped(struct prueth_emac *emac,
619 struct prueth_rx_chn *rx_chn,
620 struct page *page, u32 buf_len)
621 {
622 struct net_device *ndev = emac->ndev;
623 struct cppi5_host_desc_t *desc_rx;
624 struct prueth_swdata *swdata;
625 dma_addr_t desc_dma;
626 dma_addr_t buf_dma;
627
628 buf_dma = page_pool_get_dma_addr(page) + PRUETH_HEADROOM;
629 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);
630 if (!desc_rx) {
631 netdev_err(ndev, "rx push: failed to allocate descriptor\n");
632 return -ENOMEM;
633 }
634 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);
635
636 cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
637 PRUETH_NAV_PS_DATA_SIZE);
638 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);
639 cppi5_hdesc_attach_buf(desc_rx, buf_dma, buf_len, buf_dma, buf_len);
640
641 swdata = cppi5_hdesc_get_swdata(desc_rx);
642 swdata->type = PRUETH_SWDATA_PAGE;
643 swdata->data.page = page;
644
645 return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, PRUETH_RX_FLOW_DATA,
646 desc_rx, desc_dma);
647 }
648 EXPORT_SYMBOL_GPL(prueth_dma_rx_push_mapped);
649
icssg_ts_to_ns(u32 hi_sw,u32 hi,u32 lo,u32 cycle_time_ns)650 u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns)
651 {
652 u32 iepcount_lo, iepcount_hi, hi_rollover_count;
653 u64 ns;
654
655 iepcount_lo = lo & GENMASK(19, 0);
656 iepcount_hi = (hi & GENMASK(11, 0)) << 12 | lo >> 20;
657 hi_rollover_count = hi >> 11;
658
659 ns = ((u64)hi_rollover_count) << 23 | (iepcount_hi + hi_sw);
660 ns = ns * cycle_time_ns + iepcount_lo;
661
662 return ns;
663 }
664 EXPORT_SYMBOL_GPL(icssg_ts_to_ns);
665
emac_rx_timestamp(struct prueth_emac * emac,struct sk_buff * skb,u32 * psdata)666 void emac_rx_timestamp(struct prueth_emac *emac,
667 struct sk_buff *skb, u32 *psdata)
668 {
669 struct skb_shared_hwtstamps *ssh;
670 u64 ns;
671
672 if (emac->is_sr1) {
673 ns = (u64)psdata[1] << 32 | psdata[0];
674 } else {
675 u32 hi_sw = readl(emac->prueth->shram.va +
676 TIMESYNC_FW_WC_COUNT_HI_SW_OFFSET_OFFSET);
677 ns = icssg_ts_to_ns(hi_sw, psdata[1], psdata[0],
678 IEP_DEFAULT_CYCLE_TIME_NS);
679 }
680
681 ssh = skb_hwtstamps(skb);
682 memset(ssh, 0, sizeof(*ssh));
683 ssh->hwtstamp = ns_to_ktime(ns);
684 }
685
686 /**
687 * emac_xmit_xdp_frame - transmits an XDP frame
688 * @emac: emac device
689 * @xdpf: data to transmit
690 * @q_idx: queue id
691 * @buff_type: Type of buffer to be transmitted
692 *
693 * Return: XDP state
694 */
emac_xmit_xdp_frame(struct prueth_emac * emac,struct xdp_frame * xdpf,unsigned int q_idx,enum prueth_tx_buff_type buff_type)695 u32 emac_xmit_xdp_frame(struct prueth_emac *emac,
696 struct xdp_frame *xdpf,
697 unsigned int q_idx,
698 enum prueth_tx_buff_type buff_type)
699 {
700 struct cppi5_host_desc_t *first_desc;
701 struct net_device *ndev = emac->ndev;
702 struct netdev_queue *netif_txq;
703 struct prueth_tx_chn *tx_chn;
704 dma_addr_t desc_dma, buf_dma;
705 struct prueth_swdata *swdata;
706 struct page *page;
707 u32 dst_tag_id;
708 u32 *epib;
709 int ret;
710
711 if (q_idx >= PRUETH_MAX_TX_QUEUES) {
712 netdev_err(ndev, "xdp tx: invalid q_id %d\n", q_idx);
713 return ICSSG_XDP_CONSUMED; /* drop */
714 }
715
716 tx_chn = &emac->tx_chns[q_idx];
717
718 first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
719 if (!first_desc) {
720 netdev_dbg(ndev, "xdp tx: failed to allocate descriptor\n");
721 return ICSSG_XDP_CONSUMED; /* drop */
722 }
723
724 if (buff_type == PRUETH_TX_BUFF_TYPE_XDP_TX) { /* already DMA mapped by page_pool */
725 page = virt_to_head_page(xdpf->data);
726 if (unlikely(!page)) {
727 netdev_err(ndev, "xdp tx: failed to get page from xdpf\n");
728 goto drop_free_descs;
729 }
730 buf_dma = page_pool_get_dma_addr(page);
731 buf_dma += xdpf->headroom + sizeof(struct xdp_frame);
732 } else { /* Map the linear buffer */
733 buf_dma = dma_map_single(tx_chn->dma_dev, xdpf->data, xdpf->len, DMA_TO_DEVICE);
734 if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) {
735 netdev_err(ndev, "xdp tx: failed to map data buffer\n");
736 goto drop_free_descs; /* drop */
737 }
738 }
739
740 cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
741 PRUETH_NAV_PS_DATA_SIZE);
742 cppi5_hdesc_set_pkttype(first_desc, 0);
743 epib = first_desc->epib;
744 epib[0] = 0;
745 epib[1] = 0;
746
747 /* set dst tag to indicate internal qid at the firmware which is at
748 * bit8..bit15. bit0..bit7 indicates port num for directed
749 * packets in case of switch mode operation and port num 0
750 * for undirected packets in case of HSR offload mode.
751 *
752 * XDP_TX frames arrive on a slave port with the HSR tag already
753 * stripped by the PRU firmware. Like skb TX via hsr0, they must
754 * be sent as undirected so the PRU duplicates them to both ports
755 * and re-inserts the HSR sequence tag.
756 */
757 dst_tag_id = emac->port_id | (q_idx << 8);
758
759 if (emac->prueth->is_hsr_offload_mode &&
760 (ndev->features & NETIF_F_HW_HSR_DUP))
761 dst_tag_id = PRUETH_UNDIRECTED_PKT_DST_TAG;
762
763 if (emac->prueth->is_hsr_offload_mode &&
764 (ndev->features & NETIF_F_HW_HSR_TAG_INS))
765 epib[1] |= PRUETH_UNDIRECTED_PKT_TAG_INS;
766
767 cppi5_desc_set_tags_ids(&first_desc->hdr, 0, dst_tag_id);
768 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
769 cppi5_hdesc_attach_buf(first_desc, buf_dma, xdpf->len, buf_dma, xdpf->len);
770 swdata = cppi5_hdesc_get_swdata(first_desc);
771 swdata->type = PRUETH_SWDATA_XDPF;
772 swdata->data.xdpf = xdpf;
773
774 /* Report BQL before sending the packet */
775 netif_txq = netdev_get_tx_queue(ndev, tx_chn->id);
776 netdev_tx_sent_queue(netif_txq, xdpf->len);
777
778 cppi5_hdesc_set_pktlen(first_desc, xdpf->len);
779 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);
780
781 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
782 if (ret) {
783 netdev_err(ndev, "xdp tx: push failed: %d\n", ret);
784 netdev_tx_completed_queue(netif_txq, 1, xdpf->len);
785 goto drop_free_descs;
786 }
787
788 return ICSSG_XDP_TX;
789
790 drop_free_descs:
791 prueth_xmit_free(tx_chn, first_desc);
792 return ICSSG_XDP_CONSUMED;
793 }
794 EXPORT_SYMBOL_GPL(emac_xmit_xdp_frame);
795
796 /**
797 * emac_run_xdp - run an XDP program
798 * @emac: emac device
799 * @xdp: XDP buffer containing the frame
800 * @len: Rx descriptor packet length
801 *
802 * Return: XDP state
803 */
emac_run_xdp(struct prueth_emac * emac,struct xdp_buff * xdp,u32 * len)804 static u32 emac_run_xdp(struct prueth_emac *emac, struct xdp_buff *xdp, u32 *len)
805 {
806 struct net_device *ndev = emac->ndev;
807 struct netdev_queue *netif_txq;
808 int cpu = smp_processor_id();
809 struct bpf_prog *xdp_prog;
810 struct xdp_frame *xdpf;
811 u32 pkt_len = *len;
812 u32 act, result;
813 int q_idx, err;
814
815 xdp_prog = READ_ONCE(emac->xdp_prog);
816 act = bpf_prog_run_xdp(xdp_prog, xdp);
817 switch (act) {
818 case XDP_PASS:
819 return ICSSG_XDP_PASS;
820 case XDP_TX:
821 /* Send packet to TX ring for immediate transmission */
822 xdpf = xdp_convert_buff_to_frame(xdp);
823 if (unlikely(!xdpf)) {
824 ndev->stats.tx_dropped++;
825 goto drop;
826 }
827
828 q_idx = cpu % emac->tx_ch_num;
829 netif_txq = netdev_get_tx_queue(ndev, q_idx);
830 __netif_tx_lock(netif_txq, cpu);
831 result = emac_xmit_xdp_frame(emac, xdpf, q_idx,
832 PRUETH_TX_BUFF_TYPE_XDP_TX);
833 __netif_tx_unlock(netif_txq);
834 if (result == ICSSG_XDP_CONSUMED) {
835 ndev->stats.tx_dropped++;
836 goto drop;
837 }
838
839 dev_sw_netstats_rx_add(ndev, xdpf->len);
840 return result;
841 case XDP_REDIRECT:
842 err = xdp_do_redirect(emac->ndev, xdp, xdp_prog);
843 if (err)
844 goto drop;
845
846 dev_sw_netstats_rx_add(ndev, pkt_len);
847 return ICSSG_XDP_REDIR;
848 default:
849 bpf_warn_invalid_xdp_action(emac->ndev, xdp_prog, act);
850 fallthrough;
851 case XDP_ABORTED:
852 drop:
853 trace_xdp_exception(emac->ndev, xdp_prog, act);
854 fallthrough; /* handle aborts by dropping packet */
855 case XDP_DROP:
856 ndev->stats.rx_dropped++;
857 return ICSSG_XDP_CONSUMED;
858 }
859 }
860
prueth_dma_rx_push_mapped_zc(struct prueth_emac * emac,struct prueth_rx_chn * rx_chn,struct xdp_buff * xdp)861 static int prueth_dma_rx_push_mapped_zc(struct prueth_emac *emac,
862 struct prueth_rx_chn *rx_chn,
863 struct xdp_buff *xdp)
864 {
865 struct net_device *ndev = emac->ndev;
866 struct cppi5_host_desc_t *desc_rx;
867 struct prueth_swdata *swdata;
868 dma_addr_t desc_dma;
869 dma_addr_t buf_dma;
870 int buf_len;
871
872 buf_dma = xsk_buff_xdp_get_dma(xdp);
873 desc_rx = k3_cppi_desc_pool_alloc(rx_chn->desc_pool);
874 if (!desc_rx) {
875 netdev_err(ndev, "rx push: failed to allocate descriptor\n");
876 return -ENOMEM;
877 }
878 desc_dma = k3_cppi_desc_pool_virt2dma(rx_chn->desc_pool, desc_rx);
879
880 cppi5_hdesc_init(desc_rx, CPPI5_INFO0_HDESC_EPIB_PRESENT,
881 PRUETH_NAV_PS_DATA_SIZE);
882 k3_udma_glue_rx_dma_to_cppi5_addr(rx_chn->rx_chn, &buf_dma);
883 buf_len = xsk_pool_get_rx_frame_size(rx_chn->xsk_pool);
884 cppi5_hdesc_attach_buf(desc_rx, buf_dma, buf_len, buf_dma, buf_len);
885 swdata = cppi5_hdesc_get_swdata(desc_rx);
886 swdata->type = PRUETH_SWDATA_XSK;
887 swdata->data.xdp = xdp;
888
889 return k3_udma_glue_push_rx_chn(rx_chn->rx_chn, PRUETH_RX_FLOW_DATA,
890 desc_rx, desc_dma);
891 }
892
prueth_rx_alloc_zc(struct prueth_emac * emac,int budget)893 static int prueth_rx_alloc_zc(struct prueth_emac *emac, int budget)
894 {
895 struct prueth_rx_chn *rx_chn = &emac->rx_chns;
896 struct xdp_buff *xdp;
897 int i, ret;
898
899 for (i = 0; i < budget; i++) {
900 xdp = xsk_buff_alloc(rx_chn->xsk_pool);
901 if (!xdp)
902 break;
903
904 ret = prueth_dma_rx_push_mapped_zc(emac, rx_chn, xdp);
905 if (ret) {
906 netdev_err(emac->ndev, "rx alloc: failed to map descriptors to xdp buff\n");
907 xsk_buff_free(xdp);
908 break;
909 }
910 }
911
912 return i;
913 }
914
emac_dispatch_skb_zc(struct prueth_emac * emac,struct xdp_buff * xdp,u32 * psdata)915 static void emac_dispatch_skb_zc(struct prueth_emac *emac, struct xdp_buff *xdp, u32 *psdata)
916 {
917 unsigned int headroom = xdp->data - xdp->data_hard_start;
918 unsigned int pkt_len = xdp->data_end - xdp->data;
919 struct net_device *ndev = emac->ndev;
920 struct sk_buff *skb;
921
922 skb = napi_alloc_skb(&emac->napi_rx, xdp->data_end - xdp->data_hard_start);
923 if (unlikely(!skb)) {
924 ndev->stats.rx_dropped++;
925 return;
926 }
927
928 skb_reserve(skb, headroom);
929 skb_put(skb, pkt_len);
930 skb_copy_to_linear_data(skb, xdp->data, pkt_len);
931 skb->dev = ndev;
932
933 /* RX HW timestamp */
934 if (emac->rx_ts_enabled)
935 emac_rx_timestamp(emac, skb, psdata);
936
937 if (emac->prueth->is_switch_mode)
938 skb->offload_fwd_mark = emac->offload_fwd_mark;
939 skb->protocol = eth_type_trans(skb, ndev);
940
941 napi_gro_receive(&emac->napi_rx, skb);
942 ndev->stats.rx_bytes += pkt_len;
943 ndev->stats.rx_packets++;
944 }
945
emac_rx_packet_zc(struct prueth_emac * emac,u32 flow_id,int budget)946 static int emac_rx_packet_zc(struct prueth_emac *emac, u32 flow_id,
947 int budget)
948 {
949 struct prueth_rx_chn *rx_chn = &emac->rx_chns;
950 u32 buf_dma_len, pkt_len, port_id = 0;
951 struct net_device *ndev = emac->ndev;
952 struct cppi5_host_desc_t *desc_rx;
953 struct prueth_swdata *swdata;
954 dma_addr_t desc_dma, buf_dma;
955 int avail_desc, alloc_budget;
956 struct xdp_buff *xdp;
957 int xdp_status = 0;
958 int count = 0;
959 u32 *psdata;
960 int ret;
961
962 while (count < budget) {
963 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma);
964 if (ret) {
965 if (ret != -ENODATA)
966 netdev_err(ndev, "rx pop: failed: %d\n", ret);
967 break;
968 }
969
970 if (cppi5_desc_is_tdcm(desc_dma)) {
971 complete(&emac->tdown_complete);
972 break;
973 }
974
975 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
976 swdata = cppi5_hdesc_get_swdata(desc_rx);
977 if (swdata->type != PRUETH_SWDATA_XSK) {
978 netdev_err(ndev, "rx_pkt: invalid swdata->type %d\n", swdata->type);
979 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
980 break;
981 }
982
983 xdp = swdata->data.xdp;
984 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
985 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
986 pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
987 /* firmware adds 4 CRC bytes, strip them */
988 pkt_len -= 4;
989 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
990 psdata = cppi5_hdesc_get_psdata(desc_rx);
991 count++;
992 xsk_buff_set_size(xdp, pkt_len);
993 xsk_buff_dma_sync_for_cpu(xdp);
994
995 if (prueth_xdp_is_enabled(emac)) {
996 ret = emac_run_xdp(emac, xdp, &pkt_len);
997 switch (ret) {
998 case ICSSG_XDP_PASS:
999 /* prepare skb and send to n/w stack */
1000 emac_dispatch_skb_zc(emac, xdp, psdata);
1001 xsk_buff_free(xdp);
1002 break;
1003 case ICSSG_XDP_CONSUMED:
1004 xsk_buff_free(xdp);
1005 break;
1006 case ICSSG_XDP_TX:
1007 case ICSSG_XDP_REDIR:
1008 xdp_status |= ret;
1009 break;
1010 }
1011 } else {
1012 /* prepare skb and send to n/w stack */
1013 emac_dispatch_skb_zc(emac, xdp, psdata);
1014 xsk_buff_free(xdp);
1015 }
1016 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
1017 }
1018
1019 if (xdp_status & ICSSG_XDP_REDIR)
1020 xdp_do_flush();
1021
1022 avail_desc = k3_cppi_desc_pool_avail(rx_chn->desc_pool);
1023 alloc_budget = min_t(int, budget, avail_desc);
1024
1025 ret = prueth_rx_alloc_zc(emac, alloc_budget);
1026
1027 if (xsk_uses_need_wakeup(rx_chn->xsk_pool)) {
1028 if (ret < alloc_budget)
1029 xsk_set_rx_need_wakeup(rx_chn->xsk_pool);
1030 else
1031 xsk_clear_rx_need_wakeup(rx_chn->xsk_pool);
1032 }
1033
1034 return count;
1035 }
1036
emac_rx_packet(struct prueth_emac * emac,u32 flow_id,u32 * xdp_state)1037 static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state)
1038 {
1039 struct prueth_rx_chn *rx_chn = &emac->rx_chns;
1040 u32 buf_dma_len, pkt_len, port_id = 0;
1041 struct net_device *ndev = emac->ndev;
1042 struct cppi5_host_desc_t *desc_rx;
1043 struct prueth_swdata *swdata;
1044 dma_addr_t desc_dma, buf_dma;
1045 struct page *page, *new_page;
1046 struct page_pool *pool;
1047 struct sk_buff *skb;
1048 struct xdp_buff xdp;
1049 int headroom, ret;
1050 u32 *psdata;
1051 void *pa;
1052
1053 *xdp_state = 0;
1054 pool = rx_chn->pg_pool;
1055 ret = k3_udma_glue_pop_rx_chn(rx_chn->rx_chn, flow_id, &desc_dma);
1056 if (ret) {
1057 if (ret != -ENODATA)
1058 netdev_err(ndev, "rx pop: failed: %d\n", ret);
1059 return ret;
1060 }
1061
1062 if (cppi5_desc_is_tdcm(desc_dma)) {
1063 complete(&emac->tdown_complete);
1064 return 0;
1065 }
1066
1067 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
1068 swdata = cppi5_hdesc_get_swdata(desc_rx);
1069 if (swdata->type != PRUETH_SWDATA_PAGE) {
1070 netdev_err(ndev, "rx_pkt: invalid swdata->type %d\n", swdata->type);
1071 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
1072 return 0;
1073 }
1074
1075 page = swdata->data.page;
1076 page_pool_dma_sync_for_cpu(pool, page, 0, PAGE_SIZE);
1077 cppi5_hdesc_get_obuf(desc_rx, &buf_dma, &buf_dma_len);
1078 k3_udma_glue_rx_cppi5_to_dma_addr(rx_chn->rx_chn, &buf_dma);
1079 pkt_len = cppi5_hdesc_get_pktlen(desc_rx);
1080 /* firmware adds 4 CRC bytes, strip them */
1081 pkt_len -= 4;
1082 cppi5_desc_get_tags_ids(&desc_rx->hdr, &port_id, NULL);
1083
1084 /* if allocation fails we drop the packet but push the
1085 * descriptor back to the ring with old page to prevent a stall
1086 */
1087 new_page = page_pool_dev_alloc_pages(pool);
1088 if (unlikely(!new_page)) {
1089 new_page = page;
1090 ndev->stats.rx_dropped++;
1091 goto requeue;
1092 }
1093
1094 pa = page_address(page);
1095 if (prueth_xdp_is_enabled(emac)) {
1096 xdp_init_buff(&xdp, PAGE_SIZE, &rx_chn->xdp_rxq);
1097 xdp_prepare_buff(&xdp, pa, PRUETH_HEADROOM, pkt_len, false);
1098
1099 *xdp_state = emac_run_xdp(emac, &xdp, &pkt_len);
1100 if (*xdp_state == ICSSG_XDP_CONSUMED) {
1101 page_pool_recycle_direct(pool, page);
1102 goto requeue;
1103 }
1104
1105 if (*xdp_state != ICSSG_XDP_PASS)
1106 goto requeue;
1107 headroom = xdp.data - xdp.data_hard_start;
1108 pkt_len = xdp.data_end - xdp.data;
1109 } else {
1110 headroom = PRUETH_HEADROOM;
1111 }
1112
1113 /* prepare skb and send to n/w stack */
1114 skb = napi_build_skb(pa, PAGE_SIZE);
1115 if (!skb) {
1116 ndev->stats.rx_dropped++;
1117 page_pool_recycle_direct(pool, page);
1118 goto requeue;
1119 }
1120
1121 skb_reserve(skb, headroom);
1122 skb_put(skb, pkt_len);
1123 skb->dev = ndev;
1124
1125 psdata = cppi5_hdesc_get_psdata(desc_rx);
1126 /* RX HW timestamp */
1127 if (emac->rx_ts_enabled)
1128 emac_rx_timestamp(emac, skb, psdata);
1129
1130 if (emac->prueth->is_switch_mode)
1131 skb->offload_fwd_mark = emac->offload_fwd_mark;
1132 skb->protocol = eth_type_trans(skb, ndev);
1133
1134 skb_mark_for_recycle(skb);
1135 napi_gro_receive(&emac->napi_rx, skb);
1136 ndev->stats.rx_bytes += pkt_len;
1137 ndev->stats.rx_packets++;
1138
1139 requeue:
1140 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
1141 /* queue another RX DMA */
1142 ret = prueth_dma_rx_push_mapped(emac, &emac->rx_chns, new_page,
1143 PRUETH_MAX_PKT_SIZE);
1144 if (WARN_ON(ret < 0)) {
1145 page_pool_recycle_direct(pool, new_page);
1146 ndev->stats.rx_errors++;
1147 ndev->stats.rx_dropped++;
1148 }
1149
1150 return ret;
1151 }
1152
prueth_rx_cleanup(void * data,dma_addr_t desc_dma)1153 void prueth_rx_cleanup(void *data, dma_addr_t desc_dma)
1154 {
1155 struct prueth_rx_chn *rx_chn = data;
1156 struct cppi5_host_desc_t *desc_rx;
1157 struct prueth_swdata *swdata;
1158 struct page_pool *pool;
1159 struct xdp_buff *xdp;
1160 struct page *page;
1161
1162 pool = rx_chn->pg_pool;
1163 desc_rx = k3_cppi_desc_pool_dma2virt(rx_chn->desc_pool, desc_dma);
1164 swdata = cppi5_hdesc_get_swdata(desc_rx);
1165 if (rx_chn->xsk_pool) {
1166 xdp = swdata->data.xdp;
1167 xsk_buff_free(xdp);
1168 } else {
1169 page = swdata->data.page;
1170 page_pool_recycle_direct(pool, page);
1171 }
1172
1173 k3_cppi_desc_pool_free(rx_chn->desc_pool, desc_rx);
1174 }
1175 EXPORT_SYMBOL_GPL(prueth_rx_cleanup);
1176
prueth_tx_ts_cookie_get(struct prueth_emac * emac)1177 static int prueth_tx_ts_cookie_get(struct prueth_emac *emac)
1178 {
1179 int i;
1180
1181 /* search and get the next free slot */
1182 for (i = 0; i < PRUETH_MAX_TX_TS_REQUESTS; i++) {
1183 if (!emac->tx_ts_skb[i]) {
1184 emac->tx_ts_skb[i] = ERR_PTR(-EBUSY); /* reserve slot */
1185 return i;
1186 }
1187 }
1188
1189 return -EBUSY;
1190 }
1191
1192 /**
1193 * icssg_ndo_start_xmit - EMAC Transmit function
1194 * @skb: SKB pointer
1195 * @ndev: EMAC network adapter
1196 *
1197 * Called by the system to transmit a packet - we queue the packet in
1198 * EMAC hardware transmit queue
1199 * Doesn't wait for completion we'll check for TX completion in
1200 * emac_tx_complete_packets().
1201 *
1202 * Return: enum netdev_tx
1203 */
icssg_ndo_start_xmit(struct sk_buff * skb,struct net_device * ndev)1204 enum netdev_tx icssg_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev)
1205 {
1206 struct cppi5_host_desc_t *first_desc, *next_desc, *cur_desc;
1207 struct prueth_emac *emac = netdev_priv(ndev);
1208 struct prueth *prueth = emac->prueth;
1209 struct netdev_queue *netif_txq;
1210 struct prueth_swdata *swdata;
1211 struct prueth_tx_chn *tx_chn;
1212 dma_addr_t desc_dma, buf_dma;
1213 u32 pkt_len, dst_tag_id;
1214 int i, ret = 0, q_idx;
1215 bool in_tx_ts = 0;
1216 int tx_ts_cookie;
1217 u32 *epib;
1218
1219 pkt_len = skb_headlen(skb);
1220 q_idx = skb_get_queue_mapping(skb);
1221
1222 tx_chn = &emac->tx_chns[q_idx];
1223 netif_txq = netdev_get_tx_queue(ndev, q_idx);
1224
1225 /* Map the linear buffer */
1226 buf_dma = dma_map_single(tx_chn->dma_dev, skb->data, pkt_len, DMA_TO_DEVICE);
1227 if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) {
1228 netdev_err(ndev, "tx: failed to map skb buffer\n");
1229 ret = NETDEV_TX_OK;
1230 goto drop_free_skb;
1231 }
1232
1233 first_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1234 if (!first_desc) {
1235 netdev_dbg(ndev, "tx: failed to allocate descriptor\n");
1236 dma_unmap_single(tx_chn->dma_dev, buf_dma, pkt_len, DMA_TO_DEVICE);
1237 goto drop_stop_q_busy;
1238 }
1239
1240 cppi5_hdesc_init(first_desc, CPPI5_INFO0_HDESC_EPIB_PRESENT,
1241 PRUETH_NAV_PS_DATA_SIZE);
1242 cppi5_hdesc_set_pkttype(first_desc, 0);
1243 epib = first_desc->epib;
1244 epib[0] = 0;
1245 epib[1] = 0;
1246 if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP &&
1247 emac->tx_ts_enabled) {
1248 tx_ts_cookie = prueth_tx_ts_cookie_get(emac);
1249 if (tx_ts_cookie >= 0) {
1250 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1251 /* Request TX timestamp */
1252 epib[0] = (u32)tx_ts_cookie;
1253 epib[1] = 0x80000000; /* TX TS request */
1254 emac->tx_ts_skb[tx_ts_cookie] = skb_get(skb);
1255 in_tx_ts = 1;
1256 }
1257 }
1258
1259 /* set dst tag to indicate internal qid at the firmware which is at
1260 * bit8..bit15. bit0..bit7 indicates port num for directed
1261 * packets in case of switch mode operation and port num 0
1262 * for undirected packets in case of HSR offload mode
1263 */
1264 dst_tag_id = emac->port_id | (q_idx << 8);
1265
1266 if (prueth->is_hsr_offload_mode &&
1267 (ndev->features & NETIF_F_HW_HSR_DUP))
1268 dst_tag_id = PRUETH_UNDIRECTED_PKT_DST_TAG;
1269
1270 if (prueth->is_hsr_offload_mode &&
1271 (ndev->features & NETIF_F_HW_HSR_TAG_INS))
1272 epib[1] |= PRUETH_UNDIRECTED_PKT_TAG_INS;
1273
1274 cppi5_desc_set_tags_ids(&first_desc->hdr, 0, dst_tag_id);
1275 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1276 cppi5_hdesc_attach_buf(first_desc, buf_dma, pkt_len, buf_dma, pkt_len);
1277 swdata = cppi5_hdesc_get_swdata(first_desc);
1278 swdata->type = PRUETH_SWDATA_SKB;
1279 swdata->data.skb = skb;
1280
1281 /* Handle the case where skb is fragmented in pages */
1282 cur_desc = first_desc;
1283 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1284 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1285 u32 frag_size = skb_frag_size(frag);
1286
1287 next_desc = k3_cppi_desc_pool_alloc(tx_chn->desc_pool);
1288 if (!next_desc) {
1289 netdev_err(ndev,
1290 "tx: failed to allocate frag. descriptor\n");
1291 goto free_desc_stop_q_busy_cleanup_tx_ts;
1292 }
1293
1294 buf_dma = skb_frag_dma_map(tx_chn->dma_dev, frag, 0, frag_size,
1295 DMA_TO_DEVICE);
1296 if (dma_mapping_error(tx_chn->dma_dev, buf_dma)) {
1297 netdev_err(ndev, "tx: Failed to map skb page\n");
1298 k3_cppi_desc_pool_free(tx_chn->desc_pool, next_desc);
1299 ret = NETDEV_TX_OK;
1300 goto cleanup_tx_ts;
1301 }
1302
1303 cppi5_hdesc_reset_hbdesc(next_desc);
1304 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &buf_dma);
1305 cppi5_hdesc_attach_buf(next_desc,
1306 buf_dma, frag_size, buf_dma, frag_size);
1307
1308 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool,
1309 next_desc);
1310 k3_udma_glue_tx_dma_to_cppi5_addr(tx_chn->tx_chn, &desc_dma);
1311 cppi5_hdesc_link_hbdesc(cur_desc, desc_dma);
1312
1313 pkt_len += frag_size;
1314 cur_desc = next_desc;
1315 }
1316 WARN_ON_ONCE(pkt_len != skb->len);
1317
1318 /* report bql before sending packet */
1319 netdev_tx_sent_queue(netif_txq, pkt_len);
1320
1321 cppi5_hdesc_set_pktlen(first_desc, pkt_len);
1322 desc_dma = k3_cppi_desc_pool_virt2dma(tx_chn->desc_pool, first_desc);
1323 /* cppi5_desc_dump(first_desc, 64); */
1324
1325 skb_tx_timestamp(skb); /* SW timestamp if SKBTX_IN_PROGRESS not set */
1326 ret = k3_udma_glue_push_tx_chn(tx_chn->tx_chn, first_desc, desc_dma);
1327 if (ret) {
1328 netdev_err(ndev, "tx: push failed: %d\n", ret);
1329 netdev_tx_completed_queue(netif_txq, 1, pkt_len);
1330 goto drop_free_descs;
1331 }
1332
1333 if (in_tx_ts)
1334 atomic_inc(&emac->tx_ts_pending);
1335
1336 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) < MAX_SKB_FRAGS) {
1337 netif_tx_stop_queue(netif_txq);
1338 /* Barrier, so that stop_queue visible to other cpus */
1339 smp_mb__after_atomic();
1340
1341 if (k3_cppi_desc_pool_avail(tx_chn->desc_pool) >=
1342 MAX_SKB_FRAGS)
1343 netif_tx_wake_queue(netif_txq);
1344 }
1345
1346 return NETDEV_TX_OK;
1347
1348 cleanup_tx_ts:
1349 if (in_tx_ts) {
1350 dev_kfree_skb_any(emac->tx_ts_skb[tx_ts_cookie]);
1351 emac->tx_ts_skb[tx_ts_cookie] = NULL;
1352 }
1353
1354 drop_free_descs:
1355 prueth_xmit_free(tx_chn, first_desc);
1356
1357 drop_free_skb:
1358 dev_kfree_skb_any(skb);
1359
1360 /* error */
1361 ndev->stats.tx_dropped++;
1362 netdev_err(ndev, "tx: error: %d\n", ret);
1363
1364 return ret;
1365
1366 free_desc_stop_q_busy_cleanup_tx_ts:
1367 if (in_tx_ts) {
1368 dev_kfree_skb_any(emac->tx_ts_skb[tx_ts_cookie]);
1369 emac->tx_ts_skb[tx_ts_cookie] = NULL;
1370 }
1371 prueth_xmit_free(tx_chn, first_desc);
1372
1373 drop_stop_q_busy:
1374 netif_tx_stop_queue(netif_txq);
1375 return NETDEV_TX_BUSY;
1376 }
1377 EXPORT_SYMBOL_GPL(icssg_ndo_start_xmit);
1378
prueth_tx_cleanup(void * data,dma_addr_t desc_dma)1379 void prueth_tx_cleanup(void *data, dma_addr_t desc_dma)
1380 {
1381 struct prueth_tx_chn *tx_chn = data;
1382 struct cppi5_host_desc_t *desc_tx;
1383 struct xsk_buff_pool *xsk_pool;
1384 struct prueth_swdata *swdata;
1385 struct xdp_frame *xdpf;
1386 struct sk_buff *skb;
1387
1388 desc_tx = k3_cppi_desc_pool_dma2virt(tx_chn->desc_pool, desc_dma);
1389 swdata = cppi5_hdesc_get_swdata(desc_tx);
1390
1391 switch (swdata->type) {
1392 case PRUETH_SWDATA_SKB:
1393 skb = swdata->data.skb;
1394 dev_kfree_skb_any(skb);
1395 break;
1396 case PRUETH_SWDATA_XDPF:
1397 xdpf = swdata->data.xdpf;
1398 xdp_return_frame(xdpf);
1399 break;
1400 case PRUETH_SWDATA_XSK:
1401 xsk_pool = tx_chn->xsk_pool;
1402 xsk_tx_completed(xsk_pool, 1);
1403 break;
1404 default:
1405 break;
1406 }
1407
1408 prueth_xmit_free(tx_chn, desc_tx);
1409 }
1410 EXPORT_SYMBOL_GPL(prueth_tx_cleanup);
1411
prueth_rx_irq(int irq,void * dev_id)1412 irqreturn_t prueth_rx_irq(int irq, void *dev_id)
1413 {
1414 struct prueth_emac *emac = dev_id;
1415
1416 emac->rx_chns.irq_disabled = true;
1417 disable_irq_nosync(irq);
1418 napi_schedule(&emac->napi_rx);
1419
1420 return IRQ_HANDLED;
1421 }
1422 EXPORT_SYMBOL_GPL(prueth_rx_irq);
1423
prueth_cleanup_tx_ts(struct prueth_emac * emac)1424 void prueth_cleanup_tx_ts(struct prueth_emac *emac)
1425 {
1426 int i;
1427
1428 for (i = 0; i < PRUETH_MAX_TX_TS_REQUESTS; i++) {
1429 if (emac->tx_ts_skb[i]) {
1430 dev_kfree_skb_any(emac->tx_ts_skb[i]);
1431 emac->tx_ts_skb[i] = NULL;
1432 }
1433 }
1434 }
1435 EXPORT_SYMBOL_GPL(prueth_cleanup_tx_ts);
1436
icssg_napi_rx_poll(struct napi_struct * napi_rx,int budget)1437 int icssg_napi_rx_poll(struct napi_struct *napi_rx, int budget)
1438 {
1439 struct prueth_emac *emac = prueth_napi_to_emac(napi_rx);
1440 int rx_flow = emac->is_sr1 ?
1441 PRUETH_RX_FLOW_DATA_SR1 : PRUETH_RX_FLOW_DATA;
1442 int flow = emac->is_sr1 ?
1443 PRUETH_MAX_RX_FLOWS_SR1 : PRUETH_MAX_RX_FLOWS;
1444 struct prueth_rx_chn *rx_chn = &emac->rx_chns;
1445 int xdp_state_or = 0;
1446 int num_rx = 0;
1447 int cur_budget;
1448 u32 xdp_state;
1449 int ret;
1450
1451 while (flow--) {
1452 if (rx_chn->xsk_pool) {
1453 num_rx = emac_rx_packet_zc(emac, flow, budget);
1454 } else {
1455 cur_budget = budget - num_rx;
1456
1457 while (cur_budget--) {
1458 ret = emac_rx_packet(emac, flow, &xdp_state);
1459 xdp_state_or |= xdp_state;
1460 if (ret)
1461 break;
1462 num_rx++;
1463 }
1464 }
1465
1466 if (num_rx >= budget)
1467 break;
1468 }
1469
1470 if (xdp_state_or & ICSSG_XDP_REDIR)
1471 xdp_do_flush();
1472
1473 if (num_rx < budget && napi_complete_done(napi_rx, num_rx)) {
1474 if (unlikely(emac->rx_pace_timeout_ns)) {
1475 hrtimer_start(&emac->rx_hrtimer,
1476 ns_to_ktime(emac->rx_pace_timeout_ns),
1477 HRTIMER_MODE_REL_PINNED);
1478 } else {
1479 if (emac->rx_chns.irq_disabled) {
1480 /* re-enable the RX IRQ */
1481 emac->rx_chns.irq_disabled = false;
1482 enable_irq(emac->rx_chns.irq[rx_flow]);
1483 }
1484 }
1485 }
1486
1487 return num_rx;
1488 }
1489 EXPORT_SYMBOL_GPL(icssg_napi_rx_poll);
1490
prueth_prepare_rx_chan(struct prueth_emac * emac,struct prueth_rx_chn * chn,int buf_size)1491 int prueth_prepare_rx_chan(struct prueth_emac *emac,
1492 struct prueth_rx_chn *chn,
1493 int buf_size)
1494 {
1495 struct page *page;
1496 int desc_avail;
1497 int i, ret;
1498
1499 desc_avail = k3_cppi_desc_pool_avail(chn->desc_pool);
1500 if (desc_avail < chn->descs_num)
1501 netdev_warn(emac->ndev,
1502 "not enough RX descriptors available %d < %d\n",
1503 desc_avail, chn->descs_num);
1504
1505 if (chn->xsk_pool) {
1506 /* get pages from xsk_pool and push to RX ring
1507 * queue as much as possible
1508 */
1509 ret = prueth_rx_alloc_zc(emac, desc_avail);
1510 if (!ret)
1511 goto recycle_alloc_pg;
1512 } else {
1513 for (i = 0; i < desc_avail; i++) {
1514 /* NOTE: we're not using memory efficiently here.
1515 * 1 full page (4KB?) used here instead of
1516 * PRUETH_MAX_PKT_SIZE (~1.5KB?)
1517 */
1518 page = page_pool_dev_alloc_pages(chn->pg_pool);
1519 if (!page) {
1520 netdev_err(emac->ndev, "couldn't allocate rx page\n");
1521 ret = -ENOMEM;
1522 goto recycle_alloc_pg;
1523 }
1524
1525 ret = prueth_dma_rx_push_mapped(emac, chn, page, buf_size);
1526 if (ret < 0) {
1527 netdev_err(emac->ndev,
1528 "cannot submit page for rx chan %s ret %d\n",
1529 chn->name, ret);
1530 page_pool_recycle_direct(chn->pg_pool, page);
1531 goto recycle_alloc_pg;
1532 }
1533 }
1534 }
1535
1536 return 0;
1537
1538 recycle_alloc_pg:
1539 prueth_reset_rx_chan(&emac->rx_chns, PRUETH_MAX_RX_FLOWS, false);
1540
1541 return ret;
1542 }
1543 EXPORT_SYMBOL_GPL(prueth_prepare_rx_chan);
1544
prueth_reset_tx_chan(struct prueth_emac * emac,int ch_num,bool free_skb)1545 void prueth_reset_tx_chan(struct prueth_emac *emac, int ch_num,
1546 bool free_skb)
1547 {
1548 int i;
1549
1550 for (i = 0; i < ch_num; i++) {
1551 if (free_skb)
1552 k3_udma_glue_reset_tx_chn(emac->tx_chns[i].tx_chn,
1553 &emac->tx_chns[i],
1554 prueth_tx_cleanup);
1555 k3_udma_glue_disable_tx_chn(emac->tx_chns[i].tx_chn);
1556 }
1557 }
1558 EXPORT_SYMBOL_GPL(prueth_reset_tx_chan);
1559
prueth_reset_rx_chan(struct prueth_rx_chn * chn,int num_flows,bool disable)1560 void prueth_reset_rx_chan(struct prueth_rx_chn *chn,
1561 int num_flows, bool disable)
1562 {
1563 int i;
1564
1565 for (i = 0; i < num_flows; i++)
1566 k3_udma_glue_reset_rx_chn(chn->rx_chn, i, chn,
1567 prueth_rx_cleanup);
1568 if (disable)
1569 k3_udma_glue_disable_rx_chn(chn->rx_chn);
1570 }
1571 EXPORT_SYMBOL_GPL(prueth_reset_rx_chan);
1572
icssg_ndo_tx_timeout(struct net_device * ndev,unsigned int txqueue)1573 void icssg_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue)
1574 {
1575 ndev->stats.tx_errors++;
1576 }
1577 EXPORT_SYMBOL_GPL(icssg_ndo_tx_timeout);
1578
icssg_ndo_set_ts_config(struct net_device * ndev,struct kernel_hwtstamp_config * config,struct netlink_ext_ack * extack)1579 int icssg_ndo_set_ts_config(struct net_device *ndev,
1580 struct kernel_hwtstamp_config *config,
1581 struct netlink_ext_ack *extack)
1582 {
1583 struct prueth_emac *emac = netdev_priv(ndev);
1584
1585 switch (config->tx_type) {
1586 case HWTSTAMP_TX_OFF:
1587 emac->tx_ts_enabled = 0;
1588 break;
1589 case HWTSTAMP_TX_ON:
1590 emac->tx_ts_enabled = 1;
1591 break;
1592 default:
1593 return -ERANGE;
1594 }
1595
1596 switch (config->rx_filter) {
1597 case HWTSTAMP_FILTER_NONE:
1598 emac->rx_ts_enabled = 0;
1599 break;
1600 case HWTSTAMP_FILTER_ALL:
1601 case HWTSTAMP_FILTER_SOME:
1602 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1603 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1604 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1605 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1606 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1607 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1608 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1609 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1610 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1611 case HWTSTAMP_FILTER_PTP_V2_EVENT:
1612 case HWTSTAMP_FILTER_PTP_V2_SYNC:
1613 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1614 case HWTSTAMP_FILTER_NTP_ALL:
1615 emac->rx_ts_enabled = 1;
1616 config->rx_filter = HWTSTAMP_FILTER_ALL;
1617 break;
1618 default:
1619 return -ERANGE;
1620 }
1621
1622 return 0;
1623 }
1624 EXPORT_SYMBOL_GPL(icssg_ndo_set_ts_config);
1625
icssg_ndo_get_ts_config(struct net_device * ndev,struct kernel_hwtstamp_config * config)1626 int icssg_ndo_get_ts_config(struct net_device *ndev,
1627 struct kernel_hwtstamp_config *config)
1628 {
1629 struct prueth_emac *emac = netdev_priv(ndev);
1630
1631 config->flags = 0;
1632 config->tx_type = emac->tx_ts_enabled ? HWTSTAMP_TX_ON : HWTSTAMP_TX_OFF;
1633 config->rx_filter = emac->rx_ts_enabled ? HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1634
1635 return 0;
1636 }
1637 EXPORT_SYMBOL_GPL(icssg_ndo_get_ts_config);
1638
icssg_ndo_get_stats64(struct net_device * ndev,struct rtnl_link_stats64 * stats)1639 void icssg_ndo_get_stats64(struct net_device *ndev,
1640 struct rtnl_link_stats64 *stats)
1641 {
1642 struct prueth_emac *emac = netdev_priv(ndev);
1643
1644 emac_update_hardware_stats(emac);
1645
1646 stats->rx_packets = emac_get_stat_by_name(emac, "rx_packets");
1647 stats->rx_bytes = emac_get_stat_by_name(emac, "rx_bytes");
1648 stats->tx_packets = emac_get_stat_by_name(emac, "tx_packets");
1649 stats->tx_bytes = emac_get_stat_by_name(emac, "tx_bytes");
1650 stats->rx_crc_errors = emac_get_stat_by_name(emac, "rx_crc_errors");
1651 stats->rx_over_errors = emac_get_stat_by_name(emac, "rx_over_errors");
1652 stats->multicast = emac_get_stat_by_name(emac, "rx_multicast_frames");
1653
1654 stats->rx_errors = ndev->stats.rx_errors;
1655 stats->rx_dropped = ndev->stats.rx_dropped;
1656 stats->tx_errors = ndev->stats.tx_errors;
1657 stats->tx_dropped = ndev->stats.tx_dropped;
1658
1659 if (!emac->prueth->pa_stats)
1660 return;
1661
1662 stats->rx_errors +=
1663 emac_get_stat_by_name(emac, "FW_RX_ERROR") +
1664 emac_get_stat_by_name(emac, "FW_RX_EOF_SHORT_FRMERR") +
1665 emac_get_stat_by_name(emac, "FW_RX_B0_DROP_EARLY_EOF") +
1666 emac_get_stat_by_name(emac, "FW_RX_EXP_FRAG_Q_DROP") +
1667 emac_get_stat_by_name(emac, "FW_RX_FIFO_OVERRUN");
1668 stats->rx_dropped +=
1669 emac_get_stat_by_name(emac, "FW_DROPPED_PKT") +
1670 emac_get_stat_by_name(emac, "FW_INF_PORT_DISABLED") +
1671 emac_get_stat_by_name(emac, "FW_INF_SAV") +
1672 emac_get_stat_by_name(emac, "FW_INF_SA_DL") +
1673 emac_get_stat_by_name(emac, "FW_INF_PORT_BLOCKED") +
1674 emac_get_stat_by_name(emac, "FW_INF_DROP_TAGGED") +
1675 emac_get_stat_by_name(emac, "FW_INF_DROP_PRIOTAGGED") +
1676 emac_get_stat_by_name(emac, "FW_INF_DROP_NOTAG") +
1677 emac_get_stat_by_name(emac, "FW_INF_DROP_NOTMEMBER");
1678 stats->tx_dropped +=
1679 emac_get_stat_by_name(emac, "FW_RTU_PKT_DROP") +
1680 emac_get_stat_by_name(emac, "FW_TX_DROPPED_PACKET") +
1681 emac_get_stat_by_name(emac, "FW_TX_TS_DROPPED_PACKET") +
1682 emac_get_stat_by_name(emac, "FW_TX_JUMBO_FRM_CUTOFF");
1683 }
1684 EXPORT_SYMBOL_GPL(icssg_ndo_get_stats64);
1685
icssg_ndo_get_phys_port_name(struct net_device * ndev,char * name,size_t len)1686 int icssg_ndo_get_phys_port_name(struct net_device *ndev, char *name,
1687 size_t len)
1688 {
1689 struct prueth_emac *emac = netdev_priv(ndev);
1690 int ret;
1691
1692 ret = snprintf(name, len, "p%d", emac->port_id);
1693 if (ret >= len)
1694 return -EINVAL;
1695
1696 return 0;
1697 }
1698 EXPORT_SYMBOL_GPL(icssg_ndo_get_phys_port_name);
1699
1700 /* get emac_port corresponding to eth_node name */
prueth_node_port(struct device_node * eth_node)1701 int prueth_node_port(struct device_node *eth_node)
1702 {
1703 u32 port_id;
1704 int ret;
1705
1706 ret = of_property_read_u32(eth_node, "reg", &port_id);
1707 if (ret)
1708 return ret;
1709
1710 if (port_id == 0)
1711 return PRUETH_PORT_MII0;
1712 else if (port_id == 1)
1713 return PRUETH_PORT_MII1;
1714 else
1715 return PRUETH_PORT_INVALID;
1716 }
1717 EXPORT_SYMBOL_GPL(prueth_node_port);
1718
1719 /* get MAC instance corresponding to eth_node name */
prueth_node_mac(struct device_node * eth_node)1720 int prueth_node_mac(struct device_node *eth_node)
1721 {
1722 u32 port_id;
1723 int ret;
1724
1725 ret = of_property_read_u32(eth_node, "reg", &port_id);
1726 if (ret)
1727 return ret;
1728
1729 if (port_id == 0)
1730 return PRUETH_MAC0;
1731 else if (port_id == 1)
1732 return PRUETH_MAC1;
1733 else
1734 return PRUETH_MAC_INVALID;
1735 }
1736 EXPORT_SYMBOL_GPL(prueth_node_mac);
1737
prueth_netdev_exit(struct prueth * prueth,struct device_node * eth_node)1738 void prueth_netdev_exit(struct prueth *prueth,
1739 struct device_node *eth_node)
1740 {
1741 struct prueth_emac *emac;
1742 enum prueth_mac mac;
1743
1744 mac = prueth_node_mac(eth_node);
1745 if (mac == PRUETH_MAC_INVALID)
1746 return;
1747
1748 emac = prueth->emac[mac];
1749 if (!emac)
1750 return;
1751
1752 if (of_phy_is_fixed_link(emac->phy_node))
1753 of_phy_deregister_fixed_link(emac->phy_node);
1754
1755 netif_napi_del(&emac->napi_rx);
1756
1757 pruss_release_mem_region(prueth->pruss, &emac->dram);
1758 free_netdev(emac->ndev);
1759 prueth->emac[mac] = NULL;
1760 }
1761 EXPORT_SYMBOL_GPL(prueth_netdev_exit);
1762
prueth_get_cores(struct prueth * prueth,int slice,bool is_sr1)1763 int prueth_get_cores(struct prueth *prueth, int slice, bool is_sr1)
1764 {
1765 struct device *dev = prueth->dev;
1766 enum pruss_pru_id pruss_id;
1767 struct device_node *np;
1768 int idx = -1, ret;
1769
1770 np = dev->of_node;
1771
1772 switch (slice) {
1773 case ICSS_SLICE0:
1774 idx = 0;
1775 break;
1776 case ICSS_SLICE1:
1777 idx = is_sr1 ? 2 : 3;
1778 break;
1779 default:
1780 return -EINVAL;
1781 }
1782
1783 prueth->pru[slice] = pru_rproc_get(np, idx, &pruss_id);
1784 if (IS_ERR(prueth->pru[slice])) {
1785 ret = PTR_ERR(prueth->pru[slice]);
1786 prueth->pru[slice] = NULL;
1787 return dev_err_probe(dev, ret, "unable to get PRU%d\n", slice);
1788 }
1789 prueth->pru_id[slice] = pruss_id;
1790
1791 idx++;
1792 prueth->rtu[slice] = pru_rproc_get(np, idx, NULL);
1793 if (IS_ERR(prueth->rtu[slice])) {
1794 ret = PTR_ERR(prueth->rtu[slice]);
1795 prueth->rtu[slice] = NULL;
1796 return dev_err_probe(dev, ret, "unable to get RTU%d\n", slice);
1797 }
1798
1799 if (is_sr1)
1800 return 0;
1801
1802 idx++;
1803 prueth->txpru[slice] = pru_rproc_get(np, idx, NULL);
1804 if (IS_ERR(prueth->txpru[slice])) {
1805 ret = PTR_ERR(prueth->txpru[slice]);
1806 prueth->txpru[slice] = NULL;
1807 return dev_err_probe(dev, ret, "unable to get TX_PRU%d\n", slice);
1808 }
1809
1810 return 0;
1811 }
1812 EXPORT_SYMBOL_GPL(prueth_get_cores);
1813
prueth_put_cores(struct prueth * prueth,int slice)1814 void prueth_put_cores(struct prueth *prueth, int slice)
1815 {
1816 if (prueth->txpru[slice])
1817 pru_rproc_put(prueth->txpru[slice]);
1818
1819 if (prueth->rtu[slice])
1820 pru_rproc_put(prueth->rtu[slice]);
1821
1822 if (prueth->pru[slice])
1823 pru_rproc_put(prueth->pru[slice]);
1824 }
1825 EXPORT_SYMBOL_GPL(prueth_put_cores);
1826
1827 #ifdef CONFIG_PM_SLEEP
prueth_suspend(struct device * dev)1828 static int prueth_suspend(struct device *dev)
1829 {
1830 struct prueth *prueth = dev_get_drvdata(dev);
1831 struct net_device *ndev;
1832 int i, ret;
1833
1834 for (i = 0; i < PRUETH_NUM_MACS; i++) {
1835 ndev = prueth->registered_netdevs[i];
1836
1837 if (!ndev)
1838 continue;
1839
1840 if (netif_running(ndev)) {
1841 netif_device_detach(ndev);
1842 ret = ndev->netdev_ops->ndo_stop(ndev);
1843 if (ret < 0) {
1844 netdev_err(ndev, "failed to stop: %d", ret);
1845 return ret;
1846 }
1847 }
1848 }
1849
1850 return 0;
1851 }
1852
prueth_resume(struct device * dev)1853 static int prueth_resume(struct device *dev)
1854 {
1855 struct prueth *prueth = dev_get_drvdata(dev);
1856 struct net_device *ndev;
1857 int i, ret;
1858
1859 for (i = 0; i < PRUETH_NUM_MACS; i++) {
1860 ndev = prueth->registered_netdevs[i];
1861
1862 if (!ndev)
1863 continue;
1864
1865 if (netif_running(ndev)) {
1866 ret = ndev->netdev_ops->ndo_open(ndev);
1867 if (ret < 0) {
1868 netdev_err(ndev, "failed to start: %d", ret);
1869 return ret;
1870 }
1871 netif_device_attach(ndev);
1872 }
1873 }
1874
1875 return 0;
1876 }
1877 #endif /* CONFIG_PM_SLEEP */
1878
1879 const struct dev_pm_ops prueth_dev_pm_ops = {
1880 SET_SYSTEM_SLEEP_PM_OPS(prueth_suspend, prueth_resume)
1881 };
1882 EXPORT_SYMBOL_GPL(prueth_dev_pm_ops);
1883
1884 MODULE_AUTHOR("Roger Quadros <rogerq@ti.com>");
1885 MODULE_AUTHOR("Md Danish Anwar <danishanwar@ti.com>");
1886 MODULE_DESCRIPTION("PRUSS ICSSG Ethernet Driver Common Module");
1887 MODULE_LICENSE("GPL");
1888