1501ef306SVadym Kochan // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
2501ef306SVadym Kochan /* Copyright (c) 2019-2020 Marvell International Ltd. All rights reserved */
3501ef306SVadym Kochan
4501ef306SVadym Kochan #include <linux/bitfield.h>
5501ef306SVadym Kochan #include <linux/dmapool.h>
6501ef306SVadym Kochan #include <linux/etherdevice.h>
7501ef306SVadym Kochan #include <linux/if_vlan.h>
8501ef306SVadym Kochan #include <linux/platform_device.h>
9501ef306SVadym Kochan
10501ef306SVadym Kochan #include "prestera_dsa.h"
11501ef306SVadym Kochan #include "prestera.h"
12501ef306SVadym Kochan #include "prestera_hw.h"
13501ef306SVadym Kochan #include "prestera_rxtx.h"
140a9003f4SOleksandr Mazur #include "prestera_devlink.h"
15501ef306SVadym Kochan
16501ef306SVadym Kochan #define PRESTERA_SDMA_WAIT_MUL 10
17501ef306SVadym Kochan
18501ef306SVadym Kochan struct prestera_sdma_desc {
19501ef306SVadym Kochan __le32 word1;
20501ef306SVadym Kochan __le32 word2;
21501ef306SVadym Kochan __le32 buff;
22501ef306SVadym Kochan __le32 next;
23501ef306SVadym Kochan } __packed __aligned(16);
24501ef306SVadym Kochan
25501ef306SVadym Kochan #define PRESTERA_SDMA_BUFF_SIZE_MAX 1544
26501ef306SVadym Kochan
27501ef306SVadym Kochan #define PRESTERA_SDMA_RX_DESC_PKT_LEN(desc) \
28501ef306SVadym Kochan ((le32_to_cpu((desc)->word2) >> 16) & GENMASK(13, 0))
29501ef306SVadym Kochan
30501ef306SVadym Kochan #define PRESTERA_SDMA_RX_DESC_OWNER(desc) \
31501ef306SVadym Kochan ((le32_to_cpu((desc)->word1) & BIT(31)) >> 31)
32501ef306SVadym Kochan
33501ef306SVadym Kochan #define PRESTERA_SDMA_RX_DESC_IS_RCVD(desc) \
34501ef306SVadym Kochan (PRESTERA_SDMA_RX_DESC_OWNER(desc) == PRESTERA_SDMA_RX_DESC_CPU_OWN)
35501ef306SVadym Kochan
36501ef306SVadym Kochan #define PRESTERA_SDMA_RX_DESC_CPU_OWN 0
37501ef306SVadym Kochan #define PRESTERA_SDMA_RX_DESC_DMA_OWN 1
38501ef306SVadym Kochan
39501ef306SVadym Kochan #define PRESTERA_SDMA_RX_QUEUE_NUM 8
40501ef306SVadym Kochan
41501ef306SVadym Kochan #define PRESTERA_SDMA_RX_DESC_PER_Q 1000
42501ef306SVadym Kochan
43501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_PER_Q 1000
44501ef306SVadym Kochan #define PRESTERA_SDMA_TX_MAX_BURST 64
45501ef306SVadym Kochan
46501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_OWNER(desc) \
47501ef306SVadym Kochan ((le32_to_cpu((desc)->word1) & BIT(31)) >> 31)
48501ef306SVadym Kochan
49501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_CPU_OWN 0
50501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_DMA_OWN 1U
51501ef306SVadym Kochan
52501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_IS_SENT(desc) \
53501ef306SVadym Kochan (PRESTERA_SDMA_TX_DESC_OWNER(desc) == PRESTERA_SDMA_TX_DESC_CPU_OWN)
54501ef306SVadym Kochan
55501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_LAST BIT(20)
56501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_FIRST BIT(21)
57501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_CALC_CRC BIT(12)
58501ef306SVadym Kochan
59501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_SINGLE \
60501ef306SVadym Kochan (PRESTERA_SDMA_TX_DESC_FIRST | PRESTERA_SDMA_TX_DESC_LAST)
61501ef306SVadym Kochan
62501ef306SVadym Kochan #define PRESTERA_SDMA_TX_DESC_INIT \
63501ef306SVadym Kochan (PRESTERA_SDMA_TX_DESC_SINGLE | PRESTERA_SDMA_TX_DESC_CALC_CRC)
64501ef306SVadym Kochan
65501ef306SVadym Kochan #define PRESTERA_SDMA_RX_INTR_MASK_REG 0x2814
66501ef306SVadym Kochan #define PRESTERA_SDMA_RX_QUEUE_STATUS_REG 0x2680
67501ef306SVadym Kochan #define PRESTERA_SDMA_RX_QUEUE_DESC_REG(n) (0x260C + (n) * 16)
68501ef306SVadym Kochan
69501ef306SVadym Kochan #define PRESTERA_SDMA_TX_QUEUE_DESC_REG 0x26C0
70501ef306SVadym Kochan #define PRESTERA_SDMA_TX_QUEUE_START_REG 0x2868
71501ef306SVadym Kochan
72501ef306SVadym Kochan struct prestera_sdma_buf {
73501ef306SVadym Kochan struct prestera_sdma_desc *desc;
74501ef306SVadym Kochan dma_addr_t desc_dma;
75501ef306SVadym Kochan struct sk_buff *skb;
76501ef306SVadym Kochan dma_addr_t buf_dma;
77501ef306SVadym Kochan bool is_used;
78501ef306SVadym Kochan };
79501ef306SVadym Kochan
80501ef306SVadym Kochan struct prestera_rx_ring {
81501ef306SVadym Kochan struct prestera_sdma_buf *bufs;
82501ef306SVadym Kochan int next_rx;
83501ef306SVadym Kochan };
84501ef306SVadym Kochan
85501ef306SVadym Kochan struct prestera_tx_ring {
86501ef306SVadym Kochan struct prestera_sdma_buf *bufs;
87501ef306SVadym Kochan int next_tx;
88501ef306SVadym Kochan int max_burst;
89501ef306SVadym Kochan int burst;
90501ef306SVadym Kochan };
91501ef306SVadym Kochan
92501ef306SVadym Kochan struct prestera_sdma {
93501ef306SVadym Kochan struct prestera_rx_ring rx_ring[PRESTERA_SDMA_RX_QUEUE_NUM];
94501ef306SVadym Kochan struct prestera_tx_ring tx_ring;
95501ef306SVadym Kochan struct prestera_switch *sw;
96501ef306SVadym Kochan struct dma_pool *desc_pool;
97501ef306SVadym Kochan struct work_struct tx_work;
98501ef306SVadym Kochan struct napi_struct rx_napi;
99*ec24c06eSBreno Leitao struct net_device *napi_dev;
100501ef306SVadym Kochan u32 map_addr;
101501ef306SVadym Kochan u64 dma_mask;
102878e2eb2SJulia Lawall /* protect SDMA with concurrent access from multiple CPUs */
103501ef306SVadym Kochan spinlock_t tx_lock;
104501ef306SVadym Kochan };
105501ef306SVadym Kochan
106501ef306SVadym Kochan struct prestera_rxtx {
107501ef306SVadym Kochan struct prestera_sdma sdma;
108501ef306SVadym Kochan };
109501ef306SVadym Kochan
prestera_sdma_buf_init(struct prestera_sdma * sdma,struct prestera_sdma_buf * buf)110501ef306SVadym Kochan static int prestera_sdma_buf_init(struct prestera_sdma *sdma,
111501ef306SVadym Kochan struct prestera_sdma_buf *buf)
112501ef306SVadym Kochan {
113501ef306SVadym Kochan struct prestera_sdma_desc *desc;
114501ef306SVadym Kochan dma_addr_t dma;
115501ef306SVadym Kochan
116501ef306SVadym Kochan desc = dma_pool_alloc(sdma->desc_pool, GFP_DMA | GFP_KERNEL, &dma);
117501ef306SVadym Kochan if (!desc)
118501ef306SVadym Kochan return -ENOMEM;
119501ef306SVadym Kochan
120501ef306SVadym Kochan buf->buf_dma = DMA_MAPPING_ERROR;
121501ef306SVadym Kochan buf->desc_dma = dma;
122501ef306SVadym Kochan buf->desc = desc;
123501ef306SVadym Kochan buf->skb = NULL;
124501ef306SVadym Kochan
125501ef306SVadym Kochan return 0;
126501ef306SVadym Kochan }
127501ef306SVadym Kochan
prestera_sdma_map(struct prestera_sdma * sdma,dma_addr_t pa)128501ef306SVadym Kochan static u32 prestera_sdma_map(struct prestera_sdma *sdma, dma_addr_t pa)
129501ef306SVadym Kochan {
130501ef306SVadym Kochan return sdma->map_addr + pa;
131501ef306SVadym Kochan }
132501ef306SVadym Kochan
prestera_sdma_rx_desc_init(struct prestera_sdma * sdma,struct prestera_sdma_desc * desc,dma_addr_t buf)133501ef306SVadym Kochan static void prestera_sdma_rx_desc_init(struct prestera_sdma *sdma,
134501ef306SVadym Kochan struct prestera_sdma_desc *desc,
135501ef306SVadym Kochan dma_addr_t buf)
136501ef306SVadym Kochan {
137501ef306SVadym Kochan u32 word = le32_to_cpu(desc->word2);
138501ef306SVadym Kochan
139501ef306SVadym Kochan u32p_replace_bits(&word, PRESTERA_SDMA_BUFF_SIZE_MAX, GENMASK(15, 0));
140501ef306SVadym Kochan desc->word2 = cpu_to_le32(word);
141501ef306SVadym Kochan
142501ef306SVadym Kochan desc->buff = cpu_to_le32(prestera_sdma_map(sdma, buf));
143501ef306SVadym Kochan
144501ef306SVadym Kochan /* make sure buffer is set before reset the descriptor */
145501ef306SVadym Kochan wmb();
146501ef306SVadym Kochan
147501ef306SVadym Kochan desc->word1 = cpu_to_le32(0xA0000000);
148501ef306SVadym Kochan }
149501ef306SVadym Kochan
prestera_sdma_rx_desc_set_next(struct prestera_sdma * sdma,struct prestera_sdma_desc * desc,dma_addr_t next)150501ef306SVadym Kochan static void prestera_sdma_rx_desc_set_next(struct prestera_sdma *sdma,
151501ef306SVadym Kochan struct prestera_sdma_desc *desc,
152501ef306SVadym Kochan dma_addr_t next)
153501ef306SVadym Kochan {
154501ef306SVadym Kochan desc->next = cpu_to_le32(prestera_sdma_map(sdma, next));
155501ef306SVadym Kochan }
156501ef306SVadym Kochan
prestera_sdma_rx_skb_alloc(struct prestera_sdma * sdma,struct prestera_sdma_buf * buf)157501ef306SVadym Kochan static int prestera_sdma_rx_skb_alloc(struct prestera_sdma *sdma,
158501ef306SVadym Kochan struct prestera_sdma_buf *buf)
159501ef306SVadym Kochan {
160501ef306SVadym Kochan struct device *dev = sdma->sw->dev->dev;
161501ef306SVadym Kochan struct sk_buff *skb;
162501ef306SVadym Kochan dma_addr_t dma;
163501ef306SVadym Kochan
164501ef306SVadym Kochan skb = alloc_skb(PRESTERA_SDMA_BUFF_SIZE_MAX, GFP_DMA | GFP_ATOMIC);
165501ef306SVadym Kochan if (!skb)
166501ef306SVadym Kochan return -ENOMEM;
167501ef306SVadym Kochan
168501ef306SVadym Kochan dma = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE);
169501ef306SVadym Kochan if (dma_mapping_error(dev, dma))
170501ef306SVadym Kochan goto err_dma_map;
171501ef306SVadym Kochan
172501ef306SVadym Kochan if (buf->skb)
173501ef306SVadym Kochan dma_unmap_single(dev, buf->buf_dma, buf->skb->len,
174501ef306SVadym Kochan DMA_FROM_DEVICE);
175501ef306SVadym Kochan
176501ef306SVadym Kochan buf->buf_dma = dma;
177501ef306SVadym Kochan buf->skb = skb;
178501ef306SVadym Kochan
179501ef306SVadym Kochan return 0;
180501ef306SVadym Kochan
181501ef306SVadym Kochan err_dma_map:
182501ef306SVadym Kochan kfree_skb(skb);
183501ef306SVadym Kochan
184501ef306SVadym Kochan return -ENOMEM;
185501ef306SVadym Kochan }
186501ef306SVadym Kochan
prestera_sdma_rx_skb_get(struct prestera_sdma * sdma,struct prestera_sdma_buf * buf)187501ef306SVadym Kochan static struct sk_buff *prestera_sdma_rx_skb_get(struct prestera_sdma *sdma,
188501ef306SVadym Kochan struct prestera_sdma_buf *buf)
189501ef306SVadym Kochan {
190501ef306SVadym Kochan dma_addr_t buf_dma = buf->buf_dma;
191501ef306SVadym Kochan struct sk_buff *skb = buf->skb;
192501ef306SVadym Kochan u32 len = skb->len;
193501ef306SVadym Kochan int err;
194501ef306SVadym Kochan
195501ef306SVadym Kochan err = prestera_sdma_rx_skb_alloc(sdma, buf);
196501ef306SVadym Kochan if (err) {
197501ef306SVadym Kochan buf->buf_dma = buf_dma;
198501ef306SVadym Kochan buf->skb = skb;
199501ef306SVadym Kochan
200501ef306SVadym Kochan skb = alloc_skb(skb->len, GFP_ATOMIC);
201501ef306SVadym Kochan if (skb) {
202501ef306SVadym Kochan skb_put(skb, len);
203501ef306SVadym Kochan skb_copy_from_linear_data(buf->skb, skb->data, len);
204501ef306SVadym Kochan }
205501ef306SVadym Kochan }
206501ef306SVadym Kochan
207501ef306SVadym Kochan prestera_sdma_rx_desc_init(sdma, buf->desc, buf->buf_dma);
208501ef306SVadym Kochan
209501ef306SVadym Kochan return skb;
210501ef306SVadym Kochan }
211501ef306SVadym Kochan
prestera_rxtx_process_skb(struct prestera_sdma * sdma,struct sk_buff * skb)212501ef306SVadym Kochan static int prestera_rxtx_process_skb(struct prestera_sdma *sdma,
213501ef306SVadym Kochan struct sk_buff *skb)
214501ef306SVadym Kochan {
2150a9003f4SOleksandr Mazur struct prestera_port *port;
216501ef306SVadym Kochan struct prestera_dsa dsa;
217501ef306SVadym Kochan u32 hw_port, dev_id;
2180a9003f4SOleksandr Mazur u8 cpu_code;
219501ef306SVadym Kochan int err;
220501ef306SVadym Kochan
221501ef306SVadym Kochan skb_pull(skb, ETH_HLEN);
222501ef306SVadym Kochan
223501ef306SVadym Kochan /* ethertype field is part of the dsa header */
224501ef306SVadym Kochan err = prestera_dsa_parse(&dsa, skb->data - ETH_TLEN);
225501ef306SVadym Kochan if (err)
226501ef306SVadym Kochan return err;
227501ef306SVadym Kochan
228501ef306SVadym Kochan dev_id = dsa.hw_dev_num;
229501ef306SVadym Kochan hw_port = dsa.port_num;
230501ef306SVadym Kochan
231501ef306SVadym Kochan port = prestera_port_find_by_hwid(sdma->sw, dev_id, hw_port);
232501ef306SVadym Kochan if (unlikely(!port)) {
233501ef306SVadym Kochan dev_warn_ratelimited(prestera_dev(sdma->sw), "received pkt for non-existent port(%u, %u)\n",
234501ef306SVadym Kochan dev_id, hw_port);
235501ef306SVadym Kochan return -ENOENT;
236501ef306SVadym Kochan }
237501ef306SVadym Kochan
238501ef306SVadym Kochan if (unlikely(!pskb_may_pull(skb, PRESTERA_DSA_HLEN)))
239501ef306SVadym Kochan return -EINVAL;
240501ef306SVadym Kochan
241501ef306SVadym Kochan /* remove DSA tag and update checksum */
242501ef306SVadym Kochan skb_pull_rcsum(skb, PRESTERA_DSA_HLEN);
243501ef306SVadym Kochan
244501ef306SVadym Kochan memmove(skb->data - ETH_HLEN, skb->data - ETH_HLEN - PRESTERA_DSA_HLEN,
245501ef306SVadym Kochan ETH_ALEN * 2);
246501ef306SVadym Kochan
247501ef306SVadym Kochan skb_push(skb, ETH_HLEN);
248501ef306SVadym Kochan
249501ef306SVadym Kochan skb->protocol = eth_type_trans(skb, port->dev);
250501ef306SVadym Kochan
251501ef306SVadym Kochan if (dsa.vlan.is_tagged) {
252501ef306SVadym Kochan u16 tci = dsa.vlan.vid & VLAN_VID_MASK;
253501ef306SVadym Kochan
254501ef306SVadym Kochan tci |= dsa.vlan.vpt << VLAN_PRIO_SHIFT;
255501ef306SVadym Kochan if (dsa.vlan.cfi_bit)
256501ef306SVadym Kochan tci |= VLAN_CFI_MASK;
257501ef306SVadym Kochan
258501ef306SVadym Kochan __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tci);
259501ef306SVadym Kochan }
260501ef306SVadym Kochan
2610a9003f4SOleksandr Mazur cpu_code = dsa.cpu_code;
2620a9003f4SOleksandr Mazur prestera_devlink_trap_report(port, skb, cpu_code);
2630a9003f4SOleksandr Mazur
264501ef306SVadym Kochan return 0;
265501ef306SVadym Kochan }
266501ef306SVadym Kochan
prestera_sdma_next_rx_buf_idx(int buf_idx)267501ef306SVadym Kochan static int prestera_sdma_next_rx_buf_idx(int buf_idx)
268501ef306SVadym Kochan {
269501ef306SVadym Kochan return (buf_idx + 1) % PRESTERA_SDMA_RX_DESC_PER_Q;
270501ef306SVadym Kochan }
271501ef306SVadym Kochan
prestera_sdma_rx_poll(struct napi_struct * napi,int budget)272501ef306SVadym Kochan static int prestera_sdma_rx_poll(struct napi_struct *napi, int budget)
273501ef306SVadym Kochan {
274501ef306SVadym Kochan int qnum = PRESTERA_SDMA_RX_QUEUE_NUM;
275501ef306SVadym Kochan unsigned int rxq_done_map = 0;
276501ef306SVadym Kochan struct prestera_sdma *sdma;
277501ef306SVadym Kochan struct list_head rx_list;
278501ef306SVadym Kochan unsigned int qmask;
279501ef306SVadym Kochan int pkts_done = 0;
280501ef306SVadym Kochan int q;
281501ef306SVadym Kochan
282501ef306SVadym Kochan qnum = PRESTERA_SDMA_RX_QUEUE_NUM;
283501ef306SVadym Kochan qmask = GENMASK(qnum - 1, 0);
284501ef306SVadym Kochan
285501ef306SVadym Kochan INIT_LIST_HEAD(&rx_list);
286501ef306SVadym Kochan
287501ef306SVadym Kochan sdma = container_of(napi, struct prestera_sdma, rx_napi);
288501ef306SVadym Kochan
289501ef306SVadym Kochan while (pkts_done < budget && rxq_done_map != qmask) {
290501ef306SVadym Kochan for (q = 0; q < qnum && pkts_done < budget; q++) {
291501ef306SVadym Kochan struct prestera_rx_ring *ring = &sdma->rx_ring[q];
292501ef306SVadym Kochan struct prestera_sdma_desc *desc;
293501ef306SVadym Kochan struct prestera_sdma_buf *buf;
294501ef306SVadym Kochan int buf_idx = ring->next_rx;
295501ef306SVadym Kochan struct sk_buff *skb;
296501ef306SVadym Kochan
297501ef306SVadym Kochan buf = &ring->bufs[buf_idx];
298501ef306SVadym Kochan desc = buf->desc;
299501ef306SVadym Kochan
300501ef306SVadym Kochan if (PRESTERA_SDMA_RX_DESC_IS_RCVD(desc)) {
301501ef306SVadym Kochan rxq_done_map &= ~BIT(q);
302501ef306SVadym Kochan } else {
303501ef306SVadym Kochan rxq_done_map |= BIT(q);
304501ef306SVadym Kochan continue;
305501ef306SVadym Kochan }
306501ef306SVadym Kochan
307501ef306SVadym Kochan pkts_done++;
308501ef306SVadym Kochan
309501ef306SVadym Kochan __skb_trim(buf->skb, PRESTERA_SDMA_RX_DESC_PKT_LEN(desc));
310501ef306SVadym Kochan
311501ef306SVadym Kochan skb = prestera_sdma_rx_skb_get(sdma, buf);
312501ef306SVadym Kochan if (!skb)
313501ef306SVadym Kochan goto rx_next_buf;
314501ef306SVadym Kochan
315501ef306SVadym Kochan if (unlikely(prestera_rxtx_process_skb(sdma, skb)))
316501ef306SVadym Kochan goto rx_next_buf;
317501ef306SVadym Kochan
318501ef306SVadym Kochan list_add_tail(&skb->list, &rx_list);
319501ef306SVadym Kochan rx_next_buf:
320501ef306SVadym Kochan ring->next_rx = prestera_sdma_next_rx_buf_idx(buf_idx);
321501ef306SVadym Kochan }
322501ef306SVadym Kochan }
323501ef306SVadym Kochan
324501ef306SVadym Kochan if (pkts_done < budget && napi_complete_done(napi, pkts_done))
325501ef306SVadym Kochan prestera_write(sdma->sw, PRESTERA_SDMA_RX_INTR_MASK_REG,
326501ef306SVadym Kochan GENMASK(9, 2));
327501ef306SVadym Kochan
328501ef306SVadym Kochan netif_receive_skb_list(&rx_list);
329501ef306SVadym Kochan
330501ef306SVadym Kochan return pkts_done;
331501ef306SVadym Kochan }
332501ef306SVadym Kochan
prestera_sdma_rx_fini(struct prestera_sdma * sdma)333501ef306SVadym Kochan static void prestera_sdma_rx_fini(struct prestera_sdma *sdma)
334501ef306SVadym Kochan {
335501ef306SVadym Kochan int qnum = PRESTERA_SDMA_RX_QUEUE_NUM;
336501ef306SVadym Kochan int q, b;
337501ef306SVadym Kochan
338501ef306SVadym Kochan /* disable all rx queues */
339501ef306SVadym Kochan prestera_write(sdma->sw, PRESTERA_SDMA_RX_QUEUE_STATUS_REG,
340501ef306SVadym Kochan GENMASK(15, 8));
341501ef306SVadym Kochan
342501ef306SVadym Kochan for (q = 0; q < qnum; q++) {
343501ef306SVadym Kochan struct prestera_rx_ring *ring = &sdma->rx_ring[q];
344501ef306SVadym Kochan
345501ef306SVadym Kochan if (!ring->bufs)
346501ef306SVadym Kochan break;
347501ef306SVadym Kochan
348501ef306SVadym Kochan for (b = 0; b < PRESTERA_SDMA_RX_DESC_PER_Q; b++) {
349501ef306SVadym Kochan struct prestera_sdma_buf *buf = &ring->bufs[b];
350501ef306SVadym Kochan
351501ef306SVadym Kochan if (buf->desc_dma)
352501ef306SVadym Kochan dma_pool_free(sdma->desc_pool, buf->desc,
353501ef306SVadym Kochan buf->desc_dma);
354501ef306SVadym Kochan
355501ef306SVadym Kochan if (!buf->skb)
356501ef306SVadym Kochan continue;
357501ef306SVadym Kochan
358501ef306SVadym Kochan if (buf->buf_dma != DMA_MAPPING_ERROR)
359501ef306SVadym Kochan dma_unmap_single(sdma->sw->dev->dev,
360501ef306SVadym Kochan buf->buf_dma, buf->skb->len,
361501ef306SVadym Kochan DMA_FROM_DEVICE);
362501ef306SVadym Kochan kfree_skb(buf->skb);
363501ef306SVadym Kochan }
364501ef306SVadym Kochan }
365501ef306SVadym Kochan }
366501ef306SVadym Kochan
prestera_sdma_rx_init(struct prestera_sdma * sdma)367501ef306SVadym Kochan static int prestera_sdma_rx_init(struct prestera_sdma *sdma)
368501ef306SVadym Kochan {
369501ef306SVadym Kochan int bnum = PRESTERA_SDMA_RX_DESC_PER_Q;
370501ef306SVadym Kochan int qnum = PRESTERA_SDMA_RX_QUEUE_NUM;
371501ef306SVadym Kochan int err;
372501ef306SVadym Kochan int q;
373501ef306SVadym Kochan
374501ef306SVadym Kochan /* disable all rx queues */
375501ef306SVadym Kochan prestera_write(sdma->sw, PRESTERA_SDMA_RX_QUEUE_STATUS_REG,
376501ef306SVadym Kochan GENMASK(15, 8));
377501ef306SVadym Kochan
378501ef306SVadym Kochan for (q = 0; q < qnum; q++) {
379501ef306SVadym Kochan struct prestera_sdma_buf *head, *tail, *next, *prev;
380501ef306SVadym Kochan struct prestera_rx_ring *ring = &sdma->rx_ring[q];
381501ef306SVadym Kochan
382501ef306SVadym Kochan ring->bufs = kmalloc_array(bnum, sizeof(*head), GFP_KERNEL);
383501ef306SVadym Kochan if (!ring->bufs)
384501ef306SVadym Kochan return -ENOMEM;
385501ef306SVadym Kochan
386501ef306SVadym Kochan ring->next_rx = 0;
387501ef306SVadym Kochan
388501ef306SVadym Kochan tail = &ring->bufs[bnum - 1];
389501ef306SVadym Kochan head = &ring->bufs[0];
390501ef306SVadym Kochan next = head;
391501ef306SVadym Kochan prev = next;
392501ef306SVadym Kochan
393501ef306SVadym Kochan do {
394501ef306SVadym Kochan err = prestera_sdma_buf_init(sdma, next);
395501ef306SVadym Kochan if (err)
396501ef306SVadym Kochan return err;
397501ef306SVadym Kochan
398501ef306SVadym Kochan err = prestera_sdma_rx_skb_alloc(sdma, next);
399501ef306SVadym Kochan if (err)
400501ef306SVadym Kochan return err;
401501ef306SVadym Kochan
402501ef306SVadym Kochan prestera_sdma_rx_desc_init(sdma, next->desc,
403501ef306SVadym Kochan next->buf_dma);
404501ef306SVadym Kochan
405501ef306SVadym Kochan prestera_sdma_rx_desc_set_next(sdma, prev->desc,
406501ef306SVadym Kochan next->desc_dma);
407501ef306SVadym Kochan
408501ef306SVadym Kochan prev = next;
409501ef306SVadym Kochan next++;
410501ef306SVadym Kochan } while (prev != tail);
411501ef306SVadym Kochan
412501ef306SVadym Kochan /* join tail with head to make a circular list */
413501ef306SVadym Kochan prestera_sdma_rx_desc_set_next(sdma, tail->desc, head->desc_dma);
414501ef306SVadym Kochan
415501ef306SVadym Kochan prestera_write(sdma->sw, PRESTERA_SDMA_RX_QUEUE_DESC_REG(q),
416501ef306SVadym Kochan prestera_sdma_map(sdma, head->desc_dma));
417501ef306SVadym Kochan }
418501ef306SVadym Kochan
419501ef306SVadym Kochan /* make sure all rx descs are filled before enabling all rx queues */
420501ef306SVadym Kochan wmb();
421501ef306SVadym Kochan
422501ef306SVadym Kochan prestera_write(sdma->sw, PRESTERA_SDMA_RX_QUEUE_STATUS_REG,
423501ef306SVadym Kochan GENMASK(7, 0));
424501ef306SVadym Kochan
425501ef306SVadym Kochan return 0;
426501ef306SVadym Kochan }
427501ef306SVadym Kochan
prestera_sdma_tx_desc_init(struct prestera_sdma * sdma,struct prestera_sdma_desc * desc)428501ef306SVadym Kochan static void prestera_sdma_tx_desc_init(struct prestera_sdma *sdma,
429501ef306SVadym Kochan struct prestera_sdma_desc *desc)
430501ef306SVadym Kochan {
431501ef306SVadym Kochan desc->word1 = cpu_to_le32(PRESTERA_SDMA_TX_DESC_INIT);
432501ef306SVadym Kochan desc->word2 = 0;
433501ef306SVadym Kochan }
434501ef306SVadym Kochan
prestera_sdma_tx_desc_set_next(struct prestera_sdma * sdma,struct prestera_sdma_desc * desc,dma_addr_t next)435501ef306SVadym Kochan static void prestera_sdma_tx_desc_set_next(struct prestera_sdma *sdma,
436501ef306SVadym Kochan struct prestera_sdma_desc *desc,
437501ef306SVadym Kochan dma_addr_t next)
438501ef306SVadym Kochan {
439501ef306SVadym Kochan desc->next = cpu_to_le32(prestera_sdma_map(sdma, next));
440501ef306SVadym Kochan }
441501ef306SVadym Kochan
prestera_sdma_tx_desc_set_buf(struct prestera_sdma * sdma,struct prestera_sdma_desc * desc,dma_addr_t buf,size_t len)442501ef306SVadym Kochan static void prestera_sdma_tx_desc_set_buf(struct prestera_sdma *sdma,
443501ef306SVadym Kochan struct prestera_sdma_desc *desc,
444501ef306SVadym Kochan dma_addr_t buf, size_t len)
445501ef306SVadym Kochan {
446501ef306SVadym Kochan u32 word = le32_to_cpu(desc->word2);
447501ef306SVadym Kochan
448501ef306SVadym Kochan u32p_replace_bits(&word, len + ETH_FCS_LEN, GENMASK(30, 16));
449501ef306SVadym Kochan
450501ef306SVadym Kochan desc->buff = cpu_to_le32(prestera_sdma_map(sdma, buf));
451501ef306SVadym Kochan desc->word2 = cpu_to_le32(word);
452501ef306SVadym Kochan }
453501ef306SVadym Kochan
prestera_sdma_tx_desc_xmit(struct prestera_sdma_desc * desc)454501ef306SVadym Kochan static void prestera_sdma_tx_desc_xmit(struct prestera_sdma_desc *desc)
455501ef306SVadym Kochan {
456501ef306SVadym Kochan u32 word = le32_to_cpu(desc->word1);
457501ef306SVadym Kochan
458501ef306SVadym Kochan word |= PRESTERA_SDMA_TX_DESC_DMA_OWN << 31;
459501ef306SVadym Kochan
460501ef306SVadym Kochan /* make sure everything is written before enable xmit */
461501ef306SVadym Kochan wmb();
462501ef306SVadym Kochan
463501ef306SVadym Kochan desc->word1 = cpu_to_le32(word);
464501ef306SVadym Kochan }
465501ef306SVadym Kochan
prestera_sdma_tx_buf_map(struct prestera_sdma * sdma,struct prestera_sdma_buf * buf,struct sk_buff * skb)466501ef306SVadym Kochan static int prestera_sdma_tx_buf_map(struct prestera_sdma *sdma,
467501ef306SVadym Kochan struct prestera_sdma_buf *buf,
468501ef306SVadym Kochan struct sk_buff *skb)
469501ef306SVadym Kochan {
470501ef306SVadym Kochan struct device *dma_dev = sdma->sw->dev->dev;
471501ef306SVadym Kochan dma_addr_t dma;
472501ef306SVadym Kochan
473501ef306SVadym Kochan dma = dma_map_single(dma_dev, skb->data, skb->len, DMA_TO_DEVICE);
474501ef306SVadym Kochan if (dma_mapping_error(dma_dev, dma))
475501ef306SVadym Kochan return -ENOMEM;
476501ef306SVadym Kochan
477501ef306SVadym Kochan buf->buf_dma = dma;
478501ef306SVadym Kochan buf->skb = skb;
479501ef306SVadym Kochan
480501ef306SVadym Kochan return 0;
481501ef306SVadym Kochan }
482501ef306SVadym Kochan
prestera_sdma_tx_buf_unmap(struct prestera_sdma * sdma,struct prestera_sdma_buf * buf)483501ef306SVadym Kochan static void prestera_sdma_tx_buf_unmap(struct prestera_sdma *sdma,
484501ef306SVadym Kochan struct prestera_sdma_buf *buf)
485501ef306SVadym Kochan {
486501ef306SVadym Kochan struct device *dma_dev = sdma->sw->dev->dev;
487501ef306SVadym Kochan
488501ef306SVadym Kochan dma_unmap_single(dma_dev, buf->buf_dma, buf->skb->len, DMA_TO_DEVICE);
489501ef306SVadym Kochan }
490501ef306SVadym Kochan
prestera_sdma_tx_recycle_work_fn(struct work_struct * work)491501ef306SVadym Kochan static void prestera_sdma_tx_recycle_work_fn(struct work_struct *work)
492501ef306SVadym Kochan {
493501ef306SVadym Kochan int bnum = PRESTERA_SDMA_TX_DESC_PER_Q;
494501ef306SVadym Kochan struct prestera_tx_ring *tx_ring;
495501ef306SVadym Kochan struct prestera_sdma *sdma;
496501ef306SVadym Kochan int b;
497501ef306SVadym Kochan
498501ef306SVadym Kochan sdma = container_of(work, struct prestera_sdma, tx_work);
499501ef306SVadym Kochan
500501ef306SVadym Kochan tx_ring = &sdma->tx_ring;
501501ef306SVadym Kochan
502501ef306SVadym Kochan for (b = 0; b < bnum; b++) {
503501ef306SVadym Kochan struct prestera_sdma_buf *buf = &tx_ring->bufs[b];
504501ef306SVadym Kochan
505501ef306SVadym Kochan if (!buf->is_used)
506501ef306SVadym Kochan continue;
507501ef306SVadym Kochan
508501ef306SVadym Kochan if (!PRESTERA_SDMA_TX_DESC_IS_SENT(buf->desc))
509501ef306SVadym Kochan continue;
510501ef306SVadym Kochan
511501ef306SVadym Kochan prestera_sdma_tx_buf_unmap(sdma, buf);
512501ef306SVadym Kochan dev_consume_skb_any(buf->skb);
513501ef306SVadym Kochan buf->skb = NULL;
514501ef306SVadym Kochan
515501ef306SVadym Kochan /* make sure everything is cleaned up */
516501ef306SVadym Kochan wmb();
517501ef306SVadym Kochan
518501ef306SVadym Kochan buf->is_used = false;
519501ef306SVadym Kochan }
520501ef306SVadym Kochan }
521501ef306SVadym Kochan
prestera_sdma_tx_init(struct prestera_sdma * sdma)522501ef306SVadym Kochan static int prestera_sdma_tx_init(struct prestera_sdma *sdma)
523501ef306SVadym Kochan {
524501ef306SVadym Kochan struct prestera_sdma_buf *head, *tail, *next, *prev;
525501ef306SVadym Kochan struct prestera_tx_ring *tx_ring = &sdma->tx_ring;
526501ef306SVadym Kochan int bnum = PRESTERA_SDMA_TX_DESC_PER_Q;
527501ef306SVadym Kochan int err;
528501ef306SVadym Kochan
529501ef306SVadym Kochan INIT_WORK(&sdma->tx_work, prestera_sdma_tx_recycle_work_fn);
530501ef306SVadym Kochan spin_lock_init(&sdma->tx_lock);
531501ef306SVadym Kochan
532501ef306SVadym Kochan tx_ring->bufs = kmalloc_array(bnum, sizeof(*head), GFP_KERNEL);
533501ef306SVadym Kochan if (!tx_ring->bufs)
534501ef306SVadym Kochan return -ENOMEM;
535501ef306SVadym Kochan
536501ef306SVadym Kochan tail = &tx_ring->bufs[bnum - 1];
537501ef306SVadym Kochan head = &tx_ring->bufs[0];
538501ef306SVadym Kochan next = head;
539501ef306SVadym Kochan prev = next;
540501ef306SVadym Kochan
541501ef306SVadym Kochan tx_ring->max_burst = PRESTERA_SDMA_TX_MAX_BURST;
542501ef306SVadym Kochan tx_ring->burst = tx_ring->max_burst;
543501ef306SVadym Kochan tx_ring->next_tx = 0;
544501ef306SVadym Kochan
545501ef306SVadym Kochan do {
546501ef306SVadym Kochan err = prestera_sdma_buf_init(sdma, next);
547501ef306SVadym Kochan if (err)
548501ef306SVadym Kochan return err;
549501ef306SVadym Kochan
550501ef306SVadym Kochan next->is_used = false;
551501ef306SVadym Kochan
552501ef306SVadym Kochan prestera_sdma_tx_desc_init(sdma, next->desc);
553501ef306SVadym Kochan
554501ef306SVadym Kochan prestera_sdma_tx_desc_set_next(sdma, prev->desc,
555501ef306SVadym Kochan next->desc_dma);
556501ef306SVadym Kochan
557501ef306SVadym Kochan prev = next;
558501ef306SVadym Kochan next++;
559501ef306SVadym Kochan } while (prev != tail);
560501ef306SVadym Kochan
561501ef306SVadym Kochan /* join tail with head to make a circular list */
562501ef306SVadym Kochan prestera_sdma_tx_desc_set_next(sdma, tail->desc, head->desc_dma);
563501ef306SVadym Kochan
564501ef306SVadym Kochan /* make sure descriptors are written */
565501ef306SVadym Kochan wmb();
566501ef306SVadym Kochan
567501ef306SVadym Kochan prestera_write(sdma->sw, PRESTERA_SDMA_TX_QUEUE_DESC_REG,
568501ef306SVadym Kochan prestera_sdma_map(sdma, head->desc_dma));
569501ef306SVadym Kochan
570501ef306SVadym Kochan return 0;
571501ef306SVadym Kochan }
572501ef306SVadym Kochan
prestera_sdma_tx_fini(struct prestera_sdma * sdma)573501ef306SVadym Kochan static void prestera_sdma_tx_fini(struct prestera_sdma *sdma)
574501ef306SVadym Kochan {
575501ef306SVadym Kochan struct prestera_tx_ring *ring = &sdma->tx_ring;
576501ef306SVadym Kochan int bnum = PRESTERA_SDMA_TX_DESC_PER_Q;
577501ef306SVadym Kochan int b;
578501ef306SVadym Kochan
579501ef306SVadym Kochan cancel_work_sync(&sdma->tx_work);
580501ef306SVadym Kochan
581501ef306SVadym Kochan if (!ring->bufs)
582501ef306SVadym Kochan return;
583501ef306SVadym Kochan
584501ef306SVadym Kochan for (b = 0; b < bnum; b++) {
585501ef306SVadym Kochan struct prestera_sdma_buf *buf = &ring->bufs[b];
586501ef306SVadym Kochan
587501ef306SVadym Kochan if (buf->desc)
588501ef306SVadym Kochan dma_pool_free(sdma->desc_pool, buf->desc,
589501ef306SVadym Kochan buf->desc_dma);
590501ef306SVadym Kochan
591501ef306SVadym Kochan if (!buf->skb)
592501ef306SVadym Kochan continue;
593501ef306SVadym Kochan
594501ef306SVadym Kochan dma_unmap_single(sdma->sw->dev->dev, buf->buf_dma,
595501ef306SVadym Kochan buf->skb->len, DMA_TO_DEVICE);
596501ef306SVadym Kochan
597501ef306SVadym Kochan dev_consume_skb_any(buf->skb);
598501ef306SVadym Kochan }
599501ef306SVadym Kochan }
600501ef306SVadym Kochan
prestera_rxtx_handle_event(struct prestera_switch * sw,struct prestera_event * evt,void * arg)601501ef306SVadym Kochan static void prestera_rxtx_handle_event(struct prestera_switch *sw,
602501ef306SVadym Kochan struct prestera_event *evt,
603501ef306SVadym Kochan void *arg)
604501ef306SVadym Kochan {
605501ef306SVadym Kochan struct prestera_sdma *sdma = arg;
606501ef306SVadym Kochan
607501ef306SVadym Kochan if (evt->id != PRESTERA_RXTX_EVENT_RCV_PKT)
608501ef306SVadym Kochan return;
609501ef306SVadym Kochan
610501ef306SVadym Kochan prestera_write(sdma->sw, PRESTERA_SDMA_RX_INTR_MASK_REG, 0);
611501ef306SVadym Kochan napi_schedule(&sdma->rx_napi);
612501ef306SVadym Kochan }
613501ef306SVadym Kochan
prestera_sdma_switch_init(struct prestera_switch * sw)614501ef306SVadym Kochan static int prestera_sdma_switch_init(struct prestera_switch *sw)
615501ef306SVadym Kochan {
616501ef306SVadym Kochan struct prestera_sdma *sdma = &sw->rxtx->sdma;
617501ef306SVadym Kochan struct device *dev = sw->dev->dev;
618501ef306SVadym Kochan struct prestera_rxtx_params p;
619501ef306SVadym Kochan int err;
620501ef306SVadym Kochan
621501ef306SVadym Kochan p.use_sdma = true;
622501ef306SVadym Kochan
623501ef306SVadym Kochan err = prestera_hw_rxtx_init(sw, &p);
624501ef306SVadym Kochan if (err) {
625501ef306SVadym Kochan dev_err(dev, "failed to init rxtx by hw\n");
626501ef306SVadym Kochan return err;
627501ef306SVadym Kochan }
628501ef306SVadym Kochan
629501ef306SVadym Kochan sdma->dma_mask = dma_get_mask(dev);
630501ef306SVadym Kochan sdma->map_addr = p.map_addr;
631501ef306SVadym Kochan sdma->sw = sw;
632501ef306SVadym Kochan
633501ef306SVadym Kochan sdma->desc_pool = dma_pool_create("desc_pool", dev,
634501ef306SVadym Kochan sizeof(struct prestera_sdma_desc),
635501ef306SVadym Kochan 16, 0);
636501ef306SVadym Kochan if (!sdma->desc_pool)
637501ef306SVadym Kochan return -ENOMEM;
638501ef306SVadym Kochan
639501ef306SVadym Kochan err = prestera_sdma_rx_init(sdma);
640501ef306SVadym Kochan if (err) {
641501ef306SVadym Kochan dev_err(dev, "failed to init rx ring\n");
642501ef306SVadym Kochan goto err_rx_init;
643501ef306SVadym Kochan }
644501ef306SVadym Kochan
645501ef306SVadym Kochan err = prestera_sdma_tx_init(sdma);
646501ef306SVadym Kochan if (err) {
647501ef306SVadym Kochan dev_err(dev, "failed to init tx ring\n");
648501ef306SVadym Kochan goto err_tx_init;
649501ef306SVadym Kochan }
650501ef306SVadym Kochan
651501ef306SVadym Kochan err = prestera_hw_event_handler_register(sw, PRESTERA_EVENT_TYPE_RXTX,
652501ef306SVadym Kochan prestera_rxtx_handle_event,
653501ef306SVadym Kochan sdma);
654501ef306SVadym Kochan if (err)
655501ef306SVadym Kochan goto err_evt_register;
656501ef306SVadym Kochan
657*ec24c06eSBreno Leitao sdma->napi_dev = alloc_netdev_dummy(0);
658*ec24c06eSBreno Leitao if (!sdma->napi_dev) {
659*ec24c06eSBreno Leitao dev_err(dev, "not able to initialize dummy device\n");
660*ec24c06eSBreno Leitao err = -ENOMEM;
661*ec24c06eSBreno Leitao goto err_alloc_dummy;
662*ec24c06eSBreno Leitao }
663501ef306SVadym Kochan
664*ec24c06eSBreno Leitao netif_napi_add(sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll);
665501ef306SVadym Kochan napi_enable(&sdma->rx_napi);
666501ef306SVadym Kochan
667501ef306SVadym Kochan return 0;
668501ef306SVadym Kochan
669*ec24c06eSBreno Leitao err_alloc_dummy:
670*ec24c06eSBreno Leitao prestera_hw_event_handler_unregister(sw, PRESTERA_EVENT_TYPE_RXTX,
671*ec24c06eSBreno Leitao prestera_rxtx_handle_event);
672501ef306SVadym Kochan err_evt_register:
673501ef306SVadym Kochan err_tx_init:
674501ef306SVadym Kochan prestera_sdma_tx_fini(sdma);
675501ef306SVadym Kochan err_rx_init:
676501ef306SVadym Kochan prestera_sdma_rx_fini(sdma);
677501ef306SVadym Kochan
678501ef306SVadym Kochan dma_pool_destroy(sdma->desc_pool);
679501ef306SVadym Kochan return err;
680501ef306SVadym Kochan }
681501ef306SVadym Kochan
prestera_sdma_switch_fini(struct prestera_switch * sw)682501ef306SVadym Kochan static void prestera_sdma_switch_fini(struct prestera_switch *sw)
683501ef306SVadym Kochan {
684501ef306SVadym Kochan struct prestera_sdma *sdma = &sw->rxtx->sdma;
685501ef306SVadym Kochan
686501ef306SVadym Kochan napi_disable(&sdma->rx_napi);
687501ef306SVadym Kochan netif_napi_del(&sdma->rx_napi);
688*ec24c06eSBreno Leitao free_netdev(sdma->napi_dev);
689501ef306SVadym Kochan prestera_hw_event_handler_unregister(sw, PRESTERA_EVENT_TYPE_RXTX,
690501ef306SVadym Kochan prestera_rxtx_handle_event);
691501ef306SVadym Kochan prestera_sdma_tx_fini(sdma);
692501ef306SVadym Kochan prestera_sdma_rx_fini(sdma);
693501ef306SVadym Kochan dma_pool_destroy(sdma->desc_pool);
694501ef306SVadym Kochan }
695501ef306SVadym Kochan
prestera_sdma_is_ready(struct prestera_sdma * sdma)696501ef306SVadym Kochan static bool prestera_sdma_is_ready(struct prestera_sdma *sdma)
697501ef306SVadym Kochan {
698501ef306SVadym Kochan return !(prestera_read(sdma->sw, PRESTERA_SDMA_TX_QUEUE_START_REG) & 1);
699501ef306SVadym Kochan }
700501ef306SVadym Kochan
prestera_sdma_tx_wait(struct prestera_sdma * sdma,struct prestera_tx_ring * tx_ring)701501ef306SVadym Kochan static int prestera_sdma_tx_wait(struct prestera_sdma *sdma,
702501ef306SVadym Kochan struct prestera_tx_ring *tx_ring)
703501ef306SVadym Kochan {
704501ef306SVadym Kochan int tx_wait_num = PRESTERA_SDMA_WAIT_MUL * tx_ring->max_burst;
705501ef306SVadym Kochan
706501ef306SVadym Kochan do {
707501ef306SVadym Kochan if (prestera_sdma_is_ready(sdma))
708501ef306SVadym Kochan return 0;
709501ef306SVadym Kochan
710501ef306SVadym Kochan udelay(1);
711501ef306SVadym Kochan } while (--tx_wait_num);
712501ef306SVadym Kochan
713501ef306SVadym Kochan return -EBUSY;
714501ef306SVadym Kochan }
715501ef306SVadym Kochan
prestera_sdma_tx_start(struct prestera_sdma * sdma)716501ef306SVadym Kochan static void prestera_sdma_tx_start(struct prestera_sdma *sdma)
717501ef306SVadym Kochan {
718501ef306SVadym Kochan prestera_write(sdma->sw, PRESTERA_SDMA_TX_QUEUE_START_REG, 1);
719501ef306SVadym Kochan schedule_work(&sdma->tx_work);
720501ef306SVadym Kochan }
721501ef306SVadym Kochan
prestera_sdma_xmit(struct prestera_sdma * sdma,struct sk_buff * skb)722501ef306SVadym Kochan static netdev_tx_t prestera_sdma_xmit(struct prestera_sdma *sdma,
723501ef306SVadym Kochan struct sk_buff *skb)
724501ef306SVadym Kochan {
725501ef306SVadym Kochan struct device *dma_dev = sdma->sw->dev->dev;
726501ef306SVadym Kochan struct net_device *dev = skb->dev;
727501ef306SVadym Kochan struct prestera_tx_ring *tx_ring;
728501ef306SVadym Kochan struct prestera_sdma_buf *buf;
729501ef306SVadym Kochan int err;
730501ef306SVadym Kochan
731501ef306SVadym Kochan spin_lock(&sdma->tx_lock);
732501ef306SVadym Kochan
733501ef306SVadym Kochan tx_ring = &sdma->tx_ring;
734501ef306SVadym Kochan
735501ef306SVadym Kochan buf = &tx_ring->bufs[tx_ring->next_tx];
736501ef306SVadym Kochan if (buf->is_used) {
737501ef306SVadym Kochan schedule_work(&sdma->tx_work);
738501ef306SVadym Kochan goto drop_skb;
739501ef306SVadym Kochan }
740501ef306SVadym Kochan
741501ef306SVadym Kochan if (unlikely(eth_skb_pad(skb)))
742501ef306SVadym Kochan goto drop_skb_nofree;
743501ef306SVadym Kochan
744501ef306SVadym Kochan err = prestera_sdma_tx_buf_map(sdma, buf, skb);
745501ef306SVadym Kochan if (err)
746501ef306SVadym Kochan goto drop_skb;
747501ef306SVadym Kochan
748501ef306SVadym Kochan prestera_sdma_tx_desc_set_buf(sdma, buf->desc, buf->buf_dma, skb->len);
749501ef306SVadym Kochan
750501ef306SVadym Kochan dma_sync_single_for_device(dma_dev, buf->buf_dma, skb->len,
751501ef306SVadym Kochan DMA_TO_DEVICE);
752501ef306SVadym Kochan
753501ef306SVadym Kochan if (tx_ring->burst) {
754501ef306SVadym Kochan tx_ring->burst--;
755501ef306SVadym Kochan } else {
756501ef306SVadym Kochan tx_ring->burst = tx_ring->max_burst;
757501ef306SVadym Kochan
758501ef306SVadym Kochan err = prestera_sdma_tx_wait(sdma, tx_ring);
759501ef306SVadym Kochan if (err)
760501ef306SVadym Kochan goto drop_skb_unmap;
761501ef306SVadym Kochan }
762501ef306SVadym Kochan
763501ef306SVadym Kochan tx_ring->next_tx = (tx_ring->next_tx + 1) % PRESTERA_SDMA_TX_DESC_PER_Q;
764501ef306SVadym Kochan prestera_sdma_tx_desc_xmit(buf->desc);
765501ef306SVadym Kochan buf->is_used = true;
766501ef306SVadym Kochan
767501ef306SVadym Kochan prestera_sdma_tx_start(sdma);
768501ef306SVadym Kochan
769501ef306SVadym Kochan goto tx_done;
770501ef306SVadym Kochan
771501ef306SVadym Kochan drop_skb_unmap:
772501ef306SVadym Kochan prestera_sdma_tx_buf_unmap(sdma, buf);
773501ef306SVadym Kochan drop_skb:
774501ef306SVadym Kochan dev_consume_skb_any(skb);
775501ef306SVadym Kochan drop_skb_nofree:
776501ef306SVadym Kochan dev->stats.tx_dropped++;
777501ef306SVadym Kochan tx_done:
778501ef306SVadym Kochan spin_unlock(&sdma->tx_lock);
779501ef306SVadym Kochan return NETDEV_TX_OK;
780501ef306SVadym Kochan }
781501ef306SVadym Kochan
prestera_rxtx_switch_init(struct prestera_switch * sw)782501ef306SVadym Kochan int prestera_rxtx_switch_init(struct prestera_switch *sw)
783501ef306SVadym Kochan {
784501ef306SVadym Kochan struct prestera_rxtx *rxtx;
785519b58bbSZhengchao Shao int err;
786501ef306SVadym Kochan
787501ef306SVadym Kochan rxtx = kzalloc(sizeof(*rxtx), GFP_KERNEL);
788501ef306SVadym Kochan if (!rxtx)
789501ef306SVadym Kochan return -ENOMEM;
790501ef306SVadym Kochan
791501ef306SVadym Kochan sw->rxtx = rxtx;
792501ef306SVadym Kochan
793519b58bbSZhengchao Shao err = prestera_sdma_switch_init(sw);
794519b58bbSZhengchao Shao if (err)
795519b58bbSZhengchao Shao kfree(rxtx);
796519b58bbSZhengchao Shao
797519b58bbSZhengchao Shao return err;
798501ef306SVadym Kochan }
799501ef306SVadym Kochan
prestera_rxtx_switch_fini(struct prestera_switch * sw)800501ef306SVadym Kochan void prestera_rxtx_switch_fini(struct prestera_switch *sw)
801501ef306SVadym Kochan {
802501ef306SVadym Kochan prestera_sdma_switch_fini(sw);
803501ef306SVadym Kochan kfree(sw->rxtx);
804501ef306SVadym Kochan }
805501ef306SVadym Kochan
prestera_rxtx_port_init(struct prestera_port * port)806501ef306SVadym Kochan int prestera_rxtx_port_init(struct prestera_port *port)
807501ef306SVadym Kochan {
808501ef306SVadym Kochan port->dev->needed_headroom = PRESTERA_DSA_HLEN;
809501ef306SVadym Kochan return 0;
810501ef306SVadym Kochan }
811501ef306SVadym Kochan
prestera_rxtx_xmit(struct prestera_port * port,struct sk_buff * skb)812501ef306SVadym Kochan netdev_tx_t prestera_rxtx_xmit(struct prestera_port *port, struct sk_buff *skb)
813501ef306SVadym Kochan {
814501ef306SVadym Kochan struct prestera_dsa dsa;
815501ef306SVadym Kochan
816501ef306SVadym Kochan dsa.hw_dev_num = port->dev_id;
817501ef306SVadym Kochan dsa.port_num = port->hw_id;
818501ef306SVadym Kochan
819501ef306SVadym Kochan if (skb_cow_head(skb, PRESTERA_DSA_HLEN) < 0)
820501ef306SVadym Kochan return NET_XMIT_DROP;
821501ef306SVadym Kochan
822501ef306SVadym Kochan skb_push(skb, PRESTERA_DSA_HLEN);
823501ef306SVadym Kochan memmove(skb->data, skb->data + PRESTERA_DSA_HLEN, 2 * ETH_ALEN);
824501ef306SVadym Kochan
825501ef306SVadym Kochan if (prestera_dsa_build(&dsa, skb->data + 2 * ETH_ALEN) != 0)
826501ef306SVadym Kochan return NET_XMIT_DROP;
827501ef306SVadym Kochan
828501ef306SVadym Kochan return prestera_sdma_xmit(&port->sw->rxtx->sdma, skb);
829501ef306SVadym Kochan }
830