1 // SPDX-License-Identifier: GPL-2.0+
2 /* Microchip Sparx5 Switch driver
3 *
4 * Copyright (c) 2021 Microchip Technology Inc. and its subsidiaries.
5 *
6 * The Sparx5 Chip Register Model can be browsed at this location:
7 * https://github.com/microchip-ung/sparx-5_reginfo
8 */
9
10 #include <linux/types.h>
11 #include <linux/skbuff.h>
12 #include <linux/netdevice.h>
13 #include <linux/interrupt.h>
14 #include <linux/ip.h>
15 #include <linux/dma-mapping.h>
16
17 #include "sparx5_main_regs.h"
18 #include "sparx5_main.h"
19 #include "sparx5_port.h"
20
21 #define FDMA_XTR_CHANNEL 6
22 #define FDMA_INJ_CHANNEL 0
23
24 #define FDMA_XTR_BUFFER_SIZE 2048
25 #define FDMA_WEIGHT 4
26
sparx5_fdma_tx_dataptr_cb(struct fdma * fdma,int dcb,int db,u64 * dataptr)27 static int sparx5_fdma_tx_dataptr_cb(struct fdma *fdma, int dcb, int db,
28 u64 *dataptr)
29 {
30 *dataptr = fdma->dma + (sizeof(struct fdma_dcb) * fdma->n_dcbs) +
31 ((dcb * fdma->n_dbs + db) * fdma->db_size);
32
33 return 0;
34 }
35
sparx5_fdma_rx_dataptr_cb(struct fdma * fdma,int dcb,int db,u64 * dataptr)36 static int sparx5_fdma_rx_dataptr_cb(struct fdma *fdma, int dcb, int db,
37 u64 *dataptr)
38 {
39 struct sparx5 *sparx5 = fdma->priv;
40 struct sparx5_rx *rx = &sparx5->rx;
41 struct sk_buff *skb;
42
43 skb = __netdev_alloc_skb(rx->ndev, fdma->db_size, GFP_ATOMIC);
44 if (unlikely(!skb))
45 return -ENOMEM;
46
47 *dataptr = virt_to_phys(skb->data);
48
49 rx->skb[dcb][db] = skb;
50
51 return 0;
52 }
53
sparx5_fdma_rx_activate(struct sparx5 * sparx5,struct sparx5_rx * rx)54 static void sparx5_fdma_rx_activate(struct sparx5 *sparx5, struct sparx5_rx *rx)
55 {
56 struct fdma *fdma = &rx->fdma;
57
58 /* Write the buffer address in the LLP and LLP1 regs */
59 spx5_wr(((u64)fdma->dma) & GENMASK(31, 0), sparx5,
60 FDMA_DCB_LLP(fdma->channel_id));
61 spx5_wr(((u64)fdma->dma) >> 32, sparx5,
62 FDMA_DCB_LLP1(fdma->channel_id));
63
64 /* Set the number of RX DBs to be used, and DB end-of-frame interrupt */
65 spx5_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(fdma->n_dbs) |
66 FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) |
67 FDMA_CH_CFG_CH_INJ_PORT_SET(XTR_QUEUE),
68 sparx5, FDMA_CH_CFG(fdma->channel_id));
69
70 /* Set the RX Watermark to max */
71 spx5_rmw(FDMA_XTR_CFG_XTR_FIFO_WM_SET(31), FDMA_XTR_CFG_XTR_FIFO_WM,
72 sparx5,
73 FDMA_XTR_CFG);
74
75 /* Start RX fdma */
76 spx5_rmw(FDMA_PORT_CTRL_XTR_STOP_SET(0), FDMA_PORT_CTRL_XTR_STOP,
77 sparx5, FDMA_PORT_CTRL(0));
78
79 /* Enable RX channel DB interrupt */
80 spx5_rmw(BIT(fdma->channel_id),
81 BIT(fdma->channel_id) & FDMA_INTR_DB_ENA_INTR_DB_ENA,
82 sparx5, FDMA_INTR_DB_ENA);
83
84 /* Activate the RX channel */
85 spx5_wr(BIT(fdma->channel_id), sparx5, FDMA_CH_ACTIVATE);
86 }
87
sparx5_fdma_rx_deactivate(struct sparx5 * sparx5,struct sparx5_rx * rx)88 static void sparx5_fdma_rx_deactivate(struct sparx5 *sparx5, struct sparx5_rx *rx)
89 {
90 struct fdma *fdma = &rx->fdma;
91
92 /* Deactivate the RX channel */
93 spx5_rmw(0, BIT(fdma->channel_id) & FDMA_CH_ACTIVATE_CH_ACTIVATE,
94 sparx5, FDMA_CH_ACTIVATE);
95
96 /* Disable RX channel DB interrupt */
97 spx5_rmw(0, BIT(fdma->channel_id) & FDMA_INTR_DB_ENA_INTR_DB_ENA,
98 sparx5, FDMA_INTR_DB_ENA);
99
100 /* Stop RX fdma */
101 spx5_rmw(FDMA_PORT_CTRL_XTR_STOP_SET(1), FDMA_PORT_CTRL_XTR_STOP,
102 sparx5, FDMA_PORT_CTRL(0));
103 }
104
sparx5_fdma_tx_activate(struct sparx5 * sparx5,struct sparx5_tx * tx)105 static void sparx5_fdma_tx_activate(struct sparx5 *sparx5, struct sparx5_tx *tx)
106 {
107 struct fdma *fdma = &tx->fdma;
108
109 /* Write the buffer address in the LLP and LLP1 regs */
110 spx5_wr(((u64)fdma->dma) & GENMASK(31, 0), sparx5,
111 FDMA_DCB_LLP(fdma->channel_id));
112 spx5_wr(((u64)fdma->dma) >> 32, sparx5,
113 FDMA_DCB_LLP1(fdma->channel_id));
114
115 /* Set the number of TX DBs to be used, and DB end-of-frame interrupt */
116 spx5_wr(FDMA_CH_CFG_CH_DCB_DB_CNT_SET(fdma->n_dbs) |
117 FDMA_CH_CFG_CH_INTR_DB_EOF_ONLY_SET(1) |
118 FDMA_CH_CFG_CH_INJ_PORT_SET(INJ_QUEUE),
119 sparx5, FDMA_CH_CFG(fdma->channel_id));
120
121 /* Start TX fdma */
122 spx5_rmw(FDMA_PORT_CTRL_INJ_STOP_SET(0), FDMA_PORT_CTRL_INJ_STOP,
123 sparx5, FDMA_PORT_CTRL(0));
124
125 /* Activate the channel */
126 spx5_wr(BIT(fdma->channel_id), sparx5, FDMA_CH_ACTIVATE);
127 }
128
sparx5_fdma_tx_deactivate(struct sparx5 * sparx5,struct sparx5_tx * tx)129 static void sparx5_fdma_tx_deactivate(struct sparx5 *sparx5, struct sparx5_tx *tx)
130 {
131 /* Disable the channel */
132 spx5_rmw(0, BIT(tx->fdma.channel_id) & FDMA_CH_ACTIVATE_CH_ACTIVATE,
133 sparx5, FDMA_CH_ACTIVATE);
134 }
135
sparx5_fdma_reload(struct sparx5 * sparx5,struct fdma * fdma)136 static void sparx5_fdma_reload(struct sparx5 *sparx5, struct fdma *fdma)
137 {
138 /* Reload the RX channel */
139 spx5_wr(BIT(fdma->channel_id), sparx5, FDMA_CH_RELOAD);
140 }
141
sparx5_fdma_rx_get_frame(struct sparx5 * sparx5,struct sparx5_rx * rx)142 static bool sparx5_fdma_rx_get_frame(struct sparx5 *sparx5, struct sparx5_rx *rx)
143 {
144 struct fdma *fdma = &rx->fdma;
145 struct sparx5_port *port;
146 struct fdma_db *db_hw;
147 struct frame_info fi;
148 struct sk_buff *skb;
149
150 /* Check if the DCB is done */
151 db_hw = fdma_db_next_get(fdma);
152 if (unlikely(!fdma_db_is_done(db_hw)))
153 return false;
154 skb = rx->skb[fdma->dcb_index][fdma->db_index];
155 skb_put(skb, fdma_db_len_get(db_hw));
156 /* Now do the normal processing of the skb */
157 sparx5_ifh_parse((u32 *)skb->data, &fi);
158 /* Map to port netdev */
159 port = fi.src_port < SPX5_PORTS ? sparx5->ports[fi.src_port] : NULL;
160 if (!port || !port->ndev) {
161 dev_err(sparx5->dev, "Data on inactive port %d\n", fi.src_port);
162 sparx5_xtr_flush(sparx5, XTR_QUEUE);
163 return false;
164 }
165 skb->dev = port->ndev;
166 skb_pull(skb, IFH_LEN * sizeof(u32));
167 if (likely(!(skb->dev->features & NETIF_F_RXFCS)))
168 skb_trim(skb, skb->len - ETH_FCS_LEN);
169
170 sparx5_ptp_rxtstamp(sparx5, skb, fi.timestamp);
171 skb->protocol = eth_type_trans(skb, skb->dev);
172 /* Everything we see on an interface that is in the HW bridge
173 * has already been forwarded
174 */
175 if (test_bit(port->portno, sparx5->bridge_mask))
176 skb->offload_fwd_mark = 1;
177 skb->dev->stats.rx_bytes += skb->len;
178 skb->dev->stats.rx_packets++;
179 rx->packets++;
180 netif_receive_skb(skb);
181 return true;
182 }
183
sparx5_fdma_napi_callback(struct napi_struct * napi,int weight)184 static int sparx5_fdma_napi_callback(struct napi_struct *napi, int weight)
185 {
186 struct sparx5_rx *rx = container_of(napi, struct sparx5_rx, napi);
187 struct sparx5 *sparx5 = container_of(rx, struct sparx5, rx);
188 struct fdma *fdma = &rx->fdma;
189 int counter = 0;
190
191 while (counter < weight && sparx5_fdma_rx_get_frame(sparx5, rx)) {
192 fdma_db_advance(fdma);
193 counter++;
194 /* Check if the DCB can be reused */
195 if (fdma_dcb_is_reusable(fdma))
196 continue;
197 fdma_dcb_add(fdma, fdma->dcb_index,
198 FDMA_DCB_INFO_DATAL(fdma->db_size),
199 FDMA_DCB_STATUS_INTR);
200 fdma_db_reset(fdma);
201 fdma_dcb_advance(fdma);
202 }
203 if (counter < weight) {
204 napi_complete_done(&rx->napi, counter);
205 spx5_rmw(BIT(fdma->channel_id),
206 BIT(fdma->channel_id) & FDMA_INTR_DB_ENA_INTR_DB_ENA,
207 sparx5, FDMA_INTR_DB_ENA);
208 }
209 if (counter)
210 sparx5_fdma_reload(sparx5, fdma);
211 return counter;
212 }
213
sparx5_fdma_xmit(struct sparx5 * sparx5,u32 * ifh,struct sk_buff * skb)214 int sparx5_fdma_xmit(struct sparx5 *sparx5, u32 *ifh, struct sk_buff *skb)
215 {
216 struct sparx5_tx *tx = &sparx5->tx;
217 struct fdma *fdma = &tx->fdma;
218 static bool first_time = true;
219 void *virt_addr;
220
221 fdma_dcb_advance(fdma);
222 if (!fdma_db_is_done(fdma_db_get(fdma, fdma->dcb_index, 0)))
223 return -EINVAL;
224
225 /* Get the virtual address of the dataptr for the next DB */
226 virt_addr = ((u8 *)fdma->dcbs +
227 (sizeof(struct fdma_dcb) * fdma->n_dcbs) +
228 ((fdma->dcb_index * fdma->n_dbs) * fdma->db_size));
229
230 memcpy(virt_addr, ifh, IFH_LEN * 4);
231 memcpy(virt_addr + IFH_LEN * 4, skb->data, skb->len);
232
233 fdma_dcb_add(fdma, fdma->dcb_index, 0,
234 FDMA_DCB_STATUS_SOF |
235 FDMA_DCB_STATUS_EOF |
236 FDMA_DCB_STATUS_BLOCKO(0) |
237 FDMA_DCB_STATUS_BLOCKL(skb->len + IFH_LEN * 4 + 4));
238
239 if (first_time) {
240 sparx5_fdma_tx_activate(sparx5, tx);
241 first_time = false;
242 } else {
243 sparx5_fdma_reload(sparx5, fdma);
244 }
245 return NETDEV_TX_OK;
246 }
247
sparx5_fdma_rx_alloc(struct sparx5 * sparx5)248 static int sparx5_fdma_rx_alloc(struct sparx5 *sparx5)
249 {
250 struct sparx5_rx *rx = &sparx5->rx;
251 struct fdma *fdma = &rx->fdma;
252 int err;
253
254 err = fdma_alloc_phys(fdma);
255 if (err)
256 return err;
257
258 fdma_dcbs_init(fdma, FDMA_DCB_INFO_DATAL(fdma->db_size),
259 FDMA_DCB_STATUS_INTR);
260
261 netif_napi_add_weight(rx->ndev, &rx->napi, sparx5_fdma_napi_callback,
262 FDMA_WEIGHT);
263 napi_enable(&rx->napi);
264 sparx5_fdma_rx_activate(sparx5, rx);
265 return 0;
266 }
267
sparx5_fdma_tx_alloc(struct sparx5 * sparx5)268 static int sparx5_fdma_tx_alloc(struct sparx5 *sparx5)
269 {
270 struct sparx5_tx *tx = &sparx5->tx;
271 struct fdma *fdma = &tx->fdma;
272 int err;
273
274 err = fdma_alloc_phys(fdma);
275 if (err)
276 return err;
277
278 fdma_dcbs_init(fdma, FDMA_DCB_INFO_DATAL(fdma->db_size),
279 FDMA_DCB_STATUS_DONE);
280
281 return 0;
282 }
283
sparx5_fdma_rx_init(struct sparx5 * sparx5,struct sparx5_rx * rx,int channel)284 static void sparx5_fdma_rx_init(struct sparx5 *sparx5,
285 struct sparx5_rx *rx, int channel)
286 {
287 struct fdma *fdma = &rx->fdma;
288 int idx;
289
290 fdma->channel_id = channel;
291 fdma->n_dcbs = FDMA_DCB_MAX;
292 fdma->n_dbs = FDMA_RX_DCB_MAX_DBS;
293 fdma->priv = sparx5;
294 fdma->db_size = ALIGN(FDMA_XTR_BUFFER_SIZE, PAGE_SIZE);
295 fdma->size = fdma_get_size(&sparx5->rx.fdma);
296 fdma->ops.dataptr_cb = &sparx5_fdma_rx_dataptr_cb;
297 fdma->ops.nextptr_cb = &fdma_nextptr_cb;
298 /* Fetch a netdev for SKB and NAPI use, any will do */
299 for (idx = 0; idx < SPX5_PORTS; ++idx) {
300 struct sparx5_port *port = sparx5->ports[idx];
301
302 if (port && port->ndev) {
303 rx->ndev = port->ndev;
304 break;
305 }
306 }
307 }
308
sparx5_fdma_tx_init(struct sparx5 * sparx5,struct sparx5_tx * tx,int channel)309 static void sparx5_fdma_tx_init(struct sparx5 *sparx5,
310 struct sparx5_tx *tx, int channel)
311 {
312 struct fdma *fdma = &tx->fdma;
313
314 fdma->channel_id = channel;
315 fdma->n_dcbs = FDMA_DCB_MAX;
316 fdma->n_dbs = FDMA_TX_DCB_MAX_DBS;
317 fdma->priv = sparx5;
318 fdma->db_size = ALIGN(FDMA_XTR_BUFFER_SIZE, PAGE_SIZE);
319 fdma->size = fdma_get_size_contiguous(&sparx5->tx.fdma);
320 fdma->ops.dataptr_cb = &sparx5_fdma_tx_dataptr_cb;
321 fdma->ops.nextptr_cb = &fdma_nextptr_cb;
322 }
323
sparx5_fdma_handler(int irq,void * args)324 irqreturn_t sparx5_fdma_handler(int irq, void *args)
325 {
326 struct sparx5 *sparx5 = args;
327 u32 db = 0, err = 0;
328
329 db = spx5_rd(sparx5, FDMA_INTR_DB);
330 err = spx5_rd(sparx5, FDMA_INTR_ERR);
331 /* Clear interrupt */
332 if (db) {
333 spx5_wr(0, sparx5, FDMA_INTR_DB_ENA);
334 spx5_wr(db, sparx5, FDMA_INTR_DB);
335 napi_schedule(&sparx5->rx.napi);
336 }
337 if (err) {
338 u32 err_type = spx5_rd(sparx5, FDMA_ERRORS);
339
340 dev_err_ratelimited(sparx5->dev,
341 "ERR: int: %#x, type: %#x\n",
342 err, err_type);
343 spx5_wr(err, sparx5, FDMA_INTR_ERR);
344 spx5_wr(err_type, sparx5, FDMA_ERRORS);
345 }
346 return IRQ_HANDLED;
347 }
348
sparx5_fdma_injection_mode(struct sparx5 * sparx5)349 static void sparx5_fdma_injection_mode(struct sparx5 *sparx5)
350 {
351 const int byte_swap = 1;
352 int portno;
353 int urgency;
354
355 /* Change mode to fdma extraction and injection */
356 spx5_wr(QS_XTR_GRP_CFG_MODE_SET(2) |
357 QS_XTR_GRP_CFG_STATUS_WORD_POS_SET(1) |
358 QS_XTR_GRP_CFG_BYTE_SWAP_SET(byte_swap),
359 sparx5, QS_XTR_GRP_CFG(XTR_QUEUE));
360 spx5_wr(QS_INJ_GRP_CFG_MODE_SET(2) |
361 QS_INJ_GRP_CFG_BYTE_SWAP_SET(byte_swap),
362 sparx5, QS_INJ_GRP_CFG(INJ_QUEUE));
363
364 /* CPU ports capture setup */
365 for (portno = SPX5_PORT_CPU_0; portno <= SPX5_PORT_CPU_1; portno++) {
366 /* ASM CPU port: No preamble, IFH, enable padding */
367 spx5_wr(ASM_PORT_CFG_PAD_ENA_SET(1) |
368 ASM_PORT_CFG_NO_PREAMBLE_ENA_SET(1) |
369 ASM_PORT_CFG_INJ_FORMAT_CFG_SET(1), /* 1 = IFH */
370 sparx5, ASM_PORT_CFG(portno));
371
372 /* Reset WM cnt to unclog queued frames */
373 spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR_SET(1),
374 DSM_DEV_TX_STOP_WM_CFG_DEV_TX_CNT_CLR,
375 sparx5,
376 DSM_DEV_TX_STOP_WM_CFG(portno));
377
378 /* Set Disassembler Stop Watermark level */
379 spx5_rmw(DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM_SET(100),
380 DSM_DEV_TX_STOP_WM_CFG_DEV_TX_STOP_WM,
381 sparx5,
382 DSM_DEV_TX_STOP_WM_CFG(portno));
383
384 /* Enable port in queue system */
385 urgency = sparx5_port_fwd_urg(sparx5, SPEED_2500);
386 spx5_rmw(QFWD_SWITCH_PORT_MODE_PORT_ENA_SET(1) |
387 QFWD_SWITCH_PORT_MODE_FWD_URGENCY_SET(urgency),
388 QFWD_SWITCH_PORT_MODE_PORT_ENA |
389 QFWD_SWITCH_PORT_MODE_FWD_URGENCY,
390 sparx5,
391 QFWD_SWITCH_PORT_MODE(portno));
392
393 /* Disable Disassembler buffer underrun watchdog
394 * to avoid truncated packets in XTR
395 */
396 spx5_rmw(DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS_SET(1),
397 DSM_BUF_CFG_UNDERFLOW_WATCHDOG_DIS,
398 sparx5,
399 DSM_BUF_CFG(portno));
400
401 /* Disabling frame aging */
402 spx5_rmw(HSCH_PORT_MODE_AGE_DIS_SET(1),
403 HSCH_PORT_MODE_AGE_DIS,
404 sparx5,
405 HSCH_PORT_MODE(portno));
406 }
407 }
408
sparx5_fdma_start(struct sparx5 * sparx5)409 int sparx5_fdma_start(struct sparx5 *sparx5)
410 {
411 int err;
412
413 /* Reset FDMA state */
414 spx5_wr(FDMA_CTRL_NRESET_SET(0), sparx5, FDMA_CTRL);
415 spx5_wr(FDMA_CTRL_NRESET_SET(1), sparx5, FDMA_CTRL);
416
417 /* Force ACP caching but disable read/write allocation */
418 spx5_rmw(CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA_SET(1) |
419 CPU_PROC_CTRL_ACP_AWCACHE_SET(0) |
420 CPU_PROC_CTRL_ACP_ARCACHE_SET(0),
421 CPU_PROC_CTRL_ACP_CACHE_FORCE_ENA |
422 CPU_PROC_CTRL_ACP_AWCACHE |
423 CPU_PROC_CTRL_ACP_ARCACHE,
424 sparx5, CPU_PROC_CTRL);
425
426 sparx5_fdma_injection_mode(sparx5);
427 sparx5_fdma_rx_init(sparx5, &sparx5->rx, FDMA_XTR_CHANNEL);
428 sparx5_fdma_tx_init(sparx5, &sparx5->tx, FDMA_INJ_CHANNEL);
429 err = sparx5_fdma_rx_alloc(sparx5);
430 if (err) {
431 dev_err(sparx5->dev, "Could not allocate RX buffers: %d\n", err);
432 return err;
433 }
434 err = sparx5_fdma_tx_alloc(sparx5);
435 if (err) {
436 dev_err(sparx5->dev, "Could not allocate TX buffers: %d\n", err);
437 return err;
438 }
439 return err;
440 }
441
sparx5_fdma_port_ctrl(struct sparx5 * sparx5)442 static u32 sparx5_fdma_port_ctrl(struct sparx5 *sparx5)
443 {
444 return spx5_rd(sparx5, FDMA_PORT_CTRL(0));
445 }
446
sparx5_fdma_stop(struct sparx5 * sparx5)447 int sparx5_fdma_stop(struct sparx5 *sparx5)
448 {
449 u32 val;
450
451 napi_disable(&sparx5->rx.napi);
452 /* Stop the fdma and channel interrupts */
453 sparx5_fdma_rx_deactivate(sparx5, &sparx5->rx);
454 sparx5_fdma_tx_deactivate(sparx5, &sparx5->tx);
455 /* Wait for the RX channel to stop */
456 read_poll_timeout(sparx5_fdma_port_ctrl, val,
457 FDMA_PORT_CTRL_XTR_BUF_IS_EMPTY_GET(val) == 0,
458 500, 10000, 0, sparx5);
459 fdma_free_phys(&sparx5->rx.fdma);
460 fdma_free_phys(&sparx5->tx.fdma);
461 return 0;
462 }
463