1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3
4 #include "enetc.h"
5 #include <linux/bpf_trace.h>
6 #include <linux/clk.h>
7 #include <linux/tcp.h>
8 #include <linux/udp.h>
9 #include <linux/vmalloc.h>
10 #include <linux/ptp_classify.h>
11 #include <net/ip6_checksum.h>
12 #include <net/pkt_sched.h>
13 #include <net/tso.h>
14
enetc_port_mac_rd(struct enetc_si * si,u32 reg)15 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
16 {
17 return enetc_port_rd(&si->hw, reg);
18 }
19 EXPORT_SYMBOL_GPL(enetc_port_mac_rd);
20
enetc_port_mac_wr(struct enetc_si * si,u32 reg,u32 val)21 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
22 {
23 enetc_port_wr(&si->hw, reg, val);
24 if (si->hw_features & ENETC_SI_F_QBU)
25 enetc_port_wr(&si->hw, reg + si->drvdata->pmac_offset, val);
26 }
27 EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
28
enetc_change_preemptible_tcs(struct enetc_ndev_priv * priv,u8 preemptible_tcs)29 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
30 u8 preemptible_tcs)
31 {
32 if (!(priv->si->hw_features & ENETC_SI_F_QBU))
33 return;
34
35 priv->preemptible_tcs = preemptible_tcs;
36 enetc_mm_commit_preemptible_tcs(priv);
37 }
38
enetc_mac_addr_hash_idx(const u8 * addr)39 static int enetc_mac_addr_hash_idx(const u8 *addr)
40 {
41 u64 fold = __swab64(ether_addr_to_u64(addr)) >> 16;
42 u64 mask = 0;
43 int res = 0;
44 int i;
45
46 for (i = 0; i < 8; i++)
47 mask |= BIT_ULL(i * 6);
48
49 for (i = 0; i < 6; i++)
50 res |= (hweight64(fold & (mask << i)) & 0x1) << i;
51
52 return res;
53 }
54
enetc_add_mac_addr_ht_filter(struct enetc_mac_filter * filter,const unsigned char * addr)55 void enetc_add_mac_addr_ht_filter(struct enetc_mac_filter *filter,
56 const unsigned char *addr)
57 {
58 int idx = enetc_mac_addr_hash_idx(addr);
59
60 /* add hash table entry */
61 __set_bit(idx, filter->mac_hash_table);
62 filter->mac_addr_cnt++;
63 }
64 EXPORT_SYMBOL_GPL(enetc_add_mac_addr_ht_filter);
65
enetc_reset_mac_addr_filter(struct enetc_mac_filter * filter)66 void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter)
67 {
68 filter->mac_addr_cnt = 0;
69
70 bitmap_zero(filter->mac_hash_table,
71 ENETC_MADDR_HASH_TBL_SZ);
72 }
73 EXPORT_SYMBOL_GPL(enetc_reset_mac_addr_filter);
74
enetc_num_stack_tx_queues(struct enetc_ndev_priv * priv)75 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
76 {
77 int num_tx_rings = priv->num_tx_rings;
78
79 if (priv->xdp_prog)
80 return num_tx_rings - num_possible_cpus();
81
82 return num_tx_rings;
83 }
84
enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv * priv,struct enetc_bdr * tx_ring)85 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
86 struct enetc_bdr *tx_ring)
87 {
88 int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
89
90 return priv->rx_ring[index];
91 }
92
enetc_tx_swbd_get_skb(struct enetc_tx_swbd * tx_swbd)93 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
94 {
95 if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
96 return NULL;
97
98 return tx_swbd->skb;
99 }
100
101 static struct xdp_frame *
enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd * tx_swbd)102 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
103 {
104 if (tx_swbd->is_xdp_redirect)
105 return tx_swbd->xdp_frame;
106
107 return NULL;
108 }
109
enetc_unmap_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)110 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
111 struct enetc_tx_swbd *tx_swbd)
112 {
113 /* For XDP_TX, pages come from RX, whereas for the other contexts where
114 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
115 * to match the DMA mapping length, so we need to differentiate those.
116 */
117 if (tx_swbd->is_dma_page)
118 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
119 tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
120 tx_swbd->dir);
121 else
122 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
123 tx_swbd->len, tx_swbd->dir);
124 tx_swbd->dma = 0;
125 }
126
enetc_free_tx_frame(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)127 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
128 struct enetc_tx_swbd *tx_swbd)
129 {
130 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
131 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
132
133 if (tx_swbd->dma)
134 enetc_unmap_tx_buff(tx_ring, tx_swbd);
135
136 if (xdp_frame) {
137 xdp_return_frame(tx_swbd->xdp_frame);
138 tx_swbd->xdp_frame = NULL;
139 } else if (skb) {
140 dev_kfree_skb_any(skb);
141 tx_swbd->skb = NULL;
142 }
143 }
144
145 /* Let H/W know BD ring has been updated */
enetc_update_tx_ring_tail(struct enetc_bdr * tx_ring)146 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
147 {
148 /* includes wmb() */
149 enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
150 }
151
enetc_ptp_parse(struct sk_buff * skb,u8 * udp,u8 * msgtype,u8 * twostep,u16 * correction_offset,u16 * body_offset)152 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
153 u8 *msgtype, u8 *twostep,
154 u16 *correction_offset, u16 *body_offset)
155 {
156 unsigned int ptp_class;
157 struct ptp_header *hdr;
158 unsigned int type;
159 u8 *base;
160
161 ptp_class = ptp_classify_raw(skb);
162 if (ptp_class == PTP_CLASS_NONE)
163 return -EINVAL;
164
165 hdr = ptp_parse_header(skb, ptp_class);
166 if (!hdr)
167 return -EINVAL;
168
169 type = ptp_class & PTP_CLASS_PMASK;
170 if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
171 *udp = 1;
172 else
173 *udp = 0;
174
175 *msgtype = ptp_get_msgtype(hdr, ptp_class);
176 *twostep = hdr->flag_field[0] & 0x2;
177
178 base = skb_mac_header(skb);
179 *correction_offset = (u8 *)&hdr->correction - base;
180 *body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
181
182 return 0;
183 }
184
enetc_tx_csum_offload_check(struct sk_buff * skb)185 static bool enetc_tx_csum_offload_check(struct sk_buff *skb)
186 {
187 switch (skb->csum_offset) {
188 case offsetof(struct tcphdr, check):
189 case offsetof(struct udphdr, check):
190 return true;
191 default:
192 return false;
193 }
194 }
195
enetc_skb_is_ipv6(struct sk_buff * skb)196 static bool enetc_skb_is_ipv6(struct sk_buff *skb)
197 {
198 return vlan_get_protocol(skb) == htons(ETH_P_IPV6);
199 }
200
enetc_skb_is_tcp(struct sk_buff * skb)201 static bool enetc_skb_is_tcp(struct sk_buff *skb)
202 {
203 return skb->csum_offset == offsetof(struct tcphdr, check);
204 }
205
206 /**
207 * enetc_unwind_tx_frame() - Unwind the DMA mappings of a multi-buffer Tx frame
208 * @tx_ring: Pointer to the Tx ring on which the buffer descriptors are located
209 * @count: Number of Tx buffer descriptors which need to be unmapped
210 * @i: Index of the last successfully mapped Tx buffer descriptor
211 */
enetc_unwind_tx_frame(struct enetc_bdr * tx_ring,int count,int i)212 static void enetc_unwind_tx_frame(struct enetc_bdr *tx_ring, int count, int i)
213 {
214 while (count--) {
215 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
216
217 enetc_free_tx_frame(tx_ring, tx_swbd);
218 if (i == 0)
219 i = tx_ring->bd_count;
220 i--;
221 }
222 }
223
enetc_map_tx_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)224 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
225 {
226 bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
227 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
228 struct enetc_hw *hw = &priv->si->hw;
229 struct enetc_tx_swbd *tx_swbd;
230 int len = skb_headlen(skb);
231 union enetc_tx_bd temp_bd;
232 u8 msgtype, twostep, udp;
233 union enetc_tx_bd *txbd;
234 u16 offset1, offset2;
235 int i, count = 0;
236 skb_frag_t *frag;
237 unsigned int f;
238 dma_addr_t dma;
239 u8 flags = 0;
240
241 enetc_clear_tx_bd(&temp_bd);
242 if (skb->ip_summed == CHECKSUM_PARTIAL) {
243 /* Can not support TSD and checksum offload at the same time */
244 if (priv->active_offloads & ENETC_F_TXCSUM &&
245 enetc_tx_csum_offload_check(skb) && !tx_ring->tsd_enable) {
246 temp_bd.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START,
247 skb_network_offset(skb));
248 temp_bd.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN,
249 skb_network_header_len(skb) / 4);
250 temp_bd.l3_aux1 |= FIELD_PREP(ENETC_TX_BD_L3T,
251 enetc_skb_is_ipv6(skb));
252 if (enetc_skb_is_tcp(skb))
253 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T,
254 ENETC_TXBD_L4T_TCP);
255 else
256 temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T,
257 ENETC_TXBD_L4T_UDP);
258 flags |= ENETC_TXBD_FLAGS_CSUM_LSO | ENETC_TXBD_FLAGS_L4CS;
259 } else if (skb_checksum_help(skb)) {
260 return 0;
261 }
262 }
263
264 i = tx_ring->next_to_use;
265 txbd = ENETC_TXBD(*tx_ring, i);
266 prefetchw(txbd);
267
268 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
269 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
270 goto dma_err;
271
272 temp_bd.addr = cpu_to_le64(dma);
273 temp_bd.buf_len = cpu_to_le16(len);
274
275 tx_swbd = &tx_ring->tx_swbd[i];
276 tx_swbd->dma = dma;
277 tx_swbd->len = len;
278 tx_swbd->is_dma_page = 0;
279 tx_swbd->dir = DMA_TO_DEVICE;
280 count++;
281
282 do_vlan = skb_vlan_tag_present(skb);
283 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
284 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
285 &offset2) ||
286 msgtype != PTP_MSGTYPE_SYNC || twostep)
287 WARN_ONCE(1, "Bad packet for one-step timestamping\n");
288 else
289 do_onestep_tstamp = true;
290 } else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
291 do_twostep_tstamp = true;
292 }
293
294 tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
295 tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
296 tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
297
298 if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
299 flags |= ENETC_TXBD_FLAGS_EX;
300
301 if (tx_ring->tsd_enable)
302 flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
303
304 /* first BD needs frm_len and offload flags set */
305 temp_bd.frm_len = cpu_to_le16(skb->len);
306 temp_bd.flags = flags;
307
308 if (flags & ENETC_TXBD_FLAGS_TSE)
309 temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
310 flags);
311
312 if (flags & ENETC_TXBD_FLAGS_EX) {
313 u8 e_flags = 0;
314 *txbd = temp_bd;
315 enetc_clear_tx_bd(&temp_bd);
316
317 /* add extension BD for VLAN and/or timestamping */
318 flags = 0;
319 tx_swbd++;
320 txbd++;
321 i++;
322 if (unlikely(i == tx_ring->bd_count)) {
323 i = 0;
324 tx_swbd = tx_ring->tx_swbd;
325 txbd = ENETC_TXBD(*tx_ring, 0);
326 }
327 prefetchw(txbd);
328
329 if (do_vlan) {
330 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
331 temp_bd.ext.tpid = 0; /* < C-TAG */
332 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
333 }
334
335 if (do_onestep_tstamp) {
336 __be32 new_sec_l, new_nsec;
337 u32 lo, hi, nsec, val;
338 __be16 new_sec_h;
339 u8 *data;
340 u64 sec;
341
342 lo = enetc_rd_hot(hw, ENETC_SICTR0);
343 hi = enetc_rd_hot(hw, ENETC_SICTR1);
344 sec = (u64)hi << 32 | lo;
345 nsec = do_div(sec, 1000000000);
346
347 /* Configure extension BD */
348 temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
349 e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
350
351 /* Update originTimestamp field of Sync packet
352 * - 48 bits seconds field
353 * - 32 bits nanseconds field
354 *
355 * In addition, the UDP checksum needs to be updated
356 * by software after updating originTimestamp field,
357 * otherwise the hardware will calculate the wrong
358 * checksum when updating the correction field and
359 * update it to the packet.
360 */
361 data = skb_mac_header(skb);
362 new_sec_h = htons((sec >> 32) & 0xffff);
363 new_sec_l = htonl(sec & 0xffffffff);
364 new_nsec = htonl(nsec);
365 if (udp) {
366 struct udphdr *uh = udp_hdr(skb);
367 __be32 old_sec_l, old_nsec;
368 __be16 old_sec_h;
369
370 old_sec_h = *(__be16 *)(data + offset2);
371 inet_proto_csum_replace2(&uh->check, skb, old_sec_h,
372 new_sec_h, false);
373
374 old_sec_l = *(__be32 *)(data + offset2 + 2);
375 inet_proto_csum_replace4(&uh->check, skb, old_sec_l,
376 new_sec_l, false);
377
378 old_nsec = *(__be32 *)(data + offset2 + 6);
379 inet_proto_csum_replace4(&uh->check, skb, old_nsec,
380 new_nsec, false);
381 }
382
383 *(__be16 *)(data + offset2) = new_sec_h;
384 *(__be32 *)(data + offset2 + 2) = new_sec_l;
385 *(__be32 *)(data + offset2 + 6) = new_nsec;
386
387 /* Configure single-step register */
388 val = ENETC_PM0_SINGLE_STEP_EN;
389 val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
390 if (udp)
391 val |= ENETC_PM0_SINGLE_STEP_CH;
392
393 enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP,
394 val);
395 } else if (do_twostep_tstamp) {
396 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
397 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
398 }
399
400 temp_bd.ext.e_flags = e_flags;
401 count++;
402 }
403
404 frag = &skb_shinfo(skb)->frags[0];
405 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
406 len = skb_frag_size(frag);
407 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
408 DMA_TO_DEVICE);
409 if (dma_mapping_error(tx_ring->dev, dma))
410 goto dma_err;
411
412 *txbd = temp_bd;
413 enetc_clear_tx_bd(&temp_bd);
414
415 flags = 0;
416 tx_swbd++;
417 txbd++;
418 i++;
419 if (unlikely(i == tx_ring->bd_count)) {
420 i = 0;
421 tx_swbd = tx_ring->tx_swbd;
422 txbd = ENETC_TXBD(*tx_ring, 0);
423 }
424 prefetchw(txbd);
425
426 temp_bd.addr = cpu_to_le64(dma);
427 temp_bd.buf_len = cpu_to_le16(len);
428
429 tx_swbd->dma = dma;
430 tx_swbd->len = len;
431 tx_swbd->is_dma_page = 1;
432 tx_swbd->dir = DMA_TO_DEVICE;
433 count++;
434 }
435
436 /* last BD needs 'F' bit set */
437 flags |= ENETC_TXBD_FLAGS_F;
438 temp_bd.flags = flags;
439 *txbd = temp_bd;
440
441 tx_ring->tx_swbd[i].is_eof = true;
442 tx_ring->tx_swbd[i].skb = skb;
443
444 enetc_bdr_idx_inc(tx_ring, &i);
445 tx_ring->next_to_use = i;
446
447 skb_tx_timestamp(skb);
448
449 enetc_update_tx_ring_tail(tx_ring);
450
451 return count;
452
453 dma_err:
454 dev_err(tx_ring->dev, "DMA map error");
455
456 enetc_unwind_tx_frame(tx_ring, count, i);
457
458 return 0;
459 }
460
enetc_map_tx_tso_hdr(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,int * i,int hdr_len,int data_len)461 static int enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
462 struct enetc_tx_swbd *tx_swbd,
463 union enetc_tx_bd *txbd, int *i, int hdr_len,
464 int data_len)
465 {
466 union enetc_tx_bd txbd_tmp;
467 u8 flags = 0, e_flags = 0;
468 dma_addr_t addr;
469 int count = 1;
470
471 enetc_clear_tx_bd(&txbd_tmp);
472 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
473
474 if (skb_vlan_tag_present(skb))
475 flags |= ENETC_TXBD_FLAGS_EX;
476
477 txbd_tmp.addr = cpu_to_le64(addr);
478 txbd_tmp.buf_len = cpu_to_le16(hdr_len);
479
480 /* first BD needs frm_len and offload flags set */
481 txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
482 txbd_tmp.flags = flags;
483
484 /* For the TSO header we do not set the dma address since we do not
485 * want it unmapped when we do cleanup. We still set len so that we
486 * count the bytes sent.
487 */
488 tx_swbd->len = hdr_len;
489 tx_swbd->do_twostep_tstamp = false;
490 tx_swbd->check_wb = false;
491
492 /* Actually write the header in the BD */
493 *txbd = txbd_tmp;
494
495 /* Add extension BD for VLAN */
496 if (flags & ENETC_TXBD_FLAGS_EX) {
497 /* Get the next BD */
498 enetc_bdr_idx_inc(tx_ring, i);
499 txbd = ENETC_TXBD(*tx_ring, *i);
500 tx_swbd = &tx_ring->tx_swbd[*i];
501 prefetchw(txbd);
502
503 /* Setup the VLAN fields */
504 enetc_clear_tx_bd(&txbd_tmp);
505 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
506 txbd_tmp.ext.tpid = 0; /* < C-TAG */
507 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
508
509 /* Write the BD */
510 txbd_tmp.ext.e_flags = e_flags;
511 *txbd = txbd_tmp;
512 count++;
513 }
514
515 return count;
516 }
517
enetc_map_tx_tso_data(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,char * data,int size,bool last_bd)518 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
519 struct enetc_tx_swbd *tx_swbd,
520 union enetc_tx_bd *txbd, char *data,
521 int size, bool last_bd)
522 {
523 union enetc_tx_bd txbd_tmp;
524 dma_addr_t addr;
525 u8 flags = 0;
526
527 enetc_clear_tx_bd(&txbd_tmp);
528
529 addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
530 if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
531 netdev_err(tx_ring->ndev, "DMA map error\n");
532 return -ENOMEM;
533 }
534
535 if (last_bd) {
536 flags |= ENETC_TXBD_FLAGS_F;
537 tx_swbd->is_eof = 1;
538 }
539
540 txbd_tmp.addr = cpu_to_le64(addr);
541 txbd_tmp.buf_len = cpu_to_le16(size);
542 txbd_tmp.flags = flags;
543
544 tx_swbd->dma = addr;
545 tx_swbd->len = size;
546 tx_swbd->dir = DMA_TO_DEVICE;
547
548 *txbd = txbd_tmp;
549
550 return 0;
551 }
552
enetc_tso_hdr_csum(struct tso_t * tso,struct sk_buff * skb,char * hdr,int hdr_len,int * l4_hdr_len)553 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
554 char *hdr, int hdr_len, int *l4_hdr_len)
555 {
556 char *l4_hdr = hdr + skb_transport_offset(skb);
557 int mac_hdr_len = skb_network_offset(skb);
558
559 if (tso->tlen != sizeof(struct udphdr)) {
560 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
561
562 tcph->check = 0;
563 } else {
564 struct udphdr *udph = (struct udphdr *)(l4_hdr);
565
566 udph->check = 0;
567 }
568
569 /* Compute the IP checksum. This is necessary since tso_build_hdr()
570 * already incremented the IP ID field.
571 */
572 if (!tso->ipv6) {
573 struct iphdr *iph = (void *)(hdr + mac_hdr_len);
574
575 iph->check = 0;
576 iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
577 }
578
579 /* Compute the checksum over the L4 header. */
580 *l4_hdr_len = hdr_len - skb_transport_offset(skb);
581 return csum_partial(l4_hdr, *l4_hdr_len, 0);
582 }
583
enetc_tso_complete_csum(struct enetc_bdr * tx_ring,struct tso_t * tso,struct sk_buff * skb,char * hdr,int len,__wsum sum)584 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
585 struct sk_buff *skb, char *hdr, int len,
586 __wsum sum)
587 {
588 char *l4_hdr = hdr + skb_transport_offset(skb);
589 __sum16 csum_final;
590
591 /* Complete the L4 checksum by appending the pseudo-header to the
592 * already computed checksum.
593 */
594 if (!tso->ipv6)
595 csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
596 ip_hdr(skb)->daddr,
597 len, ip_hdr(skb)->protocol, sum);
598 else
599 csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
600 &ipv6_hdr(skb)->daddr,
601 len, ipv6_hdr(skb)->nexthdr, sum);
602
603 if (tso->tlen != sizeof(struct udphdr)) {
604 struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
605
606 tcph->check = csum_final;
607 } else {
608 struct udphdr *udph = (struct udphdr *)(l4_hdr);
609
610 udph->check = csum_final;
611 }
612 }
613
enetc_lso_count_descs(const struct sk_buff * skb)614 static int enetc_lso_count_descs(const struct sk_buff *skb)
615 {
616 /* 4 BDs: 1 BD for LSO header + 1 BD for extended BD + 1 BD
617 * for linear area data but not include LSO header, namely
618 * skb_headlen(skb) - lso_hdr_len (it may be 0, but that's
619 * okay, we only need to consider the worst case). And 1 BD
620 * for gap.
621 */
622 return skb_shinfo(skb)->nr_frags + 4;
623 }
624
enetc_lso_get_hdr_len(const struct sk_buff * skb)625 static int enetc_lso_get_hdr_len(const struct sk_buff *skb)
626 {
627 int hdr_len, tlen;
628
629 tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr);
630 hdr_len = skb_transport_offset(skb) + tlen;
631
632 return hdr_len;
633 }
634
enetc_lso_start(struct sk_buff * skb,struct enetc_lso_t * lso)635 static void enetc_lso_start(struct sk_buff *skb, struct enetc_lso_t *lso)
636 {
637 lso->lso_seg_size = skb_shinfo(skb)->gso_size;
638 lso->ipv6 = enetc_skb_is_ipv6(skb);
639 lso->tcp = skb_is_gso_tcp(skb);
640 lso->l3_hdr_len = skb_network_header_len(skb);
641 lso->l3_start = skb_network_offset(skb);
642 lso->hdr_len = enetc_lso_get_hdr_len(skb);
643 lso->total_len = skb->len - lso->hdr_len;
644 }
645
enetc_lso_map_hdr(struct enetc_bdr * tx_ring,struct sk_buff * skb,int * i,struct enetc_lso_t * lso)646 static void enetc_lso_map_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
647 int *i, struct enetc_lso_t *lso)
648 {
649 union enetc_tx_bd txbd_tmp, *txbd;
650 struct enetc_tx_swbd *tx_swbd;
651 u16 frm_len, frm_len_ext;
652 u8 flags, e_flags = 0;
653 dma_addr_t addr;
654 char *hdr;
655
656 /* Get the first BD of the LSO BDs chain */
657 txbd = ENETC_TXBD(*tx_ring, *i);
658 tx_swbd = &tx_ring->tx_swbd[*i];
659 prefetchw(txbd);
660
661 /* Prepare LSO header: MAC + IP + TCP/UDP */
662 hdr = tx_ring->tso_headers + *i * TSO_HEADER_SIZE;
663 memcpy(hdr, skb->data, lso->hdr_len);
664 addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
665
666 /* {frm_len_ext, frm_len} indicates the total length of
667 * large transmit data unit. frm_len contains the 16 least
668 * significant bits and frm_len_ext contains the 4 most
669 * significant bits.
670 */
671 frm_len = lso->total_len & 0xffff;
672 frm_len_ext = (lso->total_len >> 16) & 0xf;
673
674 /* Set the flags of the first BD */
675 flags = ENETC_TXBD_FLAGS_EX | ENETC_TXBD_FLAGS_CSUM_LSO |
676 ENETC_TXBD_FLAGS_LSO | ENETC_TXBD_FLAGS_L4CS;
677
678 enetc_clear_tx_bd(&txbd_tmp);
679 txbd_tmp.addr = cpu_to_le64(addr);
680 txbd_tmp.hdr_len = cpu_to_le16(lso->hdr_len);
681
682 /* first BD needs frm_len and offload flags set */
683 txbd_tmp.frm_len = cpu_to_le16(frm_len);
684 txbd_tmp.flags = flags;
685
686 txbd_tmp.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, lso->l3_start);
687 /* l3_hdr_size in 32-bits (4 bytes) */
688 txbd_tmp.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN,
689 lso->l3_hdr_len / 4);
690 if (lso->ipv6)
691 txbd_tmp.l3_aux1 |= ENETC_TX_BD_L3T;
692 else
693 txbd_tmp.l3_aux0 |= ENETC_TX_BD_IPCS;
694
695 txbd_tmp.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, lso->tcp ?
696 ENETC_TXBD_L4T_TCP : ENETC_TXBD_L4T_UDP);
697
698 /* For the LSO header we do not set the dma address since
699 * we do not want it unmapped when we do cleanup. We still
700 * set len so that we count the bytes sent.
701 */
702 tx_swbd->len = lso->hdr_len;
703 tx_swbd->do_twostep_tstamp = false;
704 tx_swbd->check_wb = false;
705
706 /* Actually write the header in the BD */
707 *txbd = txbd_tmp;
708
709 /* Get the next BD, and the next BD is extended BD */
710 enetc_bdr_idx_inc(tx_ring, i);
711 txbd = ENETC_TXBD(*tx_ring, *i);
712 tx_swbd = &tx_ring->tx_swbd[*i];
713 prefetchw(txbd);
714
715 enetc_clear_tx_bd(&txbd_tmp);
716 if (skb_vlan_tag_present(skb)) {
717 /* Setup the VLAN fields */
718 txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
719 txbd_tmp.ext.tpid = ENETC_TPID_8021Q;
720 e_flags = ENETC_TXBD_E_FLAGS_VLAN_INS;
721 }
722
723 /* Write the BD */
724 txbd_tmp.ext.e_flags = e_flags;
725 txbd_tmp.ext.lso_sg_size = cpu_to_le16(lso->lso_seg_size);
726 txbd_tmp.ext.frm_len_ext = cpu_to_le16(frm_len_ext);
727 *txbd = txbd_tmp;
728 }
729
enetc_lso_map_data(struct enetc_bdr * tx_ring,struct sk_buff * skb,int * i,struct enetc_lso_t * lso,int * count)730 static int enetc_lso_map_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
731 int *i, struct enetc_lso_t *lso, int *count)
732 {
733 union enetc_tx_bd txbd_tmp, *txbd = NULL;
734 struct enetc_tx_swbd *tx_swbd;
735 skb_frag_t *frag;
736 dma_addr_t dma;
737 u8 flags = 0;
738 int len, f;
739
740 len = skb_headlen(skb) - lso->hdr_len;
741 if (len > 0) {
742 dma = dma_map_single(tx_ring->dev, skb->data + lso->hdr_len,
743 len, DMA_TO_DEVICE);
744 if (dma_mapping_error(tx_ring->dev, dma))
745 return -ENOMEM;
746
747 enetc_bdr_idx_inc(tx_ring, i);
748 txbd = ENETC_TXBD(*tx_ring, *i);
749 tx_swbd = &tx_ring->tx_swbd[*i];
750 prefetchw(txbd);
751 *count += 1;
752
753 enetc_clear_tx_bd(&txbd_tmp);
754 txbd_tmp.addr = cpu_to_le64(dma);
755 txbd_tmp.buf_len = cpu_to_le16(len);
756
757 tx_swbd->dma = dma;
758 tx_swbd->len = len;
759 tx_swbd->is_dma_page = 0;
760 tx_swbd->dir = DMA_TO_DEVICE;
761 }
762
763 frag = &skb_shinfo(skb)->frags[0];
764 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
765 if (txbd)
766 *txbd = txbd_tmp;
767
768 len = skb_frag_size(frag);
769 dma = skb_frag_dma_map(tx_ring->dev, frag);
770 if (dma_mapping_error(tx_ring->dev, dma))
771 return -ENOMEM;
772
773 /* Get the next BD */
774 enetc_bdr_idx_inc(tx_ring, i);
775 txbd = ENETC_TXBD(*tx_ring, *i);
776 tx_swbd = &tx_ring->tx_swbd[*i];
777 prefetchw(txbd);
778 *count += 1;
779
780 enetc_clear_tx_bd(&txbd_tmp);
781 txbd_tmp.addr = cpu_to_le64(dma);
782 txbd_tmp.buf_len = cpu_to_le16(len);
783
784 tx_swbd->dma = dma;
785 tx_swbd->len = len;
786 tx_swbd->is_dma_page = 1;
787 tx_swbd->dir = DMA_TO_DEVICE;
788 }
789
790 /* Last BD needs 'F' bit set */
791 flags |= ENETC_TXBD_FLAGS_F;
792 txbd_tmp.flags = flags;
793 *txbd = txbd_tmp;
794
795 tx_swbd->is_eof = 1;
796 tx_swbd->skb = skb;
797
798 return 0;
799 }
800
enetc_lso_hw_offload(struct enetc_bdr * tx_ring,struct sk_buff * skb)801 static int enetc_lso_hw_offload(struct enetc_bdr *tx_ring, struct sk_buff *skb)
802 {
803 struct enetc_tx_swbd *tx_swbd;
804 struct enetc_lso_t lso = {0};
805 int err, i, count = 0;
806
807 /* Initialize the LSO handler */
808 enetc_lso_start(skb, &lso);
809 i = tx_ring->next_to_use;
810
811 enetc_lso_map_hdr(tx_ring, skb, &i, &lso);
812 /* First BD and an extend BD */
813 count += 2;
814
815 err = enetc_lso_map_data(tx_ring, skb, &i, &lso, &count);
816 if (err)
817 goto dma_err;
818
819 /* Go to the next BD */
820 enetc_bdr_idx_inc(tx_ring, &i);
821 tx_ring->next_to_use = i;
822 enetc_update_tx_ring_tail(tx_ring);
823
824 return count;
825
826 dma_err:
827 do {
828 tx_swbd = &tx_ring->tx_swbd[i];
829 enetc_free_tx_frame(tx_ring, tx_swbd);
830 if (i == 0)
831 i = tx_ring->bd_count;
832 i--;
833 } while (--count);
834
835 return 0;
836 }
837
enetc_map_tx_tso_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)838 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
839 {
840 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
841 int hdr_len, total_len, data_len;
842 struct enetc_tx_swbd *tx_swbd;
843 union enetc_tx_bd *txbd;
844 struct tso_t tso;
845 __wsum csum, csum2;
846 int count = 0, pos;
847 int err, i, bd_data_num;
848
849 /* Initialize the TSO handler, and prepare the first payload */
850 hdr_len = tso_start(skb, &tso);
851 total_len = skb->len - hdr_len;
852 i = tx_ring->next_to_use;
853
854 while (total_len > 0) {
855 char *hdr;
856
857 /* Get the BD */
858 txbd = ENETC_TXBD(*tx_ring, i);
859 tx_swbd = &tx_ring->tx_swbd[i];
860 prefetchw(txbd);
861
862 /* Determine the length of this packet */
863 data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
864 total_len -= data_len;
865
866 /* prepare packet headers: MAC + IP + TCP */
867 hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
868 tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
869
870 /* compute the csum over the L4 header */
871 csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
872 count += enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd,
873 &i, hdr_len, data_len);
874 bd_data_num = 0;
875
876 while (data_len > 0) {
877 int size;
878
879 size = min_t(int, tso.size, data_len);
880
881 /* Advance the index in the BDR */
882 enetc_bdr_idx_inc(tx_ring, &i);
883 txbd = ENETC_TXBD(*tx_ring, i);
884 tx_swbd = &tx_ring->tx_swbd[i];
885 prefetchw(txbd);
886
887 /* Compute the checksum over this segment of data and
888 * add it to the csum already computed (over the L4
889 * header and possible other data segments).
890 */
891 csum2 = csum_partial(tso.data, size, 0);
892 csum = csum_block_add(csum, csum2, pos);
893 pos += size;
894
895 err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
896 tso.data, size,
897 size == data_len);
898 if (err) {
899 if (i == 0)
900 i = tx_ring->bd_count;
901 i--;
902
903 goto err_map_data;
904 }
905
906 data_len -= size;
907 count++;
908 bd_data_num++;
909 tso_build_data(skb, &tso, size);
910
911 if (unlikely(bd_data_num >= priv->max_frags && data_len))
912 goto err_chained_bd;
913 }
914
915 enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
916
917 if (total_len == 0)
918 tx_swbd->skb = skb;
919
920 /* Go to the next BD */
921 enetc_bdr_idx_inc(tx_ring, &i);
922 }
923
924 tx_ring->next_to_use = i;
925 enetc_update_tx_ring_tail(tx_ring);
926
927 return count;
928
929 err_map_data:
930 dev_err(tx_ring->dev, "DMA map error");
931
932 err_chained_bd:
933 enetc_unwind_tx_frame(tx_ring, count, i);
934
935 return 0;
936 }
937
enetc_start_xmit(struct sk_buff * skb,struct net_device * ndev)938 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
939 struct net_device *ndev)
940 {
941 struct enetc_ndev_priv *priv = netdev_priv(ndev);
942 struct enetc_bdr *tx_ring;
943 int count;
944
945 /* Queue one-step Sync packet if already locked */
946 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
947 if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
948 &priv->flags)) {
949 skb_queue_tail(&priv->tx_skbs, skb);
950 return NETDEV_TX_OK;
951 }
952 }
953
954 tx_ring = priv->tx_ring[skb->queue_mapping];
955
956 if (skb_is_gso(skb)) {
957 /* LSO data unit lengths of up to 256KB are supported */
958 if (priv->active_offloads & ENETC_F_LSO &&
959 (skb->len - enetc_lso_get_hdr_len(skb)) <=
960 ENETC_LSO_MAX_DATA_LEN) {
961 if (enetc_bd_unused(tx_ring) < enetc_lso_count_descs(skb)) {
962 netif_stop_subqueue(ndev, tx_ring->index);
963 return NETDEV_TX_BUSY;
964 }
965
966 count = enetc_lso_hw_offload(tx_ring, skb);
967 } else {
968 if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
969 netif_stop_subqueue(ndev, tx_ring->index);
970 return NETDEV_TX_BUSY;
971 }
972
973 enetc_lock_mdio();
974 count = enetc_map_tx_tso_buffs(tx_ring, skb);
975 enetc_unlock_mdio();
976 }
977 } else {
978 if (unlikely(skb_shinfo(skb)->nr_frags > priv->max_frags))
979 if (unlikely(skb_linearize(skb)))
980 goto drop_packet_err;
981
982 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
983 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
984 netif_stop_subqueue(ndev, tx_ring->index);
985 return NETDEV_TX_BUSY;
986 }
987
988 enetc_lock_mdio();
989 count = enetc_map_tx_buffs(tx_ring, skb);
990 enetc_unlock_mdio();
991 }
992
993 if (unlikely(!count))
994 goto drop_packet_err;
995
996 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED(priv->max_frags))
997 netif_stop_subqueue(ndev, tx_ring->index);
998
999 return NETDEV_TX_OK;
1000
1001 drop_packet_err:
1002 dev_kfree_skb_any(skb);
1003 return NETDEV_TX_OK;
1004 }
1005
enetc_xmit(struct sk_buff * skb,struct net_device * ndev)1006 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
1007 {
1008 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1009 u8 udp, msgtype, twostep;
1010 u16 offset1, offset2;
1011
1012 /* Mark tx timestamp type on skb->cb[0] if requires */
1013 if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1014 (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
1015 skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
1016 } else {
1017 skb->cb[0] = 0;
1018 }
1019
1020 /* Fall back to two-step timestamp if not one-step Sync packet */
1021 if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
1022 if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
1023 &offset1, &offset2) ||
1024 msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
1025 skb->cb[0] = ENETC_F_TX_TSTAMP;
1026 }
1027
1028 return enetc_start_xmit(skb, ndev);
1029 }
1030 EXPORT_SYMBOL_GPL(enetc_xmit);
1031
enetc_msix(int irq,void * data)1032 static irqreturn_t enetc_msix(int irq, void *data)
1033 {
1034 struct enetc_int_vector *v = data;
1035 int i;
1036
1037 enetc_lock_mdio();
1038
1039 /* disable interrupts */
1040 enetc_wr_reg_hot(v->rbier, 0);
1041 enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
1042
1043 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1044 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
1045
1046 enetc_unlock_mdio();
1047
1048 napi_schedule(&v->napi);
1049
1050 return IRQ_HANDLED;
1051 }
1052
enetc_rx_dim_work(struct work_struct * w)1053 static void enetc_rx_dim_work(struct work_struct *w)
1054 {
1055 struct dim *dim = container_of(w, struct dim, work);
1056 struct dim_cq_moder moder =
1057 net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1058 struct enetc_int_vector *v =
1059 container_of(dim, struct enetc_int_vector, rx_dim);
1060 struct enetc_ndev_priv *priv = netdev_priv(v->rx_ring.ndev);
1061
1062 v->rx_ictt = enetc_usecs_to_cycles(moder.usec, priv->sysclk_freq);
1063 dim->state = DIM_START_MEASURE;
1064 }
1065
enetc_rx_net_dim(struct enetc_int_vector * v)1066 static void enetc_rx_net_dim(struct enetc_int_vector *v)
1067 {
1068 struct dim_sample dim_sample = {};
1069
1070 v->comp_cnt++;
1071
1072 if (!v->rx_napi_work)
1073 return;
1074
1075 dim_update_sample(v->comp_cnt,
1076 v->rx_ring.stats.packets,
1077 v->rx_ring.stats.bytes,
1078 &dim_sample);
1079 net_dim(&v->rx_dim, &dim_sample);
1080 }
1081
enetc_bd_ready_count(struct enetc_bdr * tx_ring,int ci)1082 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
1083 {
1084 int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
1085
1086 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
1087 }
1088
enetc_page_reusable(struct page * page)1089 static bool enetc_page_reusable(struct page *page)
1090 {
1091 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
1092 }
1093
enetc_reuse_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * old)1094 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
1095 struct enetc_rx_swbd *old)
1096 {
1097 struct enetc_rx_swbd *new;
1098
1099 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
1100
1101 /* next buf that may reuse a page */
1102 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
1103
1104 /* copy page reference */
1105 *new = *old;
1106 }
1107
enetc_get_tx_tstamp(struct enetc_hw * hw,union enetc_tx_bd * txbd,u64 * tstamp)1108 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
1109 u64 *tstamp)
1110 {
1111 u32 lo, hi, tstamp_lo;
1112
1113 lo = enetc_rd_hot(hw, ENETC_SICTR0);
1114 hi = enetc_rd_hot(hw, ENETC_SICTR1);
1115 tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
1116 if (lo <= tstamp_lo)
1117 hi -= 1;
1118 *tstamp = (u64)hi << 32 | tstamp_lo;
1119 }
1120
enetc_tstamp_tx(struct sk_buff * skb,u64 tstamp)1121 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
1122 {
1123 struct skb_shared_hwtstamps shhwtstamps;
1124
1125 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
1126 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
1127 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
1128 skb_txtime_consumed(skb);
1129 skb_tstamp_tx(skb, &shhwtstamps);
1130 }
1131 }
1132
enetc_recycle_xdp_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)1133 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
1134 struct enetc_tx_swbd *tx_swbd)
1135 {
1136 struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
1137 struct enetc_rx_swbd rx_swbd = {
1138 .dma = tx_swbd->dma,
1139 .page = tx_swbd->page,
1140 .page_offset = tx_swbd->page_offset,
1141 .dir = tx_swbd->dir,
1142 .len = tx_swbd->len,
1143 };
1144 struct enetc_bdr *rx_ring;
1145
1146 rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
1147
1148 if (likely(enetc_swbd_unused(rx_ring))) {
1149 enetc_reuse_page(rx_ring, &rx_swbd);
1150
1151 /* sync for use by the device */
1152 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
1153 rx_swbd.page_offset,
1154 ENETC_RXB_DMA_SIZE_XDP,
1155 rx_swbd.dir);
1156
1157 rx_ring->stats.recycles++;
1158 } else {
1159 /* RX ring is already full, we need to unmap and free the
1160 * page, since there's nothing useful we can do with it.
1161 */
1162 rx_ring->stats.recycle_failures++;
1163
1164 dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
1165 rx_swbd.dir);
1166 __free_page(rx_swbd.page);
1167 }
1168
1169 rx_ring->xdp.xdp_tx_in_flight--;
1170 }
1171
enetc_clean_tx_ring(struct enetc_bdr * tx_ring,int napi_budget)1172 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
1173 {
1174 int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
1175 struct net_device *ndev = tx_ring->ndev;
1176 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1177 struct enetc_tx_swbd *tx_swbd;
1178 int i, bds_to_clean;
1179 bool do_twostep_tstamp;
1180 u64 tstamp = 0;
1181
1182 i = tx_ring->next_to_clean;
1183 tx_swbd = &tx_ring->tx_swbd[i];
1184
1185 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
1186
1187 do_twostep_tstamp = false;
1188
1189 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
1190 struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
1191 struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
1192 bool is_eof = tx_swbd->is_eof;
1193
1194 if (unlikely(tx_swbd->check_wb)) {
1195 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1196
1197 if (txbd->flags & ENETC_TXBD_FLAGS_W &&
1198 tx_swbd->do_twostep_tstamp) {
1199 enetc_get_tx_tstamp(&priv->si->hw, txbd,
1200 &tstamp);
1201 do_twostep_tstamp = true;
1202 }
1203
1204 if (tx_swbd->qbv_en &&
1205 txbd->wb.status & ENETC_TXBD_STATS_WIN)
1206 tx_win_drop++;
1207 }
1208
1209 if (tx_swbd->is_xdp_tx)
1210 enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
1211 else if (likely(tx_swbd->dma))
1212 enetc_unmap_tx_buff(tx_ring, tx_swbd);
1213
1214 if (xdp_frame) {
1215 xdp_return_frame(xdp_frame);
1216 } else if (skb) {
1217 if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
1218 /* Start work to release lock for next one-step
1219 * timestamping packet. And send one skb in
1220 * tx_skbs queue if has.
1221 */
1222 schedule_work(&priv->tx_onestep_tstamp);
1223 } else if (unlikely(do_twostep_tstamp)) {
1224 enetc_tstamp_tx(skb, tstamp);
1225 do_twostep_tstamp = false;
1226 }
1227 napi_consume_skb(skb, napi_budget);
1228 }
1229
1230 tx_byte_cnt += tx_swbd->len;
1231 /* Scrub the swbd here so we don't have to do that
1232 * when we reuse it during xmit
1233 */
1234 memset(tx_swbd, 0, sizeof(*tx_swbd));
1235
1236 bds_to_clean--;
1237 tx_swbd++;
1238 i++;
1239 if (unlikely(i == tx_ring->bd_count)) {
1240 i = 0;
1241 tx_swbd = tx_ring->tx_swbd;
1242 }
1243
1244 /* BD iteration loop end */
1245 if (is_eof) {
1246 tx_frm_cnt++;
1247 /* re-arm interrupt source */
1248 enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
1249 BIT(16 + tx_ring->index));
1250 }
1251
1252 if (unlikely(!bds_to_clean))
1253 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
1254 }
1255
1256 tx_ring->next_to_clean = i;
1257 tx_ring->stats.packets += tx_frm_cnt;
1258 tx_ring->stats.bytes += tx_byte_cnt;
1259 tx_ring->stats.win_drop += tx_win_drop;
1260
1261 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
1262 __netif_subqueue_stopped(ndev, tx_ring->index) &&
1263 !test_bit(ENETC_TX_DOWN, &priv->flags) &&
1264 (enetc_bd_unused(tx_ring) >=
1265 ENETC_TXBDS_MAX_NEEDED(priv->max_frags)))) {
1266 netif_wake_subqueue(ndev, tx_ring->index);
1267 }
1268
1269 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
1270 }
1271
enetc_new_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1272 static bool enetc_new_page(struct enetc_bdr *rx_ring,
1273 struct enetc_rx_swbd *rx_swbd)
1274 {
1275 bool xdp = !!(rx_ring->xdp.prog);
1276 struct page *page;
1277 dma_addr_t addr;
1278
1279 page = dev_alloc_page();
1280 if (unlikely(!page))
1281 return false;
1282
1283 /* For XDP_TX, we forgo dma_unmap -> dma_map */
1284 rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
1285
1286 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
1287 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
1288 __free_page(page);
1289
1290 return false;
1291 }
1292
1293 rx_swbd->dma = addr;
1294 rx_swbd->page = page;
1295 rx_swbd->page_offset = rx_ring->buffer_offset;
1296
1297 return true;
1298 }
1299
enetc_refill_rx_ring(struct enetc_bdr * rx_ring,const int buff_cnt)1300 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
1301 {
1302 struct enetc_rx_swbd *rx_swbd;
1303 union enetc_rx_bd *rxbd;
1304 int i, j;
1305
1306 i = rx_ring->next_to_use;
1307 rx_swbd = &rx_ring->rx_swbd[i];
1308 rxbd = enetc_rxbd(rx_ring, i);
1309
1310 for (j = 0; j < buff_cnt; j++) {
1311 /* try reuse page */
1312 if (unlikely(!rx_swbd->page)) {
1313 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
1314 rx_ring->stats.rx_alloc_errs++;
1315 break;
1316 }
1317 }
1318
1319 /* update RxBD */
1320 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
1321 rx_swbd->page_offset);
1322 /* clear 'R" as well */
1323 rxbd->r.lstatus = 0;
1324
1325 enetc_rxbd_next(rx_ring, &rxbd, &i);
1326 rx_swbd = &rx_ring->rx_swbd[i];
1327 }
1328
1329 if (likely(j)) {
1330 rx_ring->next_to_alloc = i; /* keep track from page reuse */
1331 rx_ring->next_to_use = i;
1332
1333 /* update ENETC's consumer index */
1334 enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
1335 }
1336
1337 return j;
1338 }
1339
enetc_get_rx_tstamp(struct net_device * ndev,union enetc_rx_bd * rxbd,struct sk_buff * skb)1340 static void enetc_get_rx_tstamp(struct net_device *ndev,
1341 union enetc_rx_bd *rxbd,
1342 struct sk_buff *skb)
1343 {
1344 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
1345 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1346 struct enetc_hw *hw = &priv->si->hw;
1347 u32 lo, hi, tstamp_lo;
1348 u64 tstamp;
1349
1350 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
1351 lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
1352 hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
1353 rxbd = enetc_rxbd_ext(rxbd);
1354 tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
1355 if (lo <= tstamp_lo)
1356 hi -= 1;
1357
1358 tstamp = (u64)hi << 32 | tstamp_lo;
1359 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
1360 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
1361 }
1362 }
1363
enetc_get_offloads(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,struct sk_buff * skb)1364 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
1365 union enetc_rx_bd *rxbd, struct sk_buff *skb)
1366 {
1367 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1368
1369 /* TODO: hashing */
1370 if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
1371 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
1372
1373 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
1374 skb->ip_summed = CHECKSUM_COMPLETE;
1375 }
1376
1377 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1378 struct enetc_hw *hw = &priv->si->hw;
1379 __be16 tpid = 0;
1380
1381 switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1382 case 0:
1383 tpid = htons(ETH_P_8021Q);
1384 break;
1385 case 1:
1386 tpid = htons(ETH_P_8021AD);
1387 break;
1388 case 2:
1389 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR1) &
1390 SICVLANR_ETYPE);
1391 break;
1392 case 3:
1393 tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR2) &
1394 SICVLANR_ETYPE);
1395 }
1396
1397 __vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1398 }
1399
1400 if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK) &&
1401 (priv->active_offloads & ENETC_F_RX_TSTAMP))
1402 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1403 }
1404
1405 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1406 * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1407 * mapped buffers.
1408 */
enetc_get_rx_buff(struct enetc_bdr * rx_ring,int i,u16 size)1409 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1410 int i, u16 size)
1411 {
1412 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1413
1414 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1415 rx_swbd->page_offset,
1416 size, rx_swbd->dir);
1417 return rx_swbd;
1418 }
1419
1420 /* Reuse the current page without performing half-page buffer flipping */
enetc_put_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1421 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1422 struct enetc_rx_swbd *rx_swbd)
1423 {
1424 size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1425
1426 enetc_reuse_page(rx_ring, rx_swbd);
1427
1428 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1429 rx_swbd->page_offset,
1430 buffer_size, rx_swbd->dir);
1431
1432 rx_swbd->page = NULL;
1433 }
1434
1435 /* Reuse the current page by performing half-page buffer flipping */
enetc_flip_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1436 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1437 struct enetc_rx_swbd *rx_swbd)
1438 {
1439 if (likely(enetc_page_reusable(rx_swbd->page))) {
1440 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1441 page_ref_inc(rx_swbd->page);
1442
1443 enetc_put_rx_buff(rx_ring, rx_swbd);
1444 } else {
1445 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1446 rx_swbd->dir);
1447 rx_swbd->page = NULL;
1448 }
1449 }
1450
enetc_map_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size)1451 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1452 int i, u16 size)
1453 {
1454 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1455 struct sk_buff *skb;
1456 void *ba;
1457
1458 ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1459 skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1460 if (unlikely(!skb)) {
1461 rx_ring->stats.rx_alloc_errs++;
1462 return NULL;
1463 }
1464
1465 skb_reserve(skb, rx_ring->buffer_offset);
1466 __skb_put(skb, size);
1467
1468 enetc_flip_rx_buff(rx_ring, rx_swbd);
1469
1470 return skb;
1471 }
1472
enetc_add_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size,struct sk_buff * skb)1473 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1474 u16 size, struct sk_buff *skb)
1475 {
1476 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1477
1478 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1479 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1480
1481 enetc_flip_rx_buff(rx_ring, rx_swbd);
1482 }
1483
enetc_check_bd_errors_and_consume(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i)1484 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1485 u32 bd_status,
1486 union enetc_rx_bd **rxbd, int *i)
1487 {
1488 if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1489 return false;
1490
1491 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1492 enetc_rxbd_next(rx_ring, rxbd, i);
1493
1494 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1495 dma_rmb();
1496 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1497
1498 enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1499 enetc_rxbd_next(rx_ring, rxbd, i);
1500 }
1501
1502 rx_ring->ndev->stats.rx_dropped++;
1503 rx_ring->ndev->stats.rx_errors++;
1504
1505 return true;
1506 }
1507
enetc_build_skb(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,int buffer_size)1508 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1509 u32 bd_status, union enetc_rx_bd **rxbd,
1510 int *i, int *cleaned_cnt, int buffer_size)
1511 {
1512 struct sk_buff *skb;
1513 u16 size;
1514
1515 size = le16_to_cpu((*rxbd)->r.buf_len);
1516 skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1517 if (!skb)
1518 return NULL;
1519
1520 enetc_get_offloads(rx_ring, *rxbd, skb);
1521
1522 (*cleaned_cnt)++;
1523
1524 enetc_rxbd_next(rx_ring, rxbd, i);
1525
1526 /* not last BD in frame? */
1527 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1528 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1529 size = buffer_size;
1530
1531 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1532 dma_rmb();
1533 size = le16_to_cpu((*rxbd)->r.buf_len);
1534 }
1535
1536 enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1537
1538 (*cleaned_cnt)++;
1539
1540 enetc_rxbd_next(rx_ring, rxbd, i);
1541 }
1542
1543 skb_record_rx_queue(skb, rx_ring->index);
1544 skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1545
1546 return skb;
1547 }
1548
1549 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1550
enetc_clean_rx_ring(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit)1551 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1552 struct napi_struct *napi, int work_limit)
1553 {
1554 int rx_frm_cnt = 0, rx_byte_cnt = 0;
1555 int cleaned_cnt, i;
1556
1557 cleaned_cnt = enetc_bd_unused(rx_ring);
1558 /* next descriptor to process */
1559 i = rx_ring->next_to_clean;
1560
1561 while (likely(rx_frm_cnt < work_limit)) {
1562 union enetc_rx_bd *rxbd;
1563 struct sk_buff *skb;
1564 u32 bd_status;
1565
1566 if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1567 cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1568 cleaned_cnt);
1569
1570 rxbd = enetc_rxbd(rx_ring, i);
1571 bd_status = le32_to_cpu(rxbd->r.lstatus);
1572 if (!bd_status)
1573 break;
1574
1575 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1576 dma_rmb(); /* for reading other rxbd fields */
1577
1578 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1579 &rxbd, &i))
1580 break;
1581
1582 skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1583 &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1584 if (!skb)
1585 break;
1586
1587 /* When set, the outer VLAN header is extracted and reported
1588 * in the receive buffer descriptor. So rx_byte_cnt should
1589 * add the length of the extracted VLAN header.
1590 */
1591 if (bd_status & ENETC_RXBD_FLAG_VLAN)
1592 rx_byte_cnt += VLAN_HLEN;
1593 rx_byte_cnt += skb->len + ETH_HLEN;
1594 rx_frm_cnt++;
1595
1596 napi_gro_receive(napi, skb);
1597 }
1598
1599 rx_ring->next_to_clean = i;
1600
1601 rx_ring->stats.packets += rx_frm_cnt;
1602 rx_ring->stats.bytes += rx_byte_cnt;
1603
1604 return rx_frm_cnt;
1605 }
1606
enetc_xdp_map_tx_buff(struct enetc_bdr * tx_ring,int i,struct enetc_tx_swbd * tx_swbd,int frm_len)1607 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1608 struct enetc_tx_swbd *tx_swbd,
1609 int frm_len)
1610 {
1611 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1612
1613 prefetchw(txbd);
1614
1615 enetc_clear_tx_bd(txbd);
1616 txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1617 txbd->buf_len = cpu_to_le16(tx_swbd->len);
1618 txbd->frm_len = cpu_to_le16(frm_len);
1619
1620 memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1621 }
1622
1623 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1624 * descriptors.
1625 */
enetc_xdp_tx(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,int num_tx_swbd)1626 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1627 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1628 {
1629 struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1630 int i, k, frm_len = tmp_tx_swbd->len;
1631
1632 if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1633 return false;
1634
1635 while (unlikely(!tmp_tx_swbd->is_eof)) {
1636 tmp_tx_swbd++;
1637 frm_len += tmp_tx_swbd->len;
1638 }
1639
1640 i = tx_ring->next_to_use;
1641
1642 for (k = 0; k < num_tx_swbd; k++) {
1643 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1644
1645 enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1646
1647 /* last BD needs 'F' bit set */
1648 if (xdp_tx_swbd->is_eof) {
1649 union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1650
1651 txbd->flags = ENETC_TXBD_FLAGS_F;
1652 }
1653
1654 enetc_bdr_idx_inc(tx_ring, &i);
1655 }
1656
1657 tx_ring->next_to_use = i;
1658
1659 return true;
1660 }
1661
enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,struct xdp_frame * xdp_frame)1662 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1663 struct enetc_tx_swbd *xdp_tx_arr,
1664 struct xdp_frame *xdp_frame)
1665 {
1666 struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1667 struct skb_shared_info *shinfo;
1668 void *data = xdp_frame->data;
1669 int len = xdp_frame->len;
1670 skb_frag_t *frag;
1671 dma_addr_t dma;
1672 unsigned int f;
1673 int n = 0;
1674
1675 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1676 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1677 netdev_err(tx_ring->ndev, "DMA map error\n");
1678 return -1;
1679 }
1680
1681 xdp_tx_swbd->dma = dma;
1682 xdp_tx_swbd->dir = DMA_TO_DEVICE;
1683 xdp_tx_swbd->len = len;
1684 xdp_tx_swbd->is_xdp_redirect = true;
1685 xdp_tx_swbd->is_eof = false;
1686 xdp_tx_swbd->xdp_frame = NULL;
1687
1688 n++;
1689
1690 if (!xdp_frame_has_frags(xdp_frame))
1691 goto out;
1692
1693 xdp_tx_swbd = &xdp_tx_arr[n];
1694
1695 shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1696
1697 for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1698 f++, frag++) {
1699 data = skb_frag_address(frag);
1700 len = skb_frag_size(frag);
1701
1702 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1703 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1704 /* Undo the DMA mapping for all fragments */
1705 while (--n >= 0)
1706 enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1707
1708 netdev_err(tx_ring->ndev, "DMA map error\n");
1709 return -1;
1710 }
1711
1712 xdp_tx_swbd->dma = dma;
1713 xdp_tx_swbd->dir = DMA_TO_DEVICE;
1714 xdp_tx_swbd->len = len;
1715 xdp_tx_swbd->is_xdp_redirect = true;
1716 xdp_tx_swbd->is_eof = false;
1717 xdp_tx_swbd->xdp_frame = NULL;
1718
1719 n++;
1720 xdp_tx_swbd = &xdp_tx_arr[n];
1721 }
1722 out:
1723 xdp_tx_arr[n - 1].is_eof = true;
1724 xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1725
1726 return n;
1727 }
1728
enetc_xdp_xmit(struct net_device * ndev,int num_frames,struct xdp_frame ** frames,u32 flags)1729 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1730 struct xdp_frame **frames, u32 flags)
1731 {
1732 struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1733 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1734 struct enetc_bdr *tx_ring;
1735 int xdp_tx_bd_cnt, i, k;
1736 int xdp_tx_frm_cnt = 0;
1737
1738 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags)))
1739 return -ENETDOWN;
1740
1741 enetc_lock_mdio();
1742
1743 tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1744
1745 prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1746
1747 for (k = 0; k < num_frames; k++) {
1748 xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1749 xdp_redirect_arr,
1750 frames[k]);
1751 if (unlikely(xdp_tx_bd_cnt < 0))
1752 break;
1753
1754 if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1755 xdp_tx_bd_cnt))) {
1756 for (i = 0; i < xdp_tx_bd_cnt; i++)
1757 enetc_unmap_tx_buff(tx_ring,
1758 &xdp_redirect_arr[i]);
1759 tx_ring->stats.xdp_tx_drops++;
1760 break;
1761 }
1762
1763 xdp_tx_frm_cnt++;
1764 }
1765
1766 if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1767 enetc_update_tx_ring_tail(tx_ring);
1768
1769 tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1770
1771 enetc_unlock_mdio();
1772
1773 return xdp_tx_frm_cnt;
1774 }
1775 EXPORT_SYMBOL_GPL(enetc_xdp_xmit);
1776
enetc_map_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,struct xdp_buff * xdp_buff,u16 size)1777 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1778 struct xdp_buff *xdp_buff, u16 size)
1779 {
1780 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1781 void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1782
1783 /* To be used for XDP_TX */
1784 rx_swbd->len = size;
1785
1786 xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1787 rx_ring->buffer_offset, size, false);
1788 }
1789
enetc_add_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,u16 size,struct xdp_buff * xdp_buff)1790 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1791 u16 size, struct xdp_buff *xdp_buff)
1792 {
1793 struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1794 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1795 skb_frag_t *frag;
1796
1797 /* To be used for XDP_TX */
1798 rx_swbd->len = size;
1799
1800 if (!xdp_buff_has_frags(xdp_buff)) {
1801 xdp_buff_set_frags_flag(xdp_buff);
1802 shinfo->xdp_frags_size = size;
1803 shinfo->nr_frags = 0;
1804 } else {
1805 shinfo->xdp_frags_size += size;
1806 }
1807
1808 if (page_is_pfmemalloc(rx_swbd->page))
1809 xdp_buff_set_frag_pfmemalloc(xdp_buff);
1810
1811 frag = &shinfo->frags[shinfo->nr_frags];
1812 skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,
1813 size);
1814
1815 shinfo->nr_frags++;
1816 }
1817
enetc_build_xdp_buff(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,struct xdp_buff * xdp_buff)1818 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1819 union enetc_rx_bd **rxbd, int *i,
1820 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1821 {
1822 u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1823
1824 xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1825
1826 enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1827 (*cleaned_cnt)++;
1828 enetc_rxbd_next(rx_ring, rxbd, i);
1829
1830 /* not last BD in frame? */
1831 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1832 bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1833 size = ENETC_RXB_DMA_SIZE_XDP;
1834
1835 if (bd_status & ENETC_RXBD_LSTATUS_F) {
1836 dma_rmb();
1837 size = le16_to_cpu((*rxbd)->r.buf_len);
1838 }
1839
1840 enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1841 (*cleaned_cnt)++;
1842 enetc_rxbd_next(rx_ring, rxbd, i);
1843 }
1844 }
1845
1846 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1847 * recycled back into the RX ring in enetc_clean_tx_ring.
1848 */
enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd * xdp_tx_arr,struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1849 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1850 struct enetc_bdr *rx_ring,
1851 int rx_ring_first, int rx_ring_last)
1852 {
1853 int n = 0;
1854
1855 for (; rx_ring_first != rx_ring_last;
1856 n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1857 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1858 struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1859
1860 /* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1861 tx_swbd->dma = rx_swbd->dma;
1862 tx_swbd->dir = rx_swbd->dir;
1863 tx_swbd->page = rx_swbd->page;
1864 tx_swbd->page_offset = rx_swbd->page_offset;
1865 tx_swbd->len = rx_swbd->len;
1866 tx_swbd->is_dma_page = true;
1867 tx_swbd->is_xdp_tx = true;
1868 tx_swbd->is_eof = false;
1869 }
1870
1871 /* We rely on caller providing an rx_ring_last > rx_ring_first */
1872 xdp_tx_arr[n - 1].is_eof = true;
1873
1874 return n;
1875 }
1876
enetc_xdp_drop(struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1877 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1878 int rx_ring_last)
1879 {
1880 while (rx_ring_first != rx_ring_last) {
1881 enetc_put_rx_buff(rx_ring,
1882 &rx_ring->rx_swbd[rx_ring_first]);
1883 enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1884 }
1885 }
1886
enetc_bulk_flip_buff(struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1887 static void enetc_bulk_flip_buff(struct enetc_bdr *rx_ring, int rx_ring_first,
1888 int rx_ring_last)
1889 {
1890 while (rx_ring_first != rx_ring_last) {
1891 enetc_flip_rx_buff(rx_ring,
1892 &rx_ring->rx_swbd[rx_ring_first]);
1893 enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1894 }
1895 }
1896
enetc_clean_rx_ring_xdp(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit,struct bpf_prog * prog)1897 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1898 struct napi_struct *napi, int work_limit,
1899 struct bpf_prog *prog)
1900 {
1901 int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1902 struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1903 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1904 int rx_frm_cnt = 0, rx_byte_cnt = 0;
1905 struct enetc_bdr *tx_ring;
1906 int cleaned_cnt, i;
1907 u32 xdp_act;
1908
1909 cleaned_cnt = enetc_bd_unused(rx_ring);
1910 /* next descriptor to process */
1911 i = rx_ring->next_to_clean;
1912
1913 while (likely(rx_frm_cnt < work_limit)) {
1914 union enetc_rx_bd *rxbd, *orig_rxbd;
1915 struct xdp_buff xdp_buff;
1916 struct sk_buff *skb;
1917 int orig_i, err;
1918 u32 bd_status;
1919
1920 rxbd = enetc_rxbd(rx_ring, i);
1921 bd_status = le32_to_cpu(rxbd->r.lstatus);
1922 if (!bd_status)
1923 break;
1924
1925 enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1926 dma_rmb(); /* for reading other rxbd fields */
1927
1928 if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1929 &rxbd, &i))
1930 break;
1931
1932 orig_rxbd = rxbd;
1933 orig_i = i;
1934
1935 enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1936 &cleaned_cnt, &xdp_buff);
1937
1938 /* When set, the outer VLAN header is extracted and reported
1939 * in the receive buffer descriptor. So rx_byte_cnt should
1940 * add the length of the extracted VLAN header.
1941 */
1942 if (bd_status & ENETC_RXBD_FLAG_VLAN)
1943 rx_byte_cnt += VLAN_HLEN;
1944 rx_byte_cnt += xdp_get_buff_len(&xdp_buff);
1945
1946 xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1947
1948 switch (xdp_act) {
1949 default:
1950 bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1951 fallthrough;
1952 case XDP_ABORTED:
1953 trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1954 fallthrough;
1955 case XDP_DROP:
1956 enetc_xdp_drop(rx_ring, orig_i, i);
1957 rx_ring->stats.xdp_drops++;
1958 break;
1959 case XDP_PASS:
1960 skb = xdp_build_skb_from_buff(&xdp_buff);
1961 /* Probably under memory pressure, stop NAPI */
1962 if (unlikely(!skb)) {
1963 enetc_xdp_drop(rx_ring, orig_i, i);
1964 rx_ring->stats.xdp_drops++;
1965 goto out;
1966 }
1967
1968 enetc_get_offloads(rx_ring, orig_rxbd, skb);
1969
1970 /* These buffers are about to be owned by the stack.
1971 * Update our buffer cache (the rx_swbd array elements)
1972 * with their other page halves.
1973 */
1974 enetc_bulk_flip_buff(rx_ring, orig_i, i);
1975
1976 napi_gro_receive(napi, skb);
1977 break;
1978 case XDP_TX:
1979 tx_ring = priv->xdp_tx_ring[rx_ring->index];
1980 if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) {
1981 enetc_xdp_drop(rx_ring, orig_i, i);
1982 tx_ring->stats.xdp_tx_drops++;
1983 break;
1984 }
1985
1986 xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1987 rx_ring,
1988 orig_i, i);
1989
1990 if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1991 enetc_xdp_drop(rx_ring, orig_i, i);
1992 tx_ring->stats.xdp_tx_drops++;
1993 } else {
1994 tx_ring->stats.xdp_tx++;
1995 rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1996 xdp_tx_frm_cnt++;
1997 /* The XDP_TX enqueue was successful, so we
1998 * need to scrub the RX software BDs because
1999 * the ownership of the buffers no longer
2000 * belongs to the RX ring, and we must prevent
2001 * enetc_refill_rx_ring() from reusing
2002 * rx_swbd->page.
2003 */
2004 while (orig_i != i) {
2005 rx_ring->rx_swbd[orig_i].page = NULL;
2006 enetc_bdr_idx_inc(rx_ring, &orig_i);
2007 }
2008 }
2009 break;
2010 case XDP_REDIRECT:
2011 err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
2012 if (unlikely(err)) {
2013 enetc_xdp_drop(rx_ring, orig_i, i);
2014 rx_ring->stats.xdp_redirect_failures++;
2015 } else {
2016 enetc_bulk_flip_buff(rx_ring, orig_i, i);
2017 xdp_redirect_frm_cnt++;
2018 rx_ring->stats.xdp_redirect++;
2019 }
2020 }
2021
2022 rx_frm_cnt++;
2023 }
2024
2025 out:
2026 rx_ring->next_to_clean = i;
2027
2028 rx_ring->stats.packets += rx_frm_cnt;
2029 rx_ring->stats.bytes += rx_byte_cnt;
2030
2031 if (xdp_redirect_frm_cnt)
2032 xdp_do_flush();
2033
2034 if (xdp_tx_frm_cnt)
2035 enetc_update_tx_ring_tail(tx_ring);
2036
2037 if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
2038 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
2039 rx_ring->xdp.xdp_tx_in_flight);
2040
2041 return rx_frm_cnt;
2042 }
2043
enetc_poll(struct napi_struct * napi,int budget)2044 static int enetc_poll(struct napi_struct *napi, int budget)
2045 {
2046 struct enetc_int_vector
2047 *v = container_of(napi, struct enetc_int_vector, napi);
2048 struct enetc_bdr *rx_ring = &v->rx_ring;
2049 struct bpf_prog *prog;
2050 bool complete = true;
2051 int work_done;
2052 int i;
2053
2054 enetc_lock_mdio();
2055
2056 for (i = 0; i < v->count_tx_rings; i++)
2057 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
2058 complete = false;
2059
2060 prog = rx_ring->xdp.prog;
2061 if (prog)
2062 work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
2063 else
2064 work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
2065 if (work_done == budget)
2066 complete = false;
2067 if (work_done)
2068 v->rx_napi_work = true;
2069
2070 if (!complete) {
2071 enetc_unlock_mdio();
2072 return budget;
2073 }
2074
2075 napi_complete_done(napi, work_done);
2076
2077 if (likely(v->rx_dim_en))
2078 enetc_rx_net_dim(v);
2079
2080 v->rx_napi_work = false;
2081
2082 /* enable interrupts */
2083 enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
2084
2085 for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
2086 enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
2087 ENETC_TBIER_TXTIE);
2088
2089 enetc_unlock_mdio();
2090
2091 return work_done;
2092 }
2093
2094 /* Probing and Init */
2095 #define ENETC_MAX_RFS_SIZE 64
enetc_get_si_caps(struct enetc_si * si)2096 void enetc_get_si_caps(struct enetc_si *si)
2097 {
2098 struct enetc_hw *hw = &si->hw;
2099 u32 val;
2100
2101 /* find out how many of various resources we have to work with */
2102 val = enetc_rd(hw, ENETC_SICAPR0);
2103 si->num_rx_rings = (val >> 16) & 0xff;
2104 si->num_tx_rings = val & 0xff;
2105
2106 val = enetc_rd(hw, ENETC_SIPCAPR0);
2107 if (val & ENETC_SIPCAPR0_RFS) {
2108 val = enetc_rd(hw, ENETC_SIRFSCAPR);
2109 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
2110 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
2111 } else {
2112 /* ENETC which not supports RFS */
2113 si->num_fs_entries = 0;
2114 }
2115
2116 si->num_rss = 0;
2117 val = enetc_rd(hw, ENETC_SIPCAPR0);
2118 if (val & ENETC_SIPCAPR0_RSS) {
2119 u32 rss;
2120
2121 rss = enetc_rd(hw, ENETC_SIRSSCAPR);
2122 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
2123 }
2124
2125 if (val & ENETC_SIPCAPR0_LSO)
2126 si->hw_features |= ENETC_SI_F_LSO;
2127 }
2128 EXPORT_SYMBOL_GPL(enetc_get_si_caps);
2129
enetc_dma_alloc_bdr(struct enetc_bdr_resource * res)2130 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
2131 {
2132 size_t bd_base_size = res->bd_count * res->bd_size;
2133
2134 res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,
2135 &res->bd_dma_base, GFP_KERNEL);
2136 if (!res->bd_base)
2137 return -ENOMEM;
2138
2139 /* h/w requires 128B alignment */
2140 if (!IS_ALIGNED(res->bd_dma_base, 128)) {
2141 dma_free_coherent(res->dev, bd_base_size, res->bd_base,
2142 res->bd_dma_base);
2143 return -EINVAL;
2144 }
2145
2146 return 0;
2147 }
2148
enetc_dma_free_bdr(const struct enetc_bdr_resource * res)2149 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
2150 {
2151 size_t bd_base_size = res->bd_count * res->bd_size;
2152
2153 dma_free_coherent(res->dev, bd_base_size, res->bd_base,
2154 res->bd_dma_base);
2155 }
2156
enetc_alloc_tx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count)2157 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
2158 struct device *dev, size_t bd_count)
2159 {
2160 int err;
2161
2162 res->dev = dev;
2163 res->bd_count = bd_count;
2164 res->bd_size = sizeof(union enetc_tx_bd);
2165
2166 res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));
2167 if (!res->tx_swbd)
2168 return -ENOMEM;
2169
2170 err = enetc_dma_alloc_bdr(res);
2171 if (err)
2172 goto err_alloc_bdr;
2173
2174 res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,
2175 &res->tso_headers_dma,
2176 GFP_KERNEL);
2177 if (!res->tso_headers) {
2178 err = -ENOMEM;
2179 goto err_alloc_tso;
2180 }
2181
2182 return 0;
2183
2184 err_alloc_tso:
2185 enetc_dma_free_bdr(res);
2186 err_alloc_bdr:
2187 vfree(res->tx_swbd);
2188 res->tx_swbd = NULL;
2189
2190 return err;
2191 }
2192
enetc_free_tx_resource(const struct enetc_bdr_resource * res)2193 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
2194 {
2195 dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,
2196 res->tso_headers, res->tso_headers_dma);
2197 enetc_dma_free_bdr(res);
2198 vfree(res->tx_swbd);
2199 }
2200
2201 static struct enetc_bdr_resource *
enetc_alloc_tx_resources(struct enetc_ndev_priv * priv)2202 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
2203 {
2204 struct enetc_bdr_resource *tx_res;
2205 int i, err;
2206
2207 tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL);
2208 if (!tx_res)
2209 return ERR_PTR(-ENOMEM);
2210
2211 for (i = 0; i < priv->num_tx_rings; i++) {
2212 struct enetc_bdr *tx_ring = priv->tx_ring[i];
2213
2214 err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,
2215 tx_ring->bd_count);
2216 if (err)
2217 goto fail;
2218 }
2219
2220 return tx_res;
2221
2222 fail:
2223 while (i-- > 0)
2224 enetc_free_tx_resource(&tx_res[i]);
2225
2226 kfree(tx_res);
2227
2228 return ERR_PTR(err);
2229 }
2230
enetc_free_tx_resources(const struct enetc_bdr_resource * tx_res,size_t num_resources)2231 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
2232 size_t num_resources)
2233 {
2234 size_t i;
2235
2236 for (i = 0; i < num_resources; i++)
2237 enetc_free_tx_resource(&tx_res[i]);
2238
2239 kfree(tx_res);
2240 }
2241
enetc_alloc_rx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count,bool extended)2242 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
2243 struct device *dev, size_t bd_count,
2244 bool extended)
2245 {
2246 int err;
2247
2248 res->dev = dev;
2249 res->bd_count = bd_count;
2250 res->bd_size = sizeof(union enetc_rx_bd);
2251 if (extended)
2252 res->bd_size *= 2;
2253
2254 res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));
2255 if (!res->rx_swbd)
2256 return -ENOMEM;
2257
2258 err = enetc_dma_alloc_bdr(res);
2259 if (err) {
2260 vfree(res->rx_swbd);
2261 return err;
2262 }
2263
2264 return 0;
2265 }
2266
enetc_free_rx_resource(const struct enetc_bdr_resource * res)2267 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
2268 {
2269 enetc_dma_free_bdr(res);
2270 vfree(res->rx_swbd);
2271 }
2272
2273 static struct enetc_bdr_resource *
enetc_alloc_rx_resources(struct enetc_ndev_priv * priv,bool extended)2274 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
2275 {
2276 struct enetc_bdr_resource *rx_res;
2277 int i, err;
2278
2279 rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL);
2280 if (!rx_res)
2281 return ERR_PTR(-ENOMEM);
2282
2283 for (i = 0; i < priv->num_rx_rings; i++) {
2284 struct enetc_bdr *rx_ring = priv->rx_ring[i];
2285
2286 err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,
2287 rx_ring->bd_count, extended);
2288 if (err)
2289 goto fail;
2290 }
2291
2292 return rx_res;
2293
2294 fail:
2295 while (i-- > 0)
2296 enetc_free_rx_resource(&rx_res[i]);
2297
2298 kfree(rx_res);
2299
2300 return ERR_PTR(err);
2301 }
2302
enetc_free_rx_resources(const struct enetc_bdr_resource * rx_res,size_t num_resources)2303 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
2304 size_t num_resources)
2305 {
2306 size_t i;
2307
2308 for (i = 0; i < num_resources; i++)
2309 enetc_free_rx_resource(&rx_res[i]);
2310
2311 kfree(rx_res);
2312 }
2313
enetc_assign_tx_resource(struct enetc_bdr * tx_ring,const struct enetc_bdr_resource * res)2314 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
2315 const struct enetc_bdr_resource *res)
2316 {
2317 tx_ring->bd_base = res ? res->bd_base : NULL;
2318 tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
2319 tx_ring->tx_swbd = res ? res->tx_swbd : NULL;
2320 tx_ring->tso_headers = res ? res->tso_headers : NULL;
2321 tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;
2322 }
2323
enetc_assign_rx_resource(struct enetc_bdr * rx_ring,const struct enetc_bdr_resource * res)2324 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
2325 const struct enetc_bdr_resource *res)
2326 {
2327 rx_ring->bd_base = res ? res->bd_base : NULL;
2328 rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
2329 rx_ring->rx_swbd = res ? res->rx_swbd : NULL;
2330 }
2331
enetc_assign_tx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)2332 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
2333 const struct enetc_bdr_resource *res)
2334 {
2335 int i;
2336
2337 if (priv->tx_res)
2338 enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);
2339
2340 for (i = 0; i < priv->num_tx_rings; i++) {
2341 enetc_assign_tx_resource(priv->tx_ring[i],
2342 res ? &res[i] : NULL);
2343 }
2344
2345 priv->tx_res = res;
2346 }
2347
enetc_assign_rx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)2348 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
2349 const struct enetc_bdr_resource *res)
2350 {
2351 int i;
2352
2353 if (priv->rx_res)
2354 enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);
2355
2356 for (i = 0; i < priv->num_rx_rings; i++) {
2357 enetc_assign_rx_resource(priv->rx_ring[i],
2358 res ? &res[i] : NULL);
2359 }
2360
2361 priv->rx_res = res;
2362 }
2363
enetc_free_tx_ring(struct enetc_bdr * tx_ring)2364 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
2365 {
2366 int i;
2367
2368 for (i = 0; i < tx_ring->bd_count; i++) {
2369 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
2370
2371 enetc_free_tx_frame(tx_ring, tx_swbd);
2372 }
2373 }
2374
enetc_free_rx_ring(struct enetc_bdr * rx_ring)2375 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
2376 {
2377 int i;
2378
2379 for (i = 0; i < rx_ring->bd_count; i++) {
2380 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
2381
2382 if (!rx_swbd->page)
2383 continue;
2384
2385 dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
2386 rx_swbd->dir);
2387 __free_page(rx_swbd->page);
2388 rx_swbd->page = NULL;
2389 }
2390 }
2391
enetc_free_rxtx_rings(struct enetc_ndev_priv * priv)2392 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
2393 {
2394 int i;
2395
2396 for (i = 0; i < priv->num_rx_rings; i++)
2397 enetc_free_rx_ring(priv->rx_ring[i]);
2398
2399 for (i = 0; i < priv->num_tx_rings; i++)
2400 enetc_free_tx_ring(priv->tx_ring[i]);
2401 }
2402
enetc_setup_default_rss_table(struct enetc_si * si,int num_groups)2403 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
2404 {
2405 int *rss_table;
2406 int i;
2407
2408 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
2409 if (!rss_table)
2410 return -ENOMEM;
2411
2412 /* Set up RSS table defaults */
2413 for (i = 0; i < si->num_rss; i++)
2414 rss_table[i] = i % num_groups;
2415
2416 si->ops->set_rss_table(si, rss_table, si->num_rss);
2417
2418 kfree(rss_table);
2419
2420 return 0;
2421 }
2422
enetc_set_lso_flags_mask(struct enetc_hw * hw)2423 static void enetc_set_lso_flags_mask(struct enetc_hw *hw)
2424 {
2425 enetc_wr(hw, ENETC4_SILSOSFMR0,
2426 SILSOSFMR0_VAL_SET(ENETC4_TCP_NL_SEG_FLAGS_DMASK,
2427 ENETC4_TCP_NL_SEG_FLAGS_DMASK));
2428 enetc_wr(hw, ENETC4_SILSOSFMR1, 0);
2429 }
2430
enetc_set_rss(struct net_device * ndev,int en)2431 static void enetc_set_rss(struct net_device *ndev, int en)
2432 {
2433 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2434 struct enetc_hw *hw = &priv->si->hw;
2435 u32 reg;
2436
2437 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2438
2439 reg = enetc_rd(hw, ENETC_SIMR);
2440 reg &= ~ENETC_SIMR_RSSE;
2441 reg |= (en) ? ENETC_SIMR_RSSE : 0;
2442 enetc_wr(hw, ENETC_SIMR, reg);
2443 }
2444
enetc_configure_si(struct enetc_ndev_priv * priv)2445 int enetc_configure_si(struct enetc_ndev_priv *priv)
2446 {
2447 struct enetc_si *si = priv->si;
2448 struct enetc_hw *hw = &si->hw;
2449 int err;
2450
2451 /* set SI cache attributes */
2452 enetc_wr(hw, ENETC_SICAR0,
2453 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
2454 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
2455 /* enable SI */
2456 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
2457
2458 if (si->hw_features & ENETC_SI_F_LSO)
2459 enetc_set_lso_flags_mask(hw);
2460
2461 if (si->num_rss) {
2462 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
2463 if (err)
2464 return err;
2465
2466 if (priv->ndev->features & NETIF_F_RXHASH)
2467 enetc_set_rss(priv->ndev, true);
2468 }
2469
2470 return 0;
2471 }
2472 EXPORT_SYMBOL_GPL(enetc_configure_si);
2473
enetc_init_si_rings_params(struct enetc_ndev_priv * priv)2474 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2475 {
2476 struct enetc_si *si = priv->si;
2477 int cpus = num_online_cpus();
2478
2479 priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
2480 priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2481
2482 /* Enable all available TX rings in order to configure as many
2483 * priorities as possible, when needed.
2484 * TODO: Make # of TX rings run-time configurable
2485 */
2486 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2487 priv->num_tx_rings = si->num_tx_rings;
2488 priv->bdr_int_num = priv->num_rx_rings;
2489 priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2490 priv->tx_ictt = enetc_usecs_to_cycles(600, priv->sysclk_freq);
2491 }
2492 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);
2493
enetc_alloc_si_resources(struct enetc_ndev_priv * priv)2494 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2495 {
2496 struct enetc_si *si = priv->si;
2497
2498 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2499 GFP_KERNEL);
2500 if (!priv->cls_rules)
2501 return -ENOMEM;
2502
2503 return 0;
2504 }
2505 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);
2506
enetc_free_si_resources(struct enetc_ndev_priv * priv)2507 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2508 {
2509 kfree(priv->cls_rules);
2510 }
2511 EXPORT_SYMBOL_GPL(enetc_free_si_resources);
2512
enetc_setup_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2513 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2514 {
2515 int idx = tx_ring->index;
2516 u32 tbmr;
2517
2518 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2519 lower_32_bits(tx_ring->bd_dma_base));
2520
2521 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2522 upper_32_bits(tx_ring->bd_dma_base));
2523
2524 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2525 enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2526 ENETC_RTBLENR_LEN(tx_ring->bd_count));
2527
2528 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2529 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2530 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2531
2532 /* enable Tx ints by setting pkt thr to 1 */
2533 enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2534
2535 tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);
2536 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2537 tbmr |= ENETC_TBMR_VIH;
2538
2539 /* enable ring */
2540 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2541
2542 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2543 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2544 tx_ring->idr = hw->reg + ENETC_SITXIDR;
2545 }
2546
enetc_setup_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring,bool extended)2547 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
2548 bool extended)
2549 {
2550 int idx = rx_ring->index;
2551 u32 rbmr = 0;
2552
2553 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2554 lower_32_bits(rx_ring->bd_dma_base));
2555
2556 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2557 upper_32_bits(rx_ring->bd_dma_base));
2558
2559 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2560 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2561 ENETC_RTBLENR_LEN(rx_ring->bd_count));
2562
2563 if (rx_ring->xdp.prog)
2564 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2565 else
2566 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2567
2568 /* Also prepare the consumer index in case page allocation never
2569 * succeeds. In that case, hardware will never advance producer index
2570 * to match consumer index, and will drop all frames.
2571 */
2572 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2573 enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2574
2575 /* enable Rx ints by setting pkt thr to 1 */
2576 enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2577
2578 rx_ring->ext_en = extended;
2579 if (rx_ring->ext_en)
2580 rbmr |= ENETC_RBMR_BDS;
2581
2582 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2583 rbmr |= ENETC_RBMR_VTE;
2584
2585 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2586 rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2587
2588 rx_ring->next_to_clean = 0;
2589 rx_ring->next_to_use = 0;
2590 rx_ring->next_to_alloc = 0;
2591
2592 enetc_lock_mdio();
2593 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2594 enetc_unlock_mdio();
2595
2596 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2597 }
2598
enetc_setup_bdrs(struct enetc_ndev_priv * priv,bool extended)2599 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
2600 {
2601 struct enetc_hw *hw = &priv->si->hw;
2602 int i;
2603
2604 for (i = 0; i < priv->num_tx_rings; i++)
2605 enetc_setup_txbdr(hw, priv->tx_ring[i]);
2606
2607 for (i = 0; i < priv->num_rx_rings; i++)
2608 enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);
2609 }
2610
enetc_enable_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2611 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2612 {
2613 int idx = tx_ring->index;
2614 u32 tbmr;
2615
2616 tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);
2617 tbmr |= ENETC_TBMR_EN;
2618 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2619 }
2620
enetc_enable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2621 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2622 {
2623 int idx = rx_ring->index;
2624 u32 rbmr;
2625
2626 rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR);
2627 rbmr |= ENETC_RBMR_EN;
2628 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2629 }
2630
enetc_enable_rx_bdrs(struct enetc_ndev_priv * priv)2631 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv)
2632 {
2633 struct enetc_hw *hw = &priv->si->hw;
2634 int i;
2635
2636 for (i = 0; i < priv->num_rx_rings; i++)
2637 enetc_enable_rxbdr(hw, priv->rx_ring[i]);
2638 }
2639
enetc_enable_tx_bdrs(struct enetc_ndev_priv * priv)2640 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv)
2641 {
2642 struct enetc_hw *hw = &priv->si->hw;
2643 int i;
2644
2645 for (i = 0; i < priv->num_tx_rings; i++)
2646 enetc_enable_txbdr(hw, priv->tx_ring[i]);
2647 }
2648
enetc_disable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2649 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2650 {
2651 int idx = rx_ring->index;
2652
2653 /* disable EN bit on ring */
2654 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2655 }
2656
enetc_disable_txbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2657 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2658 {
2659 int idx = rx_ring->index;
2660
2661 /* disable EN bit on ring */
2662 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2663 }
2664
enetc_disable_rx_bdrs(struct enetc_ndev_priv * priv)2665 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv)
2666 {
2667 struct enetc_hw *hw = &priv->si->hw;
2668 int i;
2669
2670 for (i = 0; i < priv->num_rx_rings; i++)
2671 enetc_disable_rxbdr(hw, priv->rx_ring[i]);
2672 }
2673
enetc_disable_tx_bdrs(struct enetc_ndev_priv * priv)2674 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv)
2675 {
2676 struct enetc_hw *hw = &priv->si->hw;
2677 int i;
2678
2679 for (i = 0; i < priv->num_tx_rings; i++)
2680 enetc_disable_txbdr(hw, priv->tx_ring[i]);
2681 }
2682
enetc_wait_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2683 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2684 {
2685 int delay = 8, timeout = 100;
2686 int idx = tx_ring->index;
2687
2688 /* wait for busy to clear */
2689 while (delay < timeout &&
2690 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2691 msleep(delay);
2692 delay *= 2;
2693 }
2694
2695 if (delay >= timeout)
2696 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2697 idx);
2698 }
2699
enetc_wait_bdrs(struct enetc_ndev_priv * priv)2700 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
2701 {
2702 struct enetc_hw *hw = &priv->si->hw;
2703 int i;
2704
2705 for (i = 0; i < priv->num_tx_rings; i++)
2706 enetc_wait_txbdr(hw, priv->tx_ring[i]);
2707 }
2708
enetc_setup_irqs(struct enetc_ndev_priv * priv)2709 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2710 {
2711 struct pci_dev *pdev = priv->si->pdev;
2712 struct enetc_hw *hw = &priv->si->hw;
2713 int i, j, err;
2714
2715 for (i = 0; i < priv->bdr_int_num; i++) {
2716 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2717 struct enetc_int_vector *v = priv->int_vector[i];
2718 int entry = ENETC_BDR_INT_BASE_IDX + i;
2719
2720 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2721 priv->ndev->name, i);
2722 err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v);
2723 if (err) {
2724 dev_err(priv->dev, "request_irq() failed!\n");
2725 goto irq_err;
2726 }
2727
2728 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2729 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2730 v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2731
2732 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2733
2734 for (j = 0; j < v->count_tx_rings; j++) {
2735 int idx = v->tx_ring[j].index;
2736
2737 enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2738 }
2739 irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2740 }
2741
2742 return 0;
2743
2744 irq_err:
2745 while (i--) {
2746 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2747
2748 irq_set_affinity_hint(irq, NULL);
2749 free_irq(irq, priv->int_vector[i]);
2750 }
2751
2752 return err;
2753 }
2754
enetc_free_irqs(struct enetc_ndev_priv * priv)2755 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2756 {
2757 struct pci_dev *pdev = priv->si->pdev;
2758 int i;
2759
2760 for (i = 0; i < priv->bdr_int_num; i++) {
2761 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2762
2763 irq_set_affinity_hint(irq, NULL);
2764 free_irq(irq, priv->int_vector[i]);
2765 }
2766 }
2767
enetc_setup_interrupts(struct enetc_ndev_priv * priv)2768 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2769 {
2770 struct enetc_hw *hw = &priv->si->hw;
2771 u32 icpt, ictt;
2772 int i;
2773
2774 /* enable Tx & Rx event indication */
2775 if (priv->ic_mode &
2776 (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2777 icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2778 /* init to non-0 minimum, will be adjusted later */
2779 ictt = 0x1;
2780 } else {
2781 icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2782 ictt = 0;
2783 }
2784
2785 for (i = 0; i < priv->num_rx_rings; i++) {
2786 enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2787 enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2788 enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2789 }
2790
2791 if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2792 icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2793 else
2794 icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2795
2796 for (i = 0; i < priv->num_tx_rings; i++) {
2797 enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2798 enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2799 enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2800 }
2801 }
2802
enetc_clear_interrupts(struct enetc_ndev_priv * priv)2803 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2804 {
2805 struct enetc_hw *hw = &priv->si->hw;
2806 int i;
2807
2808 for (i = 0; i < priv->num_tx_rings; i++)
2809 enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2810
2811 for (i = 0; i < priv->num_rx_rings; i++)
2812 enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2813 }
2814
enetc_phylink_connect(struct net_device * ndev)2815 static int enetc_phylink_connect(struct net_device *ndev)
2816 {
2817 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2818 struct ethtool_keee edata;
2819 int err;
2820
2821 if (!priv->phylink) {
2822 /* phy-less mode */
2823 netif_carrier_on(ndev);
2824 return 0;
2825 }
2826
2827 err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2828 if (err) {
2829 dev_err(&ndev->dev, "could not attach to PHY\n");
2830 return err;
2831 }
2832
2833 /* disable EEE autoneg, until ENETC driver supports it */
2834 memset(&edata, 0, sizeof(struct ethtool_keee));
2835 phylink_ethtool_set_eee(priv->phylink, &edata);
2836
2837 phylink_start(priv->phylink);
2838
2839 return 0;
2840 }
2841
enetc_tx_onestep_tstamp(struct work_struct * work)2842 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2843 {
2844 struct enetc_ndev_priv *priv;
2845 struct sk_buff *skb;
2846
2847 priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2848
2849 netif_tx_lock_bh(priv->ndev);
2850
2851 clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2852 skb = skb_dequeue(&priv->tx_skbs);
2853 if (skb)
2854 enetc_start_xmit(skb, priv->ndev);
2855
2856 netif_tx_unlock_bh(priv->ndev);
2857 }
2858
enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv * priv)2859 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2860 {
2861 INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2862 skb_queue_head_init(&priv->tx_skbs);
2863 }
2864
enetc_start(struct net_device * ndev)2865 void enetc_start(struct net_device *ndev)
2866 {
2867 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2868 int i;
2869
2870 enetc_setup_interrupts(priv);
2871
2872 for (i = 0; i < priv->bdr_int_num; i++) {
2873 int irq = pci_irq_vector(priv->si->pdev,
2874 ENETC_BDR_INT_BASE_IDX + i);
2875
2876 napi_enable(&priv->int_vector[i]->napi);
2877 enable_irq(irq);
2878 }
2879
2880 enetc_enable_tx_bdrs(priv);
2881
2882 enetc_enable_rx_bdrs(priv);
2883
2884 netif_tx_start_all_queues(ndev);
2885
2886 clear_bit(ENETC_TX_DOWN, &priv->flags);
2887 }
2888 EXPORT_SYMBOL_GPL(enetc_start);
2889
enetc_open(struct net_device * ndev)2890 int enetc_open(struct net_device *ndev)
2891 {
2892 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2893 struct enetc_bdr_resource *tx_res, *rx_res;
2894 bool extended;
2895 int err;
2896
2897 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2898
2899 err = clk_prepare_enable(priv->ref_clk);
2900 if (err)
2901 return err;
2902
2903 err = enetc_setup_irqs(priv);
2904 if (err)
2905 goto err_setup_irqs;
2906
2907 err = enetc_phylink_connect(ndev);
2908 if (err)
2909 goto err_phy_connect;
2910
2911 tx_res = enetc_alloc_tx_resources(priv);
2912 if (IS_ERR(tx_res)) {
2913 err = PTR_ERR(tx_res);
2914 goto err_alloc_tx;
2915 }
2916
2917 rx_res = enetc_alloc_rx_resources(priv, extended);
2918 if (IS_ERR(rx_res)) {
2919 err = PTR_ERR(rx_res);
2920 goto err_alloc_rx;
2921 }
2922
2923 enetc_tx_onestep_tstamp_init(priv);
2924 enetc_assign_tx_resources(priv, tx_res);
2925 enetc_assign_rx_resources(priv, rx_res);
2926 enetc_setup_bdrs(priv, extended);
2927 enetc_start(ndev);
2928
2929 return 0;
2930
2931 err_alloc_rx:
2932 enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2933 err_alloc_tx:
2934 if (priv->phylink)
2935 phylink_disconnect_phy(priv->phylink);
2936 err_phy_connect:
2937 enetc_free_irqs(priv);
2938 err_setup_irqs:
2939 clk_disable_unprepare(priv->ref_clk);
2940
2941 return err;
2942 }
2943 EXPORT_SYMBOL_GPL(enetc_open);
2944
enetc_stop(struct net_device * ndev)2945 void enetc_stop(struct net_device *ndev)
2946 {
2947 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2948 int i;
2949
2950 set_bit(ENETC_TX_DOWN, &priv->flags);
2951
2952 netif_tx_stop_all_queues(ndev);
2953
2954 enetc_disable_rx_bdrs(priv);
2955
2956 enetc_wait_bdrs(priv);
2957
2958 enetc_disable_tx_bdrs(priv);
2959
2960 for (i = 0; i < priv->bdr_int_num; i++) {
2961 int irq = pci_irq_vector(priv->si->pdev,
2962 ENETC_BDR_INT_BASE_IDX + i);
2963
2964 disable_irq(irq);
2965 napi_synchronize(&priv->int_vector[i]->napi);
2966 napi_disable(&priv->int_vector[i]->napi);
2967 }
2968
2969 enetc_clear_interrupts(priv);
2970 }
2971 EXPORT_SYMBOL_GPL(enetc_stop);
2972
enetc_close(struct net_device * ndev)2973 int enetc_close(struct net_device *ndev)
2974 {
2975 struct enetc_ndev_priv *priv = netdev_priv(ndev);
2976
2977 enetc_stop(ndev);
2978
2979 if (priv->phylink) {
2980 phylink_stop(priv->phylink);
2981 phylink_disconnect_phy(priv->phylink);
2982 } else {
2983 netif_carrier_off(ndev);
2984 }
2985
2986 enetc_free_rxtx_rings(priv);
2987
2988 /* Avoids dangling pointers and also frees old resources */
2989 enetc_assign_rx_resources(priv, NULL);
2990 enetc_assign_tx_resources(priv, NULL);
2991
2992 enetc_free_irqs(priv);
2993 clk_disable_unprepare(priv->ref_clk);
2994
2995 return 0;
2996 }
2997 EXPORT_SYMBOL_GPL(enetc_close);
2998
enetc_reconfigure(struct enetc_ndev_priv * priv,bool extended,int (* cb)(struct enetc_ndev_priv * priv,void * ctx),void * ctx)2999 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
3000 int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
3001 void *ctx)
3002 {
3003 struct enetc_bdr_resource *tx_res, *rx_res;
3004 int err;
3005
3006 ASSERT_RTNL();
3007
3008 /* If the interface is down, run the callback right away,
3009 * without reconfiguration.
3010 */
3011 if (!netif_running(priv->ndev)) {
3012 if (cb) {
3013 err = cb(priv, ctx);
3014 if (err)
3015 return err;
3016 }
3017
3018 return 0;
3019 }
3020
3021 tx_res = enetc_alloc_tx_resources(priv);
3022 if (IS_ERR(tx_res)) {
3023 err = PTR_ERR(tx_res);
3024 goto out;
3025 }
3026
3027 rx_res = enetc_alloc_rx_resources(priv, extended);
3028 if (IS_ERR(rx_res)) {
3029 err = PTR_ERR(rx_res);
3030 goto out_free_tx_res;
3031 }
3032
3033 enetc_stop(priv->ndev);
3034 enetc_free_rxtx_rings(priv);
3035
3036 /* Interface is down, run optional callback now */
3037 if (cb) {
3038 err = cb(priv, ctx);
3039 if (err)
3040 goto out_restart;
3041 }
3042
3043 enetc_assign_tx_resources(priv, tx_res);
3044 enetc_assign_rx_resources(priv, rx_res);
3045 enetc_setup_bdrs(priv, extended);
3046 enetc_start(priv->ndev);
3047
3048 return 0;
3049
3050 out_restart:
3051 enetc_setup_bdrs(priv, extended);
3052 enetc_start(priv->ndev);
3053 enetc_free_rx_resources(rx_res, priv->num_rx_rings);
3054 out_free_tx_res:
3055 enetc_free_tx_resources(tx_res, priv->num_tx_rings);
3056 out:
3057 return err;
3058 }
3059
enetc_debug_tx_ring_prios(struct enetc_ndev_priv * priv)3060 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
3061 {
3062 int i;
3063
3064 for (i = 0; i < priv->num_tx_rings; i++)
3065 netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,
3066 priv->tx_ring[i]->prio);
3067 }
3068
enetc_reset_tc_mqprio(struct net_device * ndev)3069 void enetc_reset_tc_mqprio(struct net_device *ndev)
3070 {
3071 struct enetc_ndev_priv *priv = netdev_priv(ndev);
3072 struct enetc_hw *hw = &priv->si->hw;
3073 struct enetc_bdr *tx_ring;
3074 int num_stack_tx_queues;
3075 int i;
3076
3077 num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3078
3079 netdev_reset_tc(ndev);
3080 netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
3081 priv->min_num_stack_tx_queues = num_possible_cpus();
3082
3083 /* Reset all ring priorities to 0 */
3084 for (i = 0; i < priv->num_tx_rings; i++) {
3085 tx_ring = priv->tx_ring[i];
3086 tx_ring->prio = 0;
3087 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
3088 }
3089
3090 enetc_debug_tx_ring_prios(priv);
3091
3092 enetc_change_preemptible_tcs(priv, 0);
3093 }
3094 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);
3095
enetc_setup_tc_mqprio(struct net_device * ndev,void * type_data)3096 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
3097 {
3098 struct tc_mqprio_qopt_offload *mqprio = type_data;
3099 struct enetc_ndev_priv *priv = netdev_priv(ndev);
3100 struct tc_mqprio_qopt *qopt = &mqprio->qopt;
3101 struct enetc_hw *hw = &priv->si->hw;
3102 int num_stack_tx_queues = 0;
3103 struct enetc_bdr *tx_ring;
3104 u8 num_tc = qopt->num_tc;
3105 int offset, count;
3106 int err, tc, q;
3107
3108 if (!num_tc) {
3109 enetc_reset_tc_mqprio(ndev);
3110 return 0;
3111 }
3112
3113 err = netdev_set_num_tc(ndev, num_tc);
3114 if (err)
3115 return err;
3116
3117 for (tc = 0; tc < num_tc; tc++) {
3118 offset = qopt->offset[tc];
3119 count = qopt->count[tc];
3120 num_stack_tx_queues += count;
3121
3122 err = netdev_set_tc_queue(ndev, tc, count, offset);
3123 if (err)
3124 goto err_reset_tc;
3125
3126 for (q = offset; q < offset + count; q++) {
3127 tx_ring = priv->tx_ring[q];
3128 /* The prio_tc_map is skb_tx_hash()'s way of selecting
3129 * between TX queues based on skb->priority. As such,
3130 * there's nothing to offload based on it.
3131 * Make the mqprio "traffic class" be the priority of
3132 * this ring group, and leave the Tx IPV to traffic
3133 * class mapping as its default mapping value of 1:1.
3134 */
3135 tx_ring->prio = tc;
3136 enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
3137 }
3138 }
3139
3140 err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
3141 if (err)
3142 goto err_reset_tc;
3143
3144 priv->min_num_stack_tx_queues = num_stack_tx_queues;
3145
3146 enetc_debug_tx_ring_prios(priv);
3147
3148 enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
3149
3150 return 0;
3151
3152 err_reset_tc:
3153 enetc_reset_tc_mqprio(ndev);
3154 return err;
3155 }
3156 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);
3157
enetc_reconfigure_xdp_cb(struct enetc_ndev_priv * priv,void * ctx)3158 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
3159 {
3160 struct bpf_prog *old_prog, *prog = ctx;
3161 int num_stack_tx_queues;
3162 int err, i;
3163
3164 old_prog = xchg(&priv->xdp_prog, prog);
3165
3166 num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3167 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3168 if (err) {
3169 xchg(&priv->xdp_prog, old_prog);
3170 return err;
3171 }
3172
3173 if (old_prog)
3174 bpf_prog_put(old_prog);
3175
3176 for (i = 0; i < priv->num_rx_rings; i++) {
3177 struct enetc_bdr *rx_ring = priv->rx_ring[i];
3178
3179 rx_ring->xdp.prog = prog;
3180
3181 if (prog)
3182 rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
3183 else
3184 rx_ring->buffer_offset = ENETC_RXB_PAD;
3185 }
3186
3187 return 0;
3188 }
3189
enetc_setup_xdp_prog(struct net_device * ndev,struct bpf_prog * prog,struct netlink_ext_ack * extack)3190 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
3191 struct netlink_ext_ack *extack)
3192 {
3193 int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;
3194 struct enetc_ndev_priv *priv = netdev_priv(ndev);
3195 bool extended;
3196
3197 if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >
3198 priv->num_tx_rings) {
3199 NL_SET_ERR_MSG_FMT_MOD(extack,
3200 "Reserving %d XDP TXQs leaves under %d for stack (total %d)",
3201 num_xdp_tx_queues,
3202 priv->min_num_stack_tx_queues,
3203 priv->num_tx_rings);
3204 return -EBUSY;
3205 }
3206
3207 extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
3208
3209 /* The buffer layout is changing, so we need to drain the old
3210 * RX buffers and seed new ones.
3211 */
3212 return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);
3213 }
3214
enetc_setup_bpf(struct net_device * ndev,struct netdev_bpf * bpf)3215 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
3216 {
3217 switch (bpf->command) {
3218 case XDP_SETUP_PROG:
3219 return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);
3220 default:
3221 return -EINVAL;
3222 }
3223
3224 return 0;
3225 }
3226 EXPORT_SYMBOL_GPL(enetc_setup_bpf);
3227
enetc_get_stats(struct net_device * ndev)3228 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
3229 {
3230 struct enetc_ndev_priv *priv = netdev_priv(ndev);
3231 struct net_device_stats *stats = &ndev->stats;
3232 unsigned long packets = 0, bytes = 0;
3233 unsigned long tx_dropped = 0;
3234 int i;
3235
3236 for (i = 0; i < priv->num_rx_rings; i++) {
3237 packets += priv->rx_ring[i]->stats.packets;
3238 bytes += priv->rx_ring[i]->stats.bytes;
3239 }
3240
3241 stats->rx_packets = packets;
3242 stats->rx_bytes = bytes;
3243 bytes = 0;
3244 packets = 0;
3245
3246 for (i = 0; i < priv->num_tx_rings; i++) {
3247 packets += priv->tx_ring[i]->stats.packets;
3248 bytes += priv->tx_ring[i]->stats.bytes;
3249 tx_dropped += priv->tx_ring[i]->stats.win_drop;
3250 }
3251
3252 stats->tx_packets = packets;
3253 stats->tx_bytes = bytes;
3254 stats->tx_dropped = tx_dropped;
3255
3256 return stats;
3257 }
3258 EXPORT_SYMBOL_GPL(enetc_get_stats);
3259
enetc_enable_rxvlan(struct net_device * ndev,bool en)3260 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
3261 {
3262 struct enetc_ndev_priv *priv = netdev_priv(ndev);
3263 struct enetc_hw *hw = &priv->si->hw;
3264 int i;
3265
3266 for (i = 0; i < priv->num_rx_rings; i++)
3267 enetc_bdr_enable_rxvlan(hw, i, en);
3268 }
3269
enetc_enable_txvlan(struct net_device * ndev,bool en)3270 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
3271 {
3272 struct enetc_ndev_priv *priv = netdev_priv(ndev);
3273 struct enetc_hw *hw = &priv->si->hw;
3274 int i;
3275
3276 for (i = 0; i < priv->num_tx_rings; i++)
3277 enetc_bdr_enable_txvlan(hw, i, en);
3278 }
3279
enetc_set_features(struct net_device * ndev,netdev_features_t features)3280 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
3281 {
3282 netdev_features_t changed = ndev->features ^ features;
3283
3284 if (changed & NETIF_F_RXHASH)
3285 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
3286
3287 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
3288 enetc_enable_rxvlan(ndev,
3289 !!(features & NETIF_F_HW_VLAN_CTAG_RX));
3290
3291 if (changed & NETIF_F_HW_VLAN_CTAG_TX)
3292 enetc_enable_txvlan(ndev,
3293 !!(features & NETIF_F_HW_VLAN_CTAG_TX));
3294 }
3295 EXPORT_SYMBOL_GPL(enetc_set_features);
3296
enetc_hwtstamp_set(struct net_device * ndev,struct kernel_hwtstamp_config * config,struct netlink_ext_ack * extack)3297 int enetc_hwtstamp_set(struct net_device *ndev,
3298 struct kernel_hwtstamp_config *config,
3299 struct netlink_ext_ack *extack)
3300 {
3301 struct enetc_ndev_priv *priv = netdev_priv(ndev);
3302 int err, new_offloads = priv->active_offloads;
3303
3304 if (!IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK))
3305 return -EOPNOTSUPP;
3306
3307 switch (config->tx_type) {
3308 case HWTSTAMP_TX_OFF:
3309 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
3310 break;
3311 case HWTSTAMP_TX_ON:
3312 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
3313 new_offloads |= ENETC_F_TX_TSTAMP;
3314 break;
3315 case HWTSTAMP_TX_ONESTEP_SYNC:
3316 if (!enetc_si_is_pf(priv->si))
3317 return -EOPNOTSUPP;
3318
3319 new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
3320 new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
3321 break;
3322 default:
3323 return -ERANGE;
3324 }
3325
3326 switch (config->rx_filter) {
3327 case HWTSTAMP_FILTER_NONE:
3328 new_offloads &= ~ENETC_F_RX_TSTAMP;
3329 break;
3330 default:
3331 new_offloads |= ENETC_F_RX_TSTAMP;
3332 config->rx_filter = HWTSTAMP_FILTER_ALL;
3333 }
3334
3335 if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {
3336 bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);
3337
3338 err = enetc_reconfigure(priv, extended, NULL, NULL);
3339 if (err)
3340 return err;
3341 }
3342
3343 priv->active_offloads = new_offloads;
3344
3345 return 0;
3346 }
3347 EXPORT_SYMBOL_GPL(enetc_hwtstamp_set);
3348
enetc_hwtstamp_get(struct net_device * ndev,struct kernel_hwtstamp_config * config)3349 int enetc_hwtstamp_get(struct net_device *ndev,
3350 struct kernel_hwtstamp_config *config)
3351 {
3352 struct enetc_ndev_priv *priv = netdev_priv(ndev);
3353
3354 if (!IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK))
3355 return -EOPNOTSUPP;
3356
3357 if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
3358 config->tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
3359 else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
3360 config->tx_type = HWTSTAMP_TX_ON;
3361 else
3362 config->tx_type = HWTSTAMP_TX_OFF;
3363
3364 config->rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
3365 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
3366
3367 return 0;
3368 }
3369 EXPORT_SYMBOL_GPL(enetc_hwtstamp_get);
3370
enetc_ioctl(struct net_device * ndev,struct ifreq * rq,int cmd)3371 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
3372 {
3373 struct enetc_ndev_priv *priv = netdev_priv(ndev);
3374
3375 if (!priv->phylink)
3376 return -EOPNOTSUPP;
3377
3378 return phylink_mii_ioctl(priv->phylink, rq, cmd);
3379 }
3380 EXPORT_SYMBOL_GPL(enetc_ioctl);
3381
enetc_int_vector_init(struct enetc_ndev_priv * priv,int i,int v_tx_rings)3382 static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i,
3383 int v_tx_rings)
3384 {
3385 struct enetc_int_vector *v;
3386 struct enetc_bdr *bdr;
3387 int j, err;
3388
3389 v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
3390 if (!v)
3391 return -ENOMEM;
3392
3393 priv->int_vector[i] = v;
3394 bdr = &v->rx_ring;
3395 bdr->index = i;
3396 bdr->ndev = priv->ndev;
3397 bdr->dev = priv->dev;
3398 bdr->bd_count = priv->rx_bd_count;
3399 bdr->buffer_offset = ENETC_RXB_PAD;
3400 priv->rx_ring[i] = bdr;
3401
3402 err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0,
3403 ENETC_RXB_DMA_SIZE_XDP);
3404 if (err)
3405 goto free_vector;
3406
3407 err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, MEM_TYPE_PAGE_SHARED,
3408 NULL);
3409 if (err) {
3410 xdp_rxq_info_unreg(&bdr->xdp.rxq);
3411 goto free_vector;
3412 }
3413
3414 /* init defaults for adaptive IC */
3415 if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
3416 v->rx_ictt = 0x1;
3417 v->rx_dim_en = true;
3418 }
3419
3420 INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
3421 netif_napi_add(priv->ndev, &v->napi, enetc_poll);
3422 v->count_tx_rings = v_tx_rings;
3423
3424 for (j = 0; j < v_tx_rings; j++) {
3425 int idx;
3426
3427 /* default tx ring mapping policy */
3428 idx = priv->bdr_int_num * j + i;
3429 __set_bit(idx, &v->tx_rings_map);
3430 bdr = &v->tx_ring[j];
3431 bdr->index = idx;
3432 bdr->ndev = priv->ndev;
3433 bdr->dev = priv->dev;
3434 bdr->bd_count = priv->tx_bd_count;
3435 priv->tx_ring[idx] = bdr;
3436 }
3437
3438 return 0;
3439
3440 free_vector:
3441 priv->rx_ring[i] = NULL;
3442 priv->int_vector[i] = NULL;
3443 kfree(v);
3444
3445 return err;
3446 }
3447
enetc_int_vector_destroy(struct enetc_ndev_priv * priv,int i)3448 static void enetc_int_vector_destroy(struct enetc_ndev_priv *priv, int i)
3449 {
3450 struct enetc_int_vector *v = priv->int_vector[i];
3451 struct enetc_bdr *rx_ring = &v->rx_ring;
3452 int j, tx_ring_index;
3453
3454 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3455 xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3456 netif_napi_del(&v->napi);
3457 cancel_work_sync(&v->rx_dim.work);
3458
3459 for (j = 0; j < v->count_tx_rings; j++) {
3460 tx_ring_index = priv->bdr_int_num * j + i;
3461 priv->tx_ring[tx_ring_index] = NULL;
3462 }
3463
3464 priv->rx_ring[i] = NULL;
3465 priv->int_vector[i] = NULL;
3466 kfree(v);
3467 }
3468
enetc_alloc_msix(struct enetc_ndev_priv * priv)3469 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
3470 {
3471 struct pci_dev *pdev = priv->si->pdev;
3472 int v_tx_rings, v_remainder;
3473 int num_stack_tx_queues;
3474 int first_xdp_tx_ring;
3475 int i, n, err, nvec;
3476
3477 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
3478 /* allocate MSIX for both messaging and Rx/Tx interrupts */
3479 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
3480
3481 if (n < 0)
3482 return n;
3483
3484 if (n != nvec)
3485 return -EPERM;
3486
3487 /* # of tx rings per int vector */
3488 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
3489 v_remainder = priv->num_tx_rings % priv->bdr_int_num;
3490
3491 for (i = 0; i < priv->bdr_int_num; i++) {
3492 /* Distribute the remaining TX rings to the first v_remainder
3493 * interrupt vectors
3494 */
3495 int num_tx_rings = i < v_remainder ? v_tx_rings + 1 : v_tx_rings;
3496
3497 err = enetc_int_vector_init(priv, i, num_tx_rings);
3498 if (err)
3499 goto fail;
3500 }
3501
3502 num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3503
3504 err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3505 if (err)
3506 goto fail;
3507
3508 err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);
3509 if (err)
3510 goto fail;
3511
3512 priv->min_num_stack_tx_queues = num_possible_cpus();
3513 first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
3514 priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
3515
3516 return 0;
3517
3518 fail:
3519 while (i--)
3520 enetc_int_vector_destroy(priv, i);
3521
3522 pci_free_irq_vectors(pdev);
3523
3524 return err;
3525 }
3526 EXPORT_SYMBOL_GPL(enetc_alloc_msix);
3527
enetc_free_msix(struct enetc_ndev_priv * priv)3528 void enetc_free_msix(struct enetc_ndev_priv *priv)
3529 {
3530 int i;
3531
3532 for (i = 0; i < priv->bdr_int_num; i++)
3533 enetc_int_vector_destroy(priv, i);
3534
3535 /* disable all MSIX for this device */
3536 pci_free_irq_vectors(priv->si->pdev);
3537 }
3538 EXPORT_SYMBOL_GPL(enetc_free_msix);
3539
enetc_kfree_si(struct enetc_si * si)3540 static void enetc_kfree_si(struct enetc_si *si)
3541 {
3542 char *p = (char *)si - si->pad;
3543
3544 kfree(p);
3545 }
3546
enetc_detect_errata(struct enetc_si * si)3547 static void enetc_detect_errata(struct enetc_si *si)
3548 {
3549 if (si->pdev->revision == ENETC_REV1)
3550 si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
3551 }
3552
enetc_pci_probe(struct pci_dev * pdev,const char * name,int sizeof_priv)3553 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
3554 {
3555 struct enetc_si *si, *p;
3556 struct enetc_hw *hw;
3557 size_t alloc_size;
3558 int err, len;
3559
3560 pcie_flr(pdev);
3561 err = pci_enable_device_mem(pdev);
3562 if (err)
3563 return dev_err_probe(&pdev->dev, err, "device enable failed\n");
3564
3565 /* set up for high or low dma */
3566 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3567 if (err) {
3568 dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3569 goto err_dma;
3570 }
3571
3572 err = pci_request_mem_regions(pdev, name);
3573 if (err) {
3574 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
3575 goto err_pci_mem_reg;
3576 }
3577
3578 pci_set_master(pdev);
3579
3580 alloc_size = sizeof(struct enetc_si);
3581 if (sizeof_priv) {
3582 /* align priv to 32B */
3583 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
3584 alloc_size += sizeof_priv;
3585 }
3586 /* force 32B alignment for enetc_si */
3587 alloc_size += ENETC_SI_ALIGN - 1;
3588
3589 p = kzalloc(alloc_size, GFP_KERNEL);
3590 if (!p) {
3591 err = -ENOMEM;
3592 goto err_alloc_si;
3593 }
3594
3595 si = PTR_ALIGN(p, ENETC_SI_ALIGN);
3596 si->pad = (char *)si - (char *)p;
3597
3598 pci_set_drvdata(pdev, si);
3599 si->pdev = pdev;
3600 hw = &si->hw;
3601
3602 len = pci_resource_len(pdev, ENETC_BAR_REGS);
3603 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
3604 if (!hw->reg) {
3605 err = -ENXIO;
3606 dev_err(&pdev->dev, "ioremap() failed\n");
3607 goto err_ioremap;
3608 }
3609 if (len > ENETC_PORT_BASE)
3610 hw->port = hw->reg + ENETC_PORT_BASE;
3611 if (len > ENETC_GLOBAL_BASE)
3612 hw->global = hw->reg + ENETC_GLOBAL_BASE;
3613
3614 enetc_detect_errata(si);
3615
3616 return 0;
3617
3618 err_ioremap:
3619 enetc_kfree_si(si);
3620 err_alloc_si:
3621 pci_release_mem_regions(pdev);
3622 err_pci_mem_reg:
3623 err_dma:
3624 pci_disable_device(pdev);
3625
3626 return err;
3627 }
3628 EXPORT_SYMBOL_GPL(enetc_pci_probe);
3629
enetc_pci_remove(struct pci_dev * pdev)3630 void enetc_pci_remove(struct pci_dev *pdev)
3631 {
3632 struct enetc_si *si = pci_get_drvdata(pdev);
3633 struct enetc_hw *hw = &si->hw;
3634
3635 iounmap(hw->reg);
3636 enetc_kfree_si(si);
3637 pci_release_mem_regions(pdev);
3638 pci_disable_device(pdev);
3639 }
3640 EXPORT_SYMBOL_GPL(enetc_pci_remove);
3641
3642 static const struct enetc_drvdata enetc_pf_data = {
3643 .sysclk_freq = ENETC_CLK_400M,
3644 .pmac_offset = ENETC_PMAC_OFFSET,
3645 .max_frags = ENETC_MAX_SKB_FRAGS,
3646 .eth_ops = &enetc_pf_ethtool_ops,
3647 };
3648
3649 static const struct enetc_drvdata enetc4_pf_data = {
3650 .sysclk_freq = ENETC_CLK_333M,
3651 .tx_csum = true,
3652 .max_frags = ENETC4_MAX_SKB_FRAGS,
3653 .pmac_offset = ENETC4_PMAC_OFFSET,
3654 .eth_ops = &enetc4_pf_ethtool_ops,
3655 };
3656
3657 static const struct enetc_drvdata enetc_vf_data = {
3658 .sysclk_freq = ENETC_CLK_400M,
3659 .max_frags = ENETC_MAX_SKB_FRAGS,
3660 .eth_ops = &enetc_vf_ethtool_ops,
3661 };
3662
3663 static const struct enetc_platform_info enetc_info[] = {
3664 { .revision = ENETC_REV_1_0,
3665 .dev_id = ENETC_DEV_ID_PF,
3666 .data = &enetc_pf_data,
3667 },
3668 { .revision = ENETC_REV_4_1,
3669 .dev_id = NXP_ENETC_PF_DEV_ID,
3670 .data = &enetc4_pf_data,
3671 },
3672 { .revision = ENETC_REV_1_0,
3673 .dev_id = ENETC_DEV_ID_VF,
3674 .data = &enetc_vf_data,
3675 },
3676 };
3677
enetc_get_driver_data(struct enetc_si * si)3678 int enetc_get_driver_data(struct enetc_si *si)
3679 {
3680 u16 dev_id = si->pdev->device;
3681 int i;
3682
3683 for (i = 0; i < ARRAY_SIZE(enetc_info); i++) {
3684 if (si->revision == enetc_info[i].revision &&
3685 dev_id == enetc_info[i].dev_id) {
3686 si->drvdata = enetc_info[i].data;
3687
3688 return 0;
3689 }
3690 }
3691
3692 return -ERANGE;
3693 }
3694 EXPORT_SYMBOL_GPL(enetc_get_driver_data);
3695
3696 MODULE_DESCRIPTION("NXP ENETC Ethernet driver");
3697 MODULE_LICENSE("Dual BSD/GPL");
3698