xref: /linux/drivers/net/ethernet/freescale/enetc/enetc.c (revision 8830e65ed46de41f849eefb8ba227d4852c460f6)
1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3 
4 #include "enetc.h"
5 #include <linux/bpf_trace.h>
6 #include <linux/clk.h>
7 #include <linux/tcp.h>
8 #include <linux/udp.h>
9 #include <linux/vmalloc.h>
10 #include <linux/ptp_classify.h>
11 #include <net/ip6_checksum.h>
12 #include <net/pkt_sched.h>
13 #include <net/tso.h>
14 
15 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
16 {
17 	/* ENETC with pseudo MAC does not have Ethernet MAC
18 	 * port registers.
19 	 */
20 	if (enetc_is_pseudo_mac(si))
21 		return 0;
22 
23 	return enetc_port_rd(&si->hw, reg);
24 }
25 EXPORT_SYMBOL_GPL(enetc_port_mac_rd);
26 
27 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
28 {
29 	if (enetc_is_pseudo_mac(si))
30 		return;
31 
32 	enetc_port_wr(&si->hw, reg, val);
33 	if (si->hw_features & ENETC_SI_F_QBU)
34 		enetc_port_wr(&si->hw, reg + si->drvdata->pmac_offset, val);
35 }
36 EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
37 
38 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
39 					 u8 preemptible_tcs)
40 {
41 	if (!(priv->si->hw_features & ENETC_SI_F_QBU))
42 		return;
43 
44 	priv->preemptible_tcs = preemptible_tcs;
45 	enetc_mm_commit_preemptible_tcs(priv);
46 }
47 
48 static int enetc_mac_addr_hash_idx(const u8 *addr)
49 {
50 	u64 fold = __swab64(ether_addr_to_u64(addr)) >> 16;
51 	u64 mask = 0;
52 	int res = 0;
53 	int i;
54 
55 	for (i = 0; i < 8; i++)
56 		mask |= BIT_ULL(i * 6);
57 
58 	for (i = 0; i < 6; i++)
59 		res |= (hweight64(fold & (mask << i)) & 0x1) << i;
60 
61 	return res;
62 }
63 
64 void enetc_add_mac_addr_ht_filter(struct enetc_mac_filter *filter,
65 				  const unsigned char *addr)
66 {
67 	int idx = enetc_mac_addr_hash_idx(addr);
68 
69 	/* add hash table entry */
70 	__set_bit(idx, filter->mac_hash_table);
71 	filter->mac_addr_cnt++;
72 }
73 EXPORT_SYMBOL_GPL(enetc_add_mac_addr_ht_filter);
74 
75 void enetc_reset_mac_addr_filter(struct enetc_mac_filter *filter)
76 {
77 	filter->mac_addr_cnt = 0;
78 
79 	bitmap_zero(filter->mac_hash_table,
80 		    ENETC_MADDR_HASH_TBL_SZ);
81 }
82 EXPORT_SYMBOL_GPL(enetc_reset_mac_addr_filter);
83 
84 void enetc_set_congestion_mode(struct enetc_ndev_priv *priv, bool enable)
85 {
86 	struct enetc_si *si = priv->si;
87 	struct enetc_hw *hw = &si->hw;
88 
89 	spin_lock(&si->gen_lock);
90 
91 	assign_bit(ENETC_RXBDR_CM, &priv->flags, enable);
92 
93 	for (int i = 0; i < priv->num_rx_rings; i++) {
94 		u32 old_rbmr = enetc_rxbdr_rd(hw, i, ENETC_RBMR);
95 		u32 rbmr;
96 
97 		rbmr = u32_replace_bits(old_rbmr, enable, ENETC_RBMR_CM);
98 		if (rbmr == old_rbmr)
99 			continue;
100 
101 		enetc_rxbdr_wr(hw, i, ENETC_RBMR, rbmr);
102 	}
103 
104 	spin_unlock(&si->gen_lock);
105 }
106 EXPORT_SYMBOL_GPL(enetc_set_congestion_mode);
107 
108 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
109 {
110 	int num_tx_rings = priv->num_tx_rings;
111 
112 	if (priv->xdp_prog)
113 		return num_tx_rings - num_possible_cpus();
114 
115 	return num_tx_rings;
116 }
117 
118 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
119 							struct enetc_bdr *tx_ring)
120 {
121 	int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
122 
123 	return priv->rx_ring[index];
124 }
125 
126 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
127 {
128 	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
129 		return NULL;
130 
131 	return tx_swbd->skb;
132 }
133 
134 static struct xdp_frame *
135 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
136 {
137 	if (tx_swbd->is_xdp_redirect)
138 		return tx_swbd->xdp_frame;
139 
140 	return NULL;
141 }
142 
143 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
144 				struct enetc_tx_swbd *tx_swbd)
145 {
146 	/* For XDP_TX, pages come from RX, whereas for the other contexts where
147 	 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
148 	 * to match the DMA mapping length, so we need to differentiate those.
149 	 */
150 	if (tx_swbd->is_dma_page)
151 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,
152 			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
153 			       tx_swbd->dir);
154 	else
155 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,
156 				 tx_swbd->len, tx_swbd->dir);
157 	tx_swbd->dma = 0;
158 }
159 
160 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
161 				struct enetc_tx_swbd *tx_swbd)
162 {
163 	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
164 	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
165 
166 	if (tx_swbd->dma)
167 		enetc_unmap_tx_buff(tx_ring, tx_swbd);
168 
169 	if (xdp_frame) {
170 		xdp_return_frame(tx_swbd->xdp_frame);
171 		tx_swbd->xdp_frame = NULL;
172 	} else if (skb) {
173 		dev_kfree_skb_any(skb);
174 		tx_swbd->skb = NULL;
175 	}
176 }
177 
178 /* Let H/W know BD ring has been updated */
179 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
180 {
181 	/* includes wmb() */
182 	enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
183 }
184 
185 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
186 			   u8 *msgtype, u8 *twostep,
187 			   u16 *correction_offset, u16 *body_offset)
188 {
189 	unsigned int ptp_class;
190 	struct ptp_header *hdr;
191 	unsigned int type;
192 	u8 *base;
193 
194 	ptp_class = ptp_classify_raw(skb);
195 	if (ptp_class == PTP_CLASS_NONE)
196 		return -EINVAL;
197 
198 	hdr = ptp_parse_header(skb, ptp_class);
199 	if (!hdr)
200 		return -EINVAL;
201 
202 	type = ptp_class & PTP_CLASS_PMASK;
203 	if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
204 		*udp = 1;
205 	else
206 		*udp = 0;
207 
208 	*msgtype = ptp_get_msgtype(hdr, ptp_class);
209 	*twostep = hdr->flag_field[0] & 0x2;
210 
211 	base = skb_mac_header(skb);
212 	*correction_offset = (u8 *)&hdr->correction - base;
213 	*body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
214 
215 	return 0;
216 }
217 
218 static bool enetc_tx_csum_offload_check(struct sk_buff *skb)
219 {
220 	switch (skb->csum_offset) {
221 	case offsetof(struct tcphdr, check):
222 	case offsetof(struct udphdr, check):
223 		return true;
224 	default:
225 		return false;
226 	}
227 }
228 
229 static bool enetc_skb_is_ipv6(struct sk_buff *skb)
230 {
231 	return vlan_get_protocol(skb) == htons(ETH_P_IPV6);
232 }
233 
234 static bool enetc_skb_is_tcp(struct sk_buff *skb)
235 {
236 	return skb->csum_offset == offsetof(struct tcphdr, check);
237 }
238 
239 /**
240  * enetc_unwind_tx_frame() - Unwind the DMA mappings of a multi-buffer Tx frame
241  * @tx_ring: Pointer to the Tx ring on which the buffer descriptors are located
242  * @count: Number of Tx buffer descriptors which need to be unmapped
243  * @i: Index of the last successfully mapped Tx buffer descriptor
244  */
245 static void enetc_unwind_tx_frame(struct enetc_bdr *tx_ring, int count, int i)
246 {
247 	while (count--) {
248 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
249 
250 		enetc_free_tx_frame(tx_ring, tx_swbd);
251 		if (i == 0)
252 			i = tx_ring->bd_count;
253 		i--;
254 	}
255 }
256 
257 static void enetc_set_one_step_ts(struct enetc_si *si, bool udp, int offset)
258 {
259 	u32 val = ENETC_PM0_SINGLE_STEP_EN;
260 
261 	val |= ENETC_SET_SINGLE_STEP_OFFSET(offset);
262 	if (udp)
263 		val |= ENETC_PM0_SINGLE_STEP_CH;
264 
265 	/* The "Correction" field of a packet is updated based on the
266 	 * current time and the timestamp provided
267 	 */
268 	enetc_port_mac_wr(si, ENETC_PM0_SINGLE_STEP, val);
269 }
270 
271 static void enetc4_set_one_step_ts(struct enetc_si *si, bool udp, int offset)
272 {
273 	u32 val = PM_SINGLE_STEP_EN;
274 
275 	val |= PM_SINGLE_STEP_OFFSET_SET(offset);
276 	if (udp)
277 		val |= PM_SINGLE_STEP_CH;
278 
279 	enetc_port_mac_wr(si, ENETC4_PM_SINGLE_STEP(0), val);
280 }
281 
282 static u32 enetc_update_ptp_sync_msg(struct enetc_ndev_priv *priv,
283 				     struct sk_buff *skb, bool csum_offload)
284 {
285 	struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb);
286 	u16 tstamp_off = enetc_cb->origin_tstamp_off;
287 	u16 corr_off = enetc_cb->correction_off;
288 	struct enetc_si *si = priv->si;
289 	struct enetc_hw *hw = &si->hw;
290 	__be32 new_sec_l, new_nsec;
291 	__be16 new_sec_h;
292 	u32 lo, hi, nsec;
293 	u8 *data;
294 	u64 sec;
295 
296 	lo = enetc_rd_hot(hw, ENETC_SICTR0);
297 	hi = enetc_rd_hot(hw, ENETC_SICTR1);
298 	sec = (u64)hi << 32 | lo;
299 	nsec = do_div(sec, 1000000000);
300 
301 	/* Update originTimestamp field of Sync packet
302 	 * - 48 bits seconds field
303 	 * - 32 bits nanseconds field
304 	 *
305 	 * In addition, if csum_offload is false, the UDP checksum needs
306 	 * to be updated by software after updating originTimestamp field,
307 	 * otherwise the hardware will calculate the wrong checksum when
308 	 * updating the correction field and update it to the packet.
309 	 */
310 
311 	data = skb_mac_header(skb);
312 	new_sec_h = htons((sec >> 32) & 0xffff);
313 	new_sec_l = htonl(sec & 0xffffffff);
314 	new_nsec = htonl(nsec);
315 	if (enetc_cb->udp && !csum_offload) {
316 		struct udphdr *uh = udp_hdr(skb);
317 		__be32 old_sec_l, old_nsec;
318 		__be16 old_sec_h;
319 
320 		old_sec_h = *(__be16 *)(data + tstamp_off);
321 		inet_proto_csum_replace2(&uh->check, skb, old_sec_h,
322 					 new_sec_h, false);
323 
324 		old_sec_l = *(__be32 *)(data + tstamp_off + 2);
325 		inet_proto_csum_replace4(&uh->check, skb, old_sec_l,
326 					 new_sec_l, false);
327 
328 		old_nsec = *(__be32 *)(data + tstamp_off + 6);
329 		inet_proto_csum_replace4(&uh->check, skb, old_nsec,
330 					 new_nsec, false);
331 	}
332 
333 	*(__be16 *)(data + tstamp_off) = new_sec_h;
334 	*(__be32 *)(data + tstamp_off + 2) = new_sec_l;
335 	*(__be32 *)(data + tstamp_off + 6) = new_nsec;
336 
337 	/* Configure single-step register */
338 	if (is_enetc_rev1(si))
339 		enetc_set_one_step_ts(si, enetc_cb->udp, corr_off);
340 	else
341 		enetc4_set_one_step_ts(si, enetc_cb->udp, corr_off);
342 
343 	return lo & ENETC_TXBD_TSTAMP;
344 }
345 
346 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
347 {
348 	bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
349 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
350 	struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb);
351 	struct enetc_tx_swbd *tx_swbd;
352 	int len = skb_headlen(skb);
353 	union enetc_tx_bd temp_bd;
354 	bool csum_offload = false;
355 	union enetc_tx_bd *txbd;
356 	int i, count = 0;
357 	skb_frag_t *frag;
358 	unsigned int f;
359 	dma_addr_t dma;
360 	u8 flags = 0;
361 	u32 tstamp;
362 
363 	enetc_clear_tx_bd(&temp_bd);
364 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
365 		/* Can not support TSD and checksum offload at the same time */
366 		if (priv->active_offloads & ENETC_F_TXCSUM &&
367 		    enetc_tx_csum_offload_check(skb) && !tx_ring->tsd_enable) {
368 			temp_bd.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START,
369 						     skb_network_offset(skb));
370 			temp_bd.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN,
371 						     skb_network_header_len(skb) / 4);
372 			temp_bd.l3_aux1 |= FIELD_PREP(ENETC_TX_BD_L3T,
373 						      enetc_skb_is_ipv6(skb));
374 			if (enetc_skb_is_tcp(skb))
375 				temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T,
376 							    ENETC_TXBD_L4T_TCP);
377 			else
378 				temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T,
379 							    ENETC_TXBD_L4T_UDP);
380 			flags |= ENETC_TXBD_FLAGS_CSUM_LSO | ENETC_TXBD_FLAGS_L4CS;
381 			csum_offload = true;
382 		} else if (skb_checksum_help(skb)) {
383 			return 0;
384 		}
385 	}
386 
387 	if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
388 		do_onestep_tstamp = true;
389 		tstamp = enetc_update_ptp_sync_msg(priv, skb, csum_offload);
390 	} else if (enetc_cb->flag & ENETC_F_TX_TSTAMP) {
391 		do_twostep_tstamp = true;
392 	}
393 
394 	i = tx_ring->next_to_use;
395 	txbd = ENETC_TXBD(*tx_ring, i);
396 	prefetchw(txbd);
397 
398 	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
399 	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
400 		goto dma_err;
401 
402 	temp_bd.addr = cpu_to_le64(dma);
403 	temp_bd.buf_len = cpu_to_le16(len);
404 
405 	tx_swbd = &tx_ring->tx_swbd[i];
406 	tx_swbd->dma = dma;
407 	tx_swbd->len = len;
408 	tx_swbd->is_dma_page = 0;
409 	tx_swbd->dir = DMA_TO_DEVICE;
410 	count++;
411 
412 	do_vlan = skb_vlan_tag_present(skb);
413 	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
414 	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
415 	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
416 
417 	if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
418 		flags |= ENETC_TXBD_FLAGS_EX;
419 
420 	if (tx_ring->tsd_enable)
421 		flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
422 
423 	/* first BD needs frm_len and offload flags set */
424 	temp_bd.frm_len = cpu_to_le16(skb->len);
425 	temp_bd.flags = flags;
426 
427 	if (flags & ENETC_TXBD_FLAGS_TSE)
428 		temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
429 							  flags);
430 
431 	if (flags & ENETC_TXBD_FLAGS_EX) {
432 		u8 e_flags = 0;
433 		*txbd = temp_bd;
434 		enetc_clear_tx_bd(&temp_bd);
435 
436 		/* add extension BD for VLAN and/or timestamping */
437 		flags = 0;
438 		tx_swbd++;
439 		txbd++;
440 		i++;
441 		if (unlikely(i == tx_ring->bd_count)) {
442 			i = 0;
443 			tx_swbd = tx_ring->tx_swbd;
444 			txbd = ENETC_TXBD(*tx_ring, 0);
445 		}
446 		prefetchw(txbd);
447 
448 		if (do_vlan) {
449 			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
450 			temp_bd.ext.tpid = 0; /* < C-TAG */
451 			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
452 		}
453 
454 		if (do_onestep_tstamp) {
455 			/* Configure extension BD */
456 			temp_bd.ext.tstamp = cpu_to_le32(tstamp);
457 			e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
458 		} else if (do_twostep_tstamp) {
459 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
460 			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
461 		}
462 
463 		temp_bd.ext.e_flags = e_flags;
464 		count++;
465 	}
466 
467 	frag = &skb_shinfo(skb)->frags[0];
468 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
469 		len = skb_frag_size(frag);
470 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
471 				       DMA_TO_DEVICE);
472 		if (dma_mapping_error(tx_ring->dev, dma))
473 			goto dma_err;
474 
475 		*txbd = temp_bd;
476 		enetc_clear_tx_bd(&temp_bd);
477 
478 		flags = 0;
479 		tx_swbd++;
480 		txbd++;
481 		i++;
482 		if (unlikely(i == tx_ring->bd_count)) {
483 			i = 0;
484 			tx_swbd = tx_ring->tx_swbd;
485 			txbd = ENETC_TXBD(*tx_ring, 0);
486 		}
487 		prefetchw(txbd);
488 
489 		temp_bd.addr = cpu_to_le64(dma);
490 		temp_bd.buf_len = cpu_to_le16(len);
491 
492 		tx_swbd->dma = dma;
493 		tx_swbd->len = len;
494 		tx_swbd->is_dma_page = 1;
495 		tx_swbd->dir = DMA_TO_DEVICE;
496 		count++;
497 	}
498 
499 	/* last BD needs 'F' bit set */
500 	flags |= ENETC_TXBD_FLAGS_F;
501 	temp_bd.flags = flags;
502 	*txbd = temp_bd;
503 
504 	tx_ring->tx_swbd[i].is_eof = true;
505 	tx_ring->tx_swbd[i].skb = skb;
506 
507 	enetc_bdr_idx_inc(tx_ring, &i);
508 	tx_ring->next_to_use = i;
509 
510 	skb_tx_timestamp(skb);
511 
512 	enetc_update_tx_ring_tail(tx_ring);
513 
514 	return count;
515 
516 dma_err:
517 	dev_err(tx_ring->dev, "DMA map error");
518 
519 	enetc_unwind_tx_frame(tx_ring, count, i);
520 
521 	return 0;
522 }
523 
524 static int enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
525 				struct enetc_tx_swbd *tx_swbd,
526 				union enetc_tx_bd *txbd, int *i, int hdr_len,
527 				int data_len)
528 {
529 	union enetc_tx_bd txbd_tmp;
530 	u8 flags = 0, e_flags = 0;
531 	dma_addr_t addr;
532 	int count = 1;
533 
534 	enetc_clear_tx_bd(&txbd_tmp);
535 	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
536 
537 	if (skb_vlan_tag_present(skb))
538 		flags |= ENETC_TXBD_FLAGS_EX;
539 
540 	txbd_tmp.addr = cpu_to_le64(addr);
541 	txbd_tmp.buf_len = cpu_to_le16(hdr_len);
542 
543 	/* first BD needs frm_len and offload flags set */
544 	txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
545 	txbd_tmp.flags = flags;
546 
547 	/* For the TSO header we do not set the dma address since we do not
548 	 * want it unmapped when we do cleanup. We still set len so that we
549 	 * count the bytes sent.
550 	 */
551 	tx_swbd->len = hdr_len;
552 	tx_swbd->do_twostep_tstamp = false;
553 	tx_swbd->check_wb = false;
554 
555 	/* Actually write the header in the BD */
556 	*txbd = txbd_tmp;
557 
558 	/* Add extension BD for VLAN */
559 	if (flags & ENETC_TXBD_FLAGS_EX) {
560 		/* Get the next BD */
561 		enetc_bdr_idx_inc(tx_ring, i);
562 		txbd = ENETC_TXBD(*tx_ring, *i);
563 		tx_swbd = &tx_ring->tx_swbd[*i];
564 		prefetchw(txbd);
565 
566 		/* Setup the VLAN fields */
567 		enetc_clear_tx_bd(&txbd_tmp);
568 		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
569 		txbd_tmp.ext.tpid = 0; /* < C-TAG */
570 		e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
571 
572 		/* Write the BD */
573 		txbd_tmp.ext.e_flags = e_flags;
574 		*txbd = txbd_tmp;
575 		count++;
576 	}
577 
578 	return count;
579 }
580 
581 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
582 				 struct enetc_tx_swbd *tx_swbd,
583 				 union enetc_tx_bd *txbd, char *data,
584 				 int size, bool last_bd)
585 {
586 	union enetc_tx_bd txbd_tmp;
587 	dma_addr_t addr;
588 	u8 flags = 0;
589 
590 	enetc_clear_tx_bd(&txbd_tmp);
591 
592 	addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
593 	if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
594 		netdev_err(tx_ring->ndev, "DMA map error\n");
595 		return -ENOMEM;
596 	}
597 
598 	if (last_bd) {
599 		flags |= ENETC_TXBD_FLAGS_F;
600 		tx_swbd->is_eof = 1;
601 	}
602 
603 	txbd_tmp.addr = cpu_to_le64(addr);
604 	txbd_tmp.buf_len = cpu_to_le16(size);
605 	txbd_tmp.flags = flags;
606 
607 	tx_swbd->dma = addr;
608 	tx_swbd->len = size;
609 	tx_swbd->dir = DMA_TO_DEVICE;
610 
611 	*txbd = txbd_tmp;
612 
613 	return 0;
614 }
615 
616 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
617 				 char *hdr, int hdr_len, int *l4_hdr_len)
618 {
619 	char *l4_hdr = hdr + skb_transport_offset(skb);
620 	int mac_hdr_len = skb_network_offset(skb);
621 
622 	if (tso->tlen != sizeof(struct udphdr)) {
623 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
624 
625 		tcph->check = 0;
626 	} else {
627 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
628 
629 		udph->check = 0;
630 	}
631 
632 	/* Compute the IP checksum. This is necessary since tso_build_hdr()
633 	 * already incremented the IP ID field.
634 	 */
635 	if (!tso->ipv6) {
636 		struct iphdr *iph = (void *)(hdr + mac_hdr_len);
637 
638 		iph->check = 0;
639 		iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
640 	}
641 
642 	/* Compute the checksum over the L4 header. */
643 	*l4_hdr_len = hdr_len - skb_transport_offset(skb);
644 	return csum_partial(l4_hdr, *l4_hdr_len, 0);
645 }
646 
647 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
648 				    struct sk_buff *skb, char *hdr, int len,
649 				    __wsum sum)
650 {
651 	char *l4_hdr = hdr + skb_transport_offset(skb);
652 	__sum16 csum_final;
653 
654 	/* Complete the L4 checksum by appending the pseudo-header to the
655 	 * already computed checksum.
656 	 */
657 	if (!tso->ipv6)
658 		csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
659 					       ip_hdr(skb)->daddr,
660 					       len, ip_hdr(skb)->protocol, sum);
661 	else
662 		csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
663 					     &ipv6_hdr(skb)->daddr,
664 					     len, ipv6_hdr(skb)->nexthdr, sum);
665 
666 	if (tso->tlen != sizeof(struct udphdr)) {
667 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
668 
669 		tcph->check = csum_final;
670 	} else {
671 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
672 
673 		udph->check = csum_final;
674 	}
675 }
676 
677 static int enetc_lso_count_descs(const struct sk_buff *skb)
678 {
679 	/* 4 BDs: 1 BD for LSO header + 1 BD for extended BD + 1 BD
680 	 * for linear area data but not include LSO header, namely
681 	 * skb_headlen(skb) - lso_hdr_len (it may be 0, but that's
682 	 * okay, we only need to consider the worst case). And 1 BD
683 	 * for gap.
684 	 */
685 	return skb_shinfo(skb)->nr_frags + 4;
686 }
687 
688 static int enetc_lso_get_hdr_len(const struct sk_buff *skb)
689 {
690 	int hdr_len, tlen;
691 
692 	tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr);
693 	hdr_len = skb_transport_offset(skb) + tlen;
694 
695 	return hdr_len;
696 }
697 
698 static void enetc_lso_start(struct sk_buff *skb, struct enetc_lso_t *lso)
699 {
700 	lso->lso_seg_size = skb_shinfo(skb)->gso_size;
701 	lso->ipv6 = enetc_skb_is_ipv6(skb);
702 	lso->tcp = skb_is_gso_tcp(skb);
703 	lso->l3_hdr_len = skb_network_header_len(skb);
704 	lso->l3_start = skb_network_offset(skb);
705 	lso->hdr_len = enetc_lso_get_hdr_len(skb);
706 	lso->total_len = skb->len - lso->hdr_len;
707 }
708 
709 static void enetc_lso_map_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
710 			      int *i, struct enetc_lso_t *lso)
711 {
712 	union enetc_tx_bd txbd_tmp, *txbd;
713 	struct enetc_tx_swbd *tx_swbd;
714 	u16 frm_len, frm_len_ext;
715 	u8 flags, e_flags = 0;
716 	dma_addr_t addr;
717 	char *hdr;
718 
719 	/* Get the first BD of the LSO BDs chain */
720 	txbd = ENETC_TXBD(*tx_ring, *i);
721 	tx_swbd = &tx_ring->tx_swbd[*i];
722 	prefetchw(txbd);
723 
724 	/* Prepare LSO header: MAC + IP + TCP/UDP */
725 	hdr = tx_ring->tso_headers + *i * TSO_HEADER_SIZE;
726 	memcpy(hdr, skb->data, lso->hdr_len);
727 	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
728 
729 	/* {frm_len_ext, frm_len} indicates the total length of
730 	 * large transmit data unit. frm_len contains the 16 least
731 	 * significant bits and frm_len_ext contains the 4 most
732 	 * significant bits.
733 	 */
734 	frm_len = lso->total_len & 0xffff;
735 	frm_len_ext = (lso->total_len >> 16) & 0xf;
736 
737 	/* Set the flags of the first BD */
738 	flags = ENETC_TXBD_FLAGS_EX | ENETC_TXBD_FLAGS_CSUM_LSO |
739 		ENETC_TXBD_FLAGS_LSO | ENETC_TXBD_FLAGS_L4CS;
740 
741 	enetc_clear_tx_bd(&txbd_tmp);
742 	txbd_tmp.addr = cpu_to_le64(addr);
743 	txbd_tmp.hdr_len = cpu_to_le16(lso->hdr_len);
744 
745 	/* first BD needs frm_len and offload flags set */
746 	txbd_tmp.frm_len = cpu_to_le16(frm_len);
747 	txbd_tmp.flags = flags;
748 
749 	txbd_tmp.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, lso->l3_start);
750 	/* l3_hdr_size in 32-bits (4 bytes) */
751 	txbd_tmp.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN,
752 				      lso->l3_hdr_len / 4);
753 	if (lso->ipv6)
754 		txbd_tmp.l3_aux1 |= ENETC_TX_BD_L3T;
755 	else
756 		txbd_tmp.l3_aux0 |= ENETC_TX_BD_IPCS;
757 
758 	txbd_tmp.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, lso->tcp ?
759 				     ENETC_TXBD_L4T_TCP : ENETC_TXBD_L4T_UDP);
760 
761 	/* For the LSO header we do not set the dma address since
762 	 * we do not want it unmapped when we do cleanup. We still
763 	 * set len so that we count the bytes sent.
764 	 */
765 	tx_swbd->len = lso->hdr_len;
766 	tx_swbd->do_twostep_tstamp = false;
767 	tx_swbd->check_wb = false;
768 
769 	/* Actually write the header in the BD */
770 	*txbd = txbd_tmp;
771 
772 	/* Get the next BD, and the next BD is extended BD */
773 	enetc_bdr_idx_inc(tx_ring, i);
774 	txbd = ENETC_TXBD(*tx_ring, *i);
775 	tx_swbd = &tx_ring->tx_swbd[*i];
776 	prefetchw(txbd);
777 
778 	enetc_clear_tx_bd(&txbd_tmp);
779 	if (skb_vlan_tag_present(skb)) {
780 		/* Setup the VLAN fields */
781 		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
782 		txbd_tmp.ext.tpid = ENETC_TPID_8021Q;
783 		e_flags = ENETC_TXBD_E_FLAGS_VLAN_INS;
784 	}
785 
786 	/* Write the BD */
787 	txbd_tmp.ext.e_flags = e_flags;
788 	txbd_tmp.ext.lso_sg_size = cpu_to_le16(lso->lso_seg_size);
789 	txbd_tmp.ext.frm_len_ext = cpu_to_le16(frm_len_ext);
790 	*txbd = txbd_tmp;
791 }
792 
793 static int enetc_lso_map_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
794 			      int *i, struct enetc_lso_t *lso, int *count)
795 {
796 	union enetc_tx_bd txbd_tmp, *txbd = NULL;
797 	struct enetc_tx_swbd *tx_swbd;
798 	skb_frag_t *frag;
799 	dma_addr_t dma;
800 	u8 flags = 0;
801 	int len, f;
802 
803 	len = skb_headlen(skb) - lso->hdr_len;
804 	if (len > 0) {
805 		dma = dma_map_single(tx_ring->dev, skb->data + lso->hdr_len,
806 				     len, DMA_TO_DEVICE);
807 		if (dma_mapping_error(tx_ring->dev, dma))
808 			return -ENOMEM;
809 
810 		enetc_bdr_idx_inc(tx_ring, i);
811 		txbd = ENETC_TXBD(*tx_ring, *i);
812 		tx_swbd = &tx_ring->tx_swbd[*i];
813 		prefetchw(txbd);
814 		*count += 1;
815 
816 		enetc_clear_tx_bd(&txbd_tmp);
817 		txbd_tmp.addr = cpu_to_le64(dma);
818 		txbd_tmp.buf_len = cpu_to_le16(len);
819 
820 		tx_swbd->dma = dma;
821 		tx_swbd->len = len;
822 		tx_swbd->is_dma_page = 0;
823 		tx_swbd->dir = DMA_TO_DEVICE;
824 	}
825 
826 	frag = &skb_shinfo(skb)->frags[0];
827 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
828 		if (txbd)
829 			*txbd = txbd_tmp;
830 
831 		len = skb_frag_size(frag);
832 		dma = skb_frag_dma_map(tx_ring->dev, frag);
833 		if (dma_mapping_error(tx_ring->dev, dma))
834 			return -ENOMEM;
835 
836 		/* Get the next BD */
837 		enetc_bdr_idx_inc(tx_ring, i);
838 		txbd = ENETC_TXBD(*tx_ring, *i);
839 		tx_swbd = &tx_ring->tx_swbd[*i];
840 		prefetchw(txbd);
841 		*count += 1;
842 
843 		enetc_clear_tx_bd(&txbd_tmp);
844 		txbd_tmp.addr = cpu_to_le64(dma);
845 		txbd_tmp.buf_len = cpu_to_le16(len);
846 
847 		tx_swbd->dma = dma;
848 		tx_swbd->len = len;
849 		tx_swbd->is_dma_page = 1;
850 		tx_swbd->dir = DMA_TO_DEVICE;
851 	}
852 
853 	/* Last BD needs 'F' bit set */
854 	flags |= ENETC_TXBD_FLAGS_F;
855 	txbd_tmp.flags = flags;
856 	*txbd = txbd_tmp;
857 
858 	tx_swbd->is_eof = 1;
859 	tx_swbd->skb = skb;
860 
861 	return 0;
862 }
863 
864 static int enetc_lso_hw_offload(struct enetc_bdr *tx_ring, struct sk_buff *skb)
865 {
866 	struct enetc_tx_swbd *tx_swbd;
867 	struct enetc_lso_t lso = {0};
868 	int err, i, count = 0;
869 
870 	/* Initialize the LSO handler */
871 	enetc_lso_start(skb, &lso);
872 	i = tx_ring->next_to_use;
873 
874 	enetc_lso_map_hdr(tx_ring, skb, &i, &lso);
875 	/* First BD and an extend BD */
876 	count += 2;
877 
878 	err = enetc_lso_map_data(tx_ring, skb, &i, &lso, &count);
879 	if (err)
880 		goto dma_err;
881 
882 	/* Go to the next BD */
883 	enetc_bdr_idx_inc(tx_ring, &i);
884 	tx_ring->next_to_use = i;
885 	enetc_update_tx_ring_tail(tx_ring);
886 
887 	return count;
888 
889 dma_err:
890 	do {
891 		tx_swbd = &tx_ring->tx_swbd[i];
892 		enetc_free_tx_frame(tx_ring, tx_swbd);
893 		if (i == 0)
894 			i = tx_ring->bd_count;
895 		i--;
896 	} while (--count);
897 
898 	return 0;
899 }
900 
901 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
902 {
903 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
904 	int hdr_len, total_len, data_len;
905 	struct enetc_tx_swbd *tx_swbd;
906 	union enetc_tx_bd *txbd;
907 	struct tso_t tso;
908 	__wsum csum, csum2;
909 	int count = 0, pos;
910 	int err, i, bd_data_num;
911 
912 	/* Initialize the TSO handler, and prepare the first payload */
913 	hdr_len = tso_start(skb, &tso);
914 	total_len = skb->len - hdr_len;
915 	i = tx_ring->next_to_use;
916 
917 	while (total_len > 0) {
918 		char *hdr;
919 
920 		/* Get the BD */
921 		txbd = ENETC_TXBD(*tx_ring, i);
922 		tx_swbd = &tx_ring->tx_swbd[i];
923 		prefetchw(txbd);
924 
925 		/* Determine the length of this packet */
926 		data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
927 		total_len -= data_len;
928 
929 		/* prepare packet headers: MAC + IP + TCP */
930 		hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
931 		tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
932 
933 		/* compute the csum over the L4 header */
934 		csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
935 		count += enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd,
936 					      &i, hdr_len, data_len);
937 		bd_data_num = 0;
938 
939 		while (data_len > 0) {
940 			int size;
941 
942 			size = min_t(int, tso.size, data_len);
943 
944 			/* Advance the index in the BDR */
945 			enetc_bdr_idx_inc(tx_ring, &i);
946 			txbd = ENETC_TXBD(*tx_ring, i);
947 			tx_swbd = &tx_ring->tx_swbd[i];
948 			prefetchw(txbd);
949 
950 			/* Compute the checksum over this segment of data and
951 			 * add it to the csum already computed (over the L4
952 			 * header and possible other data segments).
953 			 */
954 			csum2 = csum_partial(tso.data, size, 0);
955 			csum = csum_block_add(csum, csum2, pos);
956 			pos += size;
957 
958 			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
959 						    tso.data, size,
960 						    size == data_len);
961 			if (err) {
962 				if (i == 0)
963 					i = tx_ring->bd_count;
964 				i--;
965 
966 				goto err_map_data;
967 			}
968 
969 			data_len -= size;
970 			count++;
971 			bd_data_num++;
972 			tso_build_data(skb, &tso, size);
973 
974 			if (unlikely(bd_data_num >= priv->max_frags && data_len))
975 				goto err_chained_bd;
976 		}
977 
978 		enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
979 
980 		if (total_len == 0)
981 			tx_swbd->skb = skb;
982 
983 		/* Go to the next BD */
984 		enetc_bdr_idx_inc(tx_ring, &i);
985 	}
986 
987 	tx_ring->next_to_use = i;
988 	enetc_update_tx_ring_tail(tx_ring);
989 
990 	return count;
991 
992 err_map_data:
993 	dev_err(tx_ring->dev, "DMA map error");
994 
995 err_chained_bd:
996 	enetc_unwind_tx_frame(tx_ring, count, i);
997 
998 	return 0;
999 }
1000 
1001 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
1002 				    struct net_device *ndev)
1003 {
1004 	struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb);
1005 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1006 	struct enetc_bdr *tx_ring;
1007 	int count;
1008 
1009 	/* Queue one-step Sync packet if already locked */
1010 	if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
1011 		if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
1012 					  &priv->flags)) {
1013 			skb_queue_tail(&priv->tx_skbs, skb);
1014 			return NETDEV_TX_OK;
1015 		}
1016 	}
1017 
1018 	tx_ring = priv->tx_ring[skb->queue_mapping];
1019 
1020 	if (skb_is_gso(skb)) {
1021 		/* LSO data unit lengths of up to 256KB are supported */
1022 		if (priv->active_offloads & ENETC_F_LSO &&
1023 		    (skb->len - enetc_lso_get_hdr_len(skb)) <=
1024 		    ENETC_LSO_MAX_DATA_LEN) {
1025 			if (enetc_bd_unused(tx_ring) < enetc_lso_count_descs(skb)) {
1026 				netif_stop_subqueue(ndev, tx_ring->index);
1027 				return NETDEV_TX_BUSY;
1028 			}
1029 
1030 			count = enetc_lso_hw_offload(tx_ring, skb);
1031 		} else {
1032 			if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
1033 				netif_stop_subqueue(ndev, tx_ring->index);
1034 				return NETDEV_TX_BUSY;
1035 			}
1036 
1037 			enetc_lock_mdio();
1038 			count = enetc_map_tx_tso_buffs(tx_ring, skb);
1039 			enetc_unlock_mdio();
1040 		}
1041 	} else {
1042 		if (unlikely(skb_shinfo(skb)->nr_frags > priv->max_frags))
1043 			if (unlikely(skb_linearize(skb)))
1044 				goto drop_packet_err;
1045 
1046 		count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
1047 		if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
1048 			netif_stop_subqueue(ndev, tx_ring->index);
1049 			return NETDEV_TX_BUSY;
1050 		}
1051 
1052 		enetc_lock_mdio();
1053 		count = enetc_map_tx_buffs(tx_ring, skb);
1054 		enetc_unlock_mdio();
1055 	}
1056 
1057 	if (unlikely(!count))
1058 		goto drop_packet_err;
1059 
1060 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED(priv->max_frags))
1061 		netif_stop_subqueue(ndev, tx_ring->index);
1062 
1063 	return NETDEV_TX_OK;
1064 
1065 drop_packet_err:
1066 	dev_kfree_skb_any(skb);
1067 	return NETDEV_TX_OK;
1068 }
1069 
1070 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
1071 {
1072 	struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb);
1073 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1074 	u8 udp, msgtype, twostep;
1075 	u16 offset1, offset2;
1076 
1077 	/* Mark tx timestamp type on enetc_cb->flag if requires */
1078 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
1079 	    (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK))
1080 		enetc_cb->flag = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
1081 	else
1082 		enetc_cb->flag = 0;
1083 
1084 	/* Fall back to two-step timestamp if not one-step Sync packet */
1085 	if (enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
1086 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
1087 				    &offset1, &offset2) ||
1088 		    msgtype != PTP_MSGTYPE_SYNC || twostep != 0) {
1089 			enetc_cb->flag = ENETC_F_TX_TSTAMP;
1090 		} else {
1091 			enetc_cb->udp = !!udp;
1092 			enetc_cb->correction_off = offset1;
1093 			enetc_cb->origin_tstamp_off = offset2;
1094 		}
1095 	}
1096 
1097 	return enetc_start_xmit(skb, ndev);
1098 }
1099 EXPORT_SYMBOL_GPL(enetc_xmit);
1100 
1101 static irqreturn_t enetc_msix(int irq, void *data)
1102 {
1103 	struct enetc_int_vector	*v = data;
1104 	int i;
1105 
1106 	enetc_lock_mdio();
1107 
1108 	/* disable interrupts */
1109 	enetc_wr_reg_hot(v->rbier, 0);
1110 	enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
1111 
1112 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1113 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
1114 
1115 	enetc_unlock_mdio();
1116 
1117 	napi_schedule(&v->napi);
1118 
1119 	return IRQ_HANDLED;
1120 }
1121 
1122 static void enetc_rx_dim_work(struct work_struct *w)
1123 {
1124 	struct dim *dim = container_of(w, struct dim, work);
1125 	struct dim_cq_moder moder =
1126 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1127 	struct enetc_int_vector	*v =
1128 		container_of(dim, struct enetc_int_vector, rx_dim);
1129 	struct enetc_ndev_priv *priv = netdev_priv(v->rx_ring.ndev);
1130 
1131 	v->rx_ictt = enetc_usecs_to_cycles(moder.usec, priv->sysclk_freq);
1132 	dim->state = DIM_START_MEASURE;
1133 }
1134 
1135 static void enetc_rx_net_dim(struct enetc_int_vector *v)
1136 {
1137 	struct dim_sample dim_sample = {};
1138 
1139 	v->comp_cnt++;
1140 
1141 	if (!v->rx_napi_work)
1142 		return;
1143 
1144 	dim_update_sample(v->comp_cnt,
1145 			  v->rx_ring.stats.packets,
1146 			  v->rx_ring.stats.bytes,
1147 			  &dim_sample);
1148 	net_dim(&v->rx_dim, &dim_sample);
1149 }
1150 
1151 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
1152 {
1153 	int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
1154 
1155 	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
1156 }
1157 
1158 static bool enetc_page_reusable(struct page *page)
1159 {
1160 	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
1161 }
1162 
1163 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
1164 			     struct enetc_rx_swbd *old)
1165 {
1166 	struct enetc_rx_swbd *new;
1167 
1168 	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
1169 
1170 	/* next buf that may reuse a page */
1171 	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
1172 
1173 	/* copy page reference */
1174 	*new = *old;
1175 }
1176 
1177 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
1178 				u64 *tstamp)
1179 {
1180 	u32 lo, hi, tstamp_lo;
1181 
1182 	lo = enetc_rd_hot(hw, ENETC_SICTR0);
1183 	hi = enetc_rd_hot(hw, ENETC_SICTR1);
1184 	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
1185 	if (lo <= tstamp_lo)
1186 		hi -= 1;
1187 	*tstamp = (u64)hi << 32 | tstamp_lo;
1188 }
1189 
1190 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
1191 {
1192 	struct skb_shared_hwtstamps shhwtstamps;
1193 
1194 	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
1195 		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
1196 		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
1197 		skb_txtime_consumed(skb);
1198 		skb_tstamp_tx(skb, &shhwtstamps);
1199 	}
1200 }
1201 
1202 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
1203 				      struct enetc_tx_swbd *tx_swbd)
1204 {
1205 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
1206 	struct enetc_rx_swbd rx_swbd = {
1207 		.dma = tx_swbd->dma,
1208 		.page = tx_swbd->page,
1209 		.page_offset = tx_swbd->page_offset,
1210 		.dir = tx_swbd->dir,
1211 		.len = tx_swbd->len,
1212 	};
1213 	struct enetc_bdr *rx_ring;
1214 
1215 	rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
1216 
1217 	if (likely(enetc_swbd_unused(rx_ring))) {
1218 		enetc_reuse_page(rx_ring, &rx_swbd);
1219 
1220 		/* sync for use by the device */
1221 		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
1222 						 rx_swbd.page_offset,
1223 						 ENETC_RXB_DMA_SIZE_XDP,
1224 						 rx_swbd.dir);
1225 
1226 		rx_ring->stats.recycles++;
1227 	} else {
1228 		/* RX ring is already full, we need to unmap and free the
1229 		 * page, since there's nothing useful we can do with it.
1230 		 */
1231 		rx_ring->stats.recycle_failures++;
1232 
1233 		dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
1234 			       rx_swbd.dir);
1235 		__free_page(rx_swbd.page);
1236 	}
1237 
1238 	rx_ring->xdp.xdp_tx_in_flight--;
1239 }
1240 
1241 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
1242 {
1243 	int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
1244 	struct net_device *ndev = tx_ring->ndev;
1245 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1246 	struct enetc_tx_swbd *tx_swbd;
1247 	int i, bds_to_clean;
1248 	bool do_twostep_tstamp;
1249 	u64 tstamp = 0;
1250 
1251 	i = tx_ring->next_to_clean;
1252 	tx_swbd = &tx_ring->tx_swbd[i];
1253 
1254 	bds_to_clean = enetc_bd_ready_count(tx_ring, i);
1255 
1256 	do_twostep_tstamp = false;
1257 
1258 	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
1259 		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
1260 		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
1261 		bool is_eof = tx_swbd->is_eof;
1262 
1263 		if (unlikely(tx_swbd->check_wb)) {
1264 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1265 
1266 			if (txbd->flags & ENETC_TXBD_FLAGS_W &&
1267 			    tx_swbd->do_twostep_tstamp) {
1268 				enetc_get_tx_tstamp(&priv->si->hw, txbd,
1269 						    &tstamp);
1270 				do_twostep_tstamp = true;
1271 			}
1272 
1273 			if (tx_swbd->qbv_en &&
1274 			    txbd->wb.status & ENETC_TXBD_STATS_WIN)
1275 				tx_win_drop++;
1276 		}
1277 
1278 		if (tx_swbd->is_xdp_tx)
1279 			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
1280 		else if (likely(tx_swbd->dma))
1281 			enetc_unmap_tx_buff(tx_ring, tx_swbd);
1282 
1283 		if (xdp_frame) {
1284 			xdp_return_frame(xdp_frame);
1285 		} else if (skb) {
1286 			struct enetc_skb_cb *enetc_cb = ENETC_SKB_CB(skb);
1287 
1288 			if (unlikely(enetc_cb->flag & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
1289 				/* Start work to release lock for next one-step
1290 				 * timestamping packet. And send one skb in
1291 				 * tx_skbs queue if has.
1292 				 */
1293 				schedule_work(&priv->tx_onestep_tstamp);
1294 			} else if (unlikely(do_twostep_tstamp)) {
1295 				enetc_tstamp_tx(skb, tstamp);
1296 				do_twostep_tstamp = false;
1297 			}
1298 			napi_consume_skb(skb, napi_budget);
1299 		}
1300 
1301 		tx_byte_cnt += tx_swbd->len;
1302 		/* Scrub the swbd here so we don't have to do that
1303 		 * when we reuse it during xmit
1304 		 */
1305 		memset(tx_swbd, 0, sizeof(*tx_swbd));
1306 
1307 		bds_to_clean--;
1308 		tx_swbd++;
1309 		i++;
1310 		if (unlikely(i == tx_ring->bd_count)) {
1311 			i = 0;
1312 			tx_swbd = tx_ring->tx_swbd;
1313 		}
1314 
1315 		/* BD iteration loop end */
1316 		if (is_eof) {
1317 			tx_frm_cnt++;
1318 			/* re-arm interrupt source */
1319 			enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
1320 					 BIT(16 + tx_ring->index));
1321 		}
1322 
1323 		if (unlikely(!bds_to_clean))
1324 			bds_to_clean = enetc_bd_ready_count(tx_ring, i);
1325 	}
1326 
1327 	tx_ring->next_to_clean = i;
1328 	tx_ring->stats.packets += tx_frm_cnt;
1329 	tx_ring->stats.bytes += tx_byte_cnt;
1330 	tx_ring->stats.win_drop += tx_win_drop;
1331 
1332 	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
1333 		     __netif_subqueue_stopped(ndev, tx_ring->index) &&
1334 		     !test_bit(ENETC_TX_DOWN, &priv->flags) &&
1335 		     (enetc_bd_unused(tx_ring) >=
1336 		      ENETC_TXBDS_MAX_NEEDED(priv->max_frags)))) {
1337 		netif_wake_subqueue(ndev, tx_ring->index);
1338 	}
1339 
1340 	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
1341 }
1342 
1343 static bool enetc_new_page(struct enetc_bdr *rx_ring,
1344 			   struct enetc_rx_swbd *rx_swbd)
1345 {
1346 	bool xdp = !!(rx_ring->xdp.prog);
1347 	struct page *page;
1348 	dma_addr_t addr;
1349 
1350 	page = dev_alloc_page();
1351 	if (unlikely(!page))
1352 		return false;
1353 
1354 	/* For XDP_TX, we forgo dma_unmap -> dma_map */
1355 	rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
1356 
1357 	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
1358 	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
1359 		__free_page(page);
1360 
1361 		return false;
1362 	}
1363 
1364 	rx_swbd->dma = addr;
1365 	rx_swbd->page = page;
1366 	rx_swbd->page_offset = rx_ring->buffer_offset;
1367 
1368 	return true;
1369 }
1370 
1371 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
1372 {
1373 	struct enetc_rx_swbd *rx_swbd;
1374 	union enetc_rx_bd *rxbd;
1375 	int i, j;
1376 
1377 	i = rx_ring->next_to_use;
1378 	rx_swbd = &rx_ring->rx_swbd[i];
1379 	rxbd = enetc_rxbd(rx_ring, i);
1380 
1381 	for (j = 0; j < buff_cnt; j++) {
1382 		/* try reuse page */
1383 		if (unlikely(!rx_swbd->page)) {
1384 			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
1385 				rx_ring->stats.rx_alloc_errs++;
1386 				break;
1387 			}
1388 		}
1389 
1390 		/* update RxBD */
1391 		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
1392 					   rx_swbd->page_offset);
1393 		/* clear 'R" as well */
1394 		rxbd->r.lstatus = 0;
1395 
1396 		enetc_rxbd_next(rx_ring, &rxbd, &i);
1397 		rx_swbd = &rx_ring->rx_swbd[i];
1398 	}
1399 
1400 	if (likely(j)) {
1401 		rx_ring->next_to_alloc = i; /* keep track from page reuse */
1402 		rx_ring->next_to_use = i;
1403 
1404 		/* update ENETC's consumer index */
1405 		enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
1406 	}
1407 
1408 	return j;
1409 }
1410 
1411 static void enetc_get_rx_tstamp(struct net_device *ndev,
1412 				union enetc_rx_bd *rxbd,
1413 				struct sk_buff *skb)
1414 {
1415 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
1416 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1417 	struct enetc_hw *hw = &priv->si->hw;
1418 	u32 lo, hi, tstamp_lo;
1419 	u64 tstamp;
1420 
1421 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
1422 		lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
1423 		hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
1424 		rxbd = enetc_rxbd_ext(rxbd);
1425 		tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
1426 		if (lo <= tstamp_lo)
1427 			hi -= 1;
1428 
1429 		tstamp = (u64)hi << 32 | tstamp_lo;
1430 		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
1431 		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
1432 	}
1433 }
1434 
1435 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
1436 			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
1437 {
1438 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1439 
1440 	/* TODO: hashing */
1441 	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
1442 		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
1443 
1444 		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
1445 		skb->ip_summed = CHECKSUM_COMPLETE;
1446 	}
1447 
1448 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1449 		struct enetc_hw *hw = &priv->si->hw;
1450 		__be16 tpid = 0;
1451 
1452 		switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1453 		case 0:
1454 			tpid = htons(ETH_P_8021Q);
1455 			break;
1456 		case 1:
1457 			tpid = htons(ETH_P_8021AD);
1458 			break;
1459 		case 2:
1460 			tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR1) &
1461 				     SICVLANR_ETYPE);
1462 			break;
1463 		case 3:
1464 			tpid = htons(enetc_rd_hot(hw, ENETC_SICVLANR2) &
1465 				     SICVLANR_ETYPE);
1466 		}
1467 
1468 		__vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1469 	}
1470 
1471 	if (priv->active_offloads & ENETC_F_RX_TSTAMP)
1472 		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1473 }
1474 
1475 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1476  * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1477  * mapped buffers.
1478  */
1479 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1480 					       int i, u16 size)
1481 {
1482 	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1483 
1484 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1485 				      rx_swbd->page_offset,
1486 				      size, rx_swbd->dir);
1487 	return rx_swbd;
1488 }
1489 
1490 /* Reuse the current page without performing half-page buffer flipping */
1491 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1492 			      struct enetc_rx_swbd *rx_swbd)
1493 {
1494 	size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1495 
1496 	enetc_reuse_page(rx_ring, rx_swbd);
1497 
1498 	dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1499 					 rx_swbd->page_offset,
1500 					 buffer_size, rx_swbd->dir);
1501 
1502 	rx_swbd->page = NULL;
1503 }
1504 
1505 /* Reuse the current page by performing half-page buffer flipping */
1506 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1507 			       struct enetc_rx_swbd *rx_swbd)
1508 {
1509 	if (likely(enetc_page_reusable(rx_swbd->page))) {
1510 		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1511 		page_ref_inc(rx_swbd->page);
1512 
1513 		enetc_put_rx_buff(rx_ring, rx_swbd);
1514 	} else {
1515 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1516 			       rx_swbd->dir);
1517 		rx_swbd->page = NULL;
1518 	}
1519 }
1520 
1521 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1522 						int i, u16 size)
1523 {
1524 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1525 	struct sk_buff *skb;
1526 	void *ba;
1527 
1528 	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1529 	skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1530 	if (unlikely(!skb)) {
1531 		rx_ring->stats.rx_alloc_errs++;
1532 		return NULL;
1533 	}
1534 
1535 	skb_reserve(skb, rx_ring->buffer_offset);
1536 	__skb_put(skb, size);
1537 
1538 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1539 
1540 	return skb;
1541 }
1542 
1543 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1544 				     u16 size, struct sk_buff *skb)
1545 {
1546 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1547 
1548 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1549 			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1550 
1551 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1552 }
1553 
1554 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1555 					      u32 bd_status,
1556 					      union enetc_rx_bd **rxbd, int *i)
1557 {
1558 	if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1559 		return false;
1560 
1561 	enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1562 	enetc_rxbd_next(rx_ring, rxbd, i);
1563 
1564 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1565 		dma_rmb();
1566 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1567 
1568 		enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1569 		enetc_rxbd_next(rx_ring, rxbd, i);
1570 	}
1571 
1572 	rx_ring->ndev->stats.rx_dropped++;
1573 	rx_ring->ndev->stats.rx_errors++;
1574 
1575 	return true;
1576 }
1577 
1578 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1579 				       u32 bd_status, union enetc_rx_bd **rxbd,
1580 				       int *i, int *cleaned_cnt, int buffer_size)
1581 {
1582 	struct sk_buff *skb;
1583 	u16 size;
1584 
1585 	size = le16_to_cpu((*rxbd)->r.buf_len);
1586 	skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1587 	if (!skb)
1588 		return NULL;
1589 
1590 	enetc_get_offloads(rx_ring, *rxbd, skb);
1591 
1592 	(*cleaned_cnt)++;
1593 
1594 	enetc_rxbd_next(rx_ring, rxbd, i);
1595 
1596 	/* not last BD in frame? */
1597 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1598 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1599 		size = buffer_size;
1600 
1601 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1602 			dma_rmb();
1603 			size = le16_to_cpu((*rxbd)->r.buf_len);
1604 		}
1605 
1606 		enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1607 
1608 		(*cleaned_cnt)++;
1609 
1610 		enetc_rxbd_next(rx_ring, rxbd, i);
1611 	}
1612 
1613 	skb_record_rx_queue(skb, rx_ring->index);
1614 	skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1615 
1616 	return skb;
1617 }
1618 
1619 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1620 
1621 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1622 			       struct napi_struct *napi, int work_limit)
1623 {
1624 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1625 	int cleaned_cnt, i;
1626 
1627 	cleaned_cnt = enetc_bd_unused(rx_ring);
1628 	/* next descriptor to process */
1629 	i = rx_ring->next_to_clean;
1630 
1631 	enetc_lock_mdio();
1632 
1633 	while (likely(rx_frm_cnt < work_limit)) {
1634 		union enetc_rx_bd *rxbd;
1635 		struct sk_buff *skb;
1636 		u32 bd_status;
1637 
1638 		if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1639 			cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1640 							    cleaned_cnt);
1641 
1642 		rxbd = enetc_rxbd(rx_ring, i);
1643 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1644 		if (!bd_status)
1645 			break;
1646 
1647 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1648 		dma_rmb(); /* for reading other rxbd fields */
1649 
1650 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1651 						      &rxbd, &i))
1652 			break;
1653 
1654 		skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1655 				      &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1656 		if (!skb)
1657 			break;
1658 
1659 		/* When set, the outer VLAN header is extracted and reported
1660 		 * in the receive buffer descriptor. So rx_byte_cnt should
1661 		 * add the length of the extracted VLAN header.
1662 		 */
1663 		if (bd_status & ENETC_RXBD_FLAG_VLAN)
1664 			rx_byte_cnt += VLAN_HLEN;
1665 		rx_byte_cnt += skb->len + ETH_HLEN;
1666 		rx_frm_cnt++;
1667 
1668 		enetc_unlock_mdio();
1669 		napi_gro_receive(napi, skb);
1670 		enetc_lock_mdio();
1671 	}
1672 
1673 	rx_ring->next_to_clean = i;
1674 
1675 	rx_ring->stats.packets += rx_frm_cnt;
1676 	rx_ring->stats.bytes += rx_byte_cnt;
1677 
1678 	enetc_unlock_mdio();
1679 
1680 	return rx_frm_cnt;
1681 }
1682 
1683 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1684 				  struct enetc_tx_swbd *tx_swbd,
1685 				  int frm_len)
1686 {
1687 	union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1688 
1689 	prefetchw(txbd);
1690 
1691 	enetc_clear_tx_bd(txbd);
1692 	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1693 	txbd->buf_len = cpu_to_le16(tx_swbd->len);
1694 	txbd->frm_len = cpu_to_le16(frm_len);
1695 
1696 	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1697 }
1698 
1699 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1700  * descriptors.
1701  */
1702 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1703 			 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1704 {
1705 	struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1706 	int i, k, frm_len = tmp_tx_swbd->len;
1707 
1708 	if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1709 		return false;
1710 
1711 	while (unlikely(!tmp_tx_swbd->is_eof)) {
1712 		tmp_tx_swbd++;
1713 		frm_len += tmp_tx_swbd->len;
1714 	}
1715 
1716 	i = tx_ring->next_to_use;
1717 
1718 	for (k = 0; k < num_tx_swbd; k++) {
1719 		struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1720 
1721 		enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1722 
1723 		/* last BD needs 'F' bit set */
1724 		if (xdp_tx_swbd->is_eof) {
1725 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1726 
1727 			txbd->flags = ENETC_TXBD_FLAGS_F;
1728 		}
1729 
1730 		enetc_bdr_idx_inc(tx_ring, &i);
1731 	}
1732 
1733 	tx_ring->next_to_use = i;
1734 
1735 	return true;
1736 }
1737 
1738 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1739 					  struct enetc_tx_swbd *xdp_tx_arr,
1740 					  struct xdp_frame *xdp_frame)
1741 {
1742 	struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1743 	struct skb_shared_info *shinfo;
1744 	void *data = xdp_frame->data;
1745 	int len = xdp_frame->len;
1746 	skb_frag_t *frag;
1747 	dma_addr_t dma;
1748 	unsigned int f;
1749 	int n = 0;
1750 
1751 	dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1752 	if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1753 		netdev_err(tx_ring->ndev, "DMA map error\n");
1754 		return -1;
1755 	}
1756 
1757 	xdp_tx_swbd->dma = dma;
1758 	xdp_tx_swbd->dir = DMA_TO_DEVICE;
1759 	xdp_tx_swbd->len = len;
1760 	xdp_tx_swbd->is_xdp_redirect = true;
1761 	xdp_tx_swbd->is_eof = false;
1762 	xdp_tx_swbd->xdp_frame = NULL;
1763 
1764 	n++;
1765 
1766 	if (!xdp_frame_has_frags(xdp_frame))
1767 		goto out;
1768 
1769 	xdp_tx_swbd = &xdp_tx_arr[n];
1770 
1771 	shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1772 
1773 	for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1774 	     f++, frag++) {
1775 		data = skb_frag_address(frag);
1776 		len = skb_frag_size(frag);
1777 
1778 		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1779 		if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1780 			/* Undo the DMA mapping for all fragments */
1781 			while (--n >= 0)
1782 				enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1783 
1784 			netdev_err(tx_ring->ndev, "DMA map error\n");
1785 			return -1;
1786 		}
1787 
1788 		xdp_tx_swbd->dma = dma;
1789 		xdp_tx_swbd->dir = DMA_TO_DEVICE;
1790 		xdp_tx_swbd->len = len;
1791 		xdp_tx_swbd->is_xdp_redirect = true;
1792 		xdp_tx_swbd->is_eof = false;
1793 		xdp_tx_swbd->xdp_frame = NULL;
1794 
1795 		n++;
1796 		xdp_tx_swbd = &xdp_tx_arr[n];
1797 	}
1798 out:
1799 	xdp_tx_arr[n - 1].is_eof = true;
1800 	xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1801 
1802 	return n;
1803 }
1804 
1805 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1806 		   struct xdp_frame **frames, u32 flags)
1807 {
1808 	struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1809 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1810 	struct skb_shared_info *shinfo;
1811 	struct enetc_bdr *tx_ring;
1812 	int xdp_tx_bd_cnt, i, k;
1813 	int xdp_tx_frm_cnt = 0;
1814 
1815 	if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags) ||
1816 		     !netif_carrier_ok(ndev)))
1817 		return -ENETDOWN;
1818 
1819 	enetc_lock_mdio();
1820 
1821 	tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1822 
1823 	prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1824 
1825 	for (k = 0; k < num_frames; k++) {
1826 		if (xdp_frame_has_frags(frames[k])) {
1827 			shinfo = xdp_get_shared_info_from_frame(frames[k]);
1828 			if (unlikely((shinfo->nr_frags + 1) > ENETC_MAX_SKB_FRAGS))
1829 				break;
1830 		}
1831 
1832 		xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1833 							       xdp_redirect_arr,
1834 							       frames[k]);
1835 		if (unlikely(xdp_tx_bd_cnt < 0))
1836 			break;
1837 
1838 		if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1839 					   xdp_tx_bd_cnt))) {
1840 			for (i = 0; i < xdp_tx_bd_cnt; i++)
1841 				enetc_unmap_tx_buff(tx_ring,
1842 						    &xdp_redirect_arr[i]);
1843 			tx_ring->stats.xdp_tx_drops++;
1844 			break;
1845 		}
1846 
1847 		xdp_tx_frm_cnt++;
1848 	}
1849 
1850 	if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1851 		enetc_update_tx_ring_tail(tx_ring);
1852 
1853 	tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1854 
1855 	enetc_unlock_mdio();
1856 
1857 	return xdp_tx_frm_cnt;
1858 }
1859 EXPORT_SYMBOL_GPL(enetc_xdp_xmit);
1860 
1861 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1862 				     struct xdp_buff *xdp_buff, u16 size)
1863 {
1864 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1865 	void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1866 
1867 	/* To be used for XDP_TX */
1868 	rx_swbd->len = size;
1869 
1870 	xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1871 			 rx_ring->buffer_offset, size, false);
1872 }
1873 
1874 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1875 				     u16 size, struct xdp_buff *xdp_buff)
1876 {
1877 	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1878 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1879 	skb_frag_t *frag;
1880 
1881 	/* To be used for XDP_TX */
1882 	rx_swbd->len = size;
1883 
1884 	if (!xdp_buff_has_frags(xdp_buff)) {
1885 		xdp_buff_set_frags_flag(xdp_buff);
1886 		shinfo->xdp_frags_size = size;
1887 		shinfo->nr_frags = 0;
1888 	} else {
1889 		shinfo->xdp_frags_size += size;
1890 	}
1891 
1892 	if (page_is_pfmemalloc(rx_swbd->page))
1893 		xdp_buff_set_frag_pfmemalloc(xdp_buff);
1894 
1895 	frag = &shinfo->frags[shinfo->nr_frags];
1896 	skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,
1897 				size);
1898 
1899 	shinfo->nr_frags++;
1900 }
1901 
1902 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1903 				 union enetc_rx_bd **rxbd, int *i,
1904 				 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1905 {
1906 	u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1907 
1908 	xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1909 
1910 	enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1911 	(*cleaned_cnt)++;
1912 	enetc_rxbd_next(rx_ring, rxbd, i);
1913 
1914 	/* not last BD in frame? */
1915 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1916 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1917 		size = ENETC_RXB_DMA_SIZE_XDP;
1918 
1919 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1920 			dma_rmb();
1921 			size = le16_to_cpu((*rxbd)->r.buf_len);
1922 		}
1923 
1924 		enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1925 		(*cleaned_cnt)++;
1926 		enetc_rxbd_next(rx_ring, rxbd, i);
1927 	}
1928 }
1929 
1930 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1931  * recycled back into the RX ring in enetc_clean_tx_ring.
1932  */
1933 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1934 					struct enetc_bdr *rx_ring,
1935 					int rx_ring_first, int rx_ring_last)
1936 {
1937 	int n = 0;
1938 
1939 	for (; rx_ring_first != rx_ring_last;
1940 	     n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1941 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1942 		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1943 
1944 		/* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1945 		tx_swbd->dma = rx_swbd->dma;
1946 		tx_swbd->dir = rx_swbd->dir;
1947 		tx_swbd->page = rx_swbd->page;
1948 		tx_swbd->page_offset = rx_swbd->page_offset;
1949 		tx_swbd->len = rx_swbd->len;
1950 		tx_swbd->is_dma_page = true;
1951 		tx_swbd->is_xdp_tx = true;
1952 		tx_swbd->is_eof = false;
1953 	}
1954 
1955 	/* We rely on caller providing an rx_ring_last > rx_ring_first */
1956 	xdp_tx_arr[n - 1].is_eof = true;
1957 
1958 	return n;
1959 }
1960 
1961 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1962 			   int rx_ring_last)
1963 {
1964 	while (rx_ring_first != rx_ring_last) {
1965 		enetc_put_rx_buff(rx_ring,
1966 				  &rx_ring->rx_swbd[rx_ring_first]);
1967 		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1968 	}
1969 }
1970 
1971 static void enetc_bulk_flip_buff(struct enetc_bdr *rx_ring, int rx_ring_first,
1972 				 int rx_ring_last)
1973 {
1974 	while (rx_ring_first != rx_ring_last) {
1975 		enetc_flip_rx_buff(rx_ring,
1976 				   &rx_ring->rx_swbd[rx_ring_first]);
1977 		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1978 	}
1979 }
1980 
1981 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1982 				   struct napi_struct *napi, int work_limit,
1983 				   struct bpf_prog *prog)
1984 {
1985 	int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1986 	struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1987 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1988 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1989 	struct enetc_bdr *tx_ring;
1990 	int cleaned_cnt, i;
1991 	u32 xdp_act;
1992 
1993 	cleaned_cnt = enetc_bd_unused(rx_ring);
1994 	/* next descriptor to process */
1995 	i = rx_ring->next_to_clean;
1996 
1997 	enetc_lock_mdio();
1998 
1999 	while (likely(rx_frm_cnt < work_limit)) {
2000 		union enetc_rx_bd *rxbd, *orig_rxbd;
2001 		struct xdp_buff xdp_buff;
2002 		struct sk_buff *skb;
2003 		int orig_i, err;
2004 		u32 bd_status;
2005 
2006 		rxbd = enetc_rxbd(rx_ring, i);
2007 		bd_status = le32_to_cpu(rxbd->r.lstatus);
2008 		if (!bd_status)
2009 			break;
2010 
2011 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
2012 		dma_rmb(); /* for reading other rxbd fields */
2013 
2014 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
2015 						      &rxbd, &i))
2016 			break;
2017 
2018 		orig_rxbd = rxbd;
2019 		orig_i = i;
2020 
2021 		enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
2022 				     &cleaned_cnt, &xdp_buff);
2023 
2024 		/* When set, the outer VLAN header is extracted and reported
2025 		 * in the receive buffer descriptor. So rx_byte_cnt should
2026 		 * add the length of the extracted VLAN header.
2027 		 */
2028 		if (bd_status & ENETC_RXBD_FLAG_VLAN)
2029 			rx_byte_cnt += VLAN_HLEN;
2030 		rx_byte_cnt += xdp_get_buff_len(&xdp_buff);
2031 
2032 		xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
2033 
2034 		switch (xdp_act) {
2035 		default:
2036 			bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
2037 			fallthrough;
2038 		case XDP_ABORTED:
2039 			trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
2040 			fallthrough;
2041 		case XDP_DROP:
2042 			enetc_xdp_drop(rx_ring, orig_i, i);
2043 			rx_ring->stats.xdp_drops++;
2044 			break;
2045 		case XDP_PASS:
2046 			skb = xdp_build_skb_from_buff(&xdp_buff);
2047 			/* Probably under memory pressure, stop NAPI */
2048 			if (unlikely(!skb)) {
2049 				enetc_xdp_drop(rx_ring, orig_i, i);
2050 				rx_ring->stats.xdp_drops++;
2051 				goto out;
2052 			}
2053 
2054 			enetc_get_offloads(rx_ring, orig_rxbd, skb);
2055 
2056 			/* These buffers are about to be owned by the stack.
2057 			 * Update our buffer cache (the rx_swbd array elements)
2058 			 * with their other page halves.
2059 			 */
2060 			enetc_bulk_flip_buff(rx_ring, orig_i, i);
2061 
2062 			enetc_unlock_mdio();
2063 			napi_gro_receive(napi, skb);
2064 			enetc_lock_mdio();
2065 			break;
2066 		case XDP_TX:
2067 			tx_ring = priv->xdp_tx_ring[rx_ring->index];
2068 			if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) {
2069 				enetc_xdp_drop(rx_ring, orig_i, i);
2070 				tx_ring->stats.xdp_tx_drops++;
2071 				break;
2072 			}
2073 
2074 			xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
2075 								     rx_ring,
2076 								     orig_i, i);
2077 
2078 			if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
2079 				enetc_xdp_drop(rx_ring, orig_i, i);
2080 				tx_ring->stats.xdp_tx_drops++;
2081 			} else {
2082 				tx_ring->stats.xdp_tx++;
2083 				rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
2084 				xdp_tx_frm_cnt++;
2085 				/* The XDP_TX enqueue was successful, so we
2086 				 * need to scrub the RX software BDs because
2087 				 * the ownership of the buffers no longer
2088 				 * belongs to the RX ring, and we must prevent
2089 				 * enetc_refill_rx_ring() from reusing
2090 				 * rx_swbd->page.
2091 				 */
2092 				while (orig_i != i) {
2093 					rx_ring->rx_swbd[orig_i].page = NULL;
2094 					enetc_bdr_idx_inc(rx_ring, &orig_i);
2095 				}
2096 			}
2097 			break;
2098 		case XDP_REDIRECT:
2099 			enetc_unlock_mdio();
2100 			err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
2101 			enetc_lock_mdio();
2102 			if (unlikely(err)) {
2103 				enetc_xdp_drop(rx_ring, orig_i, i);
2104 				rx_ring->stats.xdp_redirect_failures++;
2105 			} else {
2106 				enetc_bulk_flip_buff(rx_ring, orig_i, i);
2107 				xdp_redirect_frm_cnt++;
2108 				rx_ring->stats.xdp_redirect++;
2109 			}
2110 		}
2111 
2112 		rx_frm_cnt++;
2113 	}
2114 
2115 out:
2116 	rx_ring->next_to_clean = i;
2117 
2118 	rx_ring->stats.packets += rx_frm_cnt;
2119 	rx_ring->stats.bytes += rx_byte_cnt;
2120 
2121 	if (xdp_redirect_frm_cnt) {
2122 		enetc_unlock_mdio();
2123 		xdp_do_flush();
2124 		enetc_lock_mdio();
2125 	}
2126 
2127 	if (xdp_tx_frm_cnt)
2128 		enetc_update_tx_ring_tail(tx_ring);
2129 
2130 	if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
2131 		enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
2132 				     rx_ring->xdp.xdp_tx_in_flight);
2133 
2134 	enetc_unlock_mdio();
2135 
2136 	return rx_frm_cnt;
2137 }
2138 
2139 static int enetc_poll(struct napi_struct *napi, int budget)
2140 {
2141 	struct enetc_int_vector
2142 		*v = container_of(napi, struct enetc_int_vector, napi);
2143 	struct enetc_bdr *rx_ring = &v->rx_ring;
2144 	struct bpf_prog *prog;
2145 	bool complete = true;
2146 	int work_done;
2147 	int i;
2148 
2149 	enetc_lock_mdio();
2150 
2151 	for (i = 0; i < v->count_tx_rings; i++)
2152 		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
2153 			complete = false;
2154 	enetc_unlock_mdio();
2155 
2156 	prog = rx_ring->xdp.prog;
2157 	if (prog)
2158 		work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
2159 	else
2160 		work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
2161 	if (work_done == budget)
2162 		complete = false;
2163 	if (work_done)
2164 		v->rx_napi_work = true;
2165 
2166 	if (!complete)
2167 		return budget;
2168 
2169 	napi_complete_done(napi, work_done);
2170 
2171 	if (likely(v->rx_dim_en))
2172 		enetc_rx_net_dim(v);
2173 
2174 	v->rx_napi_work = false;
2175 
2176 	enetc_lock_mdio();
2177 	/* enable interrupts */
2178 	enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
2179 
2180 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
2181 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
2182 				 ENETC_TBIER_TXTIE);
2183 
2184 	enetc_unlock_mdio();
2185 
2186 	return work_done;
2187 }
2188 
2189 /* Probing and Init */
2190 #define ENETC_MAX_RFS_SIZE 64
2191 void enetc_get_si_caps(struct enetc_si *si)
2192 {
2193 	struct enetc_hw *hw = &si->hw;
2194 	u32 val;
2195 
2196 	/* find out how many of various resources we have to work with */
2197 	val = enetc_rd(hw, ENETC_SICAPR0);
2198 	si->num_rx_rings = (val >> 16) & 0xff;
2199 	si->num_tx_rings = val & 0xff;
2200 
2201 	val = enetc_rd(hw, ENETC_SIPCAPR0);
2202 	if (val & ENETC_SIPCAPR0_RFS) {
2203 		val = enetc_rd(hw, ENETC_SIRFSCAPR);
2204 		si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
2205 		si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
2206 	} else {
2207 		/* ENETC which not supports RFS */
2208 		si->num_fs_entries = 0;
2209 	}
2210 
2211 	si->num_rss = 0;
2212 	val = enetc_rd(hw, ENETC_SIPCAPR0);
2213 	if (val & ENETC_SIPCAPR0_RSS) {
2214 		u32 rss;
2215 
2216 		rss = enetc_rd(hw, ENETC_SIRSSCAPR);
2217 		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
2218 	}
2219 
2220 	if (val & ENETC_SIPCAPR0_LSO)
2221 		si->hw_features |= ENETC_SI_F_LSO;
2222 }
2223 EXPORT_SYMBOL_GPL(enetc_get_si_caps);
2224 
2225 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
2226 {
2227 	size_t bd_base_size = res->bd_count * res->bd_size;
2228 
2229 	res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,
2230 					  &res->bd_dma_base, GFP_KERNEL);
2231 	if (!res->bd_base)
2232 		return -ENOMEM;
2233 
2234 	/* h/w requires 128B alignment */
2235 	if (!IS_ALIGNED(res->bd_dma_base, 128)) {
2236 		dma_free_coherent(res->dev, bd_base_size, res->bd_base,
2237 				  res->bd_dma_base);
2238 		return -EINVAL;
2239 	}
2240 
2241 	return 0;
2242 }
2243 
2244 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
2245 {
2246 	size_t bd_base_size = res->bd_count * res->bd_size;
2247 
2248 	dma_free_coherent(res->dev, bd_base_size, res->bd_base,
2249 			  res->bd_dma_base);
2250 }
2251 
2252 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
2253 				   struct device *dev, size_t bd_count)
2254 {
2255 	int err;
2256 
2257 	res->dev = dev;
2258 	res->bd_count = bd_count;
2259 	res->bd_size = sizeof(union enetc_tx_bd);
2260 
2261 	res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));
2262 	if (!res->tx_swbd)
2263 		return -ENOMEM;
2264 
2265 	err = enetc_dma_alloc_bdr(res);
2266 	if (err)
2267 		goto err_alloc_bdr;
2268 
2269 	res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,
2270 					      &res->tso_headers_dma,
2271 					      GFP_KERNEL);
2272 	if (!res->tso_headers) {
2273 		err = -ENOMEM;
2274 		goto err_alloc_tso;
2275 	}
2276 
2277 	return 0;
2278 
2279 err_alloc_tso:
2280 	enetc_dma_free_bdr(res);
2281 err_alloc_bdr:
2282 	vfree(res->tx_swbd);
2283 	res->tx_swbd = NULL;
2284 
2285 	return err;
2286 }
2287 
2288 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
2289 {
2290 	dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,
2291 			  res->tso_headers, res->tso_headers_dma);
2292 	enetc_dma_free_bdr(res);
2293 	vfree(res->tx_swbd);
2294 }
2295 
2296 static struct enetc_bdr_resource *
2297 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
2298 {
2299 	struct enetc_bdr_resource *tx_res;
2300 	int i, err;
2301 
2302 	tx_res = kzalloc_objs(*tx_res, priv->num_tx_rings);
2303 	if (!tx_res)
2304 		return ERR_PTR(-ENOMEM);
2305 
2306 	for (i = 0; i < priv->num_tx_rings; i++) {
2307 		struct enetc_bdr *tx_ring = priv->tx_ring[i];
2308 
2309 		err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,
2310 					      tx_ring->bd_count);
2311 		if (err)
2312 			goto fail;
2313 	}
2314 
2315 	return tx_res;
2316 
2317 fail:
2318 	while (i-- > 0)
2319 		enetc_free_tx_resource(&tx_res[i]);
2320 
2321 	kfree(tx_res);
2322 
2323 	return ERR_PTR(err);
2324 }
2325 
2326 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
2327 				    size_t num_resources)
2328 {
2329 	size_t i;
2330 
2331 	for (i = 0; i < num_resources; i++)
2332 		enetc_free_tx_resource(&tx_res[i]);
2333 
2334 	kfree(tx_res);
2335 }
2336 
2337 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
2338 				   struct device *dev, size_t bd_count,
2339 				   bool extended)
2340 {
2341 	int err;
2342 
2343 	res->dev = dev;
2344 	res->bd_count = bd_count;
2345 	res->bd_size = sizeof(union enetc_rx_bd);
2346 	if (extended)
2347 		res->bd_size *= 2;
2348 
2349 	res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));
2350 	if (!res->rx_swbd)
2351 		return -ENOMEM;
2352 
2353 	err = enetc_dma_alloc_bdr(res);
2354 	if (err) {
2355 		vfree(res->rx_swbd);
2356 		return err;
2357 	}
2358 
2359 	return 0;
2360 }
2361 
2362 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
2363 {
2364 	enetc_dma_free_bdr(res);
2365 	vfree(res->rx_swbd);
2366 }
2367 
2368 static struct enetc_bdr_resource *
2369 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
2370 {
2371 	struct enetc_bdr_resource *rx_res;
2372 	int i, err;
2373 
2374 	rx_res = kzalloc_objs(*rx_res, priv->num_rx_rings);
2375 	if (!rx_res)
2376 		return ERR_PTR(-ENOMEM);
2377 
2378 	for (i = 0; i < priv->num_rx_rings; i++) {
2379 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
2380 
2381 		err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,
2382 					      rx_ring->bd_count, extended);
2383 		if (err)
2384 			goto fail;
2385 	}
2386 
2387 	return rx_res;
2388 
2389 fail:
2390 	while (i-- > 0)
2391 		enetc_free_rx_resource(&rx_res[i]);
2392 
2393 	kfree(rx_res);
2394 
2395 	return ERR_PTR(err);
2396 }
2397 
2398 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
2399 				    size_t num_resources)
2400 {
2401 	size_t i;
2402 
2403 	for (i = 0; i < num_resources; i++)
2404 		enetc_free_rx_resource(&rx_res[i]);
2405 
2406 	kfree(rx_res);
2407 }
2408 
2409 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
2410 				     const struct enetc_bdr_resource *res)
2411 {
2412 	tx_ring->bd_base = res ? res->bd_base : NULL;
2413 	tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
2414 	tx_ring->tx_swbd = res ? res->tx_swbd : NULL;
2415 	tx_ring->tso_headers = res ? res->tso_headers : NULL;
2416 	tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;
2417 }
2418 
2419 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
2420 				     const struct enetc_bdr_resource *res)
2421 {
2422 	rx_ring->bd_base = res ? res->bd_base : NULL;
2423 	rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
2424 	rx_ring->rx_swbd = res ? res->rx_swbd : NULL;
2425 }
2426 
2427 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
2428 				      const struct enetc_bdr_resource *res)
2429 {
2430 	int i;
2431 
2432 	if (priv->tx_res)
2433 		enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);
2434 
2435 	for (i = 0; i < priv->num_tx_rings; i++) {
2436 		enetc_assign_tx_resource(priv->tx_ring[i],
2437 					 res ? &res[i] : NULL);
2438 	}
2439 
2440 	priv->tx_res = res;
2441 }
2442 
2443 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
2444 				      const struct enetc_bdr_resource *res)
2445 {
2446 	int i;
2447 
2448 	if (priv->rx_res)
2449 		enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);
2450 
2451 	for (i = 0; i < priv->num_rx_rings; i++) {
2452 		enetc_assign_rx_resource(priv->rx_ring[i],
2453 					 res ? &res[i] : NULL);
2454 	}
2455 
2456 	priv->rx_res = res;
2457 }
2458 
2459 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
2460 {
2461 	int i;
2462 
2463 	for (i = 0; i < tx_ring->bd_count; i++) {
2464 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
2465 
2466 		enetc_free_tx_frame(tx_ring, tx_swbd);
2467 	}
2468 }
2469 
2470 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
2471 {
2472 	int i;
2473 
2474 	for (i = 0; i < rx_ring->bd_count; i++) {
2475 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
2476 
2477 		if (!rx_swbd->page)
2478 			continue;
2479 
2480 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
2481 			       rx_swbd->dir);
2482 		__free_page(rx_swbd->page);
2483 		rx_swbd->page = NULL;
2484 	}
2485 }
2486 
2487 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
2488 {
2489 	int i;
2490 
2491 	for (i = 0; i < priv->num_rx_rings; i++)
2492 		enetc_free_rx_ring(priv->rx_ring[i]);
2493 
2494 	for (i = 0; i < priv->num_tx_rings; i++)
2495 		enetc_free_tx_ring(priv->tx_ring[i]);
2496 }
2497 
2498 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
2499 {
2500 	int *rss_table;
2501 	int i;
2502 
2503 	rss_table = kmalloc_objs(*rss_table, si->num_rss);
2504 	if (!rss_table)
2505 		return -ENOMEM;
2506 
2507 	/* Set up RSS table defaults */
2508 	for (i = 0; i < si->num_rss; i++)
2509 		rss_table[i] = i % num_groups;
2510 
2511 	si->ops->set_rss_table(si, rss_table, si->num_rss);
2512 
2513 	kfree(rss_table);
2514 
2515 	return 0;
2516 }
2517 
2518 static void enetc_set_lso_flags_mask(struct enetc_hw *hw)
2519 {
2520 	enetc_wr(hw, ENETC4_SILSOSFMR0,
2521 		 SILSOSFMR0_VAL_SET(ENETC4_TCP_NL_SEG_FLAGS_DMASK,
2522 				    ENETC4_TCP_NL_SEG_FLAGS_DMASK));
2523 	enetc_wr(hw, ENETC4_SILSOSFMR1, 0);
2524 }
2525 
2526 static void enetc_set_rss(struct net_device *ndev, int en)
2527 {
2528 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2529 	struct enetc_hw *hw = &priv->si->hw;
2530 	u32 reg;
2531 
2532 	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2533 
2534 	reg = enetc_rd(hw, ENETC_SIMR);
2535 	reg &= ~ENETC_SIMR_RSSE;
2536 	reg |= (en) ? ENETC_SIMR_RSSE : 0;
2537 	enetc_wr(hw, ENETC_SIMR, reg);
2538 }
2539 
2540 int enetc_configure_si(struct enetc_ndev_priv *priv)
2541 {
2542 	struct enetc_si *si = priv->si;
2543 	struct enetc_hw *hw = &si->hw;
2544 	int err;
2545 
2546 	if (is_enetc_rev1(si)) {
2547 		/* set SI cache attributes */
2548 		enetc_wr(hw, ENETC_SICAR0,
2549 			 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
2550 		enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
2551 	}
2552 
2553 	/* enable SI */
2554 	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
2555 
2556 	if (si->hw_features & ENETC_SI_F_LSO)
2557 		enetc_set_lso_flags_mask(hw);
2558 
2559 	if (si->num_rss) {
2560 		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
2561 		if (err)
2562 			return err;
2563 
2564 		if (priv->ndev->features & NETIF_F_RXHASH)
2565 			enetc_set_rss(priv->ndev, true);
2566 	}
2567 
2568 	return 0;
2569 }
2570 EXPORT_SYMBOL_GPL(enetc_configure_si);
2571 
2572 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2573 {
2574 	struct enetc_si *si = priv->si;
2575 	int cpus = num_online_cpus();
2576 
2577 	priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
2578 	priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2579 
2580 	/* Enable all available TX rings in order to configure as many
2581 	 * priorities as possible, when needed.
2582 	 * TODO: Make # of TX rings run-time configurable
2583 	 */
2584 	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2585 	priv->num_tx_rings = si->num_tx_rings;
2586 	priv->bdr_int_num = priv->num_rx_rings;
2587 	priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2588 	priv->tx_ictt = enetc_usecs_to_cycles(600, priv->sysclk_freq);
2589 }
2590 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);
2591 
2592 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2593 {
2594 	struct enetc_si *si = priv->si;
2595 
2596 	priv->cls_rules = kzalloc_objs(*priv->cls_rules, si->num_fs_entries);
2597 	if (!priv->cls_rules)
2598 		return -ENOMEM;
2599 
2600 	return 0;
2601 }
2602 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);
2603 
2604 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2605 {
2606 	kfree(priv->cls_rules);
2607 }
2608 EXPORT_SYMBOL_GPL(enetc_free_si_resources);
2609 
2610 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2611 {
2612 	struct enetc_si *si = container_of(hw, struct enetc_si, hw);
2613 	int idx = tx_ring->index;
2614 	u32 tbmr;
2615 
2616 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2617 		       lower_32_bits(tx_ring->bd_dma_base));
2618 
2619 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2620 		       upper_32_bits(tx_ring->bd_dma_base));
2621 
2622 	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2623 	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2624 		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
2625 
2626 	/* For ENETC v1, clearing PI/CI registers for Tx not supported,
2627 	 * adjust sw indexes
2628 	 */
2629 	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2630 	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2631 
2632 	if (tx_ring->next_to_use != tx_ring->next_to_clean &&
2633 	    !is_enetc_rev1(si)) {
2634 		tx_ring->next_to_use = 0;
2635 		tx_ring->next_to_clean = 0;
2636 		enetc_txbdr_wr(hw, idx, ENETC_TBPIR, 0);
2637 		enetc_txbdr_wr(hw, idx, ENETC_TBCIR, 0);
2638 	}
2639 
2640 	/* enable Tx ints by setting pkt thr to 1 */
2641 	enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2642 
2643 	tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);
2644 	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2645 		tbmr |= ENETC_TBMR_VIH;
2646 
2647 	/* enable ring */
2648 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2649 
2650 	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2651 	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2652 	tx_ring->idr = hw->reg + ENETC_SITXIDR;
2653 }
2654 
2655 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
2656 			      bool extended)
2657 {
2658 	int idx = rx_ring->index;
2659 
2660 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2661 		       lower_32_bits(rx_ring->bd_dma_base));
2662 
2663 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2664 		       upper_32_bits(rx_ring->bd_dma_base));
2665 
2666 	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2667 	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2668 		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
2669 
2670 	if (rx_ring->xdp.prog)
2671 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2672 	else
2673 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2674 
2675 	/* Also prepare the consumer index in case page allocation never
2676 	 * succeeds. In that case, hardware will never advance producer index
2677 	 * to match consumer index, and will drop all frames.
2678 	 */
2679 	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2680 	enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2681 
2682 	/* enable Rx ints by setting pkt thr to 1 */
2683 	enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2684 
2685 	rx_ring->ext_en = extended;
2686 	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2687 	rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2688 
2689 	rx_ring->next_to_clean = 0;
2690 	rx_ring->next_to_use = 0;
2691 	rx_ring->next_to_alloc = 0;
2692 
2693 	enetc_lock_mdio();
2694 	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2695 	enetc_unlock_mdio();
2696 }
2697 
2698 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
2699 {
2700 	struct enetc_hw *hw = &priv->si->hw;
2701 	int i;
2702 
2703 	for (i = 0; i < priv->num_tx_rings; i++)
2704 		enetc_setup_txbdr(hw, priv->tx_ring[i]);
2705 
2706 	for (i = 0; i < priv->num_rx_rings; i++)
2707 		enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);
2708 }
2709 
2710 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2711 {
2712 	int idx = tx_ring->index;
2713 	u32 tbmr;
2714 
2715 	tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);
2716 	tbmr |= ENETC_TBMR_EN;
2717 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2718 }
2719 
2720 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2721 {
2722 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
2723 	int idx = rx_ring->index;
2724 	u32 rbmr = ENETC_RBMR_EN;
2725 
2726 	if (rx_ring->ext_en)
2727 		rbmr |= ENETC_RBMR_BDS;
2728 
2729 	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2730 		rbmr |= ENETC_RBMR_VTE;
2731 
2732 	if (test_bit(ENETC_RXBDR_CM, &priv->flags))
2733 		rbmr |= ENETC_RBMR_CM;
2734 
2735 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2736 }
2737 
2738 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv)
2739 {
2740 	struct enetc_si *si = priv->si;
2741 	struct enetc_hw *hw = &si->hw;
2742 	int i;
2743 
2744 	spin_lock(&si->gen_lock);
2745 
2746 	for (i = 0; i < priv->num_rx_rings; i++)
2747 		enetc_enable_rxbdr(hw, priv->rx_ring[i]);
2748 
2749 	spin_unlock(&si->gen_lock);
2750 }
2751 
2752 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv)
2753 {
2754 	struct enetc_hw *hw = &priv->si->hw;
2755 	int i;
2756 
2757 	for (i = 0; i < priv->num_tx_rings; i++)
2758 		enetc_enable_txbdr(hw, priv->tx_ring[i]);
2759 }
2760 
2761 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2762 {
2763 	int idx = rx_ring->index;
2764 
2765 	/* disable EN bit on ring */
2766 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2767 }
2768 
2769 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2770 {
2771 	int idx = rx_ring->index;
2772 
2773 	/* disable EN bit on ring */
2774 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2775 }
2776 
2777 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv)
2778 {
2779 	struct enetc_si *si = priv->si;
2780 	struct enetc_hw *hw = &si->hw;
2781 	int i;
2782 
2783 	spin_lock(&si->gen_lock);
2784 
2785 	for (i = 0; i < priv->num_rx_rings; i++)
2786 		enetc_disable_rxbdr(hw, priv->rx_ring[i]);
2787 
2788 	spin_unlock(&si->gen_lock);
2789 }
2790 
2791 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv)
2792 {
2793 	struct enetc_hw *hw = &priv->si->hw;
2794 	int i;
2795 
2796 	for (i = 0; i < priv->num_tx_rings; i++)
2797 		enetc_disable_txbdr(hw, priv->tx_ring[i]);
2798 }
2799 
2800 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2801 {
2802 	int delay = 8, timeout = 100;
2803 	int idx = tx_ring->index;
2804 
2805 	/* wait for busy to clear */
2806 	while (delay < timeout &&
2807 	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2808 		msleep(delay);
2809 		delay *= 2;
2810 	}
2811 
2812 	if (delay >= timeout)
2813 		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2814 			    idx);
2815 }
2816 
2817 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
2818 {
2819 	struct enetc_hw *hw = &priv->si->hw;
2820 	int i;
2821 
2822 	for (i = 0; i < priv->num_tx_rings; i++)
2823 		enetc_wait_txbdr(hw, priv->tx_ring[i]);
2824 }
2825 
2826 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2827 {
2828 	struct pci_dev *pdev = priv->si->pdev;
2829 	struct enetc_hw *hw = &priv->si->hw;
2830 	int i, j, err;
2831 
2832 	for (i = 0; i < priv->bdr_int_num; i++) {
2833 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2834 		struct enetc_int_vector *v = priv->int_vector[i];
2835 		int entry = ENETC_BDR_INT_BASE_IDX + i;
2836 
2837 		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2838 			 priv->ndev->name, i);
2839 		err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v);
2840 		if (err) {
2841 			dev_err(priv->dev, "request_irq() failed!\n");
2842 			goto irq_err;
2843 		}
2844 
2845 		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2846 		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2847 		v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2848 
2849 		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2850 
2851 		for (j = 0; j < v->count_tx_rings; j++) {
2852 			int idx = v->tx_ring[j].index;
2853 
2854 			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2855 		}
2856 		irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2857 	}
2858 
2859 	return 0;
2860 
2861 irq_err:
2862 	while (i--) {
2863 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2864 
2865 		irq_set_affinity_hint(irq, NULL);
2866 		free_irq(irq, priv->int_vector[i]);
2867 	}
2868 
2869 	return err;
2870 }
2871 
2872 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2873 {
2874 	struct pci_dev *pdev = priv->si->pdev;
2875 	int i;
2876 
2877 	for (i = 0; i < priv->bdr_int_num; i++) {
2878 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2879 
2880 		irq_set_affinity_hint(irq, NULL);
2881 		free_irq(irq, priv->int_vector[i]);
2882 	}
2883 }
2884 
2885 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2886 {
2887 	struct enetc_hw *hw = &priv->si->hw;
2888 	u32 icpt, ictt;
2889 	int i;
2890 
2891 	/* enable Tx & Rx event indication */
2892 	if (priv->ic_mode &
2893 	    (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2894 		icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2895 		/* init to non-0 minimum, will be adjusted later */
2896 		ictt = 0x1;
2897 	} else {
2898 		icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2899 		ictt = 0;
2900 	}
2901 
2902 	for (i = 0; i < priv->num_rx_rings; i++) {
2903 		enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2904 		enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2905 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2906 	}
2907 
2908 	if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2909 		icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2910 	else
2911 		icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2912 
2913 	for (i = 0; i < priv->num_tx_rings; i++) {
2914 		enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2915 		enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2916 		enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2917 	}
2918 }
2919 
2920 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2921 {
2922 	struct enetc_hw *hw = &priv->si->hw;
2923 	int i;
2924 
2925 	for (i = 0; i < priv->num_tx_rings; i++)
2926 		enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2927 
2928 	for (i = 0; i < priv->num_rx_rings; i++)
2929 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2930 }
2931 
2932 static int enetc_phylink_connect(struct net_device *ndev)
2933 {
2934 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2935 	struct enetc_si *si = priv->si;
2936 	struct ethtool_keee edata;
2937 	int err;
2938 
2939 	if (!priv->phylink) {
2940 		/* phy-less mode */
2941 		if (!si->ops->vf_reg_link_status_notifier)
2942 			goto carrier_on;
2943 
2944 		/* For phy-less VFs on ENETC v4, attempt to register a link
2945 		 * status notifier with the PF via the VSI-to-PSI messaging
2946 		 * channel. If registration succeeds, the PF will immediately
2947 		 * send the current link status and broadcast future link
2948 		 * transitions; carrier state is then managed in
2949 		 * enetc_vf_msg_handle_link_status(). If registration fails,
2950 		 * fall back to the LS1028A behaviour and assert carrier
2951 		 * unconditionally via netif_carrier_on().
2952 		 */
2953 		if (!si->ops->vf_reg_link_status_notifier(si))
2954 			return 0;
2955 
2956 		dev_warn(&ndev->dev,
2957 			 "Link status notifier registration failed\n");
2958 
2959 carrier_on:
2960 		netif_carrier_on(ndev);
2961 		return 0;
2962 	}
2963 
2964 	err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2965 	if (err) {
2966 		dev_err(&ndev->dev, "could not attach to PHY\n");
2967 		return err;
2968 	}
2969 
2970 	/* disable EEE autoneg, until ENETC driver supports it */
2971 	memset(&edata, 0, sizeof(struct ethtool_keee));
2972 	phylink_ethtool_set_eee(priv->phylink, &edata);
2973 
2974 	phylink_start(priv->phylink);
2975 
2976 	return 0;
2977 }
2978 
2979 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2980 {
2981 	struct enetc_ndev_priv *priv;
2982 	struct sk_buff *skb;
2983 
2984 	priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2985 
2986 	netif_tx_lock_bh(priv->ndev);
2987 
2988 	clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2989 	skb = skb_dequeue(&priv->tx_skbs);
2990 	if (skb)
2991 		enetc_start_xmit(skb, priv->ndev);
2992 
2993 	netif_tx_unlock_bh(priv->ndev);
2994 }
2995 
2996 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2997 {
2998 	INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2999 	skb_queue_head_init(&priv->tx_skbs);
3000 }
3001 
3002 void enetc_start(struct net_device *ndev)
3003 {
3004 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3005 	int i;
3006 
3007 	enetc_setup_interrupts(priv);
3008 
3009 	for (i = 0; i < priv->bdr_int_num; i++) {
3010 		int irq = pci_irq_vector(priv->si->pdev,
3011 					 ENETC_BDR_INT_BASE_IDX + i);
3012 
3013 		napi_enable(&priv->int_vector[i]->napi);
3014 		enable_irq(irq);
3015 	}
3016 
3017 	enetc_enable_tx_bdrs(priv);
3018 
3019 	enetc_enable_rx_bdrs(priv);
3020 
3021 	netif_tx_start_all_queues(ndev);
3022 
3023 	clear_bit(ENETC_TX_DOWN, &priv->flags);
3024 }
3025 EXPORT_SYMBOL_GPL(enetc_start);
3026 
3027 int enetc_open(struct net_device *ndev)
3028 {
3029 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3030 	struct enetc_bdr_resource *tx_res, *rx_res;
3031 	bool extended;
3032 	int err;
3033 
3034 	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
3035 
3036 	err = clk_prepare_enable(priv->ref_clk);
3037 	if (err)
3038 		return err;
3039 
3040 	err = enetc_setup_irqs(priv);
3041 	if (err)
3042 		goto err_setup_irqs;
3043 
3044 	tx_res = enetc_alloc_tx_resources(priv);
3045 	if (IS_ERR(tx_res)) {
3046 		err = PTR_ERR(tx_res);
3047 		goto err_alloc_tx;
3048 	}
3049 
3050 	rx_res = enetc_alloc_rx_resources(priv, extended);
3051 	if (IS_ERR(rx_res)) {
3052 		err = PTR_ERR(rx_res);
3053 		goto err_alloc_rx;
3054 	}
3055 
3056 	err = enetc_phylink_connect(ndev);
3057 	if (err)
3058 		goto err_phy_connect;
3059 
3060 	enetc_tx_onestep_tstamp_init(priv);
3061 	enetc_assign_tx_resources(priv, tx_res);
3062 	enetc_assign_rx_resources(priv, rx_res);
3063 	enetc_setup_bdrs(priv, extended);
3064 	enetc_start(ndev);
3065 
3066 	return 0;
3067 
3068 err_phy_connect:
3069 	enetc_free_rx_resources(rx_res, priv->num_rx_rings);
3070 err_alloc_rx:
3071 	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
3072 err_alloc_tx:
3073 	enetc_free_irqs(priv);
3074 err_setup_irqs:
3075 	clk_disable_unprepare(priv->ref_clk);
3076 
3077 	return err;
3078 }
3079 EXPORT_SYMBOL_GPL(enetc_open);
3080 
3081 void enetc_stop(struct net_device *ndev)
3082 {
3083 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3084 	int i;
3085 
3086 	set_bit(ENETC_TX_DOWN, &priv->flags);
3087 
3088 	netif_tx_stop_all_queues(ndev);
3089 
3090 	enetc_disable_rx_bdrs(priv);
3091 
3092 	enetc_wait_bdrs(priv);
3093 
3094 	enetc_disable_tx_bdrs(priv);
3095 
3096 	for (i = 0; i < priv->bdr_int_num; i++) {
3097 		int irq = pci_irq_vector(priv->si->pdev,
3098 					 ENETC_BDR_INT_BASE_IDX + i);
3099 
3100 		disable_irq(irq);
3101 		napi_synchronize(&priv->int_vector[i]->napi);
3102 		napi_disable(&priv->int_vector[i]->napi);
3103 	}
3104 
3105 	enetc_clear_interrupts(priv);
3106 }
3107 EXPORT_SYMBOL_GPL(enetc_stop);
3108 
3109 int enetc_close(struct net_device *ndev)
3110 {
3111 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3112 	struct enetc_si *si = priv->si;
3113 
3114 	enetc_stop(ndev);
3115 
3116 	if (priv->phylink) {
3117 		phylink_stop(priv->phylink);
3118 		phylink_disconnect_phy(priv->phylink);
3119 	} else {
3120 		if (!si->ops->vf_unreg_link_status_notifier)
3121 			goto carrier_off;
3122 
3123 		/* No need to check whether the previous registration was
3124 		 * successful. Sending the deregistration message has no
3125 		 * impact; the PF side simply clears the corresponding bit
3126 		 * in link_status_ms_mask for the VF.
3127 		 */
3128 		if (!si->ops->vf_unreg_link_status_notifier(si))
3129 			goto carrier_off;
3130 
3131 		dev_warn(&ndev->dev,
3132 			 "Link status notifier unregistration failed\n");
3133 carrier_off:
3134 		netif_carrier_off(ndev);
3135 	}
3136 
3137 	enetc_free_rxtx_rings(priv);
3138 
3139 	/* Avoids dangling pointers and also frees old resources */
3140 	enetc_assign_rx_resources(priv, NULL);
3141 	enetc_assign_tx_resources(priv, NULL);
3142 
3143 	enetc_free_irqs(priv);
3144 	clk_disable_unprepare(priv->ref_clk);
3145 
3146 	return 0;
3147 }
3148 EXPORT_SYMBOL_GPL(enetc_close);
3149 
3150 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
3151 			     int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
3152 			     void *ctx)
3153 {
3154 	struct enetc_bdr_resource *tx_res, *rx_res;
3155 	int err;
3156 
3157 	ASSERT_RTNL();
3158 
3159 	/* If the interface is down, run the callback right away,
3160 	 * without reconfiguration.
3161 	 */
3162 	if (!netif_running(priv->ndev)) {
3163 		if (cb) {
3164 			err = cb(priv, ctx);
3165 			if (err)
3166 				return err;
3167 		}
3168 
3169 		return 0;
3170 	}
3171 
3172 	tx_res = enetc_alloc_tx_resources(priv);
3173 	if (IS_ERR(tx_res)) {
3174 		err = PTR_ERR(tx_res);
3175 		goto out;
3176 	}
3177 
3178 	rx_res = enetc_alloc_rx_resources(priv, extended);
3179 	if (IS_ERR(rx_res)) {
3180 		err = PTR_ERR(rx_res);
3181 		goto out_free_tx_res;
3182 	}
3183 
3184 	enetc_stop(priv->ndev);
3185 	enetc_free_rxtx_rings(priv);
3186 
3187 	/* Interface is down, run optional callback now */
3188 	if (cb) {
3189 		err = cb(priv, ctx);
3190 		if (err)
3191 			goto out_restart;
3192 	}
3193 
3194 	enetc_assign_tx_resources(priv, tx_res);
3195 	enetc_assign_rx_resources(priv, rx_res);
3196 	enetc_setup_bdrs(priv, extended);
3197 	enetc_start(priv->ndev);
3198 
3199 	return 0;
3200 
3201 out_restart:
3202 	enetc_setup_bdrs(priv, extended);
3203 	enetc_start(priv->ndev);
3204 	enetc_free_rx_resources(rx_res, priv->num_rx_rings);
3205 out_free_tx_res:
3206 	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
3207 out:
3208 	return err;
3209 }
3210 
3211 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
3212 {
3213 	int i;
3214 
3215 	for (i = 0; i < priv->num_tx_rings; i++)
3216 		netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,
3217 			   priv->tx_ring[i]->prio);
3218 }
3219 
3220 void enetc_reset_tc_mqprio(struct net_device *ndev)
3221 {
3222 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3223 	struct enetc_hw *hw = &priv->si->hw;
3224 	struct enetc_bdr *tx_ring;
3225 	int num_stack_tx_queues;
3226 	int i;
3227 
3228 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3229 
3230 	netdev_reset_tc(ndev);
3231 	netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
3232 	priv->min_num_stack_tx_queues = num_possible_cpus();
3233 
3234 	/* Reset all ring priorities to 0 */
3235 	for (i = 0; i < priv->num_tx_rings; i++) {
3236 		tx_ring = priv->tx_ring[i];
3237 		tx_ring->prio = 0;
3238 		enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
3239 	}
3240 
3241 	enetc_debug_tx_ring_prios(priv);
3242 
3243 	enetc_change_preemptible_tcs(priv, 0);
3244 }
3245 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);
3246 
3247 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
3248 {
3249 	struct tc_mqprio_qopt_offload *mqprio = type_data;
3250 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3251 	struct tc_mqprio_qopt *qopt = &mqprio->qopt;
3252 	struct enetc_hw *hw = &priv->si->hw;
3253 	int num_stack_tx_queues = 0;
3254 	struct enetc_bdr *tx_ring;
3255 	u8 num_tc = qopt->num_tc;
3256 	int offset, count;
3257 	int err, tc, q;
3258 
3259 	if (!num_tc) {
3260 		enetc_reset_tc_mqprio(ndev);
3261 		return 0;
3262 	}
3263 
3264 	err = netdev_set_num_tc(ndev, num_tc);
3265 	if (err)
3266 		return err;
3267 
3268 	for (tc = 0; tc < num_tc; tc++) {
3269 		offset = qopt->offset[tc];
3270 		count = qopt->count[tc];
3271 		num_stack_tx_queues += count;
3272 
3273 		err = netdev_set_tc_queue(ndev, tc, count, offset);
3274 		if (err)
3275 			goto err_reset_tc;
3276 
3277 		for (q = offset; q < offset + count; q++) {
3278 			tx_ring = priv->tx_ring[q];
3279 			/* The prio_tc_map is skb_tx_hash()'s way of selecting
3280 			 * between TX queues based on skb->priority. As such,
3281 			 * there's nothing to offload based on it.
3282 			 * Make the mqprio "traffic class" be the priority of
3283 			 * this ring group, and leave the Tx IPV to traffic
3284 			 * class mapping as its default mapping value of 1:1.
3285 			 */
3286 			tx_ring->prio = tc;
3287 			enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
3288 		}
3289 	}
3290 
3291 	err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
3292 	if (err)
3293 		goto err_reset_tc;
3294 
3295 	priv->min_num_stack_tx_queues = num_stack_tx_queues;
3296 
3297 	enetc_debug_tx_ring_prios(priv);
3298 
3299 	enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
3300 
3301 	return 0;
3302 
3303 err_reset_tc:
3304 	enetc_reset_tc_mqprio(ndev);
3305 	return err;
3306 }
3307 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);
3308 
3309 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
3310 {
3311 	struct bpf_prog *old_prog, *prog = ctx;
3312 	int num_stack_tx_queues;
3313 	int err, i;
3314 
3315 	old_prog = xchg(&priv->xdp_prog, prog);
3316 
3317 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3318 	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3319 	if (err) {
3320 		xchg(&priv->xdp_prog, old_prog);
3321 		return err;
3322 	}
3323 
3324 	if (old_prog)
3325 		bpf_prog_put(old_prog);
3326 
3327 	for (i = 0; i < priv->num_rx_rings; i++) {
3328 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
3329 
3330 		rx_ring->xdp.prog = prog;
3331 
3332 		if (prog)
3333 			rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
3334 		else
3335 			rx_ring->buffer_offset = ENETC_RXB_PAD;
3336 	}
3337 
3338 	return 0;
3339 }
3340 
3341 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
3342 				struct netlink_ext_ack *extack)
3343 {
3344 	int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;
3345 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3346 	bool extended;
3347 
3348 	if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >
3349 	    priv->num_tx_rings) {
3350 		NL_SET_ERR_MSG_FMT_MOD(extack,
3351 				       "Reserving %d XDP TXQs leaves under %d for stack (total %d)",
3352 				       num_xdp_tx_queues,
3353 				       priv->min_num_stack_tx_queues,
3354 				       priv->num_tx_rings);
3355 		return -EBUSY;
3356 	}
3357 
3358 	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
3359 
3360 	/* The buffer layout is changing, so we need to drain the old
3361 	 * RX buffers and seed new ones.
3362 	 */
3363 	return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);
3364 }
3365 
3366 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
3367 {
3368 	switch (bpf->command) {
3369 	case XDP_SETUP_PROG:
3370 		return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);
3371 	default:
3372 		return -EINVAL;
3373 	}
3374 
3375 	return 0;
3376 }
3377 EXPORT_SYMBOL_GPL(enetc_setup_bpf);
3378 
3379 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
3380 {
3381 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3382 	struct net_device_stats *stats = &ndev->stats;
3383 	unsigned long packets = 0, bytes = 0;
3384 	unsigned long tx_dropped = 0;
3385 	int i;
3386 
3387 	for (i = 0; i < priv->num_rx_rings; i++) {
3388 		packets += priv->rx_ring[i]->stats.packets;
3389 		bytes	+= priv->rx_ring[i]->stats.bytes;
3390 	}
3391 
3392 	stats->rx_packets = packets;
3393 	stats->rx_bytes = bytes;
3394 	bytes = 0;
3395 	packets = 0;
3396 
3397 	for (i = 0; i < priv->num_tx_rings; i++) {
3398 		packets += priv->tx_ring[i]->stats.packets;
3399 		bytes	+= priv->tx_ring[i]->stats.bytes;
3400 		tx_dropped += priv->tx_ring[i]->stats.win_drop;
3401 	}
3402 
3403 	stats->tx_packets = packets;
3404 	stats->tx_bytes = bytes;
3405 	stats->tx_dropped = tx_dropped;
3406 
3407 	return stats;
3408 }
3409 EXPORT_SYMBOL_GPL(enetc_get_stats);
3410 
3411 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
3412 {
3413 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3414 	struct enetc_si *si = priv->si;
3415 	struct enetc_hw *hw = &si->hw;
3416 	int i;
3417 
3418 	spin_lock(&si->gen_lock);
3419 
3420 	for (i = 0; i < priv->num_rx_rings; i++)
3421 		enetc_bdr_enable_rxvlan(hw, i, en);
3422 
3423 	spin_unlock(&si->gen_lock);
3424 }
3425 
3426 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
3427 {
3428 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3429 	struct enetc_hw *hw = &priv->si->hw;
3430 	int i;
3431 
3432 	for (i = 0; i < priv->num_tx_rings; i++)
3433 		enetc_bdr_enable_txvlan(hw, i, en);
3434 }
3435 
3436 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
3437 {
3438 	netdev_features_t changed = ndev->features ^ features;
3439 
3440 	if (changed & NETIF_F_RXHASH)
3441 		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
3442 
3443 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
3444 		enetc_enable_rxvlan(ndev,
3445 				    !!(features & NETIF_F_HW_VLAN_CTAG_RX));
3446 
3447 	if (changed & NETIF_F_HW_VLAN_CTAG_TX)
3448 		enetc_enable_txvlan(ndev,
3449 				    !!(features & NETIF_F_HW_VLAN_CTAG_TX));
3450 }
3451 EXPORT_SYMBOL_GPL(enetc_set_features);
3452 
3453 int enetc_hwtstamp_set(struct net_device *ndev,
3454 		       struct kernel_hwtstamp_config *config,
3455 		       struct netlink_ext_ack *extack)
3456 {
3457 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3458 	int err, new_offloads = priv->active_offloads;
3459 
3460 	if (!enetc_ptp_clock_is_enabled(priv->si))
3461 		return -EOPNOTSUPP;
3462 
3463 	switch (config->tx_type) {
3464 	case HWTSTAMP_TX_OFF:
3465 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
3466 		break;
3467 	case HWTSTAMP_TX_ON:
3468 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
3469 		new_offloads |= ENETC_F_TX_TSTAMP;
3470 		break;
3471 	case HWTSTAMP_TX_ONESTEP_SYNC:
3472 		if (!enetc_si_is_pf(priv->si) ||
3473 		    enetc_is_pseudo_mac(priv->si))
3474 			return -EOPNOTSUPP;
3475 
3476 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
3477 		new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
3478 		break;
3479 	default:
3480 		return -ERANGE;
3481 	}
3482 
3483 	switch (config->rx_filter) {
3484 	case HWTSTAMP_FILTER_NONE:
3485 		new_offloads &= ~ENETC_F_RX_TSTAMP;
3486 		break;
3487 	default:
3488 		new_offloads |= ENETC_F_RX_TSTAMP;
3489 		config->rx_filter = HWTSTAMP_FILTER_ALL;
3490 	}
3491 
3492 	if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {
3493 		bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);
3494 
3495 		err = enetc_reconfigure(priv, extended, NULL, NULL);
3496 		if (err)
3497 			return err;
3498 	}
3499 
3500 	priv->active_offloads = new_offloads;
3501 
3502 	return 0;
3503 }
3504 EXPORT_SYMBOL_GPL(enetc_hwtstamp_set);
3505 
3506 int enetc_hwtstamp_get(struct net_device *ndev,
3507 		       struct kernel_hwtstamp_config *config)
3508 {
3509 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3510 
3511 	if (!enetc_ptp_clock_is_enabled(priv->si))
3512 		return -EOPNOTSUPP;
3513 
3514 	if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
3515 		config->tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
3516 	else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
3517 		config->tx_type = HWTSTAMP_TX_ON;
3518 	else
3519 		config->tx_type = HWTSTAMP_TX_OFF;
3520 
3521 	config->rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
3522 			     HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
3523 
3524 	return 0;
3525 }
3526 EXPORT_SYMBOL_GPL(enetc_hwtstamp_get);
3527 
3528 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
3529 {
3530 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3531 
3532 	if (!priv->phylink)
3533 		return -EOPNOTSUPP;
3534 
3535 	return phylink_mii_ioctl(priv->phylink, rq, cmd);
3536 }
3537 EXPORT_SYMBOL_GPL(enetc_ioctl);
3538 
3539 static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i,
3540 				 int v_tx_rings)
3541 {
3542 	struct enetc_int_vector *v;
3543 	struct enetc_bdr *bdr;
3544 	int j, err;
3545 
3546 	v = kzalloc_flex(*v, tx_ring, v_tx_rings);
3547 	if (!v)
3548 		return -ENOMEM;
3549 
3550 	priv->int_vector[i] = v;
3551 	bdr = &v->rx_ring;
3552 	bdr->index = i;
3553 	bdr->ndev = priv->ndev;
3554 	bdr->dev = priv->dev;
3555 	bdr->bd_count = priv->rx_bd_count;
3556 	bdr->buffer_offset = ENETC_RXB_PAD;
3557 	priv->rx_ring[i] = bdr;
3558 
3559 	err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0,
3560 				 ENETC_RXB_TRUESIZE);
3561 	if (err)
3562 		goto free_vector;
3563 
3564 	err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, MEM_TYPE_PAGE_SHARED,
3565 					 NULL);
3566 	if (err) {
3567 		xdp_rxq_info_unreg(&bdr->xdp.rxq);
3568 		goto free_vector;
3569 	}
3570 
3571 	/* init defaults for adaptive IC */
3572 	if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
3573 		v->rx_ictt = 0x1;
3574 		v->rx_dim_en = true;
3575 	}
3576 
3577 	INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
3578 	netif_napi_add(priv->ndev, &v->napi, enetc_poll);
3579 	v->count_tx_rings = v_tx_rings;
3580 
3581 	for (j = 0; j < v_tx_rings; j++) {
3582 		int idx;
3583 
3584 		/* default tx ring mapping policy */
3585 		idx = priv->bdr_int_num * j + i;
3586 		__set_bit(idx, &v->tx_rings_map);
3587 		bdr = &v->tx_ring[j];
3588 		bdr->index = idx;
3589 		bdr->ndev = priv->ndev;
3590 		bdr->dev = priv->dev;
3591 		bdr->bd_count = priv->tx_bd_count;
3592 		priv->tx_ring[idx] = bdr;
3593 	}
3594 
3595 	return 0;
3596 
3597 free_vector:
3598 	priv->rx_ring[i] = NULL;
3599 	priv->int_vector[i] = NULL;
3600 	kfree(v);
3601 
3602 	return err;
3603 }
3604 
3605 static void enetc_int_vector_destroy(struct enetc_ndev_priv *priv, int i)
3606 {
3607 	struct enetc_int_vector *v = priv->int_vector[i];
3608 	struct enetc_bdr *rx_ring = &v->rx_ring;
3609 	int j, tx_ring_index;
3610 
3611 	xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3612 	xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3613 	netif_napi_del(&v->napi);
3614 	cancel_work_sync(&v->rx_dim.work);
3615 
3616 	for (j = 0; j < v->count_tx_rings; j++) {
3617 		tx_ring_index = priv->bdr_int_num * j + i;
3618 		priv->tx_ring[tx_ring_index] = NULL;
3619 	}
3620 
3621 	priv->rx_ring[i] = NULL;
3622 	priv->int_vector[i] = NULL;
3623 	kfree(v);
3624 }
3625 
3626 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
3627 {
3628 	struct pci_dev *pdev = priv->si->pdev;
3629 	int v_tx_rings, v_remainder;
3630 	int num_stack_tx_queues;
3631 	int first_xdp_tx_ring;
3632 	int i, n, err, nvec;
3633 
3634 	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
3635 	/* allocate MSIX for both messaging and Rx/Tx interrupts */
3636 	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
3637 
3638 	if (n < 0)
3639 		return n;
3640 
3641 	if (n != nvec)
3642 		return -EPERM;
3643 
3644 	/* # of tx rings per int vector */
3645 	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
3646 	v_remainder = priv->num_tx_rings % priv->bdr_int_num;
3647 
3648 	for (i = 0; i < priv->bdr_int_num; i++) {
3649 		/* Distribute the remaining TX rings to the first v_remainder
3650 		 * interrupt vectors
3651 		 */
3652 		int num_tx_rings = i < v_remainder ? v_tx_rings + 1 : v_tx_rings;
3653 
3654 		err = enetc_int_vector_init(priv, i, num_tx_rings);
3655 		if (err)
3656 			goto fail;
3657 	}
3658 
3659 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3660 
3661 	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3662 	if (err)
3663 		goto fail;
3664 
3665 	err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);
3666 	if (err)
3667 		goto fail;
3668 
3669 	priv->min_num_stack_tx_queues = num_possible_cpus();
3670 	first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
3671 	priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
3672 
3673 	return 0;
3674 
3675 fail:
3676 	while (i--)
3677 		enetc_int_vector_destroy(priv, i);
3678 
3679 	pci_free_irq_vectors(pdev);
3680 
3681 	return err;
3682 }
3683 EXPORT_SYMBOL_GPL(enetc_alloc_msix);
3684 
3685 void enetc_free_msix(struct enetc_ndev_priv *priv)
3686 {
3687 	int i;
3688 
3689 	for (i = 0; i < priv->bdr_int_num; i++)
3690 		enetc_int_vector_destroy(priv, i);
3691 
3692 	/* disable all MSIX for this device */
3693 	pci_free_irq_vectors(priv->si->pdev);
3694 }
3695 EXPORT_SYMBOL_GPL(enetc_free_msix);
3696 
3697 static void enetc_kfree_si(struct enetc_si *si)
3698 {
3699 	char *p = (char *)si - si->pad;
3700 
3701 	kfree(p);
3702 }
3703 
3704 static void enetc_detect_errata(struct enetc_si *si)
3705 {
3706 	if (si->pdev->revision == ENETC_REV1)
3707 		si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
3708 }
3709 
3710 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
3711 {
3712 	struct enetc_si *si, *p;
3713 	struct enetc_hw *hw;
3714 	size_t alloc_size;
3715 	int err, len;
3716 
3717 	pcie_flr(pdev);
3718 	err = pci_enable_device_mem(pdev);
3719 	if (err)
3720 		return dev_err_probe(&pdev->dev, err, "device enable failed\n");
3721 
3722 	/* set up for high or low dma */
3723 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3724 	if (err) {
3725 		dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3726 		goto err_dma;
3727 	}
3728 
3729 	err = pci_request_mem_regions(pdev, name);
3730 	if (err) {
3731 		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
3732 		goto err_pci_mem_reg;
3733 	}
3734 
3735 	pci_set_master(pdev);
3736 
3737 	alloc_size = sizeof(struct enetc_si);
3738 	if (sizeof_priv) {
3739 		/* align priv to 32B */
3740 		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
3741 		alloc_size += sizeof_priv;
3742 	}
3743 	/* force 32B alignment for enetc_si */
3744 	alloc_size += ENETC_SI_ALIGN - 1;
3745 
3746 	p = kzalloc(alloc_size, GFP_KERNEL);
3747 	if (!p) {
3748 		err = -ENOMEM;
3749 		goto err_alloc_si;
3750 	}
3751 
3752 	si = PTR_ALIGN(p, ENETC_SI_ALIGN);
3753 	si->pad = (char *)si - (char *)p;
3754 	spin_lock_init(&si->gen_lock);
3755 
3756 	pci_set_drvdata(pdev, si);
3757 	si->pdev = pdev;
3758 	hw = &si->hw;
3759 
3760 	len = pci_resource_len(pdev, ENETC_BAR_REGS);
3761 	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
3762 	if (!hw->reg) {
3763 		err = -ENXIO;
3764 		dev_err(&pdev->dev, "ioremap() failed\n");
3765 		goto err_ioremap;
3766 	}
3767 	if (len > ENETC_PORT_BASE)
3768 		hw->port = hw->reg + ENETC_PORT_BASE;
3769 	if (len > ENETC_GLOBAL_BASE)
3770 		hw->global = hw->reg + ENETC_GLOBAL_BASE;
3771 
3772 	enetc_detect_errata(si);
3773 
3774 	return 0;
3775 
3776 err_ioremap:
3777 	enetc_kfree_si(si);
3778 err_alloc_si:
3779 	pci_release_mem_regions(pdev);
3780 err_pci_mem_reg:
3781 err_dma:
3782 	pci_disable_device(pdev);
3783 
3784 	return err;
3785 }
3786 EXPORT_SYMBOL_GPL(enetc_pci_probe);
3787 
3788 void enetc_pci_remove(struct pci_dev *pdev)
3789 {
3790 	struct enetc_si *si = pci_get_drvdata(pdev);
3791 	struct enetc_hw *hw = &si->hw;
3792 
3793 	iounmap(hw->reg);
3794 	enetc_kfree_si(si);
3795 	pci_release_mem_regions(pdev);
3796 	pci_disable_device(pdev);
3797 }
3798 EXPORT_SYMBOL_GPL(enetc_pci_remove);
3799 
3800 static const struct enetc_drvdata enetc_pf_data = {
3801 	.sysclk_freq = ENETC_CLK_400M,
3802 	.pmac_offset = ENETC_PMAC_OFFSET,
3803 	.max_frags = ENETC_MAX_SKB_FRAGS,
3804 	.eth_ops = &enetc_pf_ethtool_ops,
3805 };
3806 
3807 static const struct enetc_drvdata enetc4_pf_data = {
3808 	.sysclk_freq = ENETC_CLK_333M,
3809 	.tx_csum = true,
3810 	.max_frags = ENETC4_MAX_SKB_FRAGS,
3811 	.pmac_offset = ENETC4_PMAC_OFFSET,
3812 	.eth_ops = &enetc4_pf_ethtool_ops,
3813 };
3814 
3815 static const struct enetc_drvdata enetc4_ppm_data = {
3816 	.sysclk_freq = ENETC_CLK_333M,
3817 	.tx_csum = true,
3818 	.max_frags = ENETC4_MAX_SKB_FRAGS,
3819 	.eth_ops = &enetc4_ppm_ethtool_ops,
3820 };
3821 
3822 static const struct enetc_drvdata enetc_vf_data = {
3823 	.sysclk_freq = ENETC_CLK_400M,
3824 	.max_frags = ENETC_MAX_SKB_FRAGS,
3825 	.eth_ops = &enetc_vf_ethtool_ops,
3826 };
3827 
3828 static const struct enetc_drvdata enetc4_vf_data = {
3829 	.sysclk_freq = ENETC_CLK_333M,
3830 	.tx_csum = true,
3831 	.max_frags = ENETC4_MAX_SKB_FRAGS,
3832 	.eth_ops = &enetc_vf_ethtool_ops,
3833 };
3834 
3835 static const struct enetc_platform_info enetc_info[] = {
3836 	{ .revision = ENETC_REV_1_0,
3837 	  .dev_id = ENETC_DEV_ID_PF,
3838 	  .data = &enetc_pf_data,
3839 	},
3840 	{ .revision = ENETC_REV_4_1,
3841 	  .dev_id = NXP_ENETC_PF_DEV_ID,
3842 	  .data = &enetc4_pf_data,
3843 	},
3844 	{ .revision = ENETC_REV_1_0,
3845 	  .dev_id = ENETC_DEV_ID_VF,
3846 	  .data = &enetc_vf_data,
3847 	},
3848 	{ .revision = ENETC_REV_4_1,
3849 	  .dev_id = NXP_ENETC_VF_DEV_ID,
3850 	  .data = &enetc4_vf_data,
3851 	},
3852 	{
3853 	  .revision = ENETC_REV_4_3,
3854 	  .dev_id = NXP_ENETC_PPM_DEV_ID,
3855 	  .data = &enetc4_ppm_data,
3856 	},
3857 	{ .revision = ENETC_REV_4_3,
3858 	  .dev_id = NXP_ENETC_PF_DEV_ID,
3859 	  .data = &enetc4_pf_data,
3860 	},
3861 	{ .revision = ENETC_REV_4_3,
3862 	  .dev_id = NXP_ENETC_VF_DEV_ID,
3863 	  .data = &enetc4_vf_data,
3864 	},
3865 };
3866 
3867 int enetc_get_driver_data(struct enetc_si *si)
3868 {
3869 	u16 dev_id = si->pdev->device;
3870 	int i;
3871 
3872 	for (i = 0; i < ARRAY_SIZE(enetc_info); i++) {
3873 		if (si->revision == enetc_info[i].revision &&
3874 		    dev_id == enetc_info[i].dev_id) {
3875 			si->drvdata = enetc_info[i].data;
3876 
3877 			return 0;
3878 		}
3879 	}
3880 
3881 	return -ERANGE;
3882 }
3883 EXPORT_SYMBOL_GPL(enetc_get_driver_data);
3884 
3885 MODULE_DESCRIPTION("NXP ENETC Ethernet driver");
3886 MODULE_LICENSE("Dual BSD/GPL");
3887