xref: /linux/drivers/net/ethernet/freescale/enetc/enetc.c (revision 7a08cb9b4bb92fb86f5fe8a3aa0ac08a9b3d783b)
1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3 
4 #include "enetc.h"
5 #include <linux/bpf_trace.h>
6 #include <linux/tcp.h>
7 #include <linux/udp.h>
8 #include <linux/vmalloc.h>
9 #include <linux/ptp_classify.h>
10 #include <net/ip6_checksum.h>
11 #include <net/pkt_sched.h>
12 #include <net/tso.h>
13 
14 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
15 {
16 	return enetc_port_rd(&si->hw, reg);
17 }
18 EXPORT_SYMBOL_GPL(enetc_port_mac_rd);
19 
20 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
21 {
22 	enetc_port_wr(&si->hw, reg, val);
23 	if (si->hw_features & ENETC_SI_F_QBU)
24 		enetc_port_wr(&si->hw, reg + ENETC_PMAC_OFFSET, val);
25 }
26 EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
27 
28 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
29 					 u8 preemptible_tcs)
30 {
31 	priv->preemptible_tcs = preemptible_tcs;
32 	enetc_mm_commit_preemptible_tcs(priv);
33 }
34 
35 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
36 {
37 	int num_tx_rings = priv->num_tx_rings;
38 
39 	if (priv->xdp_prog)
40 		return num_tx_rings - num_possible_cpus();
41 
42 	return num_tx_rings;
43 }
44 
45 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
46 							struct enetc_bdr *tx_ring)
47 {
48 	int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
49 
50 	return priv->rx_ring[index];
51 }
52 
53 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
54 {
55 	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
56 		return NULL;
57 
58 	return tx_swbd->skb;
59 }
60 
61 static struct xdp_frame *
62 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
63 {
64 	if (tx_swbd->is_xdp_redirect)
65 		return tx_swbd->xdp_frame;
66 
67 	return NULL;
68 }
69 
70 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
71 				struct enetc_tx_swbd *tx_swbd)
72 {
73 	/* For XDP_TX, pages come from RX, whereas for the other contexts where
74 	 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
75 	 * to match the DMA mapping length, so we need to differentiate those.
76 	 */
77 	if (tx_swbd->is_dma_page)
78 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,
79 			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
80 			       tx_swbd->dir);
81 	else
82 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,
83 				 tx_swbd->len, tx_swbd->dir);
84 	tx_swbd->dma = 0;
85 }
86 
87 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
88 				struct enetc_tx_swbd *tx_swbd)
89 {
90 	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
91 	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
92 
93 	if (tx_swbd->dma)
94 		enetc_unmap_tx_buff(tx_ring, tx_swbd);
95 
96 	if (xdp_frame) {
97 		xdp_return_frame(tx_swbd->xdp_frame);
98 		tx_swbd->xdp_frame = NULL;
99 	} else if (skb) {
100 		dev_kfree_skb_any(skb);
101 		tx_swbd->skb = NULL;
102 	}
103 }
104 
105 /* Let H/W know BD ring has been updated */
106 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
107 {
108 	/* includes wmb() */
109 	enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
110 }
111 
112 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
113 			   u8 *msgtype, u8 *twostep,
114 			   u16 *correction_offset, u16 *body_offset)
115 {
116 	unsigned int ptp_class;
117 	struct ptp_header *hdr;
118 	unsigned int type;
119 	u8 *base;
120 
121 	ptp_class = ptp_classify_raw(skb);
122 	if (ptp_class == PTP_CLASS_NONE)
123 		return -EINVAL;
124 
125 	hdr = ptp_parse_header(skb, ptp_class);
126 	if (!hdr)
127 		return -EINVAL;
128 
129 	type = ptp_class & PTP_CLASS_PMASK;
130 	if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
131 		*udp = 1;
132 	else
133 		*udp = 0;
134 
135 	*msgtype = ptp_get_msgtype(hdr, ptp_class);
136 	*twostep = hdr->flag_field[0] & 0x2;
137 
138 	base = skb_mac_header(skb);
139 	*correction_offset = (u8 *)&hdr->correction - base;
140 	*body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
141 
142 	return 0;
143 }
144 
145 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
146 {
147 	bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
148 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
149 	struct enetc_hw *hw = &priv->si->hw;
150 	struct enetc_tx_swbd *tx_swbd;
151 	int len = skb_headlen(skb);
152 	union enetc_tx_bd temp_bd;
153 	u8 msgtype, twostep, udp;
154 	union enetc_tx_bd *txbd;
155 	u16 offset1, offset2;
156 	int i, count = 0;
157 	skb_frag_t *frag;
158 	unsigned int f;
159 	dma_addr_t dma;
160 	u8 flags = 0;
161 
162 	i = tx_ring->next_to_use;
163 	txbd = ENETC_TXBD(*tx_ring, i);
164 	prefetchw(txbd);
165 
166 	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
167 	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
168 		goto dma_err;
169 
170 	temp_bd.addr = cpu_to_le64(dma);
171 	temp_bd.buf_len = cpu_to_le16(len);
172 	temp_bd.lstatus = 0;
173 
174 	tx_swbd = &tx_ring->tx_swbd[i];
175 	tx_swbd->dma = dma;
176 	tx_swbd->len = len;
177 	tx_swbd->is_dma_page = 0;
178 	tx_swbd->dir = DMA_TO_DEVICE;
179 	count++;
180 
181 	do_vlan = skb_vlan_tag_present(skb);
182 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
183 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
184 				    &offset2) ||
185 		    msgtype != PTP_MSGTYPE_SYNC || twostep)
186 			WARN_ONCE(1, "Bad packet for one-step timestamping\n");
187 		else
188 			do_onestep_tstamp = true;
189 	} else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
190 		do_twostep_tstamp = true;
191 	}
192 
193 	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
194 	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
195 	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
196 
197 	if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
198 		flags |= ENETC_TXBD_FLAGS_EX;
199 
200 	if (tx_ring->tsd_enable)
201 		flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
202 
203 	/* first BD needs frm_len and offload flags set */
204 	temp_bd.frm_len = cpu_to_le16(skb->len);
205 	temp_bd.flags = flags;
206 
207 	if (flags & ENETC_TXBD_FLAGS_TSE)
208 		temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
209 							  flags);
210 
211 	if (flags & ENETC_TXBD_FLAGS_EX) {
212 		u8 e_flags = 0;
213 		*txbd = temp_bd;
214 		enetc_clear_tx_bd(&temp_bd);
215 
216 		/* add extension BD for VLAN and/or timestamping */
217 		flags = 0;
218 		tx_swbd++;
219 		txbd++;
220 		i++;
221 		if (unlikely(i == tx_ring->bd_count)) {
222 			i = 0;
223 			tx_swbd = tx_ring->tx_swbd;
224 			txbd = ENETC_TXBD(*tx_ring, 0);
225 		}
226 		prefetchw(txbd);
227 
228 		if (do_vlan) {
229 			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
230 			temp_bd.ext.tpid = 0; /* < C-TAG */
231 			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
232 		}
233 
234 		if (do_onestep_tstamp) {
235 			u32 lo, hi, val;
236 			u64 sec, nsec;
237 			u8 *data;
238 
239 			lo = enetc_rd_hot(hw, ENETC_SICTR0);
240 			hi = enetc_rd_hot(hw, ENETC_SICTR1);
241 			sec = (u64)hi << 32 | lo;
242 			nsec = do_div(sec, 1000000000);
243 
244 			/* Configure extension BD */
245 			temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
246 			e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
247 
248 			/* Update originTimestamp field of Sync packet
249 			 * - 48 bits seconds field
250 			 * - 32 bits nanseconds field
251 			 */
252 			data = skb_mac_header(skb);
253 			*(__be16 *)(data + offset2) =
254 				htons((sec >> 32) & 0xffff);
255 			*(__be32 *)(data + offset2 + 2) =
256 				htonl(sec & 0xffffffff);
257 			*(__be32 *)(data + offset2 + 6) = htonl(nsec);
258 
259 			/* Configure single-step register */
260 			val = ENETC_PM0_SINGLE_STEP_EN;
261 			val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
262 			if (udp)
263 				val |= ENETC_PM0_SINGLE_STEP_CH;
264 
265 			enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP,
266 					  val);
267 		} else if (do_twostep_tstamp) {
268 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
269 			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
270 		}
271 
272 		temp_bd.ext.e_flags = e_flags;
273 		count++;
274 	}
275 
276 	frag = &skb_shinfo(skb)->frags[0];
277 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
278 		len = skb_frag_size(frag);
279 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
280 				       DMA_TO_DEVICE);
281 		if (dma_mapping_error(tx_ring->dev, dma))
282 			goto dma_err;
283 
284 		*txbd = temp_bd;
285 		enetc_clear_tx_bd(&temp_bd);
286 
287 		flags = 0;
288 		tx_swbd++;
289 		txbd++;
290 		i++;
291 		if (unlikely(i == tx_ring->bd_count)) {
292 			i = 0;
293 			tx_swbd = tx_ring->tx_swbd;
294 			txbd = ENETC_TXBD(*tx_ring, 0);
295 		}
296 		prefetchw(txbd);
297 
298 		temp_bd.addr = cpu_to_le64(dma);
299 		temp_bd.buf_len = cpu_to_le16(len);
300 
301 		tx_swbd->dma = dma;
302 		tx_swbd->len = len;
303 		tx_swbd->is_dma_page = 1;
304 		tx_swbd->dir = DMA_TO_DEVICE;
305 		count++;
306 	}
307 
308 	/* last BD needs 'F' bit set */
309 	flags |= ENETC_TXBD_FLAGS_F;
310 	temp_bd.flags = flags;
311 	*txbd = temp_bd;
312 
313 	tx_ring->tx_swbd[i].is_eof = true;
314 	tx_ring->tx_swbd[i].skb = skb;
315 
316 	enetc_bdr_idx_inc(tx_ring, &i);
317 	tx_ring->next_to_use = i;
318 
319 	skb_tx_timestamp(skb);
320 
321 	enetc_update_tx_ring_tail(tx_ring);
322 
323 	return count;
324 
325 dma_err:
326 	dev_err(tx_ring->dev, "DMA map error");
327 
328 	do {
329 		tx_swbd = &tx_ring->tx_swbd[i];
330 		enetc_free_tx_frame(tx_ring, tx_swbd);
331 		if (i == 0)
332 			i = tx_ring->bd_count;
333 		i--;
334 	} while (count--);
335 
336 	return 0;
337 }
338 
339 static void enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
340 				 struct enetc_tx_swbd *tx_swbd,
341 				 union enetc_tx_bd *txbd, int *i, int hdr_len,
342 				 int data_len)
343 {
344 	union enetc_tx_bd txbd_tmp;
345 	u8 flags = 0, e_flags = 0;
346 	dma_addr_t addr;
347 
348 	enetc_clear_tx_bd(&txbd_tmp);
349 	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
350 
351 	if (skb_vlan_tag_present(skb))
352 		flags |= ENETC_TXBD_FLAGS_EX;
353 
354 	txbd_tmp.addr = cpu_to_le64(addr);
355 	txbd_tmp.buf_len = cpu_to_le16(hdr_len);
356 
357 	/* first BD needs frm_len and offload flags set */
358 	txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
359 	txbd_tmp.flags = flags;
360 
361 	/* For the TSO header we do not set the dma address since we do not
362 	 * want it unmapped when we do cleanup. We still set len so that we
363 	 * count the bytes sent.
364 	 */
365 	tx_swbd->len = hdr_len;
366 	tx_swbd->do_twostep_tstamp = false;
367 	tx_swbd->check_wb = false;
368 
369 	/* Actually write the header in the BD */
370 	*txbd = txbd_tmp;
371 
372 	/* Add extension BD for VLAN */
373 	if (flags & ENETC_TXBD_FLAGS_EX) {
374 		/* Get the next BD */
375 		enetc_bdr_idx_inc(tx_ring, i);
376 		txbd = ENETC_TXBD(*tx_ring, *i);
377 		tx_swbd = &tx_ring->tx_swbd[*i];
378 		prefetchw(txbd);
379 
380 		/* Setup the VLAN fields */
381 		enetc_clear_tx_bd(&txbd_tmp);
382 		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
383 		txbd_tmp.ext.tpid = 0; /* < C-TAG */
384 		e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
385 
386 		/* Write the BD */
387 		txbd_tmp.ext.e_flags = e_flags;
388 		*txbd = txbd_tmp;
389 	}
390 }
391 
392 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
393 				 struct enetc_tx_swbd *tx_swbd,
394 				 union enetc_tx_bd *txbd, char *data,
395 				 int size, bool last_bd)
396 {
397 	union enetc_tx_bd txbd_tmp;
398 	dma_addr_t addr;
399 	u8 flags = 0;
400 
401 	enetc_clear_tx_bd(&txbd_tmp);
402 
403 	addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
404 	if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
405 		netdev_err(tx_ring->ndev, "DMA map error\n");
406 		return -ENOMEM;
407 	}
408 
409 	if (last_bd) {
410 		flags |= ENETC_TXBD_FLAGS_F;
411 		tx_swbd->is_eof = 1;
412 	}
413 
414 	txbd_tmp.addr = cpu_to_le64(addr);
415 	txbd_tmp.buf_len = cpu_to_le16(size);
416 	txbd_tmp.flags = flags;
417 
418 	tx_swbd->dma = addr;
419 	tx_swbd->len = size;
420 	tx_swbd->dir = DMA_TO_DEVICE;
421 
422 	*txbd = txbd_tmp;
423 
424 	return 0;
425 }
426 
427 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
428 				 char *hdr, int hdr_len, int *l4_hdr_len)
429 {
430 	char *l4_hdr = hdr + skb_transport_offset(skb);
431 	int mac_hdr_len = skb_network_offset(skb);
432 
433 	if (tso->tlen != sizeof(struct udphdr)) {
434 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
435 
436 		tcph->check = 0;
437 	} else {
438 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
439 
440 		udph->check = 0;
441 	}
442 
443 	/* Compute the IP checksum. This is necessary since tso_build_hdr()
444 	 * already incremented the IP ID field.
445 	 */
446 	if (!tso->ipv6) {
447 		struct iphdr *iph = (void *)(hdr + mac_hdr_len);
448 
449 		iph->check = 0;
450 		iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
451 	}
452 
453 	/* Compute the checksum over the L4 header. */
454 	*l4_hdr_len = hdr_len - skb_transport_offset(skb);
455 	return csum_partial(l4_hdr, *l4_hdr_len, 0);
456 }
457 
458 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
459 				    struct sk_buff *skb, char *hdr, int len,
460 				    __wsum sum)
461 {
462 	char *l4_hdr = hdr + skb_transport_offset(skb);
463 	__sum16 csum_final;
464 
465 	/* Complete the L4 checksum by appending the pseudo-header to the
466 	 * already computed checksum.
467 	 */
468 	if (!tso->ipv6)
469 		csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
470 					       ip_hdr(skb)->daddr,
471 					       len, ip_hdr(skb)->protocol, sum);
472 	else
473 		csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
474 					     &ipv6_hdr(skb)->daddr,
475 					     len, ipv6_hdr(skb)->nexthdr, sum);
476 
477 	if (tso->tlen != sizeof(struct udphdr)) {
478 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
479 
480 		tcph->check = csum_final;
481 	} else {
482 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
483 
484 		udph->check = csum_final;
485 	}
486 }
487 
488 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
489 {
490 	int hdr_len, total_len, data_len;
491 	struct enetc_tx_swbd *tx_swbd;
492 	union enetc_tx_bd *txbd;
493 	struct tso_t tso;
494 	__wsum csum, csum2;
495 	int count = 0, pos;
496 	int err, i, bd_data_num;
497 
498 	/* Initialize the TSO handler, and prepare the first payload */
499 	hdr_len = tso_start(skb, &tso);
500 	total_len = skb->len - hdr_len;
501 	i = tx_ring->next_to_use;
502 
503 	while (total_len > 0) {
504 		char *hdr;
505 
506 		/* Get the BD */
507 		txbd = ENETC_TXBD(*tx_ring, i);
508 		tx_swbd = &tx_ring->tx_swbd[i];
509 		prefetchw(txbd);
510 
511 		/* Determine the length of this packet */
512 		data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
513 		total_len -= data_len;
514 
515 		/* prepare packet headers: MAC + IP + TCP */
516 		hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
517 		tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
518 
519 		/* compute the csum over the L4 header */
520 		csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
521 		enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd, &i, hdr_len, data_len);
522 		bd_data_num = 0;
523 		count++;
524 
525 		while (data_len > 0) {
526 			int size;
527 
528 			size = min_t(int, tso.size, data_len);
529 
530 			/* Advance the index in the BDR */
531 			enetc_bdr_idx_inc(tx_ring, &i);
532 			txbd = ENETC_TXBD(*tx_ring, i);
533 			tx_swbd = &tx_ring->tx_swbd[i];
534 			prefetchw(txbd);
535 
536 			/* Compute the checksum over this segment of data and
537 			 * add it to the csum already computed (over the L4
538 			 * header and possible other data segments).
539 			 */
540 			csum2 = csum_partial(tso.data, size, 0);
541 			csum = csum_block_add(csum, csum2, pos);
542 			pos += size;
543 
544 			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
545 						    tso.data, size,
546 						    size == data_len);
547 			if (err)
548 				goto err_map_data;
549 
550 			data_len -= size;
551 			count++;
552 			bd_data_num++;
553 			tso_build_data(skb, &tso, size);
554 
555 			if (unlikely(bd_data_num >= ENETC_MAX_SKB_FRAGS && data_len))
556 				goto err_chained_bd;
557 		}
558 
559 		enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
560 
561 		if (total_len == 0)
562 			tx_swbd->skb = skb;
563 
564 		/* Go to the next BD */
565 		enetc_bdr_idx_inc(tx_ring, &i);
566 	}
567 
568 	tx_ring->next_to_use = i;
569 	enetc_update_tx_ring_tail(tx_ring);
570 
571 	return count;
572 
573 err_map_data:
574 	dev_err(tx_ring->dev, "DMA map error");
575 
576 err_chained_bd:
577 	do {
578 		tx_swbd = &tx_ring->tx_swbd[i];
579 		enetc_free_tx_frame(tx_ring, tx_swbd);
580 		if (i == 0)
581 			i = tx_ring->bd_count;
582 		i--;
583 	} while (count--);
584 
585 	return 0;
586 }
587 
588 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
589 				    struct net_device *ndev)
590 {
591 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
592 	struct enetc_bdr *tx_ring;
593 	int count, err;
594 
595 	/* Queue one-step Sync packet if already locked */
596 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
597 		if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
598 					  &priv->flags)) {
599 			skb_queue_tail(&priv->tx_skbs, skb);
600 			return NETDEV_TX_OK;
601 		}
602 	}
603 
604 	tx_ring = priv->tx_ring[skb->queue_mapping];
605 
606 	if (skb_is_gso(skb)) {
607 		if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
608 			netif_stop_subqueue(ndev, tx_ring->index);
609 			return NETDEV_TX_BUSY;
610 		}
611 
612 		enetc_lock_mdio();
613 		count = enetc_map_tx_tso_buffs(tx_ring, skb);
614 		enetc_unlock_mdio();
615 	} else {
616 		if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
617 			if (unlikely(skb_linearize(skb)))
618 				goto drop_packet_err;
619 
620 		count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
621 		if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
622 			netif_stop_subqueue(ndev, tx_ring->index);
623 			return NETDEV_TX_BUSY;
624 		}
625 
626 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
627 			err = skb_checksum_help(skb);
628 			if (err)
629 				goto drop_packet_err;
630 		}
631 		enetc_lock_mdio();
632 		count = enetc_map_tx_buffs(tx_ring, skb);
633 		enetc_unlock_mdio();
634 	}
635 
636 	if (unlikely(!count))
637 		goto drop_packet_err;
638 
639 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
640 		netif_stop_subqueue(ndev, tx_ring->index);
641 
642 	return NETDEV_TX_OK;
643 
644 drop_packet_err:
645 	dev_kfree_skb_any(skb);
646 	return NETDEV_TX_OK;
647 }
648 
649 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
650 {
651 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
652 	u8 udp, msgtype, twostep;
653 	u16 offset1, offset2;
654 
655 	/* Mark tx timestamp type on skb->cb[0] if requires */
656 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
657 	    (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
658 		skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
659 	} else {
660 		skb->cb[0] = 0;
661 	}
662 
663 	/* Fall back to two-step timestamp if not one-step Sync packet */
664 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
665 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
666 				    &offset1, &offset2) ||
667 		    msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
668 			skb->cb[0] = ENETC_F_TX_TSTAMP;
669 	}
670 
671 	return enetc_start_xmit(skb, ndev);
672 }
673 EXPORT_SYMBOL_GPL(enetc_xmit);
674 
675 static irqreturn_t enetc_msix(int irq, void *data)
676 {
677 	struct enetc_int_vector	*v = data;
678 	int i;
679 
680 	enetc_lock_mdio();
681 
682 	/* disable interrupts */
683 	enetc_wr_reg_hot(v->rbier, 0);
684 	enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
685 
686 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
687 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
688 
689 	enetc_unlock_mdio();
690 
691 	napi_schedule(&v->napi);
692 
693 	return IRQ_HANDLED;
694 }
695 
696 static void enetc_rx_dim_work(struct work_struct *w)
697 {
698 	struct dim *dim = container_of(w, struct dim, work);
699 	struct dim_cq_moder moder =
700 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
701 	struct enetc_int_vector	*v =
702 		container_of(dim, struct enetc_int_vector, rx_dim);
703 
704 	v->rx_ictt = enetc_usecs_to_cycles(moder.usec);
705 	dim->state = DIM_START_MEASURE;
706 }
707 
708 static void enetc_rx_net_dim(struct enetc_int_vector *v)
709 {
710 	struct dim_sample dim_sample = {};
711 
712 	v->comp_cnt++;
713 
714 	if (!v->rx_napi_work)
715 		return;
716 
717 	dim_update_sample(v->comp_cnt,
718 			  v->rx_ring.stats.packets,
719 			  v->rx_ring.stats.bytes,
720 			  &dim_sample);
721 	net_dim(&v->rx_dim, dim_sample);
722 }
723 
724 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
725 {
726 	int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
727 
728 	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
729 }
730 
731 static bool enetc_page_reusable(struct page *page)
732 {
733 	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
734 }
735 
736 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
737 			     struct enetc_rx_swbd *old)
738 {
739 	struct enetc_rx_swbd *new;
740 
741 	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
742 
743 	/* next buf that may reuse a page */
744 	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
745 
746 	/* copy page reference */
747 	*new = *old;
748 }
749 
750 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
751 				u64 *tstamp)
752 {
753 	u32 lo, hi, tstamp_lo;
754 
755 	lo = enetc_rd_hot(hw, ENETC_SICTR0);
756 	hi = enetc_rd_hot(hw, ENETC_SICTR1);
757 	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
758 	if (lo <= tstamp_lo)
759 		hi -= 1;
760 	*tstamp = (u64)hi << 32 | tstamp_lo;
761 }
762 
763 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
764 {
765 	struct skb_shared_hwtstamps shhwtstamps;
766 
767 	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
768 		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
769 		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
770 		skb_txtime_consumed(skb);
771 		skb_tstamp_tx(skb, &shhwtstamps);
772 	}
773 }
774 
775 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
776 				      struct enetc_tx_swbd *tx_swbd)
777 {
778 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
779 	struct enetc_rx_swbd rx_swbd = {
780 		.dma = tx_swbd->dma,
781 		.page = tx_swbd->page,
782 		.page_offset = tx_swbd->page_offset,
783 		.dir = tx_swbd->dir,
784 		.len = tx_swbd->len,
785 	};
786 	struct enetc_bdr *rx_ring;
787 
788 	rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
789 
790 	if (likely(enetc_swbd_unused(rx_ring))) {
791 		enetc_reuse_page(rx_ring, &rx_swbd);
792 
793 		/* sync for use by the device */
794 		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
795 						 rx_swbd.page_offset,
796 						 ENETC_RXB_DMA_SIZE_XDP,
797 						 rx_swbd.dir);
798 
799 		rx_ring->stats.recycles++;
800 	} else {
801 		/* RX ring is already full, we need to unmap and free the
802 		 * page, since there's nothing useful we can do with it.
803 		 */
804 		rx_ring->stats.recycle_failures++;
805 
806 		dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
807 			       rx_swbd.dir);
808 		__free_page(rx_swbd.page);
809 	}
810 
811 	rx_ring->xdp.xdp_tx_in_flight--;
812 }
813 
814 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
815 {
816 	int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
817 	struct net_device *ndev = tx_ring->ndev;
818 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
819 	struct enetc_tx_swbd *tx_swbd;
820 	int i, bds_to_clean;
821 	bool do_twostep_tstamp;
822 	u64 tstamp = 0;
823 
824 	i = tx_ring->next_to_clean;
825 	tx_swbd = &tx_ring->tx_swbd[i];
826 
827 	bds_to_clean = enetc_bd_ready_count(tx_ring, i);
828 
829 	do_twostep_tstamp = false;
830 
831 	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
832 		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
833 		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
834 		bool is_eof = tx_swbd->is_eof;
835 
836 		if (unlikely(tx_swbd->check_wb)) {
837 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
838 
839 			if (txbd->flags & ENETC_TXBD_FLAGS_W &&
840 			    tx_swbd->do_twostep_tstamp) {
841 				enetc_get_tx_tstamp(&priv->si->hw, txbd,
842 						    &tstamp);
843 				do_twostep_tstamp = true;
844 			}
845 
846 			if (tx_swbd->qbv_en &&
847 			    txbd->wb.status & ENETC_TXBD_STATS_WIN)
848 				tx_win_drop++;
849 		}
850 
851 		if (tx_swbd->is_xdp_tx)
852 			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
853 		else if (likely(tx_swbd->dma))
854 			enetc_unmap_tx_buff(tx_ring, tx_swbd);
855 
856 		if (xdp_frame) {
857 			xdp_return_frame(xdp_frame);
858 		} else if (skb) {
859 			if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
860 				/* Start work to release lock for next one-step
861 				 * timestamping packet. And send one skb in
862 				 * tx_skbs queue if has.
863 				 */
864 				schedule_work(&priv->tx_onestep_tstamp);
865 			} else if (unlikely(do_twostep_tstamp)) {
866 				enetc_tstamp_tx(skb, tstamp);
867 				do_twostep_tstamp = false;
868 			}
869 			napi_consume_skb(skb, napi_budget);
870 		}
871 
872 		tx_byte_cnt += tx_swbd->len;
873 		/* Scrub the swbd here so we don't have to do that
874 		 * when we reuse it during xmit
875 		 */
876 		memset(tx_swbd, 0, sizeof(*tx_swbd));
877 
878 		bds_to_clean--;
879 		tx_swbd++;
880 		i++;
881 		if (unlikely(i == tx_ring->bd_count)) {
882 			i = 0;
883 			tx_swbd = tx_ring->tx_swbd;
884 		}
885 
886 		/* BD iteration loop end */
887 		if (is_eof) {
888 			tx_frm_cnt++;
889 			/* re-arm interrupt source */
890 			enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
891 					 BIT(16 + tx_ring->index));
892 		}
893 
894 		if (unlikely(!bds_to_clean))
895 			bds_to_clean = enetc_bd_ready_count(tx_ring, i);
896 	}
897 
898 	tx_ring->next_to_clean = i;
899 	tx_ring->stats.packets += tx_frm_cnt;
900 	tx_ring->stats.bytes += tx_byte_cnt;
901 	tx_ring->stats.win_drop += tx_win_drop;
902 
903 	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
904 		     __netif_subqueue_stopped(ndev, tx_ring->index) &&
905 		     (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
906 		netif_wake_subqueue(ndev, tx_ring->index);
907 	}
908 
909 	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
910 }
911 
912 static bool enetc_new_page(struct enetc_bdr *rx_ring,
913 			   struct enetc_rx_swbd *rx_swbd)
914 {
915 	bool xdp = !!(rx_ring->xdp.prog);
916 	struct page *page;
917 	dma_addr_t addr;
918 
919 	page = dev_alloc_page();
920 	if (unlikely(!page))
921 		return false;
922 
923 	/* For XDP_TX, we forgo dma_unmap -> dma_map */
924 	rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
925 
926 	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
927 	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
928 		__free_page(page);
929 
930 		return false;
931 	}
932 
933 	rx_swbd->dma = addr;
934 	rx_swbd->page = page;
935 	rx_swbd->page_offset = rx_ring->buffer_offset;
936 
937 	return true;
938 }
939 
940 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
941 {
942 	struct enetc_rx_swbd *rx_swbd;
943 	union enetc_rx_bd *rxbd;
944 	int i, j;
945 
946 	i = rx_ring->next_to_use;
947 	rx_swbd = &rx_ring->rx_swbd[i];
948 	rxbd = enetc_rxbd(rx_ring, i);
949 
950 	for (j = 0; j < buff_cnt; j++) {
951 		/* try reuse page */
952 		if (unlikely(!rx_swbd->page)) {
953 			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
954 				rx_ring->stats.rx_alloc_errs++;
955 				break;
956 			}
957 		}
958 
959 		/* update RxBD */
960 		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
961 					   rx_swbd->page_offset);
962 		/* clear 'R" as well */
963 		rxbd->r.lstatus = 0;
964 
965 		enetc_rxbd_next(rx_ring, &rxbd, &i);
966 		rx_swbd = &rx_ring->rx_swbd[i];
967 	}
968 
969 	if (likely(j)) {
970 		rx_ring->next_to_alloc = i; /* keep track from page reuse */
971 		rx_ring->next_to_use = i;
972 
973 		/* update ENETC's consumer index */
974 		enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
975 	}
976 
977 	return j;
978 }
979 
980 static void enetc_get_rx_tstamp(struct net_device *ndev,
981 				union enetc_rx_bd *rxbd,
982 				struct sk_buff *skb)
983 {
984 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
985 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
986 	struct enetc_hw *hw = &priv->si->hw;
987 	u32 lo, hi, tstamp_lo;
988 	u64 tstamp;
989 
990 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
991 		lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
992 		hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
993 		rxbd = enetc_rxbd_ext(rxbd);
994 		tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
995 		if (lo <= tstamp_lo)
996 			hi -= 1;
997 
998 		tstamp = (u64)hi << 32 | tstamp_lo;
999 		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
1000 		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
1001 	}
1002 }
1003 
1004 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
1005 			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
1006 {
1007 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1008 
1009 	/* TODO: hashing */
1010 	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
1011 		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
1012 
1013 		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
1014 		skb->ip_summed = CHECKSUM_COMPLETE;
1015 	}
1016 
1017 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1018 		__be16 tpid = 0;
1019 
1020 		switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1021 		case 0:
1022 			tpid = htons(ETH_P_8021Q);
1023 			break;
1024 		case 1:
1025 			tpid = htons(ETH_P_8021AD);
1026 			break;
1027 		case 2:
1028 			tpid = htons(enetc_port_rd(&priv->si->hw,
1029 						   ENETC_PCVLANR1));
1030 			break;
1031 		case 3:
1032 			tpid = htons(enetc_port_rd(&priv->si->hw,
1033 						   ENETC_PCVLANR2));
1034 			break;
1035 		default:
1036 			break;
1037 		}
1038 
1039 		__vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1040 	}
1041 
1042 	if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK) &&
1043 	    (priv->active_offloads & ENETC_F_RX_TSTAMP))
1044 		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1045 }
1046 
1047 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1048  * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1049  * mapped buffers.
1050  */
1051 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1052 					       int i, u16 size)
1053 {
1054 	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1055 
1056 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1057 				      rx_swbd->page_offset,
1058 				      size, rx_swbd->dir);
1059 	return rx_swbd;
1060 }
1061 
1062 /* Reuse the current page without performing half-page buffer flipping */
1063 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1064 			      struct enetc_rx_swbd *rx_swbd)
1065 {
1066 	size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1067 
1068 	enetc_reuse_page(rx_ring, rx_swbd);
1069 
1070 	dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1071 					 rx_swbd->page_offset,
1072 					 buffer_size, rx_swbd->dir);
1073 
1074 	rx_swbd->page = NULL;
1075 }
1076 
1077 /* Reuse the current page by performing half-page buffer flipping */
1078 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1079 			       struct enetc_rx_swbd *rx_swbd)
1080 {
1081 	if (likely(enetc_page_reusable(rx_swbd->page))) {
1082 		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1083 		page_ref_inc(rx_swbd->page);
1084 
1085 		enetc_put_rx_buff(rx_ring, rx_swbd);
1086 	} else {
1087 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1088 			       rx_swbd->dir);
1089 		rx_swbd->page = NULL;
1090 	}
1091 }
1092 
1093 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1094 						int i, u16 size)
1095 {
1096 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1097 	struct sk_buff *skb;
1098 	void *ba;
1099 
1100 	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1101 	skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1102 	if (unlikely(!skb)) {
1103 		rx_ring->stats.rx_alloc_errs++;
1104 		return NULL;
1105 	}
1106 
1107 	skb_reserve(skb, rx_ring->buffer_offset);
1108 	__skb_put(skb, size);
1109 
1110 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1111 
1112 	return skb;
1113 }
1114 
1115 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1116 				     u16 size, struct sk_buff *skb)
1117 {
1118 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1119 
1120 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1121 			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1122 
1123 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1124 }
1125 
1126 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1127 					      u32 bd_status,
1128 					      union enetc_rx_bd **rxbd, int *i)
1129 {
1130 	if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1131 		return false;
1132 
1133 	enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1134 	enetc_rxbd_next(rx_ring, rxbd, i);
1135 
1136 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1137 		dma_rmb();
1138 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1139 
1140 		enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1141 		enetc_rxbd_next(rx_ring, rxbd, i);
1142 	}
1143 
1144 	rx_ring->ndev->stats.rx_dropped++;
1145 	rx_ring->ndev->stats.rx_errors++;
1146 
1147 	return true;
1148 }
1149 
1150 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1151 				       u32 bd_status, union enetc_rx_bd **rxbd,
1152 				       int *i, int *cleaned_cnt, int buffer_size)
1153 {
1154 	struct sk_buff *skb;
1155 	u16 size;
1156 
1157 	size = le16_to_cpu((*rxbd)->r.buf_len);
1158 	skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1159 	if (!skb)
1160 		return NULL;
1161 
1162 	enetc_get_offloads(rx_ring, *rxbd, skb);
1163 
1164 	(*cleaned_cnt)++;
1165 
1166 	enetc_rxbd_next(rx_ring, rxbd, i);
1167 
1168 	/* not last BD in frame? */
1169 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1170 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1171 		size = buffer_size;
1172 
1173 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1174 			dma_rmb();
1175 			size = le16_to_cpu((*rxbd)->r.buf_len);
1176 		}
1177 
1178 		enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1179 
1180 		(*cleaned_cnt)++;
1181 
1182 		enetc_rxbd_next(rx_ring, rxbd, i);
1183 	}
1184 
1185 	skb_record_rx_queue(skb, rx_ring->index);
1186 	skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1187 
1188 	return skb;
1189 }
1190 
1191 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1192 
1193 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1194 			       struct napi_struct *napi, int work_limit)
1195 {
1196 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1197 	int cleaned_cnt, i;
1198 
1199 	cleaned_cnt = enetc_bd_unused(rx_ring);
1200 	/* next descriptor to process */
1201 	i = rx_ring->next_to_clean;
1202 
1203 	while (likely(rx_frm_cnt < work_limit)) {
1204 		union enetc_rx_bd *rxbd;
1205 		struct sk_buff *skb;
1206 		u32 bd_status;
1207 
1208 		if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1209 			cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1210 							    cleaned_cnt);
1211 
1212 		rxbd = enetc_rxbd(rx_ring, i);
1213 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1214 		if (!bd_status)
1215 			break;
1216 
1217 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1218 		dma_rmb(); /* for reading other rxbd fields */
1219 
1220 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1221 						      &rxbd, &i))
1222 			break;
1223 
1224 		skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1225 				      &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1226 		if (!skb)
1227 			break;
1228 
1229 		/* When set, the outer VLAN header is extracted and reported
1230 		 * in the receive buffer descriptor. So rx_byte_cnt should
1231 		 * add the length of the extracted VLAN header.
1232 		 */
1233 		if (bd_status & ENETC_RXBD_FLAG_VLAN)
1234 			rx_byte_cnt += VLAN_HLEN;
1235 		rx_byte_cnt += skb->len + ETH_HLEN;
1236 		rx_frm_cnt++;
1237 
1238 		napi_gro_receive(napi, skb);
1239 	}
1240 
1241 	rx_ring->next_to_clean = i;
1242 
1243 	rx_ring->stats.packets += rx_frm_cnt;
1244 	rx_ring->stats.bytes += rx_byte_cnt;
1245 
1246 	return rx_frm_cnt;
1247 }
1248 
1249 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1250 				  struct enetc_tx_swbd *tx_swbd,
1251 				  int frm_len)
1252 {
1253 	union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1254 
1255 	prefetchw(txbd);
1256 
1257 	enetc_clear_tx_bd(txbd);
1258 	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1259 	txbd->buf_len = cpu_to_le16(tx_swbd->len);
1260 	txbd->frm_len = cpu_to_le16(frm_len);
1261 
1262 	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1263 }
1264 
1265 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1266  * descriptors.
1267  */
1268 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1269 			 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1270 {
1271 	struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1272 	int i, k, frm_len = tmp_tx_swbd->len;
1273 
1274 	if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1275 		return false;
1276 
1277 	while (unlikely(!tmp_tx_swbd->is_eof)) {
1278 		tmp_tx_swbd++;
1279 		frm_len += tmp_tx_swbd->len;
1280 	}
1281 
1282 	i = tx_ring->next_to_use;
1283 
1284 	for (k = 0; k < num_tx_swbd; k++) {
1285 		struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1286 
1287 		enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1288 
1289 		/* last BD needs 'F' bit set */
1290 		if (xdp_tx_swbd->is_eof) {
1291 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1292 
1293 			txbd->flags = ENETC_TXBD_FLAGS_F;
1294 		}
1295 
1296 		enetc_bdr_idx_inc(tx_ring, &i);
1297 	}
1298 
1299 	tx_ring->next_to_use = i;
1300 
1301 	return true;
1302 }
1303 
1304 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1305 					  struct enetc_tx_swbd *xdp_tx_arr,
1306 					  struct xdp_frame *xdp_frame)
1307 {
1308 	struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1309 	struct skb_shared_info *shinfo;
1310 	void *data = xdp_frame->data;
1311 	int len = xdp_frame->len;
1312 	skb_frag_t *frag;
1313 	dma_addr_t dma;
1314 	unsigned int f;
1315 	int n = 0;
1316 
1317 	dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1318 	if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1319 		netdev_err(tx_ring->ndev, "DMA map error\n");
1320 		return -1;
1321 	}
1322 
1323 	xdp_tx_swbd->dma = dma;
1324 	xdp_tx_swbd->dir = DMA_TO_DEVICE;
1325 	xdp_tx_swbd->len = len;
1326 	xdp_tx_swbd->is_xdp_redirect = true;
1327 	xdp_tx_swbd->is_eof = false;
1328 	xdp_tx_swbd->xdp_frame = NULL;
1329 
1330 	n++;
1331 
1332 	if (!xdp_frame_has_frags(xdp_frame))
1333 		goto out;
1334 
1335 	xdp_tx_swbd = &xdp_tx_arr[n];
1336 
1337 	shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1338 
1339 	for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1340 	     f++, frag++) {
1341 		data = skb_frag_address(frag);
1342 		len = skb_frag_size(frag);
1343 
1344 		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1345 		if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1346 			/* Undo the DMA mapping for all fragments */
1347 			while (--n >= 0)
1348 				enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1349 
1350 			netdev_err(tx_ring->ndev, "DMA map error\n");
1351 			return -1;
1352 		}
1353 
1354 		xdp_tx_swbd->dma = dma;
1355 		xdp_tx_swbd->dir = DMA_TO_DEVICE;
1356 		xdp_tx_swbd->len = len;
1357 		xdp_tx_swbd->is_xdp_redirect = true;
1358 		xdp_tx_swbd->is_eof = false;
1359 		xdp_tx_swbd->xdp_frame = NULL;
1360 
1361 		n++;
1362 		xdp_tx_swbd = &xdp_tx_arr[n];
1363 	}
1364 out:
1365 	xdp_tx_arr[n - 1].is_eof = true;
1366 	xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1367 
1368 	return n;
1369 }
1370 
1371 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1372 		   struct xdp_frame **frames, u32 flags)
1373 {
1374 	struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1375 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1376 	struct enetc_bdr *tx_ring;
1377 	int xdp_tx_bd_cnt, i, k;
1378 	int xdp_tx_frm_cnt = 0;
1379 
1380 	enetc_lock_mdio();
1381 
1382 	tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1383 
1384 	prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1385 
1386 	for (k = 0; k < num_frames; k++) {
1387 		xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1388 							       xdp_redirect_arr,
1389 							       frames[k]);
1390 		if (unlikely(xdp_tx_bd_cnt < 0))
1391 			break;
1392 
1393 		if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1394 					   xdp_tx_bd_cnt))) {
1395 			for (i = 0; i < xdp_tx_bd_cnt; i++)
1396 				enetc_unmap_tx_buff(tx_ring,
1397 						    &xdp_redirect_arr[i]);
1398 			tx_ring->stats.xdp_tx_drops++;
1399 			break;
1400 		}
1401 
1402 		xdp_tx_frm_cnt++;
1403 	}
1404 
1405 	if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1406 		enetc_update_tx_ring_tail(tx_ring);
1407 
1408 	tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1409 
1410 	enetc_unlock_mdio();
1411 
1412 	return xdp_tx_frm_cnt;
1413 }
1414 EXPORT_SYMBOL_GPL(enetc_xdp_xmit);
1415 
1416 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1417 				     struct xdp_buff *xdp_buff, u16 size)
1418 {
1419 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1420 	void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1421 
1422 	/* To be used for XDP_TX */
1423 	rx_swbd->len = size;
1424 
1425 	xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1426 			 rx_ring->buffer_offset, size, false);
1427 }
1428 
1429 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1430 				     u16 size, struct xdp_buff *xdp_buff)
1431 {
1432 	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1433 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1434 	skb_frag_t *frag;
1435 
1436 	/* To be used for XDP_TX */
1437 	rx_swbd->len = size;
1438 
1439 	if (!xdp_buff_has_frags(xdp_buff)) {
1440 		xdp_buff_set_frags_flag(xdp_buff);
1441 		shinfo->xdp_frags_size = size;
1442 		shinfo->nr_frags = 0;
1443 	} else {
1444 		shinfo->xdp_frags_size += size;
1445 	}
1446 
1447 	if (page_is_pfmemalloc(rx_swbd->page))
1448 		xdp_buff_set_frag_pfmemalloc(xdp_buff);
1449 
1450 	frag = &shinfo->frags[shinfo->nr_frags];
1451 	skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,
1452 				size);
1453 
1454 	shinfo->nr_frags++;
1455 }
1456 
1457 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1458 				 union enetc_rx_bd **rxbd, int *i,
1459 				 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1460 {
1461 	u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1462 
1463 	xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1464 
1465 	enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1466 	(*cleaned_cnt)++;
1467 	enetc_rxbd_next(rx_ring, rxbd, i);
1468 
1469 	/* not last BD in frame? */
1470 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1471 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1472 		size = ENETC_RXB_DMA_SIZE_XDP;
1473 
1474 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1475 			dma_rmb();
1476 			size = le16_to_cpu((*rxbd)->r.buf_len);
1477 		}
1478 
1479 		enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1480 		(*cleaned_cnt)++;
1481 		enetc_rxbd_next(rx_ring, rxbd, i);
1482 	}
1483 }
1484 
1485 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1486  * recycled back into the RX ring in enetc_clean_tx_ring.
1487  */
1488 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1489 					struct enetc_bdr *rx_ring,
1490 					int rx_ring_first, int rx_ring_last)
1491 {
1492 	int n = 0;
1493 
1494 	for (; rx_ring_first != rx_ring_last;
1495 	     n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1496 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1497 		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1498 
1499 		/* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1500 		tx_swbd->dma = rx_swbd->dma;
1501 		tx_swbd->dir = rx_swbd->dir;
1502 		tx_swbd->page = rx_swbd->page;
1503 		tx_swbd->page_offset = rx_swbd->page_offset;
1504 		tx_swbd->len = rx_swbd->len;
1505 		tx_swbd->is_dma_page = true;
1506 		tx_swbd->is_xdp_tx = true;
1507 		tx_swbd->is_eof = false;
1508 	}
1509 
1510 	/* We rely on caller providing an rx_ring_last > rx_ring_first */
1511 	xdp_tx_arr[n - 1].is_eof = true;
1512 
1513 	return n;
1514 }
1515 
1516 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1517 			   int rx_ring_last)
1518 {
1519 	while (rx_ring_first != rx_ring_last) {
1520 		enetc_put_rx_buff(rx_ring,
1521 				  &rx_ring->rx_swbd[rx_ring_first]);
1522 		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1523 	}
1524 	rx_ring->stats.xdp_drops++;
1525 }
1526 
1527 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1528 				   struct napi_struct *napi, int work_limit,
1529 				   struct bpf_prog *prog)
1530 {
1531 	int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1532 	struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1533 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1534 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1535 	struct enetc_bdr *tx_ring;
1536 	int cleaned_cnt, i;
1537 	u32 xdp_act;
1538 
1539 	cleaned_cnt = enetc_bd_unused(rx_ring);
1540 	/* next descriptor to process */
1541 	i = rx_ring->next_to_clean;
1542 
1543 	while (likely(rx_frm_cnt < work_limit)) {
1544 		union enetc_rx_bd *rxbd, *orig_rxbd;
1545 		int orig_i, orig_cleaned_cnt;
1546 		struct xdp_buff xdp_buff;
1547 		struct sk_buff *skb;
1548 		u32 bd_status;
1549 		int err;
1550 
1551 		rxbd = enetc_rxbd(rx_ring, i);
1552 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1553 		if (!bd_status)
1554 			break;
1555 
1556 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1557 		dma_rmb(); /* for reading other rxbd fields */
1558 
1559 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1560 						      &rxbd, &i))
1561 			break;
1562 
1563 		orig_rxbd = rxbd;
1564 		orig_cleaned_cnt = cleaned_cnt;
1565 		orig_i = i;
1566 
1567 		enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1568 				     &cleaned_cnt, &xdp_buff);
1569 
1570 		/* When set, the outer VLAN header is extracted and reported
1571 		 * in the receive buffer descriptor. So rx_byte_cnt should
1572 		 * add the length of the extracted VLAN header.
1573 		 */
1574 		if (bd_status & ENETC_RXBD_FLAG_VLAN)
1575 			rx_byte_cnt += VLAN_HLEN;
1576 		rx_byte_cnt += xdp_get_buff_len(&xdp_buff);
1577 
1578 		xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1579 
1580 		switch (xdp_act) {
1581 		default:
1582 			bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1583 			fallthrough;
1584 		case XDP_ABORTED:
1585 			trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1586 			fallthrough;
1587 		case XDP_DROP:
1588 			enetc_xdp_drop(rx_ring, orig_i, i);
1589 			break;
1590 		case XDP_PASS:
1591 			rxbd = orig_rxbd;
1592 			cleaned_cnt = orig_cleaned_cnt;
1593 			i = orig_i;
1594 
1595 			skb = enetc_build_skb(rx_ring, bd_status, &rxbd,
1596 					      &i, &cleaned_cnt,
1597 					      ENETC_RXB_DMA_SIZE_XDP);
1598 			if (unlikely(!skb))
1599 				goto out;
1600 
1601 			napi_gro_receive(napi, skb);
1602 			break;
1603 		case XDP_TX:
1604 			tx_ring = priv->xdp_tx_ring[rx_ring->index];
1605 			xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1606 								     rx_ring,
1607 								     orig_i, i);
1608 
1609 			if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1610 				enetc_xdp_drop(rx_ring, orig_i, i);
1611 				tx_ring->stats.xdp_tx_drops++;
1612 			} else {
1613 				tx_ring->stats.xdp_tx += xdp_tx_bd_cnt;
1614 				rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1615 				xdp_tx_frm_cnt++;
1616 				/* The XDP_TX enqueue was successful, so we
1617 				 * need to scrub the RX software BDs because
1618 				 * the ownership of the buffers no longer
1619 				 * belongs to the RX ring, and we must prevent
1620 				 * enetc_refill_rx_ring() from reusing
1621 				 * rx_swbd->page.
1622 				 */
1623 				while (orig_i != i) {
1624 					rx_ring->rx_swbd[orig_i].page = NULL;
1625 					enetc_bdr_idx_inc(rx_ring, &orig_i);
1626 				}
1627 			}
1628 			break;
1629 		case XDP_REDIRECT:
1630 			err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1631 			if (unlikely(err)) {
1632 				enetc_xdp_drop(rx_ring, orig_i, i);
1633 				rx_ring->stats.xdp_redirect_failures++;
1634 			} else {
1635 				while (orig_i != i) {
1636 					enetc_flip_rx_buff(rx_ring,
1637 							   &rx_ring->rx_swbd[orig_i]);
1638 					enetc_bdr_idx_inc(rx_ring, &orig_i);
1639 				}
1640 				xdp_redirect_frm_cnt++;
1641 				rx_ring->stats.xdp_redirect++;
1642 			}
1643 		}
1644 
1645 		rx_frm_cnt++;
1646 	}
1647 
1648 out:
1649 	rx_ring->next_to_clean = i;
1650 
1651 	rx_ring->stats.packets += rx_frm_cnt;
1652 	rx_ring->stats.bytes += rx_byte_cnt;
1653 
1654 	if (xdp_redirect_frm_cnt)
1655 		xdp_do_flush();
1656 
1657 	if (xdp_tx_frm_cnt)
1658 		enetc_update_tx_ring_tail(tx_ring);
1659 
1660 	if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
1661 		enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
1662 				     rx_ring->xdp.xdp_tx_in_flight);
1663 
1664 	return rx_frm_cnt;
1665 }
1666 
1667 static int enetc_poll(struct napi_struct *napi, int budget)
1668 {
1669 	struct enetc_int_vector
1670 		*v = container_of(napi, struct enetc_int_vector, napi);
1671 	struct enetc_bdr *rx_ring = &v->rx_ring;
1672 	struct bpf_prog *prog;
1673 	bool complete = true;
1674 	int work_done;
1675 	int i;
1676 
1677 	enetc_lock_mdio();
1678 
1679 	for (i = 0; i < v->count_tx_rings; i++)
1680 		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
1681 			complete = false;
1682 
1683 	prog = rx_ring->xdp.prog;
1684 	if (prog)
1685 		work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
1686 	else
1687 		work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
1688 	if (work_done == budget)
1689 		complete = false;
1690 	if (work_done)
1691 		v->rx_napi_work = true;
1692 
1693 	if (!complete) {
1694 		enetc_unlock_mdio();
1695 		return budget;
1696 	}
1697 
1698 	napi_complete_done(napi, work_done);
1699 
1700 	if (likely(v->rx_dim_en))
1701 		enetc_rx_net_dim(v);
1702 
1703 	v->rx_napi_work = false;
1704 
1705 	/* enable interrupts */
1706 	enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
1707 
1708 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1709 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
1710 				 ENETC_TBIER_TXTIE);
1711 
1712 	enetc_unlock_mdio();
1713 
1714 	return work_done;
1715 }
1716 
1717 /* Probing and Init */
1718 #define ENETC_MAX_RFS_SIZE 64
1719 void enetc_get_si_caps(struct enetc_si *si)
1720 {
1721 	struct enetc_hw *hw = &si->hw;
1722 	u32 val;
1723 
1724 	/* find out how many of various resources we have to work with */
1725 	val = enetc_rd(hw, ENETC_SICAPR0);
1726 	si->num_rx_rings = (val >> 16) & 0xff;
1727 	si->num_tx_rings = val & 0xff;
1728 
1729 	val = enetc_rd(hw, ENETC_SIRFSCAPR);
1730 	si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
1731 	si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
1732 
1733 	si->num_rss = 0;
1734 	val = enetc_rd(hw, ENETC_SIPCAPR0);
1735 	if (val & ENETC_SIPCAPR0_RSS) {
1736 		u32 rss;
1737 
1738 		rss = enetc_rd(hw, ENETC_SIRSSCAPR);
1739 		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
1740 	}
1741 
1742 	if (val & ENETC_SIPCAPR0_QBV)
1743 		si->hw_features |= ENETC_SI_F_QBV;
1744 
1745 	if (val & ENETC_SIPCAPR0_QBU)
1746 		si->hw_features |= ENETC_SI_F_QBU;
1747 
1748 	if (val & ENETC_SIPCAPR0_PSFP)
1749 		si->hw_features |= ENETC_SI_F_PSFP;
1750 }
1751 EXPORT_SYMBOL_GPL(enetc_get_si_caps);
1752 
1753 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
1754 {
1755 	size_t bd_base_size = res->bd_count * res->bd_size;
1756 
1757 	res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,
1758 					  &res->bd_dma_base, GFP_KERNEL);
1759 	if (!res->bd_base)
1760 		return -ENOMEM;
1761 
1762 	/* h/w requires 128B alignment */
1763 	if (!IS_ALIGNED(res->bd_dma_base, 128)) {
1764 		dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1765 				  res->bd_dma_base);
1766 		return -EINVAL;
1767 	}
1768 
1769 	return 0;
1770 }
1771 
1772 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
1773 {
1774 	size_t bd_base_size = res->bd_count * res->bd_size;
1775 
1776 	dma_free_coherent(res->dev, bd_base_size, res->bd_base,
1777 			  res->bd_dma_base);
1778 }
1779 
1780 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
1781 				   struct device *dev, size_t bd_count)
1782 {
1783 	int err;
1784 
1785 	res->dev = dev;
1786 	res->bd_count = bd_count;
1787 	res->bd_size = sizeof(union enetc_tx_bd);
1788 
1789 	res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));
1790 	if (!res->tx_swbd)
1791 		return -ENOMEM;
1792 
1793 	err = enetc_dma_alloc_bdr(res);
1794 	if (err)
1795 		goto err_alloc_bdr;
1796 
1797 	res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,
1798 					      &res->tso_headers_dma,
1799 					      GFP_KERNEL);
1800 	if (!res->tso_headers) {
1801 		err = -ENOMEM;
1802 		goto err_alloc_tso;
1803 	}
1804 
1805 	return 0;
1806 
1807 err_alloc_tso:
1808 	enetc_dma_free_bdr(res);
1809 err_alloc_bdr:
1810 	vfree(res->tx_swbd);
1811 	res->tx_swbd = NULL;
1812 
1813 	return err;
1814 }
1815 
1816 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
1817 {
1818 	dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,
1819 			  res->tso_headers, res->tso_headers_dma);
1820 	enetc_dma_free_bdr(res);
1821 	vfree(res->tx_swbd);
1822 }
1823 
1824 static struct enetc_bdr_resource *
1825 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
1826 {
1827 	struct enetc_bdr_resource *tx_res;
1828 	int i, err;
1829 
1830 	tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL);
1831 	if (!tx_res)
1832 		return ERR_PTR(-ENOMEM);
1833 
1834 	for (i = 0; i < priv->num_tx_rings; i++) {
1835 		struct enetc_bdr *tx_ring = priv->tx_ring[i];
1836 
1837 		err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,
1838 					      tx_ring->bd_count);
1839 		if (err)
1840 			goto fail;
1841 	}
1842 
1843 	return tx_res;
1844 
1845 fail:
1846 	while (i-- > 0)
1847 		enetc_free_tx_resource(&tx_res[i]);
1848 
1849 	kfree(tx_res);
1850 
1851 	return ERR_PTR(err);
1852 }
1853 
1854 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
1855 				    size_t num_resources)
1856 {
1857 	size_t i;
1858 
1859 	for (i = 0; i < num_resources; i++)
1860 		enetc_free_tx_resource(&tx_res[i]);
1861 
1862 	kfree(tx_res);
1863 }
1864 
1865 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
1866 				   struct device *dev, size_t bd_count,
1867 				   bool extended)
1868 {
1869 	int err;
1870 
1871 	res->dev = dev;
1872 	res->bd_count = bd_count;
1873 	res->bd_size = sizeof(union enetc_rx_bd);
1874 	if (extended)
1875 		res->bd_size *= 2;
1876 
1877 	res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));
1878 	if (!res->rx_swbd)
1879 		return -ENOMEM;
1880 
1881 	err = enetc_dma_alloc_bdr(res);
1882 	if (err) {
1883 		vfree(res->rx_swbd);
1884 		return err;
1885 	}
1886 
1887 	return 0;
1888 }
1889 
1890 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
1891 {
1892 	enetc_dma_free_bdr(res);
1893 	vfree(res->rx_swbd);
1894 }
1895 
1896 static struct enetc_bdr_resource *
1897 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
1898 {
1899 	struct enetc_bdr_resource *rx_res;
1900 	int i, err;
1901 
1902 	rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL);
1903 	if (!rx_res)
1904 		return ERR_PTR(-ENOMEM);
1905 
1906 	for (i = 0; i < priv->num_rx_rings; i++) {
1907 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
1908 
1909 		err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,
1910 					      rx_ring->bd_count, extended);
1911 		if (err)
1912 			goto fail;
1913 	}
1914 
1915 	return rx_res;
1916 
1917 fail:
1918 	while (i-- > 0)
1919 		enetc_free_rx_resource(&rx_res[i]);
1920 
1921 	kfree(rx_res);
1922 
1923 	return ERR_PTR(err);
1924 }
1925 
1926 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
1927 				    size_t num_resources)
1928 {
1929 	size_t i;
1930 
1931 	for (i = 0; i < num_resources; i++)
1932 		enetc_free_rx_resource(&rx_res[i]);
1933 
1934 	kfree(rx_res);
1935 }
1936 
1937 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
1938 				     const struct enetc_bdr_resource *res)
1939 {
1940 	tx_ring->bd_base = res ? res->bd_base : NULL;
1941 	tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1942 	tx_ring->tx_swbd = res ? res->tx_swbd : NULL;
1943 	tx_ring->tso_headers = res ? res->tso_headers : NULL;
1944 	tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;
1945 }
1946 
1947 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
1948 				     const struct enetc_bdr_resource *res)
1949 {
1950 	rx_ring->bd_base = res ? res->bd_base : NULL;
1951 	rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
1952 	rx_ring->rx_swbd = res ? res->rx_swbd : NULL;
1953 }
1954 
1955 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
1956 				      const struct enetc_bdr_resource *res)
1957 {
1958 	int i;
1959 
1960 	if (priv->tx_res)
1961 		enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);
1962 
1963 	for (i = 0; i < priv->num_tx_rings; i++) {
1964 		enetc_assign_tx_resource(priv->tx_ring[i],
1965 					 res ? &res[i] : NULL);
1966 	}
1967 
1968 	priv->tx_res = res;
1969 }
1970 
1971 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
1972 				      const struct enetc_bdr_resource *res)
1973 {
1974 	int i;
1975 
1976 	if (priv->rx_res)
1977 		enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);
1978 
1979 	for (i = 0; i < priv->num_rx_rings; i++) {
1980 		enetc_assign_rx_resource(priv->rx_ring[i],
1981 					 res ? &res[i] : NULL);
1982 	}
1983 
1984 	priv->rx_res = res;
1985 }
1986 
1987 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
1988 {
1989 	int i;
1990 
1991 	for (i = 0; i < tx_ring->bd_count; i++) {
1992 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
1993 
1994 		enetc_free_tx_frame(tx_ring, tx_swbd);
1995 	}
1996 }
1997 
1998 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
1999 {
2000 	int i;
2001 
2002 	for (i = 0; i < rx_ring->bd_count; i++) {
2003 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
2004 
2005 		if (!rx_swbd->page)
2006 			continue;
2007 
2008 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
2009 			       rx_swbd->dir);
2010 		__free_page(rx_swbd->page);
2011 		rx_swbd->page = NULL;
2012 	}
2013 }
2014 
2015 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
2016 {
2017 	int i;
2018 
2019 	for (i = 0; i < priv->num_rx_rings; i++)
2020 		enetc_free_rx_ring(priv->rx_ring[i]);
2021 
2022 	for (i = 0; i < priv->num_tx_rings; i++)
2023 		enetc_free_tx_ring(priv->tx_ring[i]);
2024 }
2025 
2026 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
2027 {
2028 	int *rss_table;
2029 	int i;
2030 
2031 	rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
2032 	if (!rss_table)
2033 		return -ENOMEM;
2034 
2035 	/* Set up RSS table defaults */
2036 	for (i = 0; i < si->num_rss; i++)
2037 		rss_table[i] = i % num_groups;
2038 
2039 	enetc_set_rss_table(si, rss_table, si->num_rss);
2040 
2041 	kfree(rss_table);
2042 
2043 	return 0;
2044 }
2045 
2046 int enetc_configure_si(struct enetc_ndev_priv *priv)
2047 {
2048 	struct enetc_si *si = priv->si;
2049 	struct enetc_hw *hw = &si->hw;
2050 	int err;
2051 
2052 	/* set SI cache attributes */
2053 	enetc_wr(hw, ENETC_SICAR0,
2054 		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
2055 	enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
2056 	/* enable SI */
2057 	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
2058 
2059 	if (si->num_rss) {
2060 		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
2061 		if (err)
2062 			return err;
2063 	}
2064 
2065 	return 0;
2066 }
2067 EXPORT_SYMBOL_GPL(enetc_configure_si);
2068 
2069 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2070 {
2071 	struct enetc_si *si = priv->si;
2072 	int cpus = num_online_cpus();
2073 
2074 	priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
2075 	priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2076 
2077 	/* Enable all available TX rings in order to configure as many
2078 	 * priorities as possible, when needed.
2079 	 * TODO: Make # of TX rings run-time configurable
2080 	 */
2081 	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2082 	priv->num_tx_rings = si->num_tx_rings;
2083 	priv->bdr_int_num = cpus;
2084 	priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2085 	priv->tx_ictt = ENETC_TXIC_TIMETHR;
2086 }
2087 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);
2088 
2089 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2090 {
2091 	struct enetc_si *si = priv->si;
2092 
2093 	priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2094 				  GFP_KERNEL);
2095 	if (!priv->cls_rules)
2096 		return -ENOMEM;
2097 
2098 	return 0;
2099 }
2100 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);
2101 
2102 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2103 {
2104 	kfree(priv->cls_rules);
2105 }
2106 EXPORT_SYMBOL_GPL(enetc_free_si_resources);
2107 
2108 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2109 {
2110 	int idx = tx_ring->index;
2111 	u32 tbmr;
2112 
2113 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2114 		       lower_32_bits(tx_ring->bd_dma_base));
2115 
2116 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2117 		       upper_32_bits(tx_ring->bd_dma_base));
2118 
2119 	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2120 	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2121 		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
2122 
2123 	/* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2124 	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2125 	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2126 
2127 	/* enable Tx ints by setting pkt thr to 1 */
2128 	enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2129 
2130 	tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);
2131 	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2132 		tbmr |= ENETC_TBMR_VIH;
2133 
2134 	/* enable ring */
2135 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2136 
2137 	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2138 	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2139 	tx_ring->idr = hw->reg + ENETC_SITXIDR;
2140 }
2141 
2142 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
2143 			      bool extended)
2144 {
2145 	int idx = rx_ring->index;
2146 	u32 rbmr = 0;
2147 
2148 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2149 		       lower_32_bits(rx_ring->bd_dma_base));
2150 
2151 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2152 		       upper_32_bits(rx_ring->bd_dma_base));
2153 
2154 	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2155 	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2156 		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
2157 
2158 	if (rx_ring->xdp.prog)
2159 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2160 	else
2161 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2162 
2163 	/* Also prepare the consumer index in case page allocation never
2164 	 * succeeds. In that case, hardware will never advance producer index
2165 	 * to match consumer index, and will drop all frames.
2166 	 */
2167 	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2168 	enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2169 
2170 	/* enable Rx ints by setting pkt thr to 1 */
2171 	enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2172 
2173 	rx_ring->ext_en = extended;
2174 	if (rx_ring->ext_en)
2175 		rbmr |= ENETC_RBMR_BDS;
2176 
2177 	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2178 		rbmr |= ENETC_RBMR_VTE;
2179 
2180 	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2181 	rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2182 
2183 	rx_ring->next_to_clean = 0;
2184 	rx_ring->next_to_use = 0;
2185 	rx_ring->next_to_alloc = 0;
2186 
2187 	enetc_lock_mdio();
2188 	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2189 	enetc_unlock_mdio();
2190 
2191 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2192 }
2193 
2194 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
2195 {
2196 	struct enetc_hw *hw = &priv->si->hw;
2197 	int i;
2198 
2199 	for (i = 0; i < priv->num_tx_rings; i++)
2200 		enetc_setup_txbdr(hw, priv->tx_ring[i]);
2201 
2202 	for (i = 0; i < priv->num_rx_rings; i++)
2203 		enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);
2204 }
2205 
2206 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2207 {
2208 	int idx = tx_ring->index;
2209 	u32 tbmr;
2210 
2211 	tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);
2212 	tbmr |= ENETC_TBMR_EN;
2213 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2214 }
2215 
2216 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2217 {
2218 	int idx = rx_ring->index;
2219 	u32 rbmr;
2220 
2221 	rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR);
2222 	rbmr |= ENETC_RBMR_EN;
2223 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2224 }
2225 
2226 static void enetc_enable_bdrs(struct enetc_ndev_priv *priv)
2227 {
2228 	struct enetc_hw *hw = &priv->si->hw;
2229 	int i;
2230 
2231 	for (i = 0; i < priv->num_tx_rings; i++)
2232 		enetc_enable_txbdr(hw, priv->tx_ring[i]);
2233 
2234 	for (i = 0; i < priv->num_rx_rings; i++)
2235 		enetc_enable_rxbdr(hw, priv->rx_ring[i]);
2236 }
2237 
2238 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2239 {
2240 	int idx = rx_ring->index;
2241 
2242 	/* disable EN bit on ring */
2243 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2244 }
2245 
2246 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2247 {
2248 	int idx = rx_ring->index;
2249 
2250 	/* disable EN bit on ring */
2251 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2252 }
2253 
2254 static void enetc_disable_bdrs(struct enetc_ndev_priv *priv)
2255 {
2256 	struct enetc_hw *hw = &priv->si->hw;
2257 	int i;
2258 
2259 	for (i = 0; i < priv->num_tx_rings; i++)
2260 		enetc_disable_txbdr(hw, priv->tx_ring[i]);
2261 
2262 	for (i = 0; i < priv->num_rx_rings; i++)
2263 		enetc_disable_rxbdr(hw, priv->rx_ring[i]);
2264 }
2265 
2266 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2267 {
2268 	int delay = 8, timeout = 100;
2269 	int idx = tx_ring->index;
2270 
2271 	/* wait for busy to clear */
2272 	while (delay < timeout &&
2273 	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2274 		msleep(delay);
2275 		delay *= 2;
2276 	}
2277 
2278 	if (delay >= timeout)
2279 		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2280 			    idx);
2281 }
2282 
2283 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
2284 {
2285 	struct enetc_hw *hw = &priv->si->hw;
2286 	int i;
2287 
2288 	for (i = 0; i < priv->num_tx_rings; i++)
2289 		enetc_wait_txbdr(hw, priv->tx_ring[i]);
2290 }
2291 
2292 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2293 {
2294 	struct pci_dev *pdev = priv->si->pdev;
2295 	struct enetc_hw *hw = &priv->si->hw;
2296 	int i, j, err;
2297 
2298 	for (i = 0; i < priv->bdr_int_num; i++) {
2299 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2300 		struct enetc_int_vector *v = priv->int_vector[i];
2301 		int entry = ENETC_BDR_INT_BASE_IDX + i;
2302 
2303 		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2304 			 priv->ndev->name, i);
2305 		err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v);
2306 		if (err) {
2307 			dev_err(priv->dev, "request_irq() failed!\n");
2308 			goto irq_err;
2309 		}
2310 
2311 		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2312 		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2313 		v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2314 
2315 		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2316 
2317 		for (j = 0; j < v->count_tx_rings; j++) {
2318 			int idx = v->tx_ring[j].index;
2319 
2320 			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2321 		}
2322 		irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2323 	}
2324 
2325 	return 0;
2326 
2327 irq_err:
2328 	while (i--) {
2329 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2330 
2331 		irq_set_affinity_hint(irq, NULL);
2332 		free_irq(irq, priv->int_vector[i]);
2333 	}
2334 
2335 	return err;
2336 }
2337 
2338 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2339 {
2340 	struct pci_dev *pdev = priv->si->pdev;
2341 	int i;
2342 
2343 	for (i = 0; i < priv->bdr_int_num; i++) {
2344 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2345 
2346 		irq_set_affinity_hint(irq, NULL);
2347 		free_irq(irq, priv->int_vector[i]);
2348 	}
2349 }
2350 
2351 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2352 {
2353 	struct enetc_hw *hw = &priv->si->hw;
2354 	u32 icpt, ictt;
2355 	int i;
2356 
2357 	/* enable Tx & Rx event indication */
2358 	if (priv->ic_mode &
2359 	    (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2360 		icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2361 		/* init to non-0 minimum, will be adjusted later */
2362 		ictt = 0x1;
2363 	} else {
2364 		icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2365 		ictt = 0;
2366 	}
2367 
2368 	for (i = 0; i < priv->num_rx_rings; i++) {
2369 		enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2370 		enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2371 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2372 	}
2373 
2374 	if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2375 		icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2376 	else
2377 		icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2378 
2379 	for (i = 0; i < priv->num_tx_rings; i++) {
2380 		enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2381 		enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2382 		enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2383 	}
2384 }
2385 
2386 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2387 {
2388 	struct enetc_hw *hw = &priv->si->hw;
2389 	int i;
2390 
2391 	for (i = 0; i < priv->num_tx_rings; i++)
2392 		enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2393 
2394 	for (i = 0; i < priv->num_rx_rings; i++)
2395 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2396 }
2397 
2398 static int enetc_phylink_connect(struct net_device *ndev)
2399 {
2400 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2401 	struct ethtool_keee edata;
2402 	int err;
2403 
2404 	if (!priv->phylink) {
2405 		/* phy-less mode */
2406 		netif_carrier_on(ndev);
2407 		return 0;
2408 	}
2409 
2410 	err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2411 	if (err) {
2412 		dev_err(&ndev->dev, "could not attach to PHY\n");
2413 		return err;
2414 	}
2415 
2416 	/* disable EEE autoneg, until ENETC driver supports it */
2417 	memset(&edata, 0, sizeof(struct ethtool_keee));
2418 	phylink_ethtool_set_eee(priv->phylink, &edata);
2419 
2420 	phylink_start(priv->phylink);
2421 
2422 	return 0;
2423 }
2424 
2425 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2426 {
2427 	struct enetc_ndev_priv *priv;
2428 	struct sk_buff *skb;
2429 
2430 	priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2431 
2432 	netif_tx_lock_bh(priv->ndev);
2433 
2434 	clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2435 	skb = skb_dequeue(&priv->tx_skbs);
2436 	if (skb)
2437 		enetc_start_xmit(skb, priv->ndev);
2438 
2439 	netif_tx_unlock_bh(priv->ndev);
2440 }
2441 
2442 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2443 {
2444 	INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2445 	skb_queue_head_init(&priv->tx_skbs);
2446 }
2447 
2448 void enetc_start(struct net_device *ndev)
2449 {
2450 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2451 	int i;
2452 
2453 	enetc_setup_interrupts(priv);
2454 
2455 	for (i = 0; i < priv->bdr_int_num; i++) {
2456 		int irq = pci_irq_vector(priv->si->pdev,
2457 					 ENETC_BDR_INT_BASE_IDX + i);
2458 
2459 		napi_enable(&priv->int_vector[i]->napi);
2460 		enable_irq(irq);
2461 	}
2462 
2463 	enetc_enable_bdrs(priv);
2464 
2465 	netif_tx_start_all_queues(ndev);
2466 }
2467 EXPORT_SYMBOL_GPL(enetc_start);
2468 
2469 int enetc_open(struct net_device *ndev)
2470 {
2471 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2472 	struct enetc_bdr_resource *tx_res, *rx_res;
2473 	bool extended;
2474 	int err;
2475 
2476 	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2477 
2478 	err = enetc_setup_irqs(priv);
2479 	if (err)
2480 		return err;
2481 
2482 	err = enetc_phylink_connect(ndev);
2483 	if (err)
2484 		goto err_phy_connect;
2485 
2486 	tx_res = enetc_alloc_tx_resources(priv);
2487 	if (IS_ERR(tx_res)) {
2488 		err = PTR_ERR(tx_res);
2489 		goto err_alloc_tx;
2490 	}
2491 
2492 	rx_res = enetc_alloc_rx_resources(priv, extended);
2493 	if (IS_ERR(rx_res)) {
2494 		err = PTR_ERR(rx_res);
2495 		goto err_alloc_rx;
2496 	}
2497 
2498 	enetc_tx_onestep_tstamp_init(priv);
2499 	enetc_assign_tx_resources(priv, tx_res);
2500 	enetc_assign_rx_resources(priv, rx_res);
2501 	enetc_setup_bdrs(priv, extended);
2502 	enetc_start(ndev);
2503 
2504 	return 0;
2505 
2506 err_alloc_rx:
2507 	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2508 err_alloc_tx:
2509 	if (priv->phylink)
2510 		phylink_disconnect_phy(priv->phylink);
2511 err_phy_connect:
2512 	enetc_free_irqs(priv);
2513 
2514 	return err;
2515 }
2516 EXPORT_SYMBOL_GPL(enetc_open);
2517 
2518 void enetc_stop(struct net_device *ndev)
2519 {
2520 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2521 	int i;
2522 
2523 	netif_tx_stop_all_queues(ndev);
2524 
2525 	enetc_disable_bdrs(priv);
2526 
2527 	for (i = 0; i < priv->bdr_int_num; i++) {
2528 		int irq = pci_irq_vector(priv->si->pdev,
2529 					 ENETC_BDR_INT_BASE_IDX + i);
2530 
2531 		disable_irq(irq);
2532 		napi_synchronize(&priv->int_vector[i]->napi);
2533 		napi_disable(&priv->int_vector[i]->napi);
2534 	}
2535 
2536 	enetc_wait_bdrs(priv);
2537 
2538 	enetc_clear_interrupts(priv);
2539 }
2540 EXPORT_SYMBOL_GPL(enetc_stop);
2541 
2542 int enetc_close(struct net_device *ndev)
2543 {
2544 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2545 
2546 	enetc_stop(ndev);
2547 
2548 	if (priv->phylink) {
2549 		phylink_stop(priv->phylink);
2550 		phylink_disconnect_phy(priv->phylink);
2551 	} else {
2552 		netif_carrier_off(ndev);
2553 	}
2554 
2555 	enetc_free_rxtx_rings(priv);
2556 
2557 	/* Avoids dangling pointers and also frees old resources */
2558 	enetc_assign_rx_resources(priv, NULL);
2559 	enetc_assign_tx_resources(priv, NULL);
2560 
2561 	enetc_free_irqs(priv);
2562 
2563 	return 0;
2564 }
2565 EXPORT_SYMBOL_GPL(enetc_close);
2566 
2567 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
2568 			     int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
2569 			     void *ctx)
2570 {
2571 	struct enetc_bdr_resource *tx_res, *rx_res;
2572 	int err;
2573 
2574 	ASSERT_RTNL();
2575 
2576 	/* If the interface is down, run the callback right away,
2577 	 * without reconfiguration.
2578 	 */
2579 	if (!netif_running(priv->ndev)) {
2580 		if (cb) {
2581 			err = cb(priv, ctx);
2582 			if (err)
2583 				return err;
2584 		}
2585 
2586 		return 0;
2587 	}
2588 
2589 	tx_res = enetc_alloc_tx_resources(priv);
2590 	if (IS_ERR(tx_res)) {
2591 		err = PTR_ERR(tx_res);
2592 		goto out;
2593 	}
2594 
2595 	rx_res = enetc_alloc_rx_resources(priv, extended);
2596 	if (IS_ERR(rx_res)) {
2597 		err = PTR_ERR(rx_res);
2598 		goto out_free_tx_res;
2599 	}
2600 
2601 	enetc_stop(priv->ndev);
2602 	enetc_free_rxtx_rings(priv);
2603 
2604 	/* Interface is down, run optional callback now */
2605 	if (cb) {
2606 		err = cb(priv, ctx);
2607 		if (err)
2608 			goto out_restart;
2609 	}
2610 
2611 	enetc_assign_tx_resources(priv, tx_res);
2612 	enetc_assign_rx_resources(priv, rx_res);
2613 	enetc_setup_bdrs(priv, extended);
2614 	enetc_start(priv->ndev);
2615 
2616 	return 0;
2617 
2618 out_restart:
2619 	enetc_setup_bdrs(priv, extended);
2620 	enetc_start(priv->ndev);
2621 	enetc_free_rx_resources(rx_res, priv->num_rx_rings);
2622 out_free_tx_res:
2623 	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2624 out:
2625 	return err;
2626 }
2627 
2628 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
2629 {
2630 	int i;
2631 
2632 	for (i = 0; i < priv->num_tx_rings; i++)
2633 		netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,
2634 			   priv->tx_ring[i]->prio);
2635 }
2636 
2637 void enetc_reset_tc_mqprio(struct net_device *ndev)
2638 {
2639 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2640 	struct enetc_hw *hw = &priv->si->hw;
2641 	struct enetc_bdr *tx_ring;
2642 	int num_stack_tx_queues;
2643 	int i;
2644 
2645 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2646 
2647 	netdev_reset_tc(ndev);
2648 	netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2649 	priv->min_num_stack_tx_queues = num_possible_cpus();
2650 
2651 	/* Reset all ring priorities to 0 */
2652 	for (i = 0; i < priv->num_tx_rings; i++) {
2653 		tx_ring = priv->tx_ring[i];
2654 		tx_ring->prio = 0;
2655 		enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2656 	}
2657 
2658 	enetc_debug_tx_ring_prios(priv);
2659 
2660 	enetc_change_preemptible_tcs(priv, 0);
2661 }
2662 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);
2663 
2664 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
2665 {
2666 	struct tc_mqprio_qopt_offload *mqprio = type_data;
2667 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2668 	struct tc_mqprio_qopt *qopt = &mqprio->qopt;
2669 	struct enetc_hw *hw = &priv->si->hw;
2670 	int num_stack_tx_queues = 0;
2671 	struct enetc_bdr *tx_ring;
2672 	u8 num_tc = qopt->num_tc;
2673 	int offset, count;
2674 	int err, tc, q;
2675 
2676 	if (!num_tc) {
2677 		enetc_reset_tc_mqprio(ndev);
2678 		return 0;
2679 	}
2680 
2681 	err = netdev_set_num_tc(ndev, num_tc);
2682 	if (err)
2683 		return err;
2684 
2685 	for (tc = 0; tc < num_tc; tc++) {
2686 		offset = qopt->offset[tc];
2687 		count = qopt->count[tc];
2688 		num_stack_tx_queues += count;
2689 
2690 		err = netdev_set_tc_queue(ndev, tc, count, offset);
2691 		if (err)
2692 			goto err_reset_tc;
2693 
2694 		for (q = offset; q < offset + count; q++) {
2695 			tx_ring = priv->tx_ring[q];
2696 			/* The prio_tc_map is skb_tx_hash()'s way of selecting
2697 			 * between TX queues based on skb->priority. As such,
2698 			 * there's nothing to offload based on it.
2699 			 * Make the mqprio "traffic class" be the priority of
2700 			 * this ring group, and leave the Tx IPV to traffic
2701 			 * class mapping as its default mapping value of 1:1.
2702 			 */
2703 			tx_ring->prio = tc;
2704 			enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
2705 		}
2706 	}
2707 
2708 	err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
2709 	if (err)
2710 		goto err_reset_tc;
2711 
2712 	priv->min_num_stack_tx_queues = num_stack_tx_queues;
2713 
2714 	enetc_debug_tx_ring_prios(priv);
2715 
2716 	enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
2717 
2718 	return 0;
2719 
2720 err_reset_tc:
2721 	enetc_reset_tc_mqprio(ndev);
2722 	return err;
2723 }
2724 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);
2725 
2726 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
2727 {
2728 	struct bpf_prog *old_prog, *prog = ctx;
2729 	int num_stack_tx_queues;
2730 	int err, i;
2731 
2732 	old_prog = xchg(&priv->xdp_prog, prog);
2733 
2734 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
2735 	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
2736 	if (err) {
2737 		xchg(&priv->xdp_prog, old_prog);
2738 		return err;
2739 	}
2740 
2741 	if (old_prog)
2742 		bpf_prog_put(old_prog);
2743 
2744 	for (i = 0; i < priv->num_rx_rings; i++) {
2745 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
2746 
2747 		rx_ring->xdp.prog = prog;
2748 
2749 		if (prog)
2750 			rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
2751 		else
2752 			rx_ring->buffer_offset = ENETC_RXB_PAD;
2753 	}
2754 
2755 	return 0;
2756 }
2757 
2758 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
2759 				struct netlink_ext_ack *extack)
2760 {
2761 	int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;
2762 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2763 	bool extended;
2764 
2765 	if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >
2766 	    priv->num_tx_rings) {
2767 		NL_SET_ERR_MSG_FMT_MOD(extack,
2768 				       "Reserving %d XDP TXQs leaves under %d for stack (total %d)",
2769 				       num_xdp_tx_queues,
2770 				       priv->min_num_stack_tx_queues,
2771 				       priv->num_tx_rings);
2772 		return -EBUSY;
2773 	}
2774 
2775 	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2776 
2777 	/* The buffer layout is changing, so we need to drain the old
2778 	 * RX buffers and seed new ones.
2779 	 */
2780 	return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);
2781 }
2782 
2783 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
2784 {
2785 	switch (bpf->command) {
2786 	case XDP_SETUP_PROG:
2787 		return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);
2788 	default:
2789 		return -EINVAL;
2790 	}
2791 
2792 	return 0;
2793 }
2794 EXPORT_SYMBOL_GPL(enetc_setup_bpf);
2795 
2796 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
2797 {
2798 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2799 	struct net_device_stats *stats = &ndev->stats;
2800 	unsigned long packets = 0, bytes = 0;
2801 	unsigned long tx_dropped = 0;
2802 	int i;
2803 
2804 	for (i = 0; i < priv->num_rx_rings; i++) {
2805 		packets += priv->rx_ring[i]->stats.packets;
2806 		bytes	+= priv->rx_ring[i]->stats.bytes;
2807 	}
2808 
2809 	stats->rx_packets = packets;
2810 	stats->rx_bytes = bytes;
2811 	bytes = 0;
2812 	packets = 0;
2813 
2814 	for (i = 0; i < priv->num_tx_rings; i++) {
2815 		packets += priv->tx_ring[i]->stats.packets;
2816 		bytes	+= priv->tx_ring[i]->stats.bytes;
2817 		tx_dropped += priv->tx_ring[i]->stats.win_drop;
2818 	}
2819 
2820 	stats->tx_packets = packets;
2821 	stats->tx_bytes = bytes;
2822 	stats->tx_dropped = tx_dropped;
2823 
2824 	return stats;
2825 }
2826 EXPORT_SYMBOL_GPL(enetc_get_stats);
2827 
2828 static int enetc_set_rss(struct net_device *ndev, int en)
2829 {
2830 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2831 	struct enetc_hw *hw = &priv->si->hw;
2832 	u32 reg;
2833 
2834 	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
2835 
2836 	reg = enetc_rd(hw, ENETC_SIMR);
2837 	reg &= ~ENETC_SIMR_RSSE;
2838 	reg |= (en) ? ENETC_SIMR_RSSE : 0;
2839 	enetc_wr(hw, ENETC_SIMR, reg);
2840 
2841 	return 0;
2842 }
2843 
2844 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
2845 {
2846 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2847 	struct enetc_hw *hw = &priv->si->hw;
2848 	int i;
2849 
2850 	for (i = 0; i < priv->num_rx_rings; i++)
2851 		enetc_bdr_enable_rxvlan(hw, i, en);
2852 }
2853 
2854 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
2855 {
2856 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2857 	struct enetc_hw *hw = &priv->si->hw;
2858 	int i;
2859 
2860 	for (i = 0; i < priv->num_tx_rings; i++)
2861 		enetc_bdr_enable_txvlan(hw, i, en);
2862 }
2863 
2864 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
2865 {
2866 	netdev_features_t changed = ndev->features ^ features;
2867 
2868 	if (changed & NETIF_F_RXHASH)
2869 		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
2870 
2871 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2872 		enetc_enable_rxvlan(ndev,
2873 				    !!(features & NETIF_F_HW_VLAN_CTAG_RX));
2874 
2875 	if (changed & NETIF_F_HW_VLAN_CTAG_TX)
2876 		enetc_enable_txvlan(ndev,
2877 				    !!(features & NETIF_F_HW_VLAN_CTAG_TX));
2878 }
2879 EXPORT_SYMBOL_GPL(enetc_set_features);
2880 
2881 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
2882 {
2883 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2884 	int err, new_offloads = priv->active_offloads;
2885 	struct hwtstamp_config config;
2886 
2887 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2888 		return -EFAULT;
2889 
2890 	switch (config.tx_type) {
2891 	case HWTSTAMP_TX_OFF:
2892 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2893 		break;
2894 	case HWTSTAMP_TX_ON:
2895 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2896 		new_offloads |= ENETC_F_TX_TSTAMP;
2897 		break;
2898 	case HWTSTAMP_TX_ONESTEP_SYNC:
2899 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
2900 		new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
2901 		break;
2902 	default:
2903 		return -ERANGE;
2904 	}
2905 
2906 	switch (config.rx_filter) {
2907 	case HWTSTAMP_FILTER_NONE:
2908 		new_offloads &= ~ENETC_F_RX_TSTAMP;
2909 		break;
2910 	default:
2911 		new_offloads |= ENETC_F_RX_TSTAMP;
2912 		config.rx_filter = HWTSTAMP_FILTER_ALL;
2913 	}
2914 
2915 	if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {
2916 		bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);
2917 
2918 		err = enetc_reconfigure(priv, extended, NULL, NULL);
2919 		if (err)
2920 			return err;
2921 	}
2922 
2923 	priv->active_offloads = new_offloads;
2924 
2925 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2926 	       -EFAULT : 0;
2927 }
2928 
2929 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
2930 {
2931 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2932 	struct hwtstamp_config config;
2933 
2934 	config.flags = 0;
2935 
2936 	if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
2937 		config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
2938 	else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
2939 		config.tx_type = HWTSTAMP_TX_ON;
2940 	else
2941 		config.tx_type = HWTSTAMP_TX_OFF;
2942 
2943 	config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
2944 			    HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
2945 
2946 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
2947 	       -EFAULT : 0;
2948 }
2949 
2950 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
2951 {
2952 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2953 
2954 	if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK)) {
2955 		if (cmd == SIOCSHWTSTAMP)
2956 			return enetc_hwtstamp_set(ndev, rq);
2957 		if (cmd == SIOCGHWTSTAMP)
2958 			return enetc_hwtstamp_get(ndev, rq);
2959 	}
2960 
2961 	if (!priv->phylink)
2962 		return -EOPNOTSUPP;
2963 
2964 	return phylink_mii_ioctl(priv->phylink, rq, cmd);
2965 }
2966 EXPORT_SYMBOL_GPL(enetc_ioctl);
2967 
2968 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
2969 {
2970 	struct pci_dev *pdev = priv->si->pdev;
2971 	int num_stack_tx_queues;
2972 	int first_xdp_tx_ring;
2973 	int i, n, err, nvec;
2974 	int v_tx_rings;
2975 
2976 	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
2977 	/* allocate MSIX for both messaging and Rx/Tx interrupts */
2978 	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
2979 
2980 	if (n < 0)
2981 		return n;
2982 
2983 	if (n != nvec)
2984 		return -EPERM;
2985 
2986 	/* # of tx rings per int vector */
2987 	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
2988 
2989 	for (i = 0; i < priv->bdr_int_num; i++) {
2990 		struct enetc_int_vector *v;
2991 		struct enetc_bdr *bdr;
2992 		int j;
2993 
2994 		v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
2995 		if (!v) {
2996 			err = -ENOMEM;
2997 			goto fail;
2998 		}
2999 
3000 		priv->int_vector[i] = v;
3001 
3002 		bdr = &v->rx_ring;
3003 		bdr->index = i;
3004 		bdr->ndev = priv->ndev;
3005 		bdr->dev = priv->dev;
3006 		bdr->bd_count = priv->rx_bd_count;
3007 		bdr->buffer_offset = ENETC_RXB_PAD;
3008 		priv->rx_ring[i] = bdr;
3009 
3010 		err = xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0);
3011 		if (err) {
3012 			kfree(v);
3013 			goto fail;
3014 		}
3015 
3016 		err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq,
3017 						 MEM_TYPE_PAGE_SHARED, NULL);
3018 		if (err) {
3019 			xdp_rxq_info_unreg(&bdr->xdp.rxq);
3020 			kfree(v);
3021 			goto fail;
3022 		}
3023 
3024 		/* init defaults for adaptive IC */
3025 		if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
3026 			v->rx_ictt = 0x1;
3027 			v->rx_dim_en = true;
3028 		}
3029 		INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
3030 		netif_napi_add(priv->ndev, &v->napi, enetc_poll);
3031 		v->count_tx_rings = v_tx_rings;
3032 
3033 		for (j = 0; j < v_tx_rings; j++) {
3034 			int idx;
3035 
3036 			/* default tx ring mapping policy */
3037 			idx = priv->bdr_int_num * j + i;
3038 			__set_bit(idx, &v->tx_rings_map);
3039 			bdr = &v->tx_ring[j];
3040 			bdr->index = idx;
3041 			bdr->ndev = priv->ndev;
3042 			bdr->dev = priv->dev;
3043 			bdr->bd_count = priv->tx_bd_count;
3044 			priv->tx_ring[idx] = bdr;
3045 		}
3046 	}
3047 
3048 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3049 
3050 	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3051 	if (err)
3052 		goto fail;
3053 
3054 	err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);
3055 	if (err)
3056 		goto fail;
3057 
3058 	priv->min_num_stack_tx_queues = num_possible_cpus();
3059 	first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
3060 	priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
3061 
3062 	return 0;
3063 
3064 fail:
3065 	while (i--) {
3066 		struct enetc_int_vector *v = priv->int_vector[i];
3067 		struct enetc_bdr *rx_ring = &v->rx_ring;
3068 
3069 		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3070 		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3071 		netif_napi_del(&v->napi);
3072 		cancel_work_sync(&v->rx_dim.work);
3073 		kfree(v);
3074 	}
3075 
3076 	pci_free_irq_vectors(pdev);
3077 
3078 	return err;
3079 }
3080 EXPORT_SYMBOL_GPL(enetc_alloc_msix);
3081 
3082 void enetc_free_msix(struct enetc_ndev_priv *priv)
3083 {
3084 	int i;
3085 
3086 	for (i = 0; i < priv->bdr_int_num; i++) {
3087 		struct enetc_int_vector *v = priv->int_vector[i];
3088 		struct enetc_bdr *rx_ring = &v->rx_ring;
3089 
3090 		xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3091 		xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3092 		netif_napi_del(&v->napi);
3093 		cancel_work_sync(&v->rx_dim.work);
3094 	}
3095 
3096 	for (i = 0; i < priv->num_rx_rings; i++)
3097 		priv->rx_ring[i] = NULL;
3098 
3099 	for (i = 0; i < priv->num_tx_rings; i++)
3100 		priv->tx_ring[i] = NULL;
3101 
3102 	for (i = 0; i < priv->bdr_int_num; i++) {
3103 		kfree(priv->int_vector[i]);
3104 		priv->int_vector[i] = NULL;
3105 	}
3106 
3107 	/* disable all MSIX for this device */
3108 	pci_free_irq_vectors(priv->si->pdev);
3109 }
3110 EXPORT_SYMBOL_GPL(enetc_free_msix);
3111 
3112 static void enetc_kfree_si(struct enetc_si *si)
3113 {
3114 	char *p = (char *)si - si->pad;
3115 
3116 	kfree(p);
3117 }
3118 
3119 static void enetc_detect_errata(struct enetc_si *si)
3120 {
3121 	if (si->pdev->revision == ENETC_REV1)
3122 		si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
3123 }
3124 
3125 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
3126 {
3127 	struct enetc_si *si, *p;
3128 	struct enetc_hw *hw;
3129 	size_t alloc_size;
3130 	int err, len;
3131 
3132 	pcie_flr(pdev);
3133 	err = pci_enable_device_mem(pdev);
3134 	if (err)
3135 		return dev_err_probe(&pdev->dev, err, "device enable failed\n");
3136 
3137 	/* set up for high or low dma */
3138 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3139 	if (err) {
3140 		dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3141 		goto err_dma;
3142 	}
3143 
3144 	err = pci_request_mem_regions(pdev, name);
3145 	if (err) {
3146 		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
3147 		goto err_pci_mem_reg;
3148 	}
3149 
3150 	pci_set_master(pdev);
3151 
3152 	alloc_size = sizeof(struct enetc_si);
3153 	if (sizeof_priv) {
3154 		/* align priv to 32B */
3155 		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
3156 		alloc_size += sizeof_priv;
3157 	}
3158 	/* force 32B alignment for enetc_si */
3159 	alloc_size += ENETC_SI_ALIGN - 1;
3160 
3161 	p = kzalloc(alloc_size, GFP_KERNEL);
3162 	if (!p) {
3163 		err = -ENOMEM;
3164 		goto err_alloc_si;
3165 	}
3166 
3167 	si = PTR_ALIGN(p, ENETC_SI_ALIGN);
3168 	si->pad = (char *)si - (char *)p;
3169 
3170 	pci_set_drvdata(pdev, si);
3171 	si->pdev = pdev;
3172 	hw = &si->hw;
3173 
3174 	len = pci_resource_len(pdev, ENETC_BAR_REGS);
3175 	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
3176 	if (!hw->reg) {
3177 		err = -ENXIO;
3178 		dev_err(&pdev->dev, "ioremap() failed\n");
3179 		goto err_ioremap;
3180 	}
3181 	if (len > ENETC_PORT_BASE)
3182 		hw->port = hw->reg + ENETC_PORT_BASE;
3183 	if (len > ENETC_GLOBAL_BASE)
3184 		hw->global = hw->reg + ENETC_GLOBAL_BASE;
3185 
3186 	enetc_detect_errata(si);
3187 
3188 	return 0;
3189 
3190 err_ioremap:
3191 	enetc_kfree_si(si);
3192 err_alloc_si:
3193 	pci_release_mem_regions(pdev);
3194 err_pci_mem_reg:
3195 err_dma:
3196 	pci_disable_device(pdev);
3197 
3198 	return err;
3199 }
3200 EXPORT_SYMBOL_GPL(enetc_pci_probe);
3201 
3202 void enetc_pci_remove(struct pci_dev *pdev)
3203 {
3204 	struct enetc_si *si = pci_get_drvdata(pdev);
3205 	struct enetc_hw *hw = &si->hw;
3206 
3207 	iounmap(hw->reg);
3208 	enetc_kfree_si(si);
3209 	pci_release_mem_regions(pdev);
3210 	pci_disable_device(pdev);
3211 }
3212 EXPORT_SYMBOL_GPL(enetc_pci_remove);
3213 
3214 MODULE_DESCRIPTION("NXP ENETC Ethernet driver");
3215 MODULE_LICENSE("Dual BSD/GPL");
3216