xref: /linux/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_rx.c (revision a4a35f6cbebbf9466b6c412506ab89299d567f51)
1 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause
2 
3 /* Packet receive logic for Mellanox Gigabit Ethernet driver
4  *
5  * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES
6  */
7 
8 #include <linux/etherdevice.h>
9 #include <linux/skbuff.h>
10 
11 #include "mlxbf_gige.h"
12 #include "mlxbf_gige_regs.h"
13 
mlxbf_gige_enable_multicast_rx(struct mlxbf_gige * priv)14 void mlxbf_gige_enable_multicast_rx(struct mlxbf_gige *priv)
15 {
16 	void __iomem *base = priv->base;
17 	u64 data;
18 
19 	data = readq(base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);
20 	data |= MLXBF_GIGE_RX_MAC_FILTER_EN_MULTICAST;
21 	writeq(data, base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);
22 }
23 
mlxbf_gige_disable_multicast_rx(struct mlxbf_gige * priv)24 void mlxbf_gige_disable_multicast_rx(struct mlxbf_gige *priv)
25 {
26 	void __iomem *base = priv->base;
27 	u64 data;
28 
29 	data = readq(base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);
30 	data &= ~MLXBF_GIGE_RX_MAC_FILTER_EN_MULTICAST;
31 	writeq(data, base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL);
32 }
33 
mlxbf_gige_enable_mac_rx_filter(struct mlxbf_gige * priv,unsigned int index)34 void mlxbf_gige_enable_mac_rx_filter(struct mlxbf_gige *priv,
35 				     unsigned int index)
36 {
37 	void __iomem *base = priv->base;
38 	u64 control;
39 
40 	/* Enable MAC receive filter mask for specified index */
41 	control = readq(base + MLXBF_GIGE_CONTROL);
42 	control |= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);
43 	writeq(control, base + MLXBF_GIGE_CONTROL);
44 }
45 
mlxbf_gige_disable_mac_rx_filter(struct mlxbf_gige * priv,unsigned int index)46 void mlxbf_gige_disable_mac_rx_filter(struct mlxbf_gige *priv,
47 				      unsigned int index)
48 {
49 	void __iomem *base = priv->base;
50 	u64 control;
51 
52 	/* Disable MAC receive filter mask for specified index */
53 	control = readq(base + MLXBF_GIGE_CONTROL);
54 	control &= ~(MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index);
55 	writeq(control, base + MLXBF_GIGE_CONTROL);
56 }
57 
mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige * priv,unsigned int index,u64 dmac)58 void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv,
59 				  unsigned int index, u64 dmac)
60 {
61 	void __iomem *base = priv->base;
62 
63 	/* Write destination MAC to specified MAC RX filter */
64 	writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER +
65 	       (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
66 }
67 
mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige * priv,unsigned int index,u64 * dmac)68 void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv,
69 				  unsigned int index, u64 *dmac)
70 {
71 	void __iomem *base = priv->base;
72 
73 	/* Read destination MAC from specified MAC RX filter */
74 	*dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER +
75 		      (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE));
76 }
77 
mlxbf_gige_enable_promisc(struct mlxbf_gige * priv)78 void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv)
79 {
80 	void __iomem *base = priv->base;
81 	u64 control;
82 	u64 end_mac;
83 
84 	/* Enable MAC_ID_RANGE match functionality */
85 	control = readq(base + MLXBF_GIGE_CONTROL);
86 	control |= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
87 	writeq(control, base + MLXBF_GIGE_CONTROL);
88 
89 	/* Set start of destination MAC range check to 0 */
90 	writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START);
91 
92 	/* Set end of destination MAC range check to all FFs */
93 	end_mac = BCAST_MAC_ADDR;
94 	writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END);
95 }
96 
mlxbf_gige_disable_promisc(struct mlxbf_gige * priv)97 void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv)
98 {
99 	void __iomem *base = priv->base;
100 	u64 control;
101 
102 	/* Disable MAC_ID_RANGE match functionality */
103 	control = readq(base + MLXBF_GIGE_CONTROL);
104 	control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN;
105 	writeq(control, base + MLXBF_GIGE_CONTROL);
106 
107 	/* NOTE: no need to change DMAC_RANGE_START or END;
108 	 * those values are ignored since MAC_ID_RANGE_EN=0
109 	 */
110 }
111 
112 /* Receive Initialization
113  * 1) Configures RX MAC filters via MMIO registers
114  * 2) Allocates RX WQE array using coherent DMA mapping
115  * 3) Initializes each element of RX WQE array with a receive
116  *    buffer pointer (also using coherent DMA mapping)
117  * 4) Allocates RX CQE array using coherent DMA mapping
118  * 5) Completes other misc receive initialization
119  */
mlxbf_gige_rx_init(struct mlxbf_gige * priv)120 int mlxbf_gige_rx_init(struct mlxbf_gige *priv)
121 {
122 	size_t wq_size, cq_size;
123 	dma_addr_t *rx_wqe_ptr;
124 	dma_addr_t rx_buf_dma;
125 	u64 data;
126 	int i, j;
127 
128 	/* Configure MAC RX filter #0 to allow RX of broadcast pkts */
129 	mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX,
130 				     BCAST_MAC_ADDR);
131 
132 	wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
133 	priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size,
134 					       &priv->rx_wqe_base_dma,
135 					       GFP_KERNEL);
136 	if (!priv->rx_wqe_base)
137 		return -ENOMEM;
138 
139 	/* Initialize 'rx_wqe_ptr' to point to first RX WQE in array
140 	 * Each RX WQE is simply a receive buffer pointer, so walk
141 	 * the entire array, allocating a 2KB buffer for each element
142 	 */
143 	rx_wqe_ptr = priv->rx_wqe_base;
144 
145 	for (i = 0; i < priv->rx_q_entries; i++) {
146 		priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
147 						       &rx_buf_dma, DMA_FROM_DEVICE);
148 		if (!priv->rx_skb[i])
149 			goto free_wqe_and_skb;
150 		*rx_wqe_ptr++ = rx_buf_dma;
151 	}
152 
153 	/* Write RX WQE base address into MMIO reg */
154 	writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE);
155 
156 	cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
157 	priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size,
158 					       &priv->rx_cqe_base_dma,
159 					       GFP_KERNEL);
160 	if (!priv->rx_cqe_base)
161 		goto free_wqe_and_skb;
162 
163 	for (i = 0; i < priv->rx_q_entries; i++)
164 		priv->rx_cqe_base[i] |= MLXBF_GIGE_RX_CQE_VALID_MASK;
165 
166 	/* Write RX CQE base address into MMIO reg */
167 	writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE);
168 
169 	/* Write RX_WQE_PI with current number of replenished buffers */
170 	writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI);
171 
172 	/* Enable removal of CRC during RX */
173 	data = readq(priv->base + MLXBF_GIGE_RX);
174 	data |= MLXBF_GIGE_RX_STRIP_CRC_EN;
175 	writeq(data, priv->base + MLXBF_GIGE_RX);
176 
177 	/* Enable RX MAC filter pass and discard counters */
178 	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN,
179 	       priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC);
180 	writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN,
181 	       priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS);
182 
183 	writeq(ilog2(priv->rx_q_entries),
184 	       priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2);
185 
186 	/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
187 	 * indicate readiness to receive interrupts
188 	 */
189 	data = readq(priv->base + MLXBF_GIGE_INT_MASK);
190 	data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
191 	writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
192 
193 	/* Enable RX DMA to write new packets to memory */
194 	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
195 	data |= MLXBF_GIGE_RX_DMA_EN;
196 	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
197 
198 	return 0;
199 
200 free_wqe_and_skb:
201 	rx_wqe_ptr = priv->rx_wqe_base;
202 	for (j = 0; j < i; j++) {
203 		dma_unmap_single(priv->dev, *rx_wqe_ptr,
204 				 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
205 		dev_kfree_skb(priv->rx_skb[j]);
206 		rx_wqe_ptr++;
207 	}
208 	dma_free_coherent(priv->dev, wq_size,
209 			  priv->rx_wqe_base, priv->rx_wqe_base_dma);
210 	return -ENOMEM;
211 }
212 
213 /* Receive Deinitialization
214  * This routine will free allocations done by mlxbf_gige_rx_init(),
215  * namely the RX WQE and RX CQE arrays, as well as all RX buffers
216  */
mlxbf_gige_rx_deinit(struct mlxbf_gige * priv)217 void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv)
218 {
219 	dma_addr_t *rx_wqe_ptr;
220 	size_t size;
221 	u64 data;
222 	int i;
223 
224 	/* Disable RX DMA to prevent packet transfers to memory */
225 	data = readq(priv->base + MLXBF_GIGE_RX_DMA);
226 	data &= ~MLXBF_GIGE_RX_DMA_EN;
227 	writeq(data, priv->base + MLXBF_GIGE_RX_DMA);
228 
229 	rx_wqe_ptr = priv->rx_wqe_base;
230 
231 	for (i = 0; i < priv->rx_q_entries; i++) {
232 		dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ,
233 				 DMA_FROM_DEVICE);
234 		dev_kfree_skb(priv->rx_skb[i]);
235 		rx_wqe_ptr++;
236 	}
237 
238 	size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries;
239 	dma_free_coherent(priv->dev, size,
240 			  priv->rx_wqe_base, priv->rx_wqe_base_dma);
241 
242 	size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries;
243 	dma_free_coherent(priv->dev, size,
244 			  priv->rx_cqe_base, priv->rx_cqe_base_dma);
245 
246 	priv->rx_wqe_base = NULL;
247 	priv->rx_wqe_base_dma = 0;
248 	priv->rx_cqe_base = NULL;
249 	priv->rx_cqe_base_dma = 0;
250 	writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE);
251 	writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE);
252 }
253 
mlxbf_gige_rx_packet(struct mlxbf_gige * priv,int * rx_pkts)254 static bool mlxbf_gige_rx_packet(struct mlxbf_gige *priv, int *rx_pkts)
255 {
256 	struct net_device *netdev = priv->netdev;
257 	struct sk_buff *skb = NULL, *rx_skb;
258 	u16 rx_pi_rem, rx_ci_rem;
259 	dma_addr_t *rx_wqe_addr;
260 	dma_addr_t rx_buf_dma;
261 	u64 *rx_cqe_addr;
262 	u64 datalen;
263 	u64 rx_cqe;
264 	u16 rx_ci;
265 	u16 rx_pi;
266 
267 	/* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */
268 	rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI);
269 	rx_pi_rem = rx_pi % priv->rx_q_entries;
270 
271 	rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem;
272 	rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem;
273 	rx_cqe = *rx_cqe_addr;
274 
275 	if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity)
276 		return false;
277 
278 	if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) {
279 		/* Packet is OK, increment stats */
280 		datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK;
281 		netdev->stats.rx_packets++;
282 		netdev->stats.rx_bytes += datalen;
283 
284 		skb = priv->rx_skb[rx_pi_rem];
285 
286 		/* Alloc another RX SKB for this same index */
287 		rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ,
288 					      &rx_buf_dma, DMA_FROM_DEVICE);
289 		if (!rx_skb)
290 			return false;
291 		priv->rx_skb[rx_pi_rem] = rx_skb;
292 		dma_unmap_single(priv->dev, *rx_wqe_addr,
293 				 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE);
294 
295 		skb_put(skb, datalen);
296 
297 		skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */
298 
299 		skb->protocol = eth_type_trans(skb, netdev);
300 
301 		*rx_wqe_addr = rx_buf_dma;
302 	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) {
303 		priv->stats.rx_mac_errors++;
304 	} else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) {
305 		priv->stats.rx_truncate_errors++;
306 	}
307 
308 	/* Read receive consumer index before replenish so that this routine
309 	 * returns accurate return value even if packet is received into
310 	 * just-replenished buffer prior to exiting this routine.
311 	 */
312 	rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI);
313 	rx_ci_rem = rx_ci % priv->rx_q_entries;
314 
315 	/* Let hardware know we've replenished one buffer */
316 	rx_pi++;
317 
318 	/* Ensure completion of all writes before notifying HW of replenish */
319 	wmb();
320 	writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI);
321 
322 	(*rx_pkts)++;
323 
324 	rx_pi_rem = rx_pi % priv->rx_q_entries;
325 	if (rx_pi_rem == 0)
326 		priv->valid_polarity ^= 1;
327 
328 	if (skb)
329 		netif_receive_skb(skb);
330 
331 	return rx_pi_rem != rx_ci_rem;
332 }
333 
334 /* Driver poll() function called by NAPI infrastructure */
mlxbf_gige_poll(struct napi_struct * napi,int budget)335 int mlxbf_gige_poll(struct napi_struct *napi, int budget)
336 {
337 	struct mlxbf_gige *priv;
338 	bool remaining_pkts;
339 	int work_done = 0;
340 	u64 data;
341 
342 	priv = container_of(napi, struct mlxbf_gige, napi);
343 
344 	mlxbf_gige_handle_tx_complete(priv);
345 
346 	do {
347 		remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done);
348 	} while (remaining_pkts && work_done < budget);
349 
350 	/* If amount of work done < budget, turn off NAPI polling
351 	 * via napi_complete_done(napi, work_done) and then
352 	 * re-enable interrupts.
353 	 */
354 	if (work_done < budget && napi_complete_done(napi, work_done)) {
355 		/* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to
356 		 * indicate receive readiness
357 		 */
358 		data = readq(priv->base + MLXBF_GIGE_INT_MASK);
359 		data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET;
360 		writeq(data, priv->base + MLXBF_GIGE_INT_MASK);
361 	}
362 
363 	return work_done;
364 }
365