xref: /linux/drivers/infiniband/hw/hfi1/netdev_rx.c (revision c532de5a67a70f8533d495f8f2aaa9a0491c3ad0) 
1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2 /*
3  * Copyright(c) 2020 Intel Corporation.
4  *
5  */
6 
7 /*
8  * This file contains HFI1 support for netdev RX functionality
9  */
10 
11 #include "sdma.h"
12 #include "verbs.h"
13 #include "netdev.h"
14 #include "hfi.h"
15 
16 #include <linux/netdevice.h>
17 #include <linux/etherdevice.h>
18 #include <rdma/ib_verbs.h>
19 
20 static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_rx *rx,
21 				  struct hfi1_ctxtdata *uctxt)
22 {
23 	unsigned int rcvctrl_ops;
24 	struct hfi1_devdata *dd = rx->dd;
25 	int ret;
26 
27 	uctxt->rhf_rcv_function_map = netdev_rhf_rcv_functions;
28 	uctxt->do_interrupt = &handle_receive_interrupt_napi_sp;
29 
30 	/* Now allocate the RcvHdr queue and eager buffers. */
31 	ret = hfi1_create_rcvhdrq(dd, uctxt);
32 	if (ret)
33 		goto done;
34 
35 	ret = hfi1_setup_eagerbufs(uctxt);
36 	if (ret)
37 		goto done;
38 
39 	clear_rcvhdrtail(uctxt);
40 
41 	rcvctrl_ops = HFI1_RCVCTRL_CTXT_DIS;
42 	rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_DIS;
43 
44 	if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR))
45 		rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
46 	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL))
47 		rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
48 	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
49 		rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
50 	if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))
51 		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;
52 
53 	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);
54 done:
55 	return ret;
56 }
57 
58 static int hfi1_netdev_allocate_ctxt(struct hfi1_devdata *dd,
59 				     struct hfi1_ctxtdata **ctxt)
60 {
61 	struct hfi1_ctxtdata *uctxt;
62 	int ret;
63 
64 	if (dd->flags & HFI1_FROZEN)
65 		return -EIO;
66 
67 	ret = hfi1_create_ctxtdata(dd->pport, dd->node, &uctxt);
68 	if (ret < 0) {
69 		dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");
70 		return -ENOMEM;
71 	}
72 
73 	uctxt->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
74 		HFI1_CAP_KGET(NODROP_RHQ_FULL) |
75 		HFI1_CAP_KGET(NODROP_EGR_FULL) |
76 		HFI1_CAP_KGET(DMA_RTAIL);
77 	/* Netdev contexts are always NO_RDMA_RTAIL */
78 	uctxt->fast_handler = handle_receive_interrupt_napi_fp;
79 	uctxt->slow_handler = handle_receive_interrupt_napi_sp;
80 	hfi1_set_seq_cnt(uctxt, 1);
81 	uctxt->is_vnic = true;
82 
83 	hfi1_stats.sps_ctxts++;
84 
85 	dd_dev_info(dd, "created netdev context %d\n", uctxt->ctxt);
86 	*ctxt = uctxt;
87 
88 	return 0;
89 }
90 
91 static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd,
92 					struct hfi1_ctxtdata *uctxt)
93 {
94 	flush_wc();
95 
96 	/*
97 	 * Disable receive context and interrupt available, reset all
98 	 * RcvCtxtCtrl bits to default values.
99 	 */
100 	hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
101 		     HFI1_RCVCTRL_TIDFLOW_DIS |
102 		     HFI1_RCVCTRL_INTRAVAIL_DIS |
103 		     HFI1_RCVCTRL_ONE_PKT_EGR_DIS |
104 		     HFI1_RCVCTRL_NO_RHQ_DROP_DIS |
105 		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);
106 
107 	if (uctxt->msix_intr != CCE_NUM_MSIX_VECTORS)
108 		msix_free_irq(dd, uctxt->msix_intr);
109 
110 	uctxt->msix_intr = CCE_NUM_MSIX_VECTORS;
111 	uctxt->event_flags = 0;
112 
113 	hfi1_clear_tids(uctxt);
114 	hfi1_clear_ctxt_pkey(dd, uctxt);
115 
116 	hfi1_stats.sps_ctxts--;
117 
118 	hfi1_free_ctxt(uctxt);
119 }
120 
121 static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_rx *rx,
122 				  struct hfi1_ctxtdata **ctxt)
123 {
124 	int rc;
125 	struct hfi1_devdata *dd = rx->dd;
126 
127 	rc = hfi1_netdev_allocate_ctxt(dd, ctxt);
128 	if (rc) {
129 		dd_dev_err(dd, "netdev ctxt alloc failed %d\n", rc);
130 		return rc;
131 	}
132 
133 	rc = hfi1_netdev_setup_ctxt(rx, *ctxt);
134 	if (rc) {
135 		dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc);
136 		hfi1_netdev_deallocate_ctxt(dd, *ctxt);
137 		*ctxt = NULL;
138 	}
139 
140 	return rc;
141 }
142 
143 /**
144  * hfi1_num_netdev_contexts - Count of netdev recv contexts to use.
145  * @dd: device on which to allocate netdev contexts
146  * @available_contexts: count of available receive contexts
147  * @cpu_mask: mask of possible cpus to include for contexts
148  *
149  * Return: count of physical cores on a node or the remaining available recv
150  * contexts for netdev recv context usage up to the maximum of
151  * HFI1_MAX_NETDEV_CTXTS.
152  * A value of 0 can be returned when acceleration is explicitly turned off,
153  * a memory allocation error occurs or when there are no available contexts.
154  *
155  */
156 u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,
157 			     struct cpumask *cpu_mask)
158 {
159 	cpumask_var_t node_cpu_mask;
160 	unsigned int available_cpus;
161 
162 	if (!HFI1_CAP_IS_KSET(AIP))
163 		return 0;
164 
165 	/* Always give user contexts priority over netdev contexts */
166 	if (available_contexts == 0) {
167 		dd_dev_info(dd, "No receive contexts available for netdevs.\n");
168 		return 0;
169 	}
170 
171 	if (!zalloc_cpumask_var(&node_cpu_mask, GFP_KERNEL)) {
172 		dd_dev_err(dd, "Unable to allocate cpu_mask for netdevs.\n");
173 		return 0;
174 	}
175 
176 	cpumask_and(node_cpu_mask, cpu_mask, cpumask_of_node(dd->node));
177 
178 	available_cpus = cpumask_weight(node_cpu_mask);
179 
180 	free_cpumask_var(node_cpu_mask);
181 
182 	return min3(available_cpus, available_contexts,
183 		    (u32)HFI1_MAX_NETDEV_CTXTS);
184 }
185 
186 static int hfi1_netdev_rxq_init(struct hfi1_netdev_rx *rx)
187 {
188 	int i;
189 	int rc;
190 	struct hfi1_devdata *dd = rx->dd;
191 	struct net_device *dev = rx->rx_napi;
192 
193 	rx->num_rx_q = dd->num_netdev_contexts;
194 	rx->rxq = kcalloc_node(rx->num_rx_q, sizeof(*rx->rxq),
195 			       GFP_KERNEL, dd->node);
196 
197 	if (!rx->rxq) {
198 		dd_dev_err(dd, "Unable to allocate netdev queue data\n");
199 		return (-ENOMEM);
200 	}
201 
202 	for (i = 0; i < rx->num_rx_q; i++) {
203 		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
204 
205 		rc = hfi1_netdev_allot_ctxt(rx, &rxq->rcd);
206 		if (rc)
207 			goto bail_context_irq_failure;
208 
209 		hfi1_rcd_get(rxq->rcd);
210 		rxq->rx = rx;
211 		rxq->rcd->napi = &rxq->napi;
212 		dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n",
213 			    i, rxq->rcd->ctxt);
214 		/*
215 		 * Disable BUSY_POLL on this NAPI as this is not supported
216 		 * right now.
217 		 */
218 		set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state);
219 		netif_napi_add(dev, &rxq->napi, hfi1_netdev_rx_napi);
220 		rc = msix_netdev_request_rcd_irq(rxq->rcd);
221 		if (rc)
222 			goto bail_context_irq_failure;
223 	}
224 
225 	return 0;
226 
227 bail_context_irq_failure:
228 	dd_dev_err(dd, "Unable to allot receive context\n");
229 	for (; i >= 0; i--) {
230 		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
231 
232 		if (rxq->rcd) {
233 			hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
234 			hfi1_rcd_put(rxq->rcd);
235 			rxq->rcd = NULL;
236 		}
237 	}
238 	kfree(rx->rxq);
239 	rx->rxq = NULL;
240 
241 	return rc;
242 }
243 
244 static void hfi1_netdev_rxq_deinit(struct hfi1_netdev_rx *rx)
245 {
246 	int i;
247 	struct hfi1_devdata *dd = rx->dd;
248 
249 	for (i = 0; i < rx->num_rx_q; i++) {
250 		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
251 
252 		netif_napi_del(&rxq->napi);
253 		hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
254 		hfi1_rcd_put(rxq->rcd);
255 		rxq->rcd = NULL;
256 	}
257 
258 	kfree(rx->rxq);
259 	rx->rxq = NULL;
260 	rx->num_rx_q = 0;
261 }
262 
263 static void enable_queues(struct hfi1_netdev_rx *rx)
264 {
265 	int i;
266 
267 	for (i = 0; i < rx->num_rx_q; i++) {
268 		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
269 
270 		dd_dev_info(rx->dd, "enabling queue %d on context %d\n", i,
271 			    rxq->rcd->ctxt);
272 		napi_enable(&rxq->napi);
273 		hfi1_rcvctrl(rx->dd,
274 			     HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB,
275 			     rxq->rcd);
276 	}
277 }
278 
279 static void disable_queues(struct hfi1_netdev_rx *rx)
280 {
281 	int i;
282 
283 	msix_netdev_synchronize_irq(rx->dd);
284 
285 	for (i = 0; i < rx->num_rx_q; i++) {
286 		struct hfi1_netdev_rxq *rxq = &rx->rxq[i];
287 
288 		dd_dev_info(rx->dd, "disabling queue %d on context %d\n", i,
289 			    rxq->rcd->ctxt);
290 
291 		/* wait for napi if it was scheduled */
292 		hfi1_rcvctrl(rx->dd,
293 			     HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS,
294 			     rxq->rcd);
295 		napi_synchronize(&rxq->napi);
296 		napi_disable(&rxq->napi);
297 	}
298 }
299 
300 /**
301  * hfi1_netdev_rx_init - Incrememnts netdevs counter. When called first time,
302  * it allocates receive queue data and calls netif_napi_add
303  * for each queue.
304  *
305  * @dd: hfi1 dev data
306  */
307 int hfi1_netdev_rx_init(struct hfi1_devdata *dd)
308 {
309 	struct hfi1_netdev_rx *rx = dd->netdev_rx;
310 	int res;
311 
312 	if (atomic_fetch_inc(&rx->netdevs))
313 		return 0;
314 
315 	mutex_lock(&hfi1_mutex);
316 	res = hfi1_netdev_rxq_init(rx);
317 	mutex_unlock(&hfi1_mutex);
318 	return res;
319 }
320 
321 /**
322  * hfi1_netdev_rx_destroy - Decrements netdevs counter, when it reaches 0
323  * napi is deleted and receive queses memory is freed.
324  *
325  * @dd: hfi1 dev data
326  */
327 int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd)
328 {
329 	struct hfi1_netdev_rx *rx = dd->netdev_rx;
330 
331 	/* destroy the RX queues only if it is the last netdev going away */
332 	if (atomic_fetch_add_unless(&rx->netdevs, -1, 0) == 1) {
333 		mutex_lock(&hfi1_mutex);
334 		hfi1_netdev_rxq_deinit(rx);
335 		mutex_unlock(&hfi1_mutex);
336 	}
337 
338 	return 0;
339 }
340 
341 /**
342  * hfi1_alloc_rx - Allocates the rx support structure
343  * @dd: hfi1 dev data
344  *
345  * Allocate the rx structure to support gathering the receive
346  * resources and the dummy netdev.
347  *
348  * Updates dd struct pointer upon success.
349  *
350  * Return: 0 (success) -error on failure
351  *
352  */
353 int hfi1_alloc_rx(struct hfi1_devdata *dd)
354 {
355 	struct hfi1_netdev_rx *rx;
356 
357 	dd_dev_info(dd, "allocating rx size %ld\n", sizeof(*rx));
358 	rx = kzalloc_node(sizeof(*rx), GFP_KERNEL, dd->node);
359 
360 	if (!rx)
361 		return -ENOMEM;
362 	rx->dd = dd;
363 	rx->rx_napi = alloc_netdev_dummy(0);
364 	if (!rx->rx_napi) {
365 		kfree(rx);
366 		return -ENOMEM;
367 	}
368 
369 	xa_init(&rx->dev_tbl);
370 	atomic_set(&rx->enabled, 0);
371 	atomic_set(&rx->netdevs, 0);
372 	dd->netdev_rx = rx;
373 
374 	return 0;
375 }
376 
377 void hfi1_free_rx(struct hfi1_devdata *dd)
378 {
379 	if (dd->netdev_rx) {
380 		dd_dev_info(dd, "hfi1 rx freed\n");
381 		free_netdev(dd->netdev_rx->rx_napi);
382 		kfree(dd->netdev_rx);
383 		dd->netdev_rx = NULL;
384 	}
385 }
386 
387 /**
388  * hfi1_netdev_enable_queues - This is napi enable function.
389  * It enables napi objects associated with queues.
390  * When at least one device has called it it increments atomic counter.
391  * Disable function decrements counter and when it is 0,
392  * calls napi_disable for every queue.
393  *
394  * @dd: hfi1 dev data
395  */
396 void hfi1_netdev_enable_queues(struct hfi1_devdata *dd)
397 {
398 	struct hfi1_netdev_rx *rx;
399 
400 	if (!dd->netdev_rx)
401 		return;
402 
403 	rx = dd->netdev_rx;
404 	if (atomic_fetch_inc(&rx->enabled))
405 		return;
406 
407 	mutex_lock(&hfi1_mutex);
408 	enable_queues(rx);
409 	mutex_unlock(&hfi1_mutex);
410 }
411 
412 void hfi1_netdev_disable_queues(struct hfi1_devdata *dd)
413 {
414 	struct hfi1_netdev_rx *rx;
415 
416 	if (!dd->netdev_rx)
417 		return;
418 
419 	rx = dd->netdev_rx;
420 	if (atomic_dec_if_positive(&rx->enabled))
421 		return;
422 
423 	mutex_lock(&hfi1_mutex);
424 	disable_queues(rx);
425 	mutex_unlock(&hfi1_mutex);
426 }
427 
428 /**
429  * hfi1_netdev_add_data - Registers data with unique identifier
430  * to be requested later this is needed for VNIC and IPoIB VLANs
431  * implementations.
432  * This call is protected by mutex idr_lock.
433  *
434  * @dd: hfi1 dev data
435  * @id: requested integer id up to INT_MAX
436  * @data: data to be associated with index
437  */
438 int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data)
439 {
440 	struct hfi1_netdev_rx *rx = dd->netdev_rx;
441 
442 	return xa_insert(&rx->dev_tbl, id, data, GFP_NOWAIT);
443 }
444 
445 /**
446  * hfi1_netdev_remove_data - Removes data with previously given id.
447  * Returns the reference to removed entry.
448  *
449  * @dd: hfi1 dev data
450  * @id: requested integer id up to INT_MAX
451  */
452 void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id)
453 {
454 	struct hfi1_netdev_rx *rx = dd->netdev_rx;
455 
456 	return xa_erase(&rx->dev_tbl, id);
457 }
458 
459 /**
460  * hfi1_netdev_get_data - Gets data with given id
461  *
462  * @dd: hfi1 dev data
463  * @id: requested integer id up to INT_MAX
464  */
465 void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id)
466 {
467 	struct hfi1_netdev_rx *rx = dd->netdev_rx;
468 
469 	return xa_load(&rx->dev_tbl, id);
470 }
471 
472 /**
473  * hfi1_netdev_get_first_data - Gets first entry with greater or equal id.
474  *
475  * @dd: hfi1 dev data
476  * @start_id: requested integer id up to INT_MAX
477  */
478 void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id)
479 {
480 	struct hfi1_netdev_rx *rx = dd->netdev_rx;
481 	unsigned long index = *start_id;
482 	void *ret;
483 
484 	ret = xa_find(&rx->dev_tbl, &index, UINT_MAX, XA_PRESENT);
485 	*start_id = (int)index;
486 	return ret;
487 }
488