1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2020 Intel Corporation. 4 * 5 */ 6 7 /* 8 * This file contains HFI1 support for netdev RX functionality 9 */ 10 11 #include "sdma.h" 12 #include "verbs.h" 13 #include "netdev.h" 14 #include "hfi.h" 15 16 #include <linux/netdevice.h> 17 #include <linux/etherdevice.h> 18 #include <rdma/ib_verbs.h> 19 20 static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_priv *priv, 21 struct hfi1_ctxtdata *uctxt) 22 { 23 unsigned int rcvctrl_ops; 24 struct hfi1_devdata *dd = priv->dd; 25 int ret; 26 27 uctxt->rhf_rcv_function_map = netdev_rhf_rcv_functions; 28 uctxt->do_interrupt = &handle_receive_interrupt_napi_sp; 29 30 /* Now allocate the RcvHdr queue and eager buffers. */ 31 ret = hfi1_create_rcvhdrq(dd, uctxt); 32 if (ret) 33 goto done; 34 35 ret = hfi1_setup_eagerbufs(uctxt); 36 if (ret) 37 goto done; 38 39 clear_rcvhdrtail(uctxt); 40 41 rcvctrl_ops = HFI1_RCVCTRL_CTXT_DIS; 42 rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_DIS; 43 44 if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR)) 45 rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB; 46 if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL)) 47 rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB; 48 if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL)) 49 rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB; 50 if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL)) 51 rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB; 52 53 hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt); 54 done: 55 return ret; 56 } 57 58 static int hfi1_netdev_allocate_ctxt(struct hfi1_devdata *dd, 59 struct hfi1_ctxtdata **ctxt) 60 { 61 struct hfi1_ctxtdata *uctxt; 62 int ret; 63 64 if (dd->flags & HFI1_FROZEN) 65 return -EIO; 66 67 ret = hfi1_create_ctxtdata(dd->pport, dd->node, &uctxt); 68 if (ret < 0) { 69 dd_dev_err(dd, "Unable to create ctxtdata, failing open\n"); 70 return -ENOMEM; 71 } 72 73 uctxt->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) | 74 HFI1_CAP_KGET(NODROP_RHQ_FULL) | 75 HFI1_CAP_KGET(NODROP_EGR_FULL) | 76 HFI1_CAP_KGET(DMA_RTAIL); 77 /* Netdev contexts are always NO_RDMA_RTAIL */ 78 uctxt->fast_handler = handle_receive_interrupt_napi_fp; 79 uctxt->slow_handler = handle_receive_interrupt_napi_sp; 80 hfi1_set_seq_cnt(uctxt, 1); 81 uctxt->is_vnic = true; 82 83 hfi1_stats.sps_ctxts++; 84 85 dd_dev_info(dd, "created netdev context %d\n", uctxt->ctxt); 86 *ctxt = uctxt; 87 88 return 0; 89 } 90 91 static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd, 92 struct hfi1_ctxtdata *uctxt) 93 { 94 flush_wc(); 95 96 /* 97 * Disable receive context and interrupt available, reset all 98 * RcvCtxtCtrl bits to default values. 99 */ 100 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS | 101 HFI1_RCVCTRL_TIDFLOW_DIS | 102 HFI1_RCVCTRL_INTRAVAIL_DIS | 103 HFI1_RCVCTRL_ONE_PKT_EGR_DIS | 104 HFI1_RCVCTRL_NO_RHQ_DROP_DIS | 105 HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt); 106 107 if (uctxt->msix_intr != CCE_NUM_MSIX_VECTORS) 108 msix_free_irq(dd, uctxt->msix_intr); 109 110 uctxt->msix_intr = CCE_NUM_MSIX_VECTORS; 111 uctxt->event_flags = 0; 112 113 hfi1_clear_tids(uctxt); 114 hfi1_clear_ctxt_pkey(dd, uctxt); 115 116 hfi1_stats.sps_ctxts--; 117 118 hfi1_free_ctxt(uctxt); 119 } 120 121 static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_priv *priv, 122 struct hfi1_ctxtdata **ctxt) 123 { 124 int rc; 125 struct hfi1_devdata *dd = priv->dd; 126 127 rc = hfi1_netdev_allocate_ctxt(dd, ctxt); 128 if (rc) { 129 dd_dev_err(dd, "netdev ctxt alloc failed %d\n", rc); 130 return rc; 131 } 132 133 rc = hfi1_netdev_setup_ctxt(priv, *ctxt); 134 if (rc) { 135 dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc); 136 hfi1_netdev_deallocate_ctxt(dd, *ctxt); 137 *ctxt = NULL; 138 } 139 140 return rc; 141 } 142 143 /** 144 * hfi1_num_netdev_contexts - Count of netdev recv contexts to use. 145 * @dd: device on which to allocate netdev contexts 146 * @available_contexts: count of available receive contexts 147 * @cpu_mask: mask of possible cpus to include for contexts 148 * 149 * Return: count of physical cores on a node or the remaining available recv 150 * contexts for netdev recv context usage up to the maximum of 151 * HFI1_MAX_NETDEV_CTXTS. 152 * A value of 0 can be returned when acceleration is explicitly turned off, 153 * a memory allocation error occurs or when there are no available contexts. 154 * 155 */ 156 u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts, 157 struct cpumask *cpu_mask) 158 { 159 cpumask_var_t node_cpu_mask; 160 unsigned int available_cpus; 161 162 if (!HFI1_CAP_IS_KSET(AIP)) 163 return 0; 164 165 /* Always give user contexts priority over netdev contexts */ 166 if (available_contexts == 0) { 167 dd_dev_info(dd, "No receive contexts available for netdevs.\n"); 168 return 0; 169 } 170 171 if (!zalloc_cpumask_var(&node_cpu_mask, GFP_KERNEL)) { 172 dd_dev_err(dd, "Unable to allocate cpu_mask for netdevs.\n"); 173 return 0; 174 } 175 176 cpumask_and(node_cpu_mask, cpu_mask, cpumask_of_node(dd->node)); 177 178 available_cpus = cpumask_weight(node_cpu_mask); 179 180 free_cpumask_var(node_cpu_mask); 181 182 return min3(available_cpus, available_contexts, 183 (u32)HFI1_MAX_NETDEV_CTXTS); 184 } 185 186 static int hfi1_netdev_rxq_init(struct net_device *dev) 187 { 188 int i; 189 int rc; 190 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev); 191 struct hfi1_devdata *dd = priv->dd; 192 193 priv->num_rx_q = dd->num_netdev_contexts; 194 priv->rxq = kcalloc_node(priv->num_rx_q, sizeof(struct hfi1_netdev_rxq), 195 GFP_KERNEL, dd->node); 196 197 if (!priv->rxq) { 198 dd_dev_err(dd, "Unable to allocate netdev queue data\n"); 199 return (-ENOMEM); 200 } 201 202 for (i = 0; i < priv->num_rx_q; i++) { 203 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 204 205 rc = hfi1_netdev_allot_ctxt(priv, &rxq->rcd); 206 if (rc) 207 goto bail_context_irq_failure; 208 209 hfi1_rcd_get(rxq->rcd); 210 rxq->priv = priv; 211 rxq->rcd->napi = &rxq->napi; 212 dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n", 213 i, rxq->rcd->ctxt); 214 /* 215 * Disable BUSY_POLL on this NAPI as this is not supported 216 * right now. 217 */ 218 set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state); 219 netif_napi_add(dev, &rxq->napi, hfi1_netdev_rx_napi, 64); 220 rc = msix_netdev_request_rcd_irq(rxq->rcd); 221 if (rc) 222 goto bail_context_irq_failure; 223 } 224 225 return 0; 226 227 bail_context_irq_failure: 228 dd_dev_err(dd, "Unable to allot receive context\n"); 229 for (; i >= 0; i--) { 230 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 231 232 if (rxq->rcd) { 233 hfi1_netdev_deallocate_ctxt(dd, rxq->rcd); 234 hfi1_rcd_put(rxq->rcd); 235 rxq->rcd = NULL; 236 } 237 } 238 kfree(priv->rxq); 239 priv->rxq = NULL; 240 241 return rc; 242 } 243 244 static void hfi1_netdev_rxq_deinit(struct net_device *dev) 245 { 246 int i; 247 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev); 248 struct hfi1_devdata *dd = priv->dd; 249 250 for (i = 0; i < priv->num_rx_q; i++) { 251 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 252 253 netif_napi_del(&rxq->napi); 254 hfi1_netdev_deallocate_ctxt(dd, rxq->rcd); 255 hfi1_rcd_put(rxq->rcd); 256 rxq->rcd = NULL; 257 } 258 259 kfree(priv->rxq); 260 priv->rxq = NULL; 261 priv->num_rx_q = 0; 262 } 263 264 static void enable_queues(struct hfi1_netdev_priv *priv) 265 { 266 int i; 267 268 for (i = 0; i < priv->num_rx_q; i++) { 269 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 270 271 dd_dev_info(priv->dd, "enabling queue %d on context %d\n", i, 272 rxq->rcd->ctxt); 273 napi_enable(&rxq->napi); 274 hfi1_rcvctrl(priv->dd, 275 HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB, 276 rxq->rcd); 277 } 278 } 279 280 static void disable_queues(struct hfi1_netdev_priv *priv) 281 { 282 int i; 283 284 msix_netdev_synchronize_irq(priv->dd); 285 286 for (i = 0; i < priv->num_rx_q; i++) { 287 struct hfi1_netdev_rxq *rxq = &priv->rxq[i]; 288 289 dd_dev_info(priv->dd, "disabling queue %d on context %d\n", i, 290 rxq->rcd->ctxt); 291 292 /* wait for napi if it was scheduled */ 293 hfi1_rcvctrl(priv->dd, 294 HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS, 295 rxq->rcd); 296 napi_synchronize(&rxq->napi); 297 napi_disable(&rxq->napi); 298 } 299 } 300 301 /** 302 * hfi1_netdev_rx_init - Incrememnts netdevs counter. When called first time, 303 * it allocates receive queue data and calls netif_napi_add 304 * for each queue. 305 * 306 * @dd: hfi1 dev data 307 */ 308 int hfi1_netdev_rx_init(struct hfi1_devdata *dd) 309 { 310 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 311 int res; 312 313 if (atomic_fetch_inc(&priv->netdevs)) 314 return 0; 315 316 mutex_lock(&hfi1_mutex); 317 init_dummy_netdev(dd->dummy_netdev); 318 res = hfi1_netdev_rxq_init(dd->dummy_netdev); 319 mutex_unlock(&hfi1_mutex); 320 return res; 321 } 322 323 /** 324 * hfi1_netdev_rx_destroy - Decrements netdevs counter, when it reaches 0 325 * napi is deleted and receive queses memory is freed. 326 * 327 * @dd: hfi1 dev data 328 */ 329 int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd) 330 { 331 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 332 333 /* destroy the RX queues only if it is the last netdev going away */ 334 if (atomic_fetch_add_unless(&priv->netdevs, -1, 0) == 1) { 335 mutex_lock(&hfi1_mutex); 336 hfi1_netdev_rxq_deinit(dd->dummy_netdev); 337 mutex_unlock(&hfi1_mutex); 338 } 339 340 return 0; 341 } 342 343 /** 344 * hfi1_netdev_alloc - Allocates netdev and private data. It is required 345 * because RMT index and MSI-X interrupt can be set only 346 * during driver initialization. 347 * 348 * @dd: hfi1 dev data 349 */ 350 int hfi1_netdev_alloc(struct hfi1_devdata *dd) 351 { 352 struct hfi1_netdev_priv *priv; 353 const int netdev_size = sizeof(*dd->dummy_netdev) + 354 sizeof(struct hfi1_netdev_priv); 355 356 dd_dev_info(dd, "allocating netdev size %d\n", netdev_size); 357 dd->dummy_netdev = kcalloc_node(1, netdev_size, GFP_KERNEL, dd->node); 358 359 if (!dd->dummy_netdev) 360 return -ENOMEM; 361 362 priv = hfi1_netdev_priv(dd->dummy_netdev); 363 priv->dd = dd; 364 xa_init(&priv->dev_tbl); 365 atomic_set(&priv->enabled, 0); 366 atomic_set(&priv->netdevs, 0); 367 368 return 0; 369 } 370 371 void hfi1_netdev_free(struct hfi1_devdata *dd) 372 { 373 if (dd->dummy_netdev) { 374 dd_dev_info(dd, "hfi1 netdev freed\n"); 375 kfree(dd->dummy_netdev); 376 dd->dummy_netdev = NULL; 377 } 378 } 379 380 /** 381 * hfi1_netdev_enable_queues - This is napi enable function. 382 * It enables napi objects associated with queues. 383 * When at least one device has called it it increments atomic counter. 384 * Disable function decrements counter and when it is 0, 385 * calls napi_disable for every queue. 386 * 387 * @dd: hfi1 dev data 388 */ 389 void hfi1_netdev_enable_queues(struct hfi1_devdata *dd) 390 { 391 struct hfi1_netdev_priv *priv; 392 393 if (!dd->dummy_netdev) 394 return; 395 396 priv = hfi1_netdev_priv(dd->dummy_netdev); 397 if (atomic_fetch_inc(&priv->enabled)) 398 return; 399 400 mutex_lock(&hfi1_mutex); 401 enable_queues(priv); 402 mutex_unlock(&hfi1_mutex); 403 } 404 405 void hfi1_netdev_disable_queues(struct hfi1_devdata *dd) 406 { 407 struct hfi1_netdev_priv *priv; 408 409 if (!dd->dummy_netdev) 410 return; 411 412 priv = hfi1_netdev_priv(dd->dummy_netdev); 413 if (atomic_dec_if_positive(&priv->enabled)) 414 return; 415 416 mutex_lock(&hfi1_mutex); 417 disable_queues(priv); 418 mutex_unlock(&hfi1_mutex); 419 } 420 421 /** 422 * hfi1_netdev_add_data - Registers data with unique identifier 423 * to be requested later this is needed for VNIC and IPoIB VLANs 424 * implementations. 425 * This call is protected by mutex idr_lock. 426 * 427 * @dd: hfi1 dev data 428 * @id: requested integer id up to INT_MAX 429 * @data: data to be associated with index 430 */ 431 int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data) 432 { 433 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 434 435 return xa_insert(&priv->dev_tbl, id, data, GFP_NOWAIT); 436 } 437 438 /** 439 * hfi1_netdev_remove_data - Removes data with previously given id. 440 * Returns the reference to removed entry. 441 * 442 * @dd: hfi1 dev data 443 * @id: requested integer id up to INT_MAX 444 */ 445 void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id) 446 { 447 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 448 449 return xa_erase(&priv->dev_tbl, id); 450 } 451 452 /** 453 * hfi1_netdev_get_data - Gets data with given id 454 * 455 * @dd: hfi1 dev data 456 * @id: requested integer id up to INT_MAX 457 */ 458 void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id) 459 { 460 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 461 462 return xa_load(&priv->dev_tbl, id); 463 } 464 465 /** 466 * hfi1_netdev_get_first_dat - Gets first entry with greater or equal id. 467 * 468 * @dd: hfi1 dev data 469 * @start_id: requested integer id up to INT_MAX 470 */ 471 void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id) 472 { 473 struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev); 474 unsigned long index = *start_id; 475 void *ret; 476 477 ret = xa_find(&priv->dev_tbl, &index, UINT_MAX, XA_PRESENT); 478 *start_id = (int)index; 479 return ret; 480 } 481