1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/dim.h> 34 #include <net/tc_act/tc_gact.h> 35 #include <linux/mlx5/fs.h> 36 #include <net/vxlan.h> 37 #include <net/geneve.h> 38 #include <linux/bpf.h> 39 #include <linux/debugfs.h> 40 #include <linux/if_bridge.h> 41 #include <linux/filter.h> 42 #include <net/netdev_lock.h> 43 #include <net/netdev_queues.h> 44 #include <net/netdev_rx_queue.h> 45 #include <net/page_pool/types.h> 46 #include <net/pkt_sched.h> 47 #include <net/xdp_sock_drv.h> 48 #include "eswitch.h" 49 #include "en.h" 50 #include "en/dim.h" 51 #include "en/txrx.h" 52 #include "en_tc.h" 53 #include "en_rep.h" 54 #include "en_accel/ipsec.h" 55 #include "en_accel/psp.h" 56 #include "en_accel/macsec.h" 57 #include "en_accel/en_accel.h" 58 #include "en_accel/ktls.h" 59 #include "lib/vxlan.h" 60 #include "lib/clock.h" 61 #include "en/port.h" 62 #include "en/xdp.h" 63 #include "lib/eq.h" 64 #include "en/monitor_stats.h" 65 #include "en/health.h" 66 #include "en/params.h" 67 #include "en/xsk/pool.h" 68 #include "en/xsk/setup.h" 69 #include "en/xsk/rx.h" 70 #include "en/xsk/tx.h" 71 #include "en/hv_vhca_stats.h" 72 #include "en/devlink.h" 73 #include "lib/mlx5.h" 74 #include "en/ptp.h" 75 #include "en/htb.h" 76 #include "qos.h" 77 #include "en/trap.h" 78 #include "lib/devcom.h" 79 #include "lib/sd.h" 80 #include "en/pcie_cong_event.h" 81 82 static bool mlx5e_hw_gro_supported(struct mlx5_core_dev *mdev) 83 { 84 if (!MLX5_CAP_GEN(mdev, shampo) || 85 !MLX5_CAP_SHAMPO(mdev, shampo_header_split_data_merge)) 86 return false; 87 88 /* Our HW-GRO implementation relies on "KSM Mkey" for 89 * SHAMPO headers buffer mapping 90 */ 91 if (!MLX5_CAP_GEN(mdev, fixed_buffer_size)) 92 return false; 93 94 if (!MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer_valid)) 95 return false; 96 97 if (MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer) > 98 MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE) 99 return false; 100 101 return true; 102 } 103 104 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, 105 enum mlx5e_mpwrq_umr_mode umr_mode) 106 { 107 u16 umr_wqebbs, max_wqebbs; 108 bool striding_rq_umr; 109 110 striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) && 111 MLX5_CAP_ETH(mdev, reg_umr_sq); 112 if (!striding_rq_umr) 113 return false; 114 115 umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode); 116 max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 117 /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is 118 * calculated from mlx5e_get_max_sq_aligned_wqebbs. 119 */ 120 if (WARN_ON(umr_wqebbs > max_wqebbs)) 121 return false; 122 123 return true; 124 } 125 126 void mlx5e_update_carrier(struct mlx5e_priv *priv) 127 { 128 struct mlx5_core_dev *mdev = priv->mdev; 129 u8 port_state; 130 bool up; 131 132 port_state = mlx5_query_vport_state(mdev, 133 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 134 0); 135 136 up = port_state == VPORT_STATE_UP; 137 if (up == netif_carrier_ok(priv->netdev)) 138 netif_carrier_event(priv->netdev); 139 if (up) { 140 netdev_info(priv->netdev, "Link up\n"); 141 netif_carrier_on(priv->netdev); 142 } else { 143 netdev_info(priv->netdev, "Link down\n"); 144 netif_carrier_off(priv->netdev); 145 } 146 } 147 148 static void mlx5e_update_carrier_work(struct work_struct *work) 149 { 150 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 151 update_carrier_work); 152 153 mutex_lock(&priv->state_lock); 154 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 155 if (priv->profile->update_carrier) 156 priv->profile->update_carrier(priv); 157 mutex_unlock(&priv->state_lock); 158 } 159 160 static void mlx5e_update_stats_work(struct work_struct *work) 161 { 162 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 163 update_stats_work); 164 165 mutex_lock(&priv->state_lock); 166 priv->profile->update_stats(priv); 167 mutex_unlock(&priv->state_lock); 168 } 169 170 void mlx5e_queue_update_stats(struct mlx5e_priv *priv) 171 { 172 if (!priv->profile->update_stats) 173 return; 174 175 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state))) 176 return; 177 178 queue_work(priv->wq, &priv->update_stats_work); 179 } 180 181 static int async_event(struct notifier_block *nb, unsigned long event, void *data) 182 { 183 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 184 struct mlx5_eqe *eqe = data; 185 186 if (event != MLX5_EVENT_TYPE_PORT_CHANGE) 187 return NOTIFY_DONE; 188 189 switch (eqe->sub_type) { 190 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 191 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 192 queue_work(priv->wq, &priv->update_carrier_work); 193 break; 194 default: 195 return NOTIFY_DONE; 196 } 197 198 return NOTIFY_OK; 199 } 200 201 static void mlx5e_enable_async_events(struct mlx5e_priv *priv) 202 { 203 priv->events_nb.notifier_call = async_event; 204 mlx5_notifier_register(priv->mdev, &priv->events_nb); 205 } 206 207 static void mlx5e_disable_async_events(struct mlx5e_priv *priv) 208 { 209 mlx5_notifier_unregister(priv->mdev, &priv->events_nb); 210 } 211 212 static int mlx5e_devcom_event_mpv(int event, void *my_data, void *event_data) 213 { 214 struct mlx5e_priv *master_priv = event_data; 215 216 switch (event) { 217 case MPV_DEVCOM_MASTER_UP: 218 mlx5_devcom_comp_set_ready(master_priv->devcom, true); 219 break; 220 case MPV_DEVCOM_MASTER_DOWN: 221 /* no need for comp set ready false since we unregister after 222 * and it hurts cleanup flow. 223 */ 224 break; 225 case MPV_DEVCOM_IPSEC_MASTER_UP: 226 case MPV_DEVCOM_IPSEC_MASTER_DOWN: 227 mlx5e_ipsec_handle_mpv_event(event, my_data, event_data); 228 break; 229 } 230 231 return 0; 232 } 233 234 static int mlx5e_devcom_init_mpv(struct mlx5e_priv *priv, u64 *data) 235 { 236 struct mlx5_devcom_match_attr attr = { 237 .key.val = *data, 238 }; 239 240 priv->devcom = mlx5_devcom_register_component(priv->mdev->priv.devc, 241 MLX5_DEVCOM_MPV, 242 &attr, 243 mlx5e_devcom_event_mpv, 244 priv); 245 if (!priv->devcom) 246 return -EINVAL; 247 248 if (mlx5_core_is_mp_master(priv->mdev)) { 249 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_UP, 250 MPV_DEVCOM_MASTER_UP, priv); 251 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_UP); 252 } 253 254 return 0; 255 } 256 257 static void mlx5e_devcom_cleanup_mpv(struct mlx5e_priv *priv) 258 { 259 if (!priv->devcom) 260 return; 261 262 if (mlx5_core_is_mp_master(priv->mdev)) { 263 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_DOWN, 264 MPV_DEVCOM_MASTER_DOWN, priv); 265 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_DOWN); 266 } 267 268 mlx5_devcom_unregister_component(priv->devcom); 269 priv->devcom = NULL; 270 } 271 272 static int blocking_event(struct notifier_block *nb, unsigned long event, void *data) 273 { 274 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb); 275 struct mlx5_devlink_trap_event_ctx *trap_event_ctx = data; 276 int err; 277 278 switch (event) { 279 case MLX5_DRIVER_EVENT_TYPE_TRAP: 280 err = mlx5e_handle_trap_event(priv, trap_event_ctx->trap); 281 if (err) { 282 trap_event_ctx->err = err; 283 return NOTIFY_BAD; 284 } 285 break; 286 case MLX5_DRIVER_EVENT_AFFILIATION_DONE: 287 if (mlx5e_devcom_init_mpv(priv, data)) 288 return NOTIFY_BAD; 289 break; 290 case MLX5_DRIVER_EVENT_AFFILIATION_REMOVED: 291 mlx5e_devcom_cleanup_mpv(priv); 292 break; 293 default: 294 return NOTIFY_DONE; 295 } 296 return NOTIFY_OK; 297 } 298 299 static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv) 300 { 301 priv->blocking_events_nb.notifier_call = blocking_event; 302 mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb); 303 } 304 305 static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv) 306 { 307 mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb); 308 } 309 310 static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode) 311 { 312 u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); 313 u32 sz; 314 315 sz = ALIGN(entries * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT); 316 317 return sz / MLX5_OCTWORD; 318 } 319 320 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, 321 struct mlx5e_icosq *sq, 322 struct mlx5e_umr_wqe *wqe) 323 { 324 struct mlx5_wqe_ctrl_seg *cseg = &wqe->hdr.ctrl; 325 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->hdr.uctrl; 326 u16 octowords; 327 u8 ds_cnt; 328 329 ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift, 330 rq->mpwqe.umr_mode), 331 MLX5_SEND_WQE_DS); 332 333 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | 334 ds_cnt); 335 cseg->umr_mkey = rq->mpwqe.umr_mkey_be; 336 337 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 338 octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode); 339 ucseg->xlt_octowords = cpu_to_be16(octowords); 340 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 341 } 342 343 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) 344 { 345 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 346 size_t alloc_size; 347 348 alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, 349 alloc_units.frag_pages, 350 rq->mpwqe.pages_per_wqe)); 351 352 rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node); 353 if (!rq->mpwqe.info) 354 return -ENOMEM; 355 356 /* For deferred page release (release right before alloc), make sure 357 * that on first round release is not called. 358 */ 359 for (int i = 0; i < wq_sz; i++) { 360 struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, i); 361 362 bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe); 363 } 364 365 mlx5e_build_umr_wqe(rq, rq->icosq, 366 container_of(&rq->mpwqe.umr_wqe, 367 struct mlx5e_umr_wqe, hdr)); 368 369 return 0; 370 } 371 372 static int mlx5e_rq_alloc_mpwqe_linear_info(struct mlx5e_rq *rq, int node, 373 struct mlx5e_params *params, 374 struct mlx5e_rq_opt_param *rqo) 375 { 376 struct mlx5_core_dev *mdev = rq->mdev; 377 struct mlx5e_mpw_linear_info *li; 378 u32 linear_frag_count; 379 380 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo) || 381 !params->xdp_prog) 382 return 0; 383 384 li = kvzalloc_node(sizeof(*li), GFP_KERNEL, node); 385 if (!li) 386 return -ENOMEM; 387 388 linear_frag_count = 389 BIT(rq->mpwqe.page_shift - MLX5E_XDP_LOG_MAX_LINEAR_SZ); 390 if (linear_frag_count > U16_MAX) { 391 netdev_warn(rq->netdev, 392 "rq %d: linear_frag_count (%u) larger than expected (%u), page_shift: %u, log_max_linear_sz: %u\n", 393 rq->ix, linear_frag_count, U16_MAX, 394 rq->mpwqe.page_shift, MLX5E_XDP_LOG_MAX_LINEAR_SZ); 395 kvfree(li); 396 return -EINVAL; 397 } 398 399 li->max_frags = linear_frag_count; 400 rq->mpwqe.linear_info = li; 401 402 /* Set to max to force allocation on first run. */ 403 li->frag_page.frags = li->max_frags; 404 405 return 0; 406 } 407 408 static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode) 409 { 410 switch (umr_mode) { 411 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 412 return MLX5_MKC_ACCESS_MODE_MTT; 413 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 414 return MLX5_MKC_ACCESS_MODE_KSM; 415 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 416 return MLX5_MKC_ACCESS_MODE_KLMS; 417 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 418 return MLX5_MKC_ACCESS_MODE_KSM; 419 } 420 WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode); 421 return 0; 422 } 423 424 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, 425 u32 npages, u8 page_shift, u32 *umr_mkey, 426 dma_addr_t filler_addr, 427 enum mlx5e_mpwrq_umr_mode umr_mode, 428 u32 xsk_chunk_size) 429 { 430 struct mlx5_mtt *mtt; 431 struct mlx5_ksm *ksm; 432 struct mlx5_klm *klm; 433 u32 octwords; 434 int inlen; 435 void *mkc; 436 u32 *in; 437 int err; 438 int i; 439 440 if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED || 441 umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) && 442 !MLX5_CAP_GEN(mdev, fixed_buffer_size)) { 443 mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n"); 444 return -EINVAL; 445 } 446 447 octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode); 448 449 inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), 450 MLX5_OCTWORD, octwords); 451 if (inlen < 0) 452 return inlen; 453 454 in = kvzalloc(inlen, GFP_KERNEL); 455 if (!in) 456 return -ENOMEM; 457 458 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 459 460 MLX5_SET(mkc, mkc, free, 1); 461 MLX5_SET(mkc, mkc, umr_en, 1); 462 MLX5_SET(mkc, mkc, lw, 1); 463 MLX5_SET(mkc, mkc, lr, 1); 464 MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode)); 465 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 466 MLX5_SET(mkc, mkc, qpn, 0xffffff); 467 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 468 MLX5_SET64(mkc, mkc, len, npages << page_shift); 469 MLX5_SET(mkc, mkc, translations_octword_size, octwords); 470 if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) 471 MLX5_SET(mkc, mkc, log_page_size, page_shift - 2); 472 else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED) 473 MLX5_SET(mkc, mkc, log_page_size, page_shift); 474 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords); 475 476 /* Initialize the mkey with all MTTs pointing to a default 477 * page (filler_addr). When the channels are activated, UMR 478 * WQEs will redirect the RX WQEs to the actual memory from 479 * the RQ's pool, while the gaps (wqe_overflow) remain mapped 480 * to the default page. 481 */ 482 switch (umr_mode) { 483 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 484 klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 485 for (i = 0; i < npages; i++) { 486 klm[i << 1] = (struct mlx5_klm) { 487 .va = cpu_to_be64(filler_addr), 488 .bcount = cpu_to_be32(xsk_chunk_size), 489 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 490 }; 491 klm[(i << 1) + 1] = (struct mlx5_klm) { 492 .va = cpu_to_be64(filler_addr), 493 .bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size), 494 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 495 }; 496 } 497 break; 498 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 499 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 500 for (i = 0; i < npages; i++) 501 ksm[i] = (struct mlx5_ksm) { 502 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 503 .va = cpu_to_be64(filler_addr), 504 }; 505 break; 506 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 507 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 508 for (i = 0; i < npages; i++) 509 mtt[i] = (struct mlx5_mtt) { 510 .ptag = cpu_to_be64(filler_addr), 511 }; 512 break; 513 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 514 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 515 for (i = 0; i < npages * 4; i++) { 516 ksm[i] = (struct mlx5_ksm) { 517 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 518 .va = cpu_to_be64(filler_addr), 519 }; 520 } 521 break; 522 } 523 524 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 525 526 kvfree(in); 527 return err; 528 } 529 530 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) 531 { 532 u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0; 533 u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 534 u32 num_entries, max_num_entries; 535 u32 umr_mkey; 536 int err; 537 538 max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode); 539 540 /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */ 541 if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe, 542 &num_entries) || 543 num_entries > max_num_entries)) 544 mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n", 545 __func__, wq_size, rq->mpwqe.mtts_per_wqe, 546 max_num_entries); 547 548 err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift, 549 &umr_mkey, rq->wqe_overflow.addr, 550 rq->mpwqe.umr_mode, xsk_chunk_size); 551 rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); 552 return err; 553 } 554 555 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) 556 { 557 struct mlx5e_wqe_frag_info next_frag = {}; 558 struct mlx5e_wqe_frag_info *prev = NULL; 559 int i; 560 561 WARN_ON(rq->xsk_pool); 562 563 next_frag.frag_page = &rq->wqe.alloc_units->frag_pages[0]; 564 565 /* Skip first release due to deferred release. */ 566 next_frag.flags = BIT(MLX5E_WQE_FRAG_SKIP_RELEASE); 567 568 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 569 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; 570 struct mlx5e_wqe_frag_info *frag = 571 &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; 572 int f; 573 574 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { 575 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { 576 /* Pages are assigned at runtime. */ 577 next_frag.frag_page++; 578 next_frag.offset = 0; 579 if (prev) 580 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE); 581 } 582 *frag = next_frag; 583 584 /* prepare next */ 585 next_frag.offset += frag_info[f].frag_stride; 586 prev = frag; 587 } 588 } 589 590 if (prev) 591 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE); 592 } 593 594 static void mlx5e_init_xsk_buffs(struct mlx5e_rq *rq) 595 { 596 int i; 597 598 /* Assumptions used by XSK batched allocator. */ 599 WARN_ON(rq->wqe.info.num_frags != 1); 600 WARN_ON(rq->wqe.info.log_num_frags != 0); 601 WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE); 602 603 /* Considering the above assumptions a fragment maps to a single 604 * xsk_buff. 605 */ 606 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 607 rq->wqe.frags[i].xskp = &rq->wqe.alloc_units->xsk_buffs[i]; 608 609 /* Skip first release due to deferred release as WQES are 610 * not allocated yet. 611 */ 612 rq->wqe.frags[i].flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE); 613 } 614 } 615 616 static int mlx5e_init_wqe_alloc_info(struct mlx5e_rq *rq, int node) 617 { 618 int wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 619 int len = wq_sz << rq->wqe.info.log_num_frags; 620 struct mlx5e_wqe_frag_info *frags; 621 union mlx5e_alloc_units *aus; 622 int aus_sz; 623 624 if (rq->xsk_pool) 625 aus_sz = sizeof(*aus->xsk_buffs); 626 else 627 aus_sz = sizeof(*aus->frag_pages); 628 629 aus = kvzalloc_node(array_size(len, aus_sz), GFP_KERNEL, node); 630 if (!aus) 631 return -ENOMEM; 632 633 frags = kvzalloc_node(array_size(len, sizeof(*frags)), GFP_KERNEL, node); 634 if (!frags) { 635 kvfree(aus); 636 return -ENOMEM; 637 } 638 639 rq->wqe.alloc_units = aus; 640 rq->wqe.frags = frags; 641 642 if (rq->xsk_pool) 643 mlx5e_init_xsk_buffs(rq); 644 else 645 mlx5e_init_frags_partition(rq); 646 647 return 0; 648 } 649 650 static void mlx5e_free_wqe_alloc_info(struct mlx5e_rq *rq) 651 { 652 kvfree(rq->wqe.frags); 653 kvfree(rq->wqe.alloc_units); 654 } 655 656 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work) 657 { 658 struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work); 659 660 mlx5e_reporter_rq_cqe_err(rq); 661 } 662 663 static void mlx5e_rq_timeout_work(struct work_struct *timeout_work) 664 { 665 struct mlx5e_rq *rq = container_of(timeout_work, 666 struct mlx5e_rq, 667 rx_timeout_work); 668 669 mlx5e_reporter_rx_timeout(rq); 670 } 671 672 static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 673 { 674 /* xsk can have page_shift < PAGE_SHIFT */ 675 u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0); 676 u32 page_size = BIT(PAGE_SHIFT + page_order); 677 678 rq->wqe_overflow.page = alloc_pages(GFP_KERNEL, page_order); 679 if (!rq->wqe_overflow.page) 680 return -ENOMEM; 681 682 rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0, 683 page_size, rq->buff.map_dir); 684 if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) { 685 __free_pages(rq->wqe_overflow.page, page_order); 686 return -ENOMEM; 687 } 688 return 0; 689 } 690 691 static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 692 { 693 u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0); 694 u32 page_size = BIT(PAGE_SHIFT + page_order); 695 696 dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, page_size, 697 rq->buff.map_dir); 698 __free_pages(rq->wqe_overflow.page, page_order); 699 } 700 701 static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 702 u32 xdp_frag_size, struct mlx5e_rq *rq) 703 { 704 struct mlx5_core_dev *mdev = c->mdev; 705 int err; 706 707 rq->wq_type = params->rq_wq_type; 708 rq->pdev = c->pdev; 709 rq->netdev = c->netdev; 710 rq->priv = c->priv; 711 rq->hwtstamp_config = &c->priv->hwtstamp_config; 712 rq->clock = mdev->clock; 713 rq->icosq = &c->icosq; 714 rq->ix = c->ix; 715 rq->channel = c; 716 rq->mdev = mdev; 717 rq->hw_mtu = 718 MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN * !params->scatter_fcs_en; 719 rq->xdpsq = &c->rq_xdpsq; 720 rq->stats = &c->priv->channel_stats[c->ix]->rq; 721 rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev); 722 err = mlx5e_rq_set_handlers(rq, params, NULL); 723 if (err) 724 return err; 725 726 return __xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id, 727 xdp_frag_size); 728 } 729 730 static void mlx5e_release_rq_hd_pages(struct mlx5e_rq *rq, 731 struct mlx5e_shampo_hd *shampo) 732 733 { 734 for (int i = 0; i < shampo->nentries; i++) { 735 struct mlx5e_dma_info *info = &shampo->hd_buf_pages[i]; 736 737 if (!info->page) 738 continue; 739 740 dma_unmap_page(rq->pdev, info->addr, PAGE_SIZE, 741 rq->buff.map_dir); 742 __free_page(info->page); 743 } 744 } 745 746 static int mlx5e_alloc_rq_hd_pages(struct mlx5e_rq *rq, int node, 747 struct mlx5e_shampo_hd *shampo) 748 { 749 int err, i; 750 751 for (i = 0; i < shampo->nentries; i++) { 752 struct page *page = alloc_pages_node(node, GFP_KERNEL, 0); 753 dma_addr_t addr; 754 755 if (!page) { 756 err = -ENOMEM; 757 goto err_free_pages; 758 } 759 760 addr = dma_map_page(rq->pdev, page, 0, PAGE_SIZE, 761 rq->buff.map_dir); 762 err = dma_mapping_error(rq->pdev, addr); 763 if (err) { 764 __free_page(page); 765 goto err_free_pages; 766 } 767 768 shampo->hd_buf_pages[i].page = page; 769 shampo->hd_buf_pages[i].addr = addr; 770 } 771 772 return 0; 773 774 err_free_pages: 775 mlx5e_release_rq_hd_pages(rq, shampo); 776 777 return err; 778 } 779 780 static int mlx5e_create_rq_hd_mkey(struct mlx5_core_dev *mdev, 781 struct mlx5e_shampo_hd *shampo) 782 { 783 enum mlx5e_mpwrq_umr_mode umr_mode = MLX5E_MPWRQ_UMR_MODE_ALIGNED; 784 struct mlx5_mtt *mtt; 785 void *mkc, *in; 786 int inlen, err; 787 u32 octwords; 788 789 octwords = mlx5e_mpwrq_umr_octowords(shampo->nentries, umr_mode); 790 inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), 791 MLX5_OCTWORD, octwords); 792 if (inlen < 0) 793 return inlen; 794 795 in = kvzalloc(inlen, GFP_KERNEL); 796 if (!in) 797 return -ENOMEM; 798 799 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 800 801 MLX5_SET(mkc, mkc, lw, 1); 802 MLX5_SET(mkc, mkc, lr, 1); 803 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 804 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 805 MLX5_SET(mkc, mkc, qpn, 0xffffff); 806 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 807 MLX5_SET64(mkc, mkc, len, shampo->hd_buf_size); 808 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 809 MLX5_SET(mkc, mkc, translations_octword_size, octwords); 810 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 811 octwords); 812 813 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 814 for (int i = 0; i < shampo->nentries; i++) 815 mtt[i].ptag = cpu_to_be64(shampo->hd_buf_pages[i].addr); 816 817 err = mlx5_core_create_mkey(mdev, &shampo->mkey, in, inlen); 818 819 kvfree(in); 820 return err; 821 } 822 823 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, 824 struct mlx5e_params *params, 825 struct mlx5e_rq_param *rq_param, 826 struct mlx5e_rq *rq, 827 int node) 828 { 829 struct mlx5e_shampo_hd *shampo; 830 int nentries, err, shampo_sz; 831 u32 hd_per_wq, hd_buf_size; 832 833 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 834 return 0; 835 836 hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rq_param); 837 hd_buf_size = hd_per_wq * BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE); 838 nentries = hd_buf_size / PAGE_SIZE; 839 if (!nentries) { 840 mlx5_core_err(mdev, "SHAMPO header buffer size %u < %lu\n", 841 hd_buf_size, PAGE_SIZE); 842 return -EINVAL; 843 } 844 845 shampo_sz = struct_size(shampo, hd_buf_pages, nentries); 846 shampo = kvzalloc_node(shampo_sz, GFP_KERNEL, node); 847 if (!shampo) 848 return -ENOMEM; 849 850 shampo->hd_per_wq = hd_per_wq; 851 shampo->hd_buf_size = hd_buf_size; 852 shampo->nentries = nentries; 853 err = mlx5e_alloc_rq_hd_pages(rq, node, shampo); 854 if (err) 855 goto err_free; 856 857 err = mlx5e_create_rq_hd_mkey(mdev, shampo); 858 if (err) 859 goto err_release_pages; 860 861 /* gro only data structures */ 862 rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node); 863 if (!rq->hw_gro_data) { 864 err = -ENOMEM; 865 goto err_destroy_mkey; 866 } 867 868 rq->mpwqe.shampo = shampo; 869 870 return 0; 871 872 err_destroy_mkey: 873 mlx5_core_destroy_mkey(mdev, shampo->mkey); 874 err_release_pages: 875 mlx5e_release_rq_hd_pages(rq, shampo); 876 err_free: 877 kvfree(shampo); 878 879 return err; 880 } 881 882 static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) 883 { 884 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 885 886 if (!shampo) 887 return; 888 889 kvfree(rq->hw_gro_data); 890 mlx5_core_destroy_mkey(rq->mdev, shampo->mkey); 891 mlx5e_release_rq_hd_pages(rq, shampo); 892 kvfree(shampo); 893 } 894 895 static int mlx5e_alloc_rq(struct mlx5e_params *params, 896 struct mlx5e_rq_param *rq_param, 897 struct mlx5e_rq_opt_param *rqo, 898 int node, struct mlx5e_rq *rq) 899 { 900 void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); 901 struct mlx5_core_dev *mdev = rq->mdev; 902 u32 pool_order = 0; 903 u32 pool_size; 904 int wq_sz; 905 int err; 906 int i; 907 908 rq_param->wq.db_numa_node = node; 909 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work); 910 INIT_WORK(&rq->rx_timeout_work, mlx5e_rq_timeout_work); 911 912 if (params->xdp_prog) 913 bpf_prog_inc(params->xdp_prog); 914 RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog); 915 916 rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 917 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, rqo); 918 pool_size = 1 << params->log_rq_mtu_frames; 919 920 rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); 921 922 switch (rq->wq_type) { 923 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 924 err = mlx5_wq_ll_create(mdev, &rq_param->wq, rqc_wq, 925 &rq->mpwqe.wq, &rq->wq_ctrl); 926 if (err) 927 goto err_rq_xdp_prog; 928 929 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; 930 931 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 932 933 rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); 934 err = mlx5e_alloc_mpwqe_rq_drop_page(rq); 935 if (err) 936 goto err_rq_wq_destroy; 937 938 rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); 939 rq->mpwqe.pages_per_wqe = 940 mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift, 941 rq->mpwqe.umr_mode); 942 rq->mpwqe.umr_wqebbs = 943 mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift, 944 rq->mpwqe.umr_mode); 945 rq->mpwqe.mtts_per_wqe = 946 mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift, 947 rq->mpwqe.umr_mode); 948 949 pool_size = rq->mpwqe.pages_per_wqe << 950 mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo); 951 pool_order = rq->mpwqe.page_shift - PAGE_SHIFT; 952 953 rq->mpwqe.log_stride_sz = 954 mlx5e_mpwqe_get_log_stride_size(mdev, params, 955 rqo); 956 rq->mpwqe.num_strides = 957 BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo)); 958 rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz); 959 960 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz); 961 962 err = mlx5e_create_rq_umr_mkey(mdev, rq); 963 if (err) 964 goto err_rq_drop_page; 965 966 err = mlx5e_rq_alloc_mpwqe_info(rq, node); 967 if (err) 968 goto err_rq_mkey; 969 970 err = mlx5e_rq_alloc_mpwqe_linear_info(rq, node, params, rqo); 971 if (err) 972 goto err_free_mpwqe_info; 973 974 err = mlx5_rq_shampo_alloc(mdev, params, rq_param, rq, node); 975 if (err) 976 goto err_free_mpwqe_linear_info; 977 978 break; 979 default: /* MLX5_WQ_TYPE_CYCLIC */ 980 err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, 981 &rq->wqe.wq, &rq->wq_ctrl); 982 if (err) 983 goto err_rq_xdp_prog; 984 985 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; 986 987 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 988 989 rq->wqe.info = rq_param->frags_info; 990 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride; 991 992 err = mlx5e_init_wqe_alloc_info(rq, node); 993 if (err) 994 goto err_rq_wq_destroy; 995 } 996 997 if (mlx5e_rqo_xsk_param(rqo)) { 998 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 999 MEM_TYPE_XSK_BUFF_POOL, NULL); 1000 if (err) 1001 goto err_free_by_rq_type; 1002 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq); 1003 } else { 1004 /* Create a page_pool and register it with rxq */ 1005 struct page_pool_params pp_params = { 0 }; 1006 1007 if (WARN_ON(BIT(PAGE_SHIFT + pool_order) / 64 > 1008 MLX5E_PAGECNT_BIAS_MAX)) { 1009 err = -E2BIG; 1010 goto err_free_by_rq_type; 1011 } 1012 1013 pp_params.order = pool_order; 1014 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 1015 pp_params.pool_size = pool_size; 1016 pp_params.nid = node; 1017 pp_params.dev = rq->pdev; 1018 pp_params.napi = rq->cq.napi; 1019 pp_params.netdev = rq->netdev; 1020 pp_params.dma_dir = rq->buff.map_dir; 1021 pp_params.max_len = BIT(PAGE_SHIFT + pool_order); 1022 pp_params.queue_idx = rq->ix; 1023 1024 /* Shampo header data split allow for unreadable netmem */ 1025 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 1026 pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; 1027 1028 /* page_pool can be used even when there is no rq->xdp_prog, 1029 * given page_pool does not handle DMA mapping there is no 1030 * required state to clear. And page_pool gracefully handle 1031 * elevated refcnt. 1032 */ 1033 rq->page_pool = page_pool_create(&pp_params); 1034 if (IS_ERR(rq->page_pool)) { 1035 err = PTR_ERR(rq->page_pool); 1036 rq->page_pool = NULL; 1037 goto err_free_by_rq_type; 1038 } 1039 if (!rq->hd_page_pool) 1040 rq->hd_page_pool = rq->page_pool; 1041 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) { 1042 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 1043 MEM_TYPE_PAGE_POOL, rq->page_pool); 1044 if (err) 1045 goto err_destroy_page_pool; 1046 } 1047 } 1048 1049 for (i = 0; i < wq_sz; i++) { 1050 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 1051 struct mlx5e_rx_wqe_ll *wqe = 1052 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i); 1053 u32 byte_count = 1054 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz; 1055 u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) << 1056 rq->mpwqe.page_shift; 1057 u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ? 1058 0 : rq->buff.headroom; 1059 1060 wqe->data[0].addr = cpu_to_be64(dma_offset + headroom); 1061 wqe->data[0].byte_count = cpu_to_be32(byte_count); 1062 wqe->data[0].lkey = rq->mpwqe.umr_mkey_be; 1063 } else { 1064 struct mlx5e_rx_wqe_cyc *wqe = 1065 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); 1066 int f; 1067 1068 for (f = 0; f < rq->wqe.info.num_frags; f++) { 1069 u32 frag_size = rq->wqe.info.arr[f].frag_size | 1070 MLX5_HW_START_PADDING; 1071 1072 wqe->data[f].byte_count = cpu_to_be32(frag_size); 1073 wqe->data[f].lkey = rq->mkey_be; 1074 } 1075 /* check if num_frags is not a pow of two */ 1076 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { 1077 wqe->data[f].byte_count = 0; 1078 wqe->data[f].lkey = params->terminate_lkey_be; 1079 wqe->data[f].addr = 0; 1080 } 1081 } 1082 } 1083 1084 return 0; 1085 1086 err_destroy_page_pool: 1087 page_pool_destroy(rq->page_pool); 1088 err_free_by_rq_type: 1089 switch (rq->wq_type) { 1090 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 1091 mlx5e_rq_free_shampo(rq); 1092 err_free_mpwqe_linear_info: 1093 kvfree(rq->mpwqe.linear_info); 1094 err_free_mpwqe_info: 1095 kvfree(rq->mpwqe.info); 1096 err_rq_mkey: 1097 mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 1098 err_rq_drop_page: 1099 mlx5e_free_mpwqe_rq_drop_page(rq); 1100 break; 1101 default: /* MLX5_WQ_TYPE_CYCLIC */ 1102 mlx5e_free_wqe_alloc_info(rq); 1103 } 1104 err_rq_wq_destroy: 1105 mlx5_wq_destroy(&rq->wq_ctrl); 1106 err_rq_xdp_prog: 1107 if (params->xdp_prog) 1108 bpf_prog_put(params->xdp_prog); 1109 1110 return err; 1111 } 1112 1113 static void mlx5e_free_rq(struct mlx5e_rq *rq) 1114 { 1115 kvfree(rq->dim); 1116 page_pool_destroy(rq->page_pool); 1117 1118 switch (rq->wq_type) { 1119 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 1120 mlx5e_rq_free_shampo(rq); 1121 kvfree(rq->mpwqe.linear_info); 1122 kvfree(rq->mpwqe.info); 1123 mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 1124 mlx5e_free_mpwqe_rq_drop_page(rq); 1125 break; 1126 default: /* MLX5_WQ_TYPE_CYCLIC */ 1127 mlx5e_free_wqe_alloc_info(rq); 1128 } 1129 1130 mlx5_wq_destroy(&rq->wq_ctrl); 1131 1132 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) { 1133 struct bpf_prog *old_prog; 1134 1135 old_prog = rcu_dereference_protected(rq->xdp_prog, 1136 lockdep_is_held(&rq->priv->state_lock)); 1137 if (old_prog) 1138 bpf_prog_put(old_prog); 1139 } 1140 xdp_rxq_info_unreg(&rq->xdp_rxq); 1141 } 1142 1143 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param, 1144 u16 q_counter) 1145 { 1146 struct mlx5_core_dev *mdev = rq->mdev; 1147 u8 ts_format; 1148 void *in; 1149 void *rqc; 1150 void *wq; 1151 int inlen; 1152 int err; 1153 1154 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + 1155 sizeof(u64) * rq->wq_ctrl.buf.npages; 1156 in = kvzalloc(inlen, GFP_KERNEL); 1157 if (!in) 1158 return -ENOMEM; 1159 1160 ts_format = mlx5_is_real_time_rq(mdev) ? 1161 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 1162 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 1163 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 1164 wq = MLX5_ADDR_OF(rqc, rqc, wq); 1165 1166 memcpy(rqc, rq_param->rqc, sizeof(rq_param->rqc)); 1167 1168 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); 1169 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 1170 MLX5_SET(rqc, rqc, ts_format, ts_format); 1171 MLX5_SET(rqc, rqc, counter_set_id, q_counter); 1172 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - 1173 MLX5_ADAPTER_PAGE_SHIFT); 1174 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); 1175 1176 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1177 MLX5_SET(wq, wq, log_headers_buffer_entry_num, 1178 order_base_2(rq->mpwqe.shampo->hd_per_wq)); 1179 MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey); 1180 } 1181 1182 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, 1183 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1184 1185 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); 1186 1187 kvfree(in); 1188 1189 return err; 1190 } 1191 1192 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) 1193 { 1194 struct mlx5_core_dev *mdev = rq->mdev; 1195 1196 void *in; 1197 void *rqc; 1198 int inlen; 1199 int err; 1200 1201 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1202 in = kvzalloc(inlen, GFP_KERNEL); 1203 if (!in) 1204 return -ENOMEM; 1205 1206 if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY) 1207 mlx5e_rqwq_reset(rq); 1208 1209 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1210 1211 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 1212 MLX5_SET(rqc, rqc, state, next_state); 1213 1214 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1215 1216 kvfree(in); 1217 1218 return err; 1219 } 1220 1221 static void mlx5e_flush_rq_cq(struct mlx5e_rq *rq) 1222 { 1223 struct mlx5_cqwq *cqwq = &rq->cq.wq; 1224 struct mlx5_cqe64 *cqe; 1225 1226 if (test_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state)) { 1227 while ((cqe = mlx5_cqwq_get_cqe_enhanced_comp(cqwq))) 1228 mlx5_cqwq_pop(cqwq); 1229 } else { 1230 while ((cqe = mlx5_cqwq_get_cqe(cqwq))) 1231 mlx5_cqwq_pop(cqwq); 1232 } 1233 1234 mlx5_cqwq_update_db_record(cqwq); 1235 } 1236 1237 int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state) 1238 { 1239 struct net_device *dev = rq->netdev; 1240 int err; 1241 1242 err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST); 1243 if (err) { 1244 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn); 1245 return err; 1246 } 1247 1248 mlx5e_free_rx_descs(rq); 1249 mlx5e_flush_rq_cq(rq); 1250 1251 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1252 if (err) { 1253 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn); 1254 return err; 1255 } 1256 1257 return 0; 1258 } 1259 1260 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) 1261 { 1262 struct mlx5_core_dev *mdev = rq->mdev; 1263 void *in; 1264 void *rqc; 1265 int inlen; 1266 int err; 1267 1268 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1269 in = kvzalloc(inlen, GFP_KERNEL); 1270 if (!in) 1271 return -ENOMEM; 1272 1273 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1274 1275 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 1276 MLX5_SET64(modify_rq_in, in, modify_bitmask, 1277 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD); 1278 MLX5_SET(rqc, rqc, vsd, vsd); 1279 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 1280 1281 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1282 1283 kvfree(in); 1284 1285 return err; 1286 } 1287 1288 void mlx5e_destroy_rq(struct mlx5e_rq *rq) 1289 { 1290 mlx5_core_destroy_rq(rq->mdev, rq->rqn); 1291 } 1292 1293 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time) 1294 { 1295 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time); 1296 1297 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq)); 1298 1299 do { 1300 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes) 1301 return 0; 1302 1303 msleep(20); 1304 } while (time_before(jiffies, exp_time)); 1305 1306 netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n", 1307 rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes); 1308 1309 queue_work(rq->priv->wq, &rq->rx_timeout_work); 1310 1311 return -ETIMEDOUT; 1312 } 1313 1314 void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq) 1315 { 1316 struct mlx5_wq_ll *wq; 1317 u16 head; 1318 int i; 1319 1320 if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) 1321 return; 1322 1323 wq = &rq->mpwqe.wq; 1324 head = wq->head; 1325 1326 /* Release WQEs that are in missing state: they have been 1327 * popped from the list after completion but were not freed 1328 * due to deferred release. 1329 * Also free the linked-list reserved entry, hence the "+ 1". 1330 */ 1331 for (i = 0; i < mlx5_wq_ll_missing(wq) + 1; i++) { 1332 rq->dealloc_wqe(rq, head); 1333 head = mlx5_wq_ll_get_wqe_next_ix(wq, head); 1334 } 1335 1336 rq->mpwqe.actual_wq_head = wq->head; 1337 rq->mpwqe.umr_in_progress = 0; 1338 rq->mpwqe.umr_completed = 0; 1339 } 1340 1341 void mlx5e_free_rx_descs(struct mlx5e_rq *rq) 1342 { 1343 __be16 wqe_ix_be; 1344 u16 wqe_ix; 1345 1346 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 1347 struct mlx5_wq_ll *wq = &rq->mpwqe.wq; 1348 1349 mlx5e_free_rx_missing_descs(rq); 1350 1351 while (!mlx5_wq_ll_is_empty(wq)) { 1352 struct mlx5e_rx_wqe_ll *wqe; 1353 1354 wqe_ix_be = *wq->tail_next; 1355 wqe_ix = be16_to_cpu(wqe_ix_be); 1356 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix); 1357 rq->dealloc_wqe(rq, wqe_ix); 1358 mlx5_wq_ll_pop(wq, wqe_ix_be, 1359 &wqe->next.next_wqe_index); 1360 } 1361 1362 mlx5e_mpwqe_dealloc_linear_page(rq); 1363 } else { 1364 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 1365 u16 missing = mlx5_wq_cyc_missing(wq); 1366 u16 head = mlx5_wq_cyc_get_head(wq); 1367 1368 while (!mlx5_wq_cyc_is_empty(wq)) { 1369 wqe_ix = mlx5_wq_cyc_get_tail(wq); 1370 rq->dealloc_wqe(rq, wqe_ix); 1371 mlx5_wq_cyc_pop(wq); 1372 } 1373 /* Missing slots might also contain unreleased pages due to 1374 * deferred release. 1375 */ 1376 while (missing--) { 1377 wqe_ix = mlx5_wq_cyc_ctr2ix(wq, head++); 1378 rq->dealloc_wqe(rq, wqe_ix); 1379 } 1380 } 1381 1382 } 1383 1384 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param, 1385 struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter, 1386 struct mlx5e_rq *rq) 1387 { 1388 struct mlx5_core_dev *mdev = rq->mdev; 1389 int err; 1390 1391 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) 1392 __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state); 1393 1394 err = mlx5e_alloc_rq(params, rq_param, rqo, node, rq); 1395 if (err) 1396 return err; 1397 1398 err = mlx5e_create_rq(rq, rq_param, q_counter); 1399 if (err) 1400 goto err_free_rq; 1401 1402 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1403 if (err) 1404 goto err_destroy_rq; 1405 1406 if (MLX5_CAP_ETH(mdev, cqe_checksum_full)) 1407 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state); 1408 1409 if (rq->channel && !params->rx_dim_enabled) { 1410 rq->channel->rx_cq_moder = params->rx_cq_moderation; 1411 } else if (rq->channel) { 1412 u8 cq_period_mode; 1413 1414 cq_period_mode = params->rx_moder_use_cqe_mode ? 1415 DIM_CQ_PERIOD_MODE_START_FROM_CQE : 1416 DIM_CQ_PERIOD_MODE_START_FROM_EQE; 1417 mlx5e_reset_rx_moderation(&rq->channel->rx_cq_moder, cq_period_mode, 1418 params->rx_dim_enabled); 1419 1420 err = mlx5e_dim_rx_change(rq, params->rx_dim_enabled); 1421 if (err) 1422 goto err_destroy_rq; 1423 } 1424 1425 /* We disable csum_complete when XDP is enabled since 1426 * XDP programs might manipulate packets which will render 1427 * skb->checksum incorrect. 1428 */ 1429 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog) 1430 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state); 1431 1432 /* For CQE compression on striding RQ, use stride index provided by 1433 * HW if capability is supported. 1434 */ 1435 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) && 1436 MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) 1437 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state); 1438 1439 /* For enhanced CQE compression packet processing. decompress 1440 * session according to the enhanced layout. 1441 */ 1442 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) && 1443 MLX5_CAP_GEN(mdev, enhanced_cqe_compression)) 1444 __set_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state); 1445 1446 return 0; 1447 1448 err_destroy_rq: 1449 mlx5e_destroy_rq(rq); 1450 err_free_rq: 1451 mlx5e_free_rq(rq); 1452 1453 return err; 1454 } 1455 1456 void mlx5e_activate_rq(struct mlx5e_rq *rq) 1457 { 1458 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1459 } 1460 1461 void mlx5e_deactivate_rq(struct mlx5e_rq *rq) 1462 { 1463 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1464 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */ 1465 } 1466 1467 void mlx5e_close_rq(struct mlx5e_rq *rq) 1468 { 1469 if (rq->dim) 1470 cancel_work_sync(&rq->dim->work); 1471 cancel_work_sync(&rq->recover_work); 1472 cancel_work_sync(&rq->rx_timeout_work); 1473 mlx5e_destroy_rq(rq); 1474 mlx5e_free_rx_descs(rq); 1475 mlx5e_free_rq(rq); 1476 } 1477 1478 u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev, 1479 struct mlx5e_priv *priv, 1480 const struct mlx5e_profile *profile, 1481 u8 lag_port, u8 tc) 1482 { 1483 if (profile->get_tisn) 1484 return profile->get_tisn(mdev, priv, lag_port, tc); 1485 1486 return mdev->mlx5e_res.hw_objs.tisn[lag_port][tc]; 1487 } 1488 1489 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) 1490 { 1491 kvfree(sq->db.xdpi_fifo.xi); 1492 kvfree(sq->db.wqe_info); 1493 } 1494 1495 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa) 1496 { 1497 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 1498 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1499 int entries; 1500 size_t size; 1501 1502 /* upper bound for maximum num of entries of all xmit_modes. */ 1503 entries = roundup_pow_of_two(wq_sz * MLX5_SEND_WQEBB_NUM_DS * 1504 MLX5E_XDP_FIFO_ENTRIES2DS_MAX_RATIO); 1505 1506 size = array_size(sizeof(*xdpi_fifo->xi), entries); 1507 xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa); 1508 if (!xdpi_fifo->xi) 1509 return -ENOMEM; 1510 1511 xdpi_fifo->pc = &sq->xdpi_fifo_pc; 1512 xdpi_fifo->cc = &sq->xdpi_fifo_cc; 1513 xdpi_fifo->mask = entries - 1; 1514 1515 return 0; 1516 } 1517 1518 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) 1519 { 1520 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1521 size_t size; 1522 int err; 1523 1524 size = array_size(sizeof(*sq->db.wqe_info), wq_sz); 1525 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1526 if (!sq->db.wqe_info) 1527 return -ENOMEM; 1528 1529 err = mlx5e_alloc_xdpsq_fifo(sq, numa); 1530 if (err) { 1531 mlx5e_free_xdpsq_db(sq); 1532 return err; 1533 } 1534 1535 return 0; 1536 } 1537 1538 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 1539 struct mlx5e_params *params, 1540 struct xsk_buff_pool *xsk_pool, 1541 struct mlx5e_sq_param *param, 1542 struct mlx5e_xdpsq *sq, 1543 bool is_redirect) 1544 { 1545 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1546 struct mlx5_core_dev *mdev = c->mdev; 1547 struct mlx5_wq_cyc *wq = &sq->wq; 1548 int err; 1549 1550 sq->pdev = c->pdev; 1551 sq->mkey_be = c->mkey_be; 1552 sq->channel = c; 1553 sq->uar_map = c->bfreg->map; 1554 sq->min_inline_mode = params->tx_min_inline_mode; 1555 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN; 1556 sq->xsk_pool = xsk_pool; 1557 1558 sq->stats = sq->xsk_pool ? 1559 &c->priv->channel_stats[c->ix]->xsksq : 1560 is_redirect ? 1561 &c->priv->channel_stats[c->ix]->xdpsq : 1562 &c->priv->channel_stats[c->ix]->rq_xdpsq; 1563 sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) : 1564 mlx5e_stop_room_for_max_wqe(mdev); 1565 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1566 1567 param->wq.db_numa_node = cpu_to_node(c->cpu); 1568 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1569 if (err) 1570 return err; 1571 wq->db = &wq->db[MLX5_SND_DBR]; 1572 1573 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu)); 1574 if (err) 1575 goto err_sq_wq_destroy; 1576 1577 return 0; 1578 1579 err_sq_wq_destroy: 1580 mlx5_wq_destroy(&sq->wq_ctrl); 1581 1582 return err; 1583 } 1584 1585 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq) 1586 { 1587 mlx5e_free_xdpsq_db(sq); 1588 mlx5_wq_destroy(&sq->wq_ctrl); 1589 } 1590 1591 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) 1592 { 1593 kvfree(sq->db.wqe_info); 1594 } 1595 1596 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa) 1597 { 1598 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1599 size_t size; 1600 1601 size = array_size(wq_sz, sizeof(*sq->db.wqe_info)); 1602 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1603 if (!sq->db.wqe_info) 1604 return -ENOMEM; 1605 1606 return 0; 1607 } 1608 1609 static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work) 1610 { 1611 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1612 recover_work); 1613 1614 mlx5e_reporter_icosq_cqe_err(sq); 1615 } 1616 1617 static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work) 1618 { 1619 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1620 recover_work); 1621 1622 /* Not implemented yet. */ 1623 1624 netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n"); 1625 } 1626 1627 static int mlx5e_alloc_icosq(struct mlx5e_channel *c, 1628 struct mlx5e_sq_param *param, 1629 struct mlx5e_icosq *sq, 1630 work_func_t recover_work_func) 1631 { 1632 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1633 struct mlx5_core_dev *mdev = c->mdev; 1634 struct mlx5_wq_cyc *wq = &sq->wq; 1635 int err; 1636 1637 sq->channel = c; 1638 sq->uar_map = c->bfreg->map; 1639 sq->reserved_room = param->stop_room; 1640 1641 param->wq.db_numa_node = cpu_to_node(c->cpu); 1642 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1643 if (err) 1644 return err; 1645 wq->db = &wq->db[MLX5_SND_DBR]; 1646 1647 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu)); 1648 if (err) 1649 goto err_sq_wq_destroy; 1650 1651 INIT_WORK(&sq->recover_work, recover_work_func); 1652 1653 return 0; 1654 1655 err_sq_wq_destroy: 1656 mlx5_wq_destroy(&sq->wq_ctrl); 1657 1658 return err; 1659 } 1660 1661 static void mlx5e_free_icosq(struct mlx5e_icosq *sq) 1662 { 1663 mlx5e_free_icosq_db(sq); 1664 mlx5_wq_destroy(&sq->wq_ctrl); 1665 } 1666 1667 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq) 1668 { 1669 kvfree(sq->db.wqe_info); 1670 kvfree(sq->db.skb_fifo.fifo); 1671 kvfree(sq->db.dma_fifo); 1672 } 1673 1674 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa) 1675 { 1676 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1677 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 1678 1679 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz, 1680 sizeof(*sq->db.dma_fifo)), 1681 GFP_KERNEL, numa); 1682 sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz, 1683 sizeof(*sq->db.skb_fifo.fifo)), 1684 GFP_KERNEL, numa); 1685 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz, 1686 sizeof(*sq->db.wqe_info)), 1687 GFP_KERNEL, numa); 1688 if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) { 1689 mlx5e_free_txqsq_db(sq); 1690 return -ENOMEM; 1691 } 1692 1693 sq->dma_fifo_mask = df_sz - 1; 1694 1695 sq->db.skb_fifo.pc = &sq->skb_fifo_pc; 1696 sq->db.skb_fifo.cc = &sq->skb_fifo_cc; 1697 sq->db.skb_fifo.mask = df_sz - 1; 1698 1699 return 0; 1700 } 1701 1702 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 1703 int txq_ix, 1704 struct mlx5e_params *params, 1705 struct mlx5e_sq_param *param, 1706 struct mlx5e_txqsq *sq, 1707 int tc) 1708 { 1709 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1710 struct mlx5_core_dev *mdev = c->mdev; 1711 struct mlx5_wq_cyc *wq = &sq->wq; 1712 int err; 1713 1714 sq->pdev = c->pdev; 1715 sq->clock = mdev->clock; 1716 sq->mkey_be = c->mkey_be; 1717 sq->netdev = c->netdev; 1718 sq->mdev = c->mdev; 1719 sq->channel = c; 1720 sq->priv = c->priv; 1721 sq->ch_ix = c->ix; 1722 sq->txq_ix = txq_ix; 1723 sq->uar_map = c->bfreg->map; 1724 sq->min_inline_mode = params->tx_min_inline_mode; 1725 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 1726 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1727 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); 1728 if (mlx5_ipsec_device_caps(c->priv->mdev)) 1729 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state); 1730 if (param->is_mpw) 1731 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state); 1732 sq->stop_room = param->stop_room; 1733 sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev); 1734 1735 param->wq.db_numa_node = cpu_to_node(c->cpu); 1736 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1737 if (err) 1738 return err; 1739 wq->db = &wq->db[MLX5_SND_DBR]; 1740 1741 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu)); 1742 if (err) 1743 goto err_sq_wq_destroy; 1744 1745 return 0; 1746 1747 err_sq_wq_destroy: 1748 mlx5_wq_destroy(&sq->wq_ctrl); 1749 1750 return err; 1751 } 1752 1753 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq) 1754 { 1755 kvfree(sq->dim); 1756 mlx5e_free_txqsq_db(sq); 1757 mlx5_wq_destroy(&sq->wq_ctrl); 1758 } 1759 1760 static int mlx5e_create_sq(struct mlx5_core_dev *mdev, 1761 struct mlx5e_sq_param *param, 1762 struct mlx5e_create_sq_param *csp, 1763 u32 *sqn) 1764 { 1765 u8 ts_format; 1766 void *in; 1767 void *sqc; 1768 void *wq; 1769 int inlen; 1770 int err; 1771 1772 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + 1773 sizeof(u64) * csp->wq_ctrl->buf.npages; 1774 in = kvzalloc(inlen, GFP_KERNEL); 1775 if (!in) 1776 return -ENOMEM; 1777 1778 ts_format = mlx5_is_real_time_sq(mdev) ? 1779 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 1780 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 1781 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 1782 wq = MLX5_ADDR_OF(sqc, sqc, wq); 1783 1784 memcpy(sqc, param->sqc, sizeof(param->sqc)); 1785 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz); 1786 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn); 1787 MLX5_SET(sqc, sqc, cqn, csp->cqn); 1788 MLX5_SET(sqc, sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn); 1789 MLX5_SET(sqc, sqc, ts_format, ts_format); 1790 1791 1792 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) 1793 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode); 1794 1795 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 1796 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 1797 1798 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1799 MLX5_SET(wq, wq, uar_page, csp->uar_page); 1800 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift - 1801 MLX5_ADAPTER_PAGE_SHIFT); 1802 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma); 1803 1804 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf, 1805 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1806 1807 err = mlx5_core_create_sq(mdev, in, inlen, sqn); 1808 1809 kvfree(in); 1810 1811 return err; 1812 } 1813 1814 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, 1815 struct mlx5e_modify_sq_param *p) 1816 { 1817 u64 bitmask = 0; 1818 void *in; 1819 void *sqc; 1820 int inlen; 1821 int err; 1822 1823 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 1824 in = kvzalloc(inlen, GFP_KERNEL); 1825 if (!in) 1826 return -ENOMEM; 1827 1828 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 1829 1830 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state); 1831 MLX5_SET(sqc, sqc, state, p->next_state); 1832 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) { 1833 bitmask |= 1; 1834 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); 1835 } 1836 if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) { 1837 bitmask |= 1 << 2; 1838 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id); 1839 } 1840 MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask); 1841 1842 err = mlx5_core_modify_sq(mdev, sqn, in); 1843 1844 kvfree(in); 1845 1846 return err; 1847 } 1848 1849 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) 1850 { 1851 mlx5_core_destroy_sq(mdev, sqn); 1852 } 1853 1854 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, 1855 struct mlx5e_sq_param *param, 1856 struct mlx5e_create_sq_param *csp, 1857 u16 qos_queue_group_id, 1858 u32 *sqn) 1859 { 1860 struct mlx5e_modify_sq_param msp = {0}; 1861 int err; 1862 1863 err = mlx5e_create_sq(mdev, param, csp, sqn); 1864 if (err) 1865 return err; 1866 1867 msp.curr_state = MLX5_SQC_STATE_RST; 1868 msp.next_state = MLX5_SQC_STATE_RDY; 1869 if (qos_queue_group_id) { 1870 msp.qos_update = true; 1871 msp.qos_queue_group_id = qos_queue_group_id; 1872 } 1873 err = mlx5e_modify_sq(mdev, *sqn, &msp); 1874 if (err) 1875 mlx5e_destroy_sq(mdev, *sqn); 1876 1877 return err; 1878 } 1879 1880 static int mlx5e_set_sq_maxrate(struct net_device *dev, 1881 struct mlx5e_txqsq *sq, u32 rate); 1882 1883 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix, 1884 struct mlx5e_params *params, struct mlx5e_sq_param *param, 1885 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, 1886 struct mlx5e_sq_stats *sq_stats) 1887 { 1888 struct mlx5e_create_sq_param csp = {}; 1889 u32 tx_rate; 1890 int err; 1891 1892 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc); 1893 if (err) 1894 return err; 1895 1896 sq->stats = sq_stats; 1897 1898 csp.tisn = tisn; 1899 csp.tis_lst_sz = 1; 1900 csp.cqn = sq->cq.mcq.cqn; 1901 csp.wq_ctrl = &sq->wq_ctrl; 1902 csp.min_inline_mode = sq->min_inline_mode; 1903 csp.uar_page = c->bfreg->index; 1904 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn); 1905 if (err) 1906 goto err_free_txqsq; 1907 1908 tx_rate = c->priv->tx_rates[sq->txq_ix]; 1909 if (tx_rate) 1910 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate); 1911 1912 if (sq->channel && !params->tx_dim_enabled) { 1913 sq->channel->tx_cq_moder = params->tx_cq_moderation; 1914 } else if (sq->channel) { 1915 u8 cq_period_mode; 1916 1917 cq_period_mode = params->tx_moder_use_cqe_mode ? 1918 DIM_CQ_PERIOD_MODE_START_FROM_CQE : 1919 DIM_CQ_PERIOD_MODE_START_FROM_EQE; 1920 mlx5e_reset_tx_moderation(&sq->channel->tx_cq_moder, 1921 cq_period_mode, 1922 params->tx_dim_enabled); 1923 1924 err = mlx5e_dim_tx_change(sq, params->tx_dim_enabled); 1925 if (err) 1926 goto err_destroy_sq; 1927 } 1928 1929 return 0; 1930 1931 err_destroy_sq: 1932 mlx5e_destroy_sq(c->mdev, sq->sqn); 1933 err_free_txqsq: 1934 mlx5e_free_txqsq(sq); 1935 1936 return err; 1937 } 1938 1939 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) 1940 { 1941 sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix); 1942 /* Reset BQL only when the SQ has no bytes in flight. */ 1943 if (sq->cc == sq->pc) 1944 netdev_tx_reset_queue(sq->txq); 1945 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1946 netif_tx_start_queue(sq->txq); 1947 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, sq->cq.napi); 1948 } 1949 1950 void mlx5e_tx_disable_queue(struct netdev_queue *txq) 1951 { 1952 __netif_tx_lock_bh(txq); 1953 netif_tx_stop_queue(txq); 1954 __netif_tx_unlock_bh(txq); 1955 } 1956 1957 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) 1958 { 1959 struct mlx5_wq_cyc *wq = &sq->wq; 1960 1961 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, NULL); 1962 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1963 synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */ 1964 1965 mlx5e_tx_disable_queue(sq->txq); 1966 1967 /* last doorbell out, godspeed .. */ 1968 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) { 1969 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1970 struct mlx5e_tx_wqe *nop; 1971 1972 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) { 1973 .num_wqebbs = 1, 1974 }; 1975 1976 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc); 1977 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl); 1978 } 1979 } 1980 1981 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) 1982 { 1983 struct mlx5_core_dev *mdev = sq->mdev; 1984 struct mlx5_rate_limit rl = {0}; 1985 1986 if (sq->dim) 1987 cancel_work_sync(&sq->dim->work); 1988 cancel_work_sync(&sq->recover_work); 1989 mlx5e_destroy_sq(mdev, sq->sqn); 1990 if (sq->rate_limit) { 1991 rl.rate = sq->rate_limit; 1992 mlx5_rl_remove_rate(mdev, &rl); 1993 } 1994 mlx5e_free_txqsq_descs(sq); 1995 mlx5e_free_txqsq(sq); 1996 } 1997 1998 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) 1999 { 2000 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, 2001 recover_work); 2002 2003 mlx5e_reporter_tx_err_cqe(sq); 2004 } 2005 2006 static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) 2007 { 2008 return (struct dim_cq_moder) { 2009 .cq_period_mode = cq_period_mode, 2010 .pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS, 2011 .usec = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? 2012 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE : 2013 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC, 2014 }; 2015 } 2016 2017 bool mlx5e_reset_tx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode, 2018 bool dim_enabled) 2019 { 2020 bool reset_needed = cq_moder->cq_period_mode != cq_period_mode; 2021 2022 if (dim_enabled) 2023 *cq_moder = net_dim_get_def_tx_moderation(cq_period_mode); 2024 else 2025 *cq_moder = mlx5e_get_def_tx_moderation(cq_period_mode); 2026 2027 return reset_needed; 2028 } 2029 2030 bool mlx5e_reset_tx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode, 2031 bool dim_enabled, bool keep_dim_state) 2032 { 2033 bool reset = false; 2034 int i, tc; 2035 2036 for (i = 0; i < chs->num; i++) { 2037 for (tc = 0; tc < mlx5e_get_dcb_num_tc(&chs->params); tc++) { 2038 if (keep_dim_state) 2039 dim_enabled = !!chs->c[i]->sq[tc].dim; 2040 2041 reset |= mlx5e_reset_tx_moderation(&chs->c[i]->tx_cq_moder, 2042 cq_period_mode, dim_enabled); 2043 } 2044 } 2045 2046 return reset; 2047 } 2048 2049 static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, 2050 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq, 2051 work_func_t recover_work_func) 2052 { 2053 struct mlx5e_create_sq_param csp = {}; 2054 int err; 2055 2056 err = mlx5e_alloc_icosq(c, param, sq, recover_work_func); 2057 if (err) 2058 return err; 2059 2060 csp.cqn = sq->cq.mcq.cqn; 2061 csp.wq_ctrl = &sq->wq_ctrl; 2062 csp.min_inline_mode = params->tx_min_inline_mode; 2063 csp.uar_page = c->bfreg->index; 2064 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 2065 if (err) 2066 goto err_free_icosq; 2067 2068 spin_lock_init(&sq->lock); 2069 2070 if (param->is_tls) { 2071 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list(); 2072 if (IS_ERR(sq->ktls_resync)) { 2073 err = PTR_ERR(sq->ktls_resync); 2074 goto err_destroy_icosq; 2075 } 2076 } 2077 return 0; 2078 2079 err_destroy_icosq: 2080 mlx5e_destroy_sq(c->mdev, sq->sqn); 2081 err_free_icosq: 2082 mlx5e_free_icosq(sq); 2083 2084 return err; 2085 } 2086 2087 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq) 2088 { 2089 set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 2090 } 2091 2092 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq) 2093 { 2094 clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 2095 synchronize_net(); /* Sync with NAPI. */ 2096 } 2097 2098 static void mlx5e_close_icosq(struct mlx5e_icosq *sq) 2099 { 2100 struct mlx5e_channel *c = sq->channel; 2101 2102 if (sq->ktls_resync) 2103 mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync); 2104 mlx5e_destroy_sq(c->mdev, sq->sqn); 2105 mlx5e_free_icosq_descs(sq); 2106 mlx5e_free_icosq(sq); 2107 } 2108 2109 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, 2110 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool, 2111 struct mlx5e_xdpsq *sq, bool is_redirect) 2112 { 2113 struct mlx5e_create_sq_param csp = {}; 2114 int err; 2115 2116 err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect); 2117 if (err) 2118 return err; 2119 2120 csp.tis_lst_sz = 1; 2121 csp.tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, 2122 c->lag_port, 0); /* tc = 0 */ 2123 csp.cqn = sq->cq.mcq.cqn; 2124 csp.wq_ctrl = &sq->wq_ctrl; 2125 csp.min_inline_mode = sq->min_inline_mode; 2126 csp.uar_page = c->bfreg->index; 2127 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 2128 2129 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 2130 if (err) 2131 goto err_free_xdpsq; 2132 2133 mlx5e_set_xmit_fp(sq, param->is_mpw); 2134 2135 return 0; 2136 2137 err_free_xdpsq: 2138 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 2139 mlx5e_free_xdpsq(sq); 2140 2141 return err; 2142 } 2143 2144 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq) 2145 { 2146 struct mlx5e_channel *c = sq->channel; 2147 2148 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 2149 synchronize_net(); /* Sync with NAPI. */ 2150 2151 mlx5e_destroy_sq(c->mdev, sq->sqn); 2152 mlx5e_free_xdpsq_descs(sq); 2153 mlx5e_free_xdpsq(sq); 2154 } 2155 2156 static struct mlx5e_xdpsq *mlx5e_open_xdpredirect_sq(struct mlx5e_channel *c, 2157 struct mlx5e_params *params, 2158 struct mlx5e_channel_param *cparam, 2159 struct mlx5e_create_cq_param *ccp) 2160 { 2161 struct mlx5e_xdpsq *xdpsq; 2162 int err; 2163 2164 xdpsq = kvzalloc_node(sizeof(*xdpsq), GFP_KERNEL, cpu_to_node(c->cpu)); 2165 if (!xdpsq) 2166 return ERR_PTR(-ENOMEM); 2167 2168 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, 2169 &cparam->xdp_sq.cqp, ccp, &xdpsq->cq); 2170 if (err) 2171 goto err_free_xdpsq; 2172 2173 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, xdpsq, true); 2174 if (err) 2175 goto err_close_xdpsq_cq; 2176 2177 return xdpsq; 2178 2179 err_close_xdpsq_cq: 2180 mlx5e_close_cq(&xdpsq->cq); 2181 err_free_xdpsq: 2182 kvfree(xdpsq); 2183 2184 return ERR_PTR(err); 2185 } 2186 2187 static void mlx5e_close_xdpredirect_sq(struct mlx5e_xdpsq *xdpsq) 2188 { 2189 mlx5e_close_xdpsq(xdpsq); 2190 mlx5e_close_cq(&xdpsq->cq); 2191 kvfree(xdpsq); 2192 } 2193 2194 static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, 2195 struct net_device *netdev, 2196 struct workqueue_struct *workqueue, 2197 struct mlx5_uars_page *uar, 2198 struct mlx5e_cq_param *param, 2199 struct mlx5e_cq *cq) 2200 { 2201 struct mlx5_core_cq *mcq = &cq->mcq; 2202 int err; 2203 u32 i; 2204 2205 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, 2206 &cq->wq_ctrl); 2207 if (err) 2208 return err; 2209 2210 mcq->cqe_sz = 64; 2211 mcq->set_ci_db = cq->wq_ctrl.db.db; 2212 mcq->arm_db = cq->wq_ctrl.db.db + 1; 2213 *mcq->set_ci_db = 0; 2214 mcq->vector = param->eq_ix; 2215 mcq->comp = mlx5e_completion_event; 2216 mcq->event = mlx5e_cq_error_event; 2217 2218 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 2219 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 2220 2221 cqe->op_own = 0xf1; 2222 cqe->validity_iteration_count = 0xff; 2223 } 2224 2225 cq->mdev = mdev; 2226 cq->netdev = netdev; 2227 cq->workqueue = workqueue; 2228 cq->uar = uar; 2229 2230 return 0; 2231 } 2232 2233 static int mlx5e_alloc_cq(struct mlx5_core_dev *mdev, 2234 struct mlx5e_cq_param *param, 2235 struct mlx5e_create_cq_param *ccp, 2236 struct mlx5e_cq *cq) 2237 { 2238 int err; 2239 2240 param->wq.buf_numa_node = ccp->node; 2241 param->wq.db_numa_node = ccp->node; 2242 param->eq_ix = ccp->ix; 2243 2244 err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq, 2245 ccp->uar, param, cq); 2246 2247 cq->napi = ccp->napi; 2248 cq->ch_stats = ccp->ch_stats; 2249 2250 return err; 2251 } 2252 2253 static void mlx5e_free_cq(struct mlx5e_cq *cq) 2254 { 2255 mlx5_wq_destroy(&cq->wq_ctrl); 2256 } 2257 2258 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 2259 { 2260 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 2261 struct mlx5_core_dev *mdev = cq->mdev; 2262 struct mlx5_core_cq *mcq = &cq->mcq; 2263 2264 void *in; 2265 void *cqc; 2266 int inlen; 2267 int eqn; 2268 int err; 2269 2270 err = mlx5_comp_eqn_get(mdev, param->eq_ix, &eqn); 2271 if (err) 2272 return err; 2273 2274 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 2275 sizeof(u64) * cq->wq_ctrl.buf.npages; 2276 in = kvzalloc(inlen, GFP_KERNEL); 2277 if (!in) 2278 return -ENOMEM; 2279 2280 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 2281 2282 memcpy(cqc, param->cqc, sizeof(param->cqc)); 2283 2284 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, 2285 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); 2286 2287 MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(param->cq_period_mode)); 2288 2289 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 2290 MLX5_SET(cqc, cqc, uar_page, cq->uar->index); 2291 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 2292 MLX5_ADAPTER_PAGE_SHIFT); 2293 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 2294 2295 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); 2296 2297 kvfree(in); 2298 2299 if (err) 2300 return err; 2301 2302 mlx5e_cq_arm(cq); 2303 2304 return 0; 2305 } 2306 2307 static void mlx5e_destroy_cq(struct mlx5e_cq *cq) 2308 { 2309 mlx5_core_destroy_cq(cq->mdev, &cq->mcq); 2310 } 2311 2312 int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder, 2313 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp, 2314 struct mlx5e_cq *cq) 2315 { 2316 int err; 2317 2318 err = mlx5e_alloc_cq(mdev, param, ccp, cq); 2319 if (err) 2320 return err; 2321 2322 err = mlx5e_create_cq(cq, param); 2323 if (err) 2324 goto err_free_cq; 2325 2326 if (MLX5_CAP_GEN(mdev, cq_moderation) && 2327 MLX5_CAP_GEN(mdev, cq_period_mode_modify)) 2328 mlx5e_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts, 2329 mlx5e_cq_period_mode(moder.cq_period_mode)); 2330 return 0; 2331 2332 err_free_cq: 2333 mlx5e_free_cq(cq); 2334 2335 return err; 2336 } 2337 2338 void mlx5e_close_cq(struct mlx5e_cq *cq) 2339 { 2340 mlx5e_destroy_cq(cq); 2341 mlx5e_free_cq(cq); 2342 } 2343 2344 int mlx5e_modify_cq_period_mode(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 2345 u8 cq_period_mode) 2346 { 2347 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {}; 2348 void *cqc; 2349 2350 MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 2351 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 2352 MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(cq_period_mode)); 2353 MLX5_SET(modify_cq_in, in, 2354 modify_field_select_resize_field_select.modify_field_select.modify_field_select, 2355 MLX5_CQ_MODIFY_PERIOD_MODE); 2356 2357 return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 2358 } 2359 2360 int mlx5e_modify_cq_moderation(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 2361 u16 cq_period, u16 cq_max_count, u8 cq_period_mode) 2362 { 2363 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {}; 2364 void *cqc; 2365 2366 MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 2367 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 2368 MLX5_SET(cqc, cqc, cq_period, cq_period); 2369 MLX5_SET(cqc, cqc, cq_max_count, cq_max_count); 2370 MLX5_SET(cqc, cqc, cq_period_mode, cq_period_mode); 2371 MLX5_SET(modify_cq_in, in, 2372 modify_field_select_resize_field_select.modify_field_select.modify_field_select, 2373 MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT | MLX5_CQ_MODIFY_PERIOD_MODE); 2374 2375 return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 2376 } 2377 2378 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, 2379 struct mlx5e_params *params, 2380 struct mlx5e_create_cq_param *ccp, 2381 struct mlx5e_channel_param *cparam) 2382 { 2383 int err; 2384 int tc; 2385 2386 for (tc = 0; tc < c->num_tc; tc++) { 2387 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->txq_sq.cqp, 2388 ccp, &c->sq[tc].cq); 2389 if (err) 2390 goto err_close_tx_cqs; 2391 } 2392 2393 return 0; 2394 2395 err_close_tx_cqs: 2396 for (tc--; tc >= 0; tc--) 2397 mlx5e_close_cq(&c->sq[tc].cq); 2398 2399 return err; 2400 } 2401 2402 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c) 2403 { 2404 int tc; 2405 2406 for (tc = 0; tc < c->num_tc; tc++) 2407 mlx5e_close_cq(&c->sq[tc].cq); 2408 } 2409 2410 static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq) 2411 { 2412 int tc; 2413 2414 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 2415 if (txq - tc_to_txq[tc].offset < tc_to_txq[tc].count) 2416 return tc; 2417 2418 WARN(1, "Unexpected TCs configuration. No match found for txq %u", txq); 2419 return -ENOENT; 2420 } 2421 2422 static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, 2423 u32 *hw_id) 2424 { 2425 int tc; 2426 2427 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) { 2428 *hw_id = 0; 2429 return 0; 2430 } 2431 2432 tc = mlx5e_mqprio_txq_to_tc(params->mqprio.tc_to_txq, txq_ix); 2433 if (tc < 0) 2434 return tc; 2435 2436 if (tc >= params->mqprio.num_tc) { 2437 WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u", 2438 tc, params->mqprio.num_tc); 2439 return -EINVAL; 2440 } 2441 2442 *hw_id = params->mqprio.channel.hw_id[tc]; 2443 return 0; 2444 } 2445 2446 static int mlx5e_open_sqs(struct mlx5e_channel *c, 2447 struct mlx5e_params *params, 2448 struct mlx5e_channel_param *cparam) 2449 { 2450 int err, tc; 2451 2452 for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) { 2453 int txq_ix = c->ix + tc * params->num_channels; 2454 u32 qos_queue_group_id; 2455 u32 tisn; 2456 2457 tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, 2458 c->lag_port, tc); 2459 err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id); 2460 if (err) 2461 goto err_close_sqs; 2462 2463 err = mlx5e_open_txqsq(c, tisn, txq_ix, 2464 params, &cparam->txq_sq, &c->sq[tc], tc, 2465 qos_queue_group_id, 2466 &c->priv->channel_stats[c->ix]->sq[tc]); 2467 if (err) 2468 goto err_close_sqs; 2469 } 2470 2471 return 0; 2472 2473 err_close_sqs: 2474 for (tc--; tc >= 0; tc--) 2475 mlx5e_close_txqsq(&c->sq[tc]); 2476 2477 return err; 2478 } 2479 2480 static void mlx5e_close_sqs(struct mlx5e_channel *c) 2481 { 2482 int tc; 2483 2484 for (tc = 0; tc < c->num_tc; tc++) 2485 mlx5e_close_txqsq(&c->sq[tc]); 2486 } 2487 2488 static int mlx5e_set_sq_maxrate(struct net_device *dev, 2489 struct mlx5e_txqsq *sq, u32 rate) 2490 { 2491 struct mlx5e_priv *priv = netdev_priv(dev); 2492 struct mlx5_core_dev *mdev = priv->mdev; 2493 struct mlx5e_modify_sq_param msp = {0}; 2494 struct mlx5_rate_limit rl = {0}; 2495 u16 rl_index = 0; 2496 int err; 2497 2498 if (rate == sq->rate_limit) 2499 /* nothing to do */ 2500 return 0; 2501 2502 if (sq->rate_limit) { 2503 rl.rate = sq->rate_limit; 2504 /* remove current rl index to free space to next ones */ 2505 mlx5_rl_remove_rate(mdev, &rl); 2506 } 2507 2508 sq->rate_limit = 0; 2509 2510 if (rate) { 2511 rl.rate = rate; 2512 err = mlx5_rl_add_rate(mdev, &rl_index, &rl); 2513 if (err) { 2514 netdev_err(dev, "Failed configuring rate %u: %d\n", 2515 rate, err); 2516 return err; 2517 } 2518 } 2519 2520 msp.curr_state = MLX5_SQC_STATE_RDY; 2521 msp.next_state = MLX5_SQC_STATE_RDY; 2522 msp.rl_index = rl_index; 2523 msp.rl_update = true; 2524 err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 2525 if (err) { 2526 netdev_err(dev, "Failed configuring rate %u: %d\n", 2527 rate, err); 2528 /* remove the rate from the table */ 2529 if (rate) 2530 mlx5_rl_remove_rate(mdev, &rl); 2531 return err; 2532 } 2533 2534 sq->rate_limit = rate; 2535 return 0; 2536 } 2537 2538 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 2539 { 2540 struct mlx5e_priv *priv = netdev_priv(dev); 2541 struct mlx5_core_dev *mdev = priv->mdev; 2542 struct mlx5e_txqsq *sq = priv->txq2sq[index]; 2543 int err = 0; 2544 2545 if (!mlx5_rl_is_supported(mdev)) { 2546 netdev_err(dev, "Rate limiting is not supported on this device\n"); 2547 return -EINVAL; 2548 } 2549 2550 /* rate is given in Mb/sec, HW config is in Kb/sec */ 2551 rate = rate << 10; 2552 2553 /* Check whether rate in valid range, 0 is always valid */ 2554 if (rate && !mlx5_rl_is_in_range(mdev, rate)) { 2555 netdev_err(dev, "TX rate %u, is not in range\n", rate); 2556 return -ERANGE; 2557 } 2558 2559 mutex_lock(&priv->state_lock); 2560 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 2561 err = mlx5e_set_sq_maxrate(dev, sq, rate); 2562 if (!err) 2563 priv->tx_rates[index] = rate; 2564 mutex_unlock(&priv->state_lock); 2565 2566 return err; 2567 } 2568 2569 static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, 2570 struct mlx5e_params *params, 2571 struct mlx5e_rq_param *rq_param, 2572 struct mlx5e_rq_opt_param *rqo) 2573 { 2574 u16 q_counter = c->priv->q_counter[c->sd_ix]; 2575 int err; 2576 2577 err = mlx5e_init_rxq_rq(c, params, rq_param->xdp_frag_size, &c->rq); 2578 if (err) 2579 return err; 2580 2581 return mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu), 2582 q_counter, &c->rq); 2583 } 2584 2585 static struct mlx5e_icosq * 2586 mlx5e_open_async_icosq(struct mlx5e_channel *c, 2587 struct mlx5e_params *params, 2588 struct mlx5e_channel_param *cparam, 2589 struct mlx5e_create_cq_param *ccp) 2590 { 2591 struct dim_cq_moder icocq_moder = {0, 0}; 2592 struct mlx5e_icosq *async_icosq; 2593 int err; 2594 2595 async_icosq = kvzalloc_node(sizeof(*async_icosq), GFP_KERNEL, 2596 cpu_to_node(c->cpu)); 2597 if (!async_icosq) 2598 return ERR_PTR(-ENOMEM); 2599 2600 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->async_icosq.cqp, ccp, 2601 &async_icosq->cq); 2602 if (err) 2603 goto err_free_async_icosq; 2604 2605 err = mlx5e_open_icosq(c, params, &cparam->async_icosq, async_icosq, 2606 mlx5e_async_icosq_err_cqe_work); 2607 if (err) 2608 goto err_close_async_icosq_cq; 2609 2610 return async_icosq; 2611 2612 err_close_async_icosq_cq: 2613 mlx5e_close_cq(&async_icosq->cq); 2614 err_free_async_icosq: 2615 kvfree(async_icosq); 2616 return ERR_PTR(err); 2617 } 2618 2619 static void mlx5e_close_async_icosq(struct mlx5e_icosq *async_icosq) 2620 { 2621 mlx5e_close_icosq(async_icosq); 2622 mlx5e_close_cq(&async_icosq->cq); 2623 kvfree(async_icosq); 2624 } 2625 2626 static int mlx5e_open_queues(struct mlx5e_channel *c, 2627 struct mlx5e_params *params, 2628 struct mlx5e_channel_param *cparam, 2629 bool async_icosq_needed) 2630 { 2631 const struct net_device_ops *netdev_ops = c->netdev->netdev_ops; 2632 struct dim_cq_moder icocq_moder = {0, 0}; 2633 struct mlx5e_create_cq_param ccp; 2634 int err; 2635 2636 mlx5e_build_create_cq_param(&ccp, c); 2637 2638 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->icosq.cqp, &ccp, 2639 &c->icosq.cq); 2640 if (err) 2641 return err; 2642 2643 err = mlx5e_open_tx_cqs(c, params, &ccp, cparam); 2644 if (err) 2645 goto err_close_icosq_cq; 2646 2647 if (netdev_ops->ndo_xdp_xmit && c->xdp) { 2648 c->xdpsq = mlx5e_open_xdpredirect_sq(c, params, cparam, &ccp); 2649 if (IS_ERR(c->xdpsq)) { 2650 err = PTR_ERR(c->xdpsq); 2651 goto err_close_tx_cqs; 2652 } 2653 } 2654 2655 err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, 2656 &c->rq.cq); 2657 if (err) 2658 goto err_close_xdpredirect_sq; 2659 2660 err = c->xdp ? mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, 2661 &ccp, &c->rq_xdpsq.cq) : 0; 2662 if (err) 2663 goto err_close_rx_cq; 2664 2665 if (async_icosq_needed) { 2666 c->async_icosq = mlx5e_open_async_icosq(c, params, cparam, 2667 &ccp); 2668 if (IS_ERR(c->async_icosq)) { 2669 err = PTR_ERR(c->async_icosq); 2670 goto err_close_rq_xdpsq_cq; 2671 } 2672 } 2673 2674 mutex_init(&c->icosq_recovery_lock); 2675 2676 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq, 2677 mlx5e_icosq_err_cqe_work); 2678 if (err) 2679 goto err_close_async_icosq; 2680 2681 err = mlx5e_open_sqs(c, params, cparam); 2682 if (err) 2683 goto err_close_icosq; 2684 2685 err = mlx5e_open_rxq_rq(c, params, &cparam->rq, &cparam->rq_opt); 2686 if (err) 2687 goto err_close_sqs; 2688 2689 if (c->xdp) { 2690 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, 2691 &c->rq_xdpsq, false); 2692 if (err) 2693 goto err_close_rq; 2694 } 2695 2696 return 0; 2697 2698 err_close_rq: 2699 mlx5e_close_rq(&c->rq); 2700 2701 err_close_sqs: 2702 mlx5e_close_sqs(c); 2703 2704 err_close_icosq: 2705 mlx5e_close_icosq(&c->icosq); 2706 2707 err_close_async_icosq: 2708 if (c->async_icosq) 2709 mlx5e_close_async_icosq(c->async_icosq); 2710 2711 err_close_rq_xdpsq_cq: 2712 if (c->xdp) 2713 mlx5e_close_cq(&c->rq_xdpsq.cq); 2714 2715 err_close_rx_cq: 2716 mlx5e_close_cq(&c->rq.cq); 2717 2718 err_close_xdpredirect_sq: 2719 if (c->xdpsq) 2720 mlx5e_close_xdpredirect_sq(c->xdpsq); 2721 2722 err_close_tx_cqs: 2723 mlx5e_close_tx_cqs(c); 2724 2725 err_close_icosq_cq: 2726 mlx5e_close_cq(&c->icosq.cq); 2727 2728 return err; 2729 } 2730 2731 static void mlx5e_close_queues(struct mlx5e_channel *c) 2732 { 2733 if (c->xdp) 2734 mlx5e_close_xdpsq(&c->rq_xdpsq); 2735 /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */ 2736 cancel_work_sync(&c->icosq.recover_work); 2737 mlx5e_close_rq(&c->rq); 2738 mlx5e_close_sqs(c); 2739 mlx5e_close_icosq(&c->icosq); 2740 mutex_destroy(&c->icosq_recovery_lock); 2741 if (c->async_icosq) 2742 mlx5e_close_async_icosq(c->async_icosq); 2743 if (c->xdp) 2744 mlx5e_close_cq(&c->rq_xdpsq.cq); 2745 mlx5e_close_cq(&c->rq.cq); 2746 if (c->xdpsq) 2747 mlx5e_close_xdpredirect_sq(c->xdpsq); 2748 mlx5e_close_tx_cqs(c); 2749 mlx5e_close_cq(&c->icosq.cq); 2750 } 2751 2752 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) 2753 { 2754 u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id); 2755 2756 return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev); 2757 } 2758 2759 static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu) 2760 { 2761 if (ix > priv->stats_nch) { 2762 netdev_warn(priv->netdev, "Unexpected channel stats index %d > %d\n", ix, 2763 priv->stats_nch); 2764 return -EINVAL; 2765 } 2766 2767 if (priv->channel_stats[ix]) 2768 return 0; 2769 2770 /* Asymmetric dynamic memory allocation. 2771 * Freed in mlx5e_priv_arrays_free, not on channel closure. 2772 */ 2773 netdev_dbg(priv->netdev, "Creating channel stats %d\n", ix); 2774 priv->channel_stats[ix] = kvzalloc_node(sizeof(**priv->channel_stats), 2775 GFP_KERNEL, cpu_to_node(cpu)); 2776 if (!priv->channel_stats[ix]) 2777 return -ENOMEM; 2778 mlx5e_stats_nch_write(priv, priv->stats_nch + 1); 2779 2780 return 0; 2781 } 2782 2783 void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c) 2784 { 2785 struct mlx5e_icosq *sq = &c->icosq; 2786 bool locked; 2787 2788 set_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &sq->state); 2789 synchronize_net(); 2790 2791 locked = mlx5e_icosq_sync_lock(sq); 2792 mlx5e_trigger_irq(sq); 2793 mlx5e_icosq_sync_unlock(sq, locked); 2794 2795 clear_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &sq->state); 2796 } 2797 2798 void mlx5e_trigger_napi_async_icosq(struct mlx5e_channel *c) 2799 { 2800 struct mlx5e_icosq *sq = c->async_icosq; 2801 2802 spin_lock_bh(&sq->lock); 2803 mlx5e_trigger_irq(sq); 2804 spin_unlock_bh(&sq->lock); 2805 } 2806 2807 void mlx5e_trigger_napi_sched(struct napi_struct *napi) 2808 { 2809 local_bh_disable(); 2810 napi_schedule(napi); 2811 local_bh_enable(); 2812 } 2813 2814 static void mlx5e_channel_pick_doorbell(struct mlx5e_channel *c) 2815 { 2816 struct mlx5e_hw_objs *hw_objs = &c->mdev->mlx5e_res.hw_objs; 2817 2818 /* No dedicated Ethernet doorbells, use the global one. */ 2819 if (hw_objs->num_bfregs == 0) { 2820 c->bfreg = &c->mdev->priv.bfreg; 2821 return; 2822 } 2823 2824 /* Round-robin between doorbells. */ 2825 c->bfreg = hw_objs->bfregs + c->vec_ix % hw_objs->num_bfregs; 2826 } 2827 2828 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 2829 struct mlx5e_params *params, 2830 struct netdev_queue_config *qcfg, 2831 struct xsk_buff_pool *xsk_pool, 2832 struct mlx5e_channel **cp) 2833 { 2834 struct net_device *netdev = priv->netdev; 2835 struct mlx5e_channel_param *cparam; 2836 struct mlx5_core_dev *mdev; 2837 struct mlx5e_xsk_param xsk; 2838 bool async_icosq_needed; 2839 struct mlx5e_channel *c; 2840 unsigned int irq; 2841 int vec_ix; 2842 int cpu; 2843 int err; 2844 2845 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); 2846 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); 2847 cpu = mlx5_comp_vector_get_cpu(mdev, vec_ix); 2848 2849 err = mlx5_comp_irqn_get(mdev, vec_ix, &irq); 2850 if (err) 2851 return err; 2852 2853 err = mlx5e_channel_stats_alloc(priv, ix, cpu); 2854 if (err) 2855 return err; 2856 2857 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu)); 2858 cparam = kvzalloc_obj(*cparam); 2859 if (!c || !cparam) { 2860 err = -ENOMEM; 2861 goto err_free; 2862 } 2863 2864 err = mlx5e_build_channel_param(mdev, params, qcfg, cparam); 2865 if (err) 2866 goto err_free; 2867 2868 c->priv = priv; 2869 c->mdev = mdev; 2870 c->ix = ix; 2871 c->vec_ix = vec_ix; 2872 c->sd_ix = mlx5_sd_ch_ix_get_dev_ix(mdev, ix); 2873 c->cpu = cpu; 2874 c->pdev = mlx5_core_dma_dev(mdev); 2875 c->netdev = priv->netdev; 2876 c->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); 2877 c->num_tc = mlx5e_get_dcb_num_tc(params); 2878 c->xdp = !!params->xdp_prog; 2879 c->stats = &priv->channel_stats[ix]->ch; 2880 c->aff_mask = irq_get_effective_affinity_mask(irq); 2881 c->lag_port = mlx5e_enumerate_lag_port(mdev, ix); 2882 2883 mlx5e_channel_pick_doorbell(c); 2884 2885 netif_napi_add_config_locked(netdev, &c->napi, mlx5e_napi_poll, ix); 2886 netif_napi_set_irq_locked(&c->napi, irq); 2887 2888 async_icosq_needed = !!params->xdp_prog || priv->ktls_rx_was_enabled; 2889 err = mlx5e_open_queues(c, params, cparam, async_icosq_needed); 2890 if (unlikely(err)) 2891 goto err_napi_del; 2892 2893 if (xsk_pool) { 2894 mlx5e_build_xsk_param(xsk_pool, &xsk); 2895 mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam); 2896 err = mlx5e_open_xsk(priv, params, cparam, xsk_pool, c); 2897 if (unlikely(err)) 2898 goto err_close_queues; 2899 } 2900 2901 *cp = c; 2902 2903 kvfree(cparam); 2904 return 0; 2905 2906 err_close_queues: 2907 mlx5e_close_queues(c); 2908 2909 err_napi_del: 2910 netif_napi_del_locked(&c->napi); 2911 2912 err_free: 2913 kvfree(cparam); 2914 kvfree(c); 2915 2916 return err; 2917 } 2918 2919 static void mlx5e_activate_channel(struct mlx5e_channel *c) 2920 { 2921 int tc; 2922 2923 napi_enable_locked(&c->napi); 2924 2925 for (tc = 0; tc < c->num_tc; tc++) 2926 mlx5e_activate_txqsq(&c->sq[tc]); 2927 mlx5e_activate_icosq(&c->icosq); 2928 if (c->async_icosq) 2929 mlx5e_activate_icosq(c->async_icosq); 2930 2931 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2932 mlx5e_activate_xsk(c); 2933 else 2934 mlx5e_activate_rq(&c->rq); 2935 2936 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, &c->napi); 2937 } 2938 2939 static void mlx5e_deactivate_channel(struct mlx5e_channel *c) 2940 { 2941 int tc; 2942 2943 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, NULL); 2944 2945 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2946 mlx5e_deactivate_xsk(c); 2947 else 2948 mlx5e_deactivate_rq(&c->rq); 2949 2950 if (c->async_icosq) 2951 mlx5e_deactivate_icosq(c->async_icosq); 2952 mlx5e_deactivate_icosq(&c->icosq); 2953 for (tc = 0; tc < c->num_tc; tc++) 2954 mlx5e_deactivate_txqsq(&c->sq[tc]); 2955 mlx5e_qos_deactivate_queues(c); 2956 2957 napi_disable_locked(&c->napi); 2958 } 2959 2960 static void mlx5e_close_channel(struct mlx5e_channel *c) 2961 { 2962 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2963 mlx5e_close_xsk(c); 2964 mlx5e_close_queues(c); 2965 mlx5e_qos_close_queues(c); 2966 netif_napi_del_locked(&c->napi); 2967 2968 kvfree(c); 2969 } 2970 2971 int mlx5e_open_channels(struct mlx5e_priv *priv, 2972 struct mlx5e_channels *chs) 2973 { 2974 int err = -ENOMEM; 2975 int i; 2976 2977 chs->num = chs->params.num_channels; 2978 2979 chs->c = kzalloc_objs(struct mlx5e_channel *, chs->num); 2980 if (!chs->c) 2981 goto err_out; 2982 2983 for (i = 0; i < chs->num; i++) { 2984 struct xsk_buff_pool *xsk_pool = NULL; 2985 2986 if (chs->params.xdp_prog) 2987 xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i); 2988 2989 err = mlx5e_open_channel(priv, i, &chs->params, NULL, 2990 xsk_pool, &chs->c[i]); 2991 if (err) 2992 goto err_close_channels; 2993 } 2994 2995 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) { 2996 err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp); 2997 if (err) 2998 goto err_close_channels; 2999 } 3000 3001 if (priv->htb) { 3002 err = mlx5e_qos_open_queues(priv, chs); 3003 if (err) 3004 goto err_close_ptp; 3005 } 3006 3007 mlx5e_health_channels_update(priv); 3008 return 0; 3009 3010 err_close_ptp: 3011 if (chs->ptp) 3012 mlx5e_ptp_close(chs->ptp); 3013 3014 err_close_channels: 3015 for (i--; i >= 0; i--) 3016 mlx5e_close_channel(chs->c[i]); 3017 3018 kfree(chs->c); 3019 err_out: 3020 chs->num = 0; 3021 return err; 3022 } 3023 3024 static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 3025 { 3026 int i; 3027 3028 for (i = 0; i < chs->num; i++) 3029 mlx5e_activate_channel(chs->c[i]); 3030 3031 if (priv->htb) 3032 mlx5e_qos_activate_queues(priv); 3033 3034 for (i = 0; i < chs->num; i++) 3035 mlx5e_trigger_napi_icosq(chs->c[i]); 3036 3037 if (chs->ptp) 3038 mlx5e_ptp_activate_channel(chs->ptp); 3039 } 3040 3041 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) 3042 { 3043 int err = 0; 3044 int i; 3045 3046 for (i = 0; i < chs->num; i++) { 3047 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; 3048 struct mlx5e_channel *c = chs->c[i]; 3049 3050 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 3051 continue; 3052 3053 err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout); 3054 3055 /* Don't wait on the XSK RQ, because the newer xdpsock sample 3056 * doesn't provide any Fill Ring entries at the setup stage. 3057 */ 3058 } 3059 3060 return err ? -ETIMEDOUT : 0; 3061 } 3062 3063 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs) 3064 { 3065 int i; 3066 3067 if (chs->ptp) 3068 mlx5e_ptp_deactivate_channel(chs->ptp); 3069 3070 for (i = 0; i < chs->num; i++) 3071 mlx5e_deactivate_channel(chs->c[i]); 3072 } 3073 3074 void mlx5e_close_channels(struct mlx5e_channels *chs) 3075 { 3076 int i; 3077 3078 if (chs->ptp) { 3079 mlx5e_ptp_close(chs->ptp); 3080 chs->ptp = NULL; 3081 } 3082 for (i = 0; i < chs->num; i++) 3083 mlx5e_close_channel(chs->c[i]); 3084 3085 kfree(chs->c); 3086 chs->num = 0; 3087 } 3088 3089 static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv) 3090 { 3091 struct mlx5e_rx_res *res = priv->rx_res; 3092 3093 return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge); 3094 } 3095 3096 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge); 3097 3098 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev, 3099 struct mlx5e_params *params, u16 mtu) 3100 { 3101 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu); 3102 int err; 3103 3104 err = mlx5_set_port_mtu(mdev, hw_mtu, 1); 3105 if (err) 3106 return err; 3107 3108 /* Update vport context MTU */ 3109 mlx5_modify_nic_vport_mtu(mdev, hw_mtu); 3110 return 0; 3111 } 3112 3113 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev, 3114 struct mlx5e_params *params, u16 *mtu) 3115 { 3116 u16 hw_mtu = 0; 3117 int err; 3118 3119 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 3120 if (err || !hw_mtu) /* fallback to port oper mtu */ 3121 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1); 3122 3123 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu); 3124 } 3125 3126 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) 3127 { 3128 struct mlx5e_params *params = &priv->channels.params; 3129 struct net_device *netdev = priv->netdev; 3130 struct mlx5_core_dev *mdev = priv->mdev; 3131 u16 mtu; 3132 int err; 3133 3134 err = mlx5e_set_mtu(mdev, params, params->sw_mtu); 3135 if (err) 3136 return err; 3137 3138 mlx5e_query_mtu(mdev, params, &mtu); 3139 if (mtu != params->sw_mtu) 3140 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", 3141 __func__, mtu, params->sw_mtu); 3142 3143 params->sw_mtu = mtu; 3144 return 0; 3145 } 3146 3147 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu); 3148 3149 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv) 3150 { 3151 struct mlx5e_params *params = &priv->channels.params; 3152 struct net_device *netdev = priv->netdev; 3153 struct mlx5_core_dev *mdev = priv->mdev; 3154 u16 max_mtu; 3155 3156 /* MTU range: 68 - hw-specific max */ 3157 netdev->min_mtu = ETH_MIN_MTU; 3158 3159 mlx5_query_port_max_mtu(mdev, &max_mtu, 1); 3160 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu), 3161 ETH_MAX_MTU); 3162 } 3163 3164 static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc, 3165 struct netdev_tc_txq *tc_to_txq) 3166 { 3167 int tc, err; 3168 3169 netdev_reset_tc(netdev); 3170 3171 if (ntc == 1) 3172 return 0; 3173 3174 err = netdev_set_num_tc(netdev, ntc); 3175 if (err) { 3176 netdev_WARN(netdev, "netdev_set_num_tc failed (%d), ntc = %d\n", err, ntc); 3177 return err; 3178 } 3179 3180 for (tc = 0; tc < ntc; tc++) { 3181 u16 count, offset; 3182 3183 count = tc_to_txq[tc].count; 3184 offset = tc_to_txq[tc].offset; 3185 netdev_set_tc_queue(netdev, tc, count, offset); 3186 } 3187 3188 return 0; 3189 } 3190 3191 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv) 3192 { 3193 int nch, ntc, num_txqs, err; 3194 int qos_queues = 0; 3195 3196 if (priv->htb) 3197 qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb); 3198 3199 nch = priv->channels.params.num_channels; 3200 ntc = mlx5e_get_dcb_num_tc(&priv->channels.params); 3201 num_txqs = nch * ntc + qos_queues; 3202 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS)) 3203 num_txqs += ntc; 3204 3205 netdev_dbg(priv->netdev, "Setting num_txqs %d\n", num_txqs); 3206 err = netif_set_real_num_tx_queues(priv->netdev, num_txqs); 3207 if (err) 3208 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err); 3209 3210 return err; 3211 } 3212 3213 static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv, 3214 struct mlx5e_params *params) 3215 { 3216 int ix; 3217 3218 for (ix = 0; ix < params->num_channels; ix++) { 3219 int num_comp_vectors, irq, vec_ix; 3220 struct mlx5_core_dev *mdev; 3221 3222 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); 3223 num_comp_vectors = mlx5_comp_vectors_max(mdev); 3224 cpumask_clear(priv->scratchpad.cpumask); 3225 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); 3226 3227 for (irq = vec_ix; irq < num_comp_vectors; irq += params->num_channels) { 3228 int cpu = mlx5_comp_vector_get_cpu(mdev, irq); 3229 3230 cpumask_set_cpu(cpu, priv->scratchpad.cpumask); 3231 } 3232 3233 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix); 3234 } 3235 } 3236 3237 static int mlx5e_update_tc_and_tx_queues(struct mlx5e_priv *priv) 3238 { 3239 struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq; 3240 struct net_device *netdev = priv->netdev; 3241 int old_num_txqs, old_ntc; 3242 int nch, ntc; 3243 int err; 3244 int i; 3245 3246 old_num_txqs = netdev->real_num_tx_queues; 3247 old_ntc = netdev_get_num_tc(netdev) ? : 1; 3248 for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++) 3249 old_tc_to_txq[i].combined = READ_ONCE(netdev->tc_to_txq[i].combined); 3250 3251 nch = priv->channels.params.num_channels; 3252 ntc = priv->channels.params.mqprio.num_tc; 3253 tc_to_txq = priv->channels.params.mqprio.tc_to_txq; 3254 3255 err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq); 3256 if (err) 3257 goto err_out; 3258 err = mlx5e_update_tx_netdev_queues(priv); 3259 if (err) 3260 goto err_tcs; 3261 mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params); 3262 3263 return 0; 3264 3265 err_tcs: 3266 WARN_ON_ONCE(mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc, 3267 old_tc_to_txq)); 3268 err_out: 3269 return err; 3270 } 3271 3272 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_tc_and_tx_queues); 3273 3274 static int mlx5e_num_channels_changed(struct mlx5e_priv *priv) 3275 { 3276 u16 count = priv->channels.params.num_channels; 3277 struct net_device *netdev = priv->netdev; 3278 int old_num_rxqs; 3279 int err; 3280 3281 old_num_rxqs = netdev->real_num_rx_queues; 3282 err = netif_set_real_num_rx_queues(netdev, count); 3283 if (err) { 3284 netdev_warn(netdev, "%s: netif_set_real_num_rx_queues failed, %d\n", 3285 __func__, err); 3286 return err; 3287 } 3288 err = mlx5e_update_tc_and_tx_queues(priv); 3289 if (err) { 3290 /* mlx5e_update_tc_and_tx_queues can fail if channels or TCs number increases. 3291 * Since channel number changed, it increased. That means, the call to 3292 * netif_set_real_num_rx_queues below should not fail, because it 3293 * decreases the number of RX queues. 3294 */ 3295 WARN_ON_ONCE(netif_set_real_num_rx_queues(netdev, old_num_rxqs)); 3296 return err; 3297 } 3298 3299 /* This function may be called on attach, before priv->rx_res is created. */ 3300 if (priv->rx_res) 3301 mlx5e_rx_res_rss_update_num_channels(priv->rx_res, count, 3302 netdev); 3303 3304 return 0; 3305 } 3306 3307 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed); 3308 3309 static void mlx5e_build_txq_maps(struct mlx5e_priv *priv) 3310 { 3311 int i, ch, tc, num_tc; 3312 3313 ch = priv->channels.num; 3314 num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params); 3315 3316 for (i = 0; i < ch; i++) { 3317 for (tc = 0; tc < num_tc; tc++) { 3318 struct mlx5e_channel *c = priv->channels.c[i]; 3319 struct mlx5e_txqsq *sq = &c->sq[tc]; 3320 3321 priv->txq2sq[sq->txq_ix] = sq; 3322 priv->txq2sq_stats[sq->txq_ix] = sq->stats; 3323 } 3324 } 3325 3326 if (!priv->channels.ptp) 3327 goto out; 3328 3329 if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state)) 3330 goto out; 3331 3332 for (tc = 0; tc < num_tc; tc++) { 3333 struct mlx5e_ptp *c = priv->channels.ptp; 3334 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq; 3335 3336 priv->txq2sq[sq->txq_ix] = sq; 3337 priv->txq2sq_stats[sq->txq_ix] = sq->stats; 3338 } 3339 3340 out: 3341 /* Make the change to txq2sq visible before the queue is started. 3342 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 3343 * which pairs with this barrier. 3344 */ 3345 smp_wmb(); 3346 } 3347 3348 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) 3349 { 3350 mlx5e_build_txq_maps(priv); 3351 mlx5e_activate_channels(priv, &priv->channels); 3352 mlx5e_xdp_tx_enable(priv); 3353 3354 /* dev_watchdog() wants all TX queues to be started when the carrier is 3355 * OK, including the ones in range real_num_tx_queues..num_tx_queues-1. 3356 * Make it happy to avoid TX timeout false alarms. 3357 */ 3358 netif_tx_start_all_queues(priv->netdev); 3359 3360 if (mlx5e_is_vport_rep(priv)) 3361 mlx5e_rep_activate_channels(priv); 3362 3363 set_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state); 3364 3365 mlx5e_wait_channels_min_rx_wqes(&priv->channels); 3366 3367 if (priv->rx_res) 3368 mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels); 3369 } 3370 3371 static void mlx5e_cancel_tx_timeout_work(struct mlx5e_priv *priv) 3372 { 3373 WARN_ON_ONCE(test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)); 3374 if (current_work() != &priv->tx_timeout_work) 3375 cancel_work_sync(&priv->tx_timeout_work); 3376 } 3377 3378 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) 3379 { 3380 if (priv->rx_res) 3381 mlx5e_rx_res_channels_deactivate(priv->rx_res); 3382 3383 clear_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state); 3384 mlx5e_cancel_tx_timeout_work(priv); 3385 3386 if (mlx5e_is_vport_rep(priv)) 3387 mlx5e_rep_deactivate_channels(priv); 3388 3389 /* The results of ndo_select_queue are unreliable, while netdev config 3390 * is being changed (real_num_tx_queues, num_tc). Stop all queues to 3391 * prevent ndo_start_xmit from being called, so that it can assume that 3392 * the selected queue is always valid. 3393 */ 3394 netif_tx_disable(priv->netdev); 3395 3396 mlx5e_xdp_tx_disable(priv); 3397 mlx5e_deactivate_channels(&priv->channels); 3398 } 3399 3400 static int mlx5e_switch_priv_params(struct mlx5e_priv *priv, 3401 struct mlx5e_params *new_params, 3402 mlx5e_fp_preactivate preactivate, 3403 void *context) 3404 { 3405 struct mlx5e_params old_params; 3406 3407 old_params = priv->channels.params; 3408 priv->channels.params = *new_params; 3409 3410 if (preactivate) { 3411 int err; 3412 3413 err = preactivate(priv, context); 3414 if (err) { 3415 priv->channels.params = old_params; 3416 return err; 3417 } 3418 } 3419 3420 mlx5e_set_xdp_feature(priv); 3421 return 0; 3422 } 3423 3424 static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv, 3425 struct mlx5e_channels *old_chs, 3426 struct mlx5e_channels *new_chs, 3427 mlx5e_fp_preactivate preactivate, 3428 void *context) 3429 { 3430 struct net_device *netdev = priv->netdev; 3431 int carrier_ok; 3432 int err = 0; 3433 3434 carrier_ok = netif_carrier_ok(netdev); 3435 netif_carrier_off(netdev); 3436 3437 mlx5e_deactivate_priv_channels(priv); 3438 3439 priv->channels = *new_chs; 3440 3441 /* New channels are ready to roll, call the preactivate hook if needed 3442 * to modify HW settings or update kernel parameters. 3443 */ 3444 if (preactivate) { 3445 err = preactivate(priv, context); 3446 if (err) { 3447 priv->channels = *old_chs; 3448 goto out; 3449 } 3450 } 3451 3452 mlx5e_set_xdp_feature(priv); 3453 if (!MLX5_CAP_GEN(priv->mdev, tis_tir_td_order)) 3454 mlx5e_close_channels(old_chs); 3455 priv->profile->update_rx(priv); 3456 3457 mlx5e_selq_apply(&priv->selq); 3458 out: 3459 mlx5e_activate_priv_channels(priv); 3460 3461 /* return carrier back if needed */ 3462 if (carrier_ok) 3463 netif_carrier_on(netdev); 3464 3465 return err; 3466 } 3467 3468 int mlx5e_safe_switch_params(struct mlx5e_priv *priv, 3469 struct mlx5e_params *params, 3470 mlx5e_fp_preactivate preactivate, 3471 void *context, bool reset) 3472 { 3473 struct mlx5e_channels *old_chs, *new_chs; 3474 int err; 3475 3476 reset &= test_bit(MLX5E_STATE_OPENED, &priv->state); 3477 if (!reset) 3478 return mlx5e_switch_priv_params(priv, params, preactivate, context); 3479 3480 old_chs = kzalloc_obj(*old_chs); 3481 new_chs = kzalloc_obj(*new_chs); 3482 if (!old_chs || !new_chs) { 3483 err = -ENOMEM; 3484 goto err_free_chs; 3485 } 3486 3487 new_chs->params = *params; 3488 3489 mlx5e_selq_prepare_params(&priv->selq, &new_chs->params); 3490 3491 err = mlx5e_open_channels(priv, new_chs); 3492 if (err) 3493 goto err_cancel_selq; 3494 3495 *old_chs = priv->channels; 3496 3497 err = mlx5e_switch_priv_channels(priv, old_chs, new_chs, 3498 preactivate, context); 3499 if (err) 3500 goto err_close; 3501 3502 if (MLX5_CAP_GEN(priv->mdev, tis_tir_td_order)) 3503 mlx5e_close_channels(old_chs); 3504 3505 kfree(new_chs); 3506 kfree(old_chs); 3507 return 0; 3508 3509 err_close: 3510 mlx5e_close_channels(new_chs); 3511 3512 err_cancel_selq: 3513 mlx5e_selq_cancel(&priv->selq); 3514 err_free_chs: 3515 kfree(new_chs); 3516 kfree(old_chs); 3517 return err; 3518 } 3519 3520 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv) 3521 { 3522 return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true); 3523 } 3524 3525 void mlx5e_timestamp_init(struct mlx5e_priv *priv) 3526 { 3527 priv->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; 3528 priv->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; 3529 } 3530 3531 static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev, 3532 enum mlx5_port_status state) 3533 { 3534 struct mlx5_eswitch *esw = mdev->priv.eswitch; 3535 int vport_admin_state; 3536 3537 mlx5_set_port_admin_status(mdev, state); 3538 3539 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS || 3540 !MLX5_CAP_GEN(mdev, uplink_follow)) 3541 return; 3542 3543 if (state == MLX5_PORT_UP) 3544 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO; 3545 else 3546 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3547 3548 mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state); 3549 } 3550 3551 int mlx5e_open_locked(struct net_device *netdev) 3552 { 3553 struct mlx5e_priv *priv = netdev_priv(netdev); 3554 int err; 3555 3556 mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params); 3557 3558 set_bit(MLX5E_STATE_OPENED, &priv->state); 3559 3560 err = mlx5e_open_channels(priv, &priv->channels); 3561 if (err) 3562 goto err_clear_state_opened_flag; 3563 3564 err = priv->profile->update_rx(priv); 3565 if (err) 3566 goto err_close_channels; 3567 3568 mlx5e_selq_apply(&priv->selq); 3569 mlx5e_activate_priv_channels(priv); 3570 mlx5e_apply_traps(priv, true); 3571 if (priv->profile->update_carrier) 3572 priv->profile->update_carrier(priv); 3573 3574 mlx5e_queue_update_stats(priv); 3575 return 0; 3576 3577 err_close_channels: 3578 mlx5e_close_channels(&priv->channels); 3579 err_clear_state_opened_flag: 3580 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3581 mlx5e_selq_cancel(&priv->selq); 3582 return err; 3583 } 3584 3585 int mlx5e_open(struct net_device *netdev) 3586 { 3587 struct mlx5e_priv *priv = netdev_priv(netdev); 3588 int err; 3589 3590 mutex_lock(&priv->state_lock); 3591 err = mlx5e_open_locked(netdev); 3592 if (!err) 3593 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP); 3594 mutex_unlock(&priv->state_lock); 3595 3596 return err; 3597 } 3598 3599 int mlx5e_close_locked(struct net_device *netdev) 3600 { 3601 struct mlx5e_priv *priv = netdev_priv(netdev); 3602 3603 /* May already be CLOSED in case a previous configuration operation 3604 * (e.g RX/TX queue size change) that involves close&open failed. 3605 */ 3606 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3607 return 0; 3608 3609 mlx5e_apply_traps(priv, false); 3610 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3611 3612 netif_carrier_off(priv->netdev); 3613 mlx5e_deactivate_priv_channels(priv); 3614 mlx5e_close_channels(&priv->channels); 3615 3616 return 0; 3617 } 3618 3619 int mlx5e_close(struct net_device *netdev) 3620 { 3621 struct mlx5e_priv *priv = netdev_priv(netdev); 3622 int err; 3623 3624 if (!netif_device_present(netdev)) 3625 return -ENODEV; 3626 3627 mutex_lock(&priv->state_lock); 3628 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN); 3629 err = mlx5e_close_locked(netdev); 3630 mutex_unlock(&priv->state_lock); 3631 3632 return err; 3633 } 3634 3635 static void mlx5e_free_drop_rq(struct mlx5e_rq *rq) 3636 { 3637 mlx5_wq_destroy(&rq->wq_ctrl); 3638 } 3639 3640 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, 3641 struct mlx5e_rq *rq, 3642 struct mlx5e_rq_param *rq_param) 3643 { 3644 void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); 3645 int err; 3646 3647 rq_param->wq.db_numa_node = rq_param->wq.buf_numa_node; 3648 3649 err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, &rq->wqe.wq, 3650 &rq->wq_ctrl); 3651 if (err) 3652 return err; 3653 3654 /* Mark as unused given "Drop-RQ" packets never reach XDP */ 3655 xdp_rxq_info_unused(&rq->xdp_rxq); 3656 3657 rq->mdev = mdev; 3658 3659 return 0; 3660 } 3661 3662 static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv, 3663 struct mlx5e_cq *cq, 3664 struct mlx5e_cq_param *param) 3665 { 3666 struct mlx5_core_dev *mdev = priv->mdev; 3667 3668 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3669 param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3670 3671 return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq, 3672 mdev->priv.bfreg.up, param, cq); 3673 } 3674 3675 int mlx5e_open_drop_rq(struct mlx5e_priv *priv, 3676 struct mlx5e_rq *drop_rq) 3677 { 3678 struct mlx5_core_dev *mdev = priv->mdev; 3679 struct mlx5e_cq_param cq_param = {}; 3680 struct mlx5e_rq_param rq_param = {}; 3681 struct mlx5e_cq *cq = &drop_rq->cq; 3682 int err; 3683 3684 mlx5e_build_drop_rq_param(mdev, &rq_param); 3685 3686 err = mlx5e_alloc_drop_cq(priv, cq, &cq_param); 3687 if (err) 3688 return err; 3689 3690 err = mlx5e_create_cq(cq, &cq_param); 3691 if (err) 3692 goto err_free_cq; 3693 3694 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param); 3695 if (err) 3696 goto err_destroy_cq; 3697 3698 err = mlx5e_create_rq(drop_rq, &rq_param, priv->drop_rq_q_counter); 3699 if (err) 3700 goto err_free_rq; 3701 3702 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 3703 if (err) 3704 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err); 3705 3706 return 0; 3707 3708 err_free_rq: 3709 mlx5e_free_drop_rq(drop_rq); 3710 3711 err_destroy_cq: 3712 mlx5e_destroy_cq(cq); 3713 3714 err_free_cq: 3715 mlx5e_free_cq(cq); 3716 3717 return err; 3718 } 3719 3720 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) 3721 { 3722 mlx5e_destroy_rq(drop_rq); 3723 mlx5e_free_drop_rq(drop_rq); 3724 mlx5e_destroy_cq(&drop_rq->cq); 3725 mlx5e_free_cq(&drop_rq->cq); 3726 } 3727 3728 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) 3729 { 3730 if (priv->mqprio_rl) { 3731 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3732 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3733 priv->mqprio_rl = NULL; 3734 } 3735 mlx5e_accel_cleanup_tx(priv); 3736 } 3737 3738 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) 3739 { 3740 int err; 3741 int i; 3742 3743 for (i = 0; i < chs->num; i++) { 3744 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd); 3745 if (err) 3746 return err; 3747 } 3748 if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state)) 3749 return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd); 3750 3751 return 0; 3752 } 3753 3754 static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3755 int ntc, int nch) 3756 { 3757 int tc; 3758 3759 memset(tc_to_txq, 0, sizeof(*tc_to_txq) * TC_MAX_QUEUE); 3760 3761 /* Map netdev TCs to offset 0. 3762 * We have our own UP to TXQ mapping for DCB mode of QoS 3763 */ 3764 for (tc = 0; tc < ntc; tc++) { 3765 tc_to_txq[tc] = (struct netdev_tc_txq) { 3766 .count = nch, 3767 .offset = 0, 3768 }; 3769 } 3770 } 3771 3772 static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3773 struct tc_mqprio_qopt *qopt) 3774 { 3775 int tc; 3776 3777 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3778 tc_to_txq[tc] = (struct netdev_tc_txq) { 3779 .count = qopt->count[tc], 3780 .offset = qopt->offset[tc], 3781 }; 3782 } 3783 } 3784 3785 static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc) 3786 { 3787 params->mqprio.mode = TC_MQPRIO_MODE_DCB; 3788 params->mqprio.num_tc = num_tc; 3789 mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc, 3790 params->num_channels); 3791 } 3792 3793 static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params, 3794 struct mlx5e_mqprio_rl *rl) 3795 { 3796 int tc; 3797 3798 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3799 u32 hw_id = 0; 3800 3801 if (rl) 3802 mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id); 3803 params->mqprio.channel.hw_id[tc] = hw_id; 3804 } 3805 } 3806 3807 static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params, 3808 struct tc_mqprio_qopt_offload *mqprio, 3809 struct mlx5e_mqprio_rl *rl) 3810 { 3811 int tc; 3812 3813 params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL; 3814 params->mqprio.num_tc = mqprio->qopt.num_tc; 3815 3816 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 3817 params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc]; 3818 3819 mlx5e_mqprio_rl_update_params(params, rl); 3820 mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt); 3821 } 3822 3823 static void mlx5e_params_mqprio_reset(struct mlx5e_params *params) 3824 { 3825 mlx5e_params_mqprio_dcb_set(params, 1); 3826 } 3827 3828 static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, 3829 struct tc_mqprio_qopt *mqprio) 3830 { 3831 struct mlx5e_params new_params; 3832 u8 tc = mqprio->num_tc; 3833 int err; 3834 3835 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 3836 3837 if (tc && tc != MLX5_MAX_NUM_TC) 3838 return -EINVAL; 3839 3840 new_params = priv->channels.params; 3841 mlx5e_params_mqprio_dcb_set(&new_params, tc ? tc : 1); 3842 3843 err = mlx5e_safe_switch_params(priv, &new_params, 3844 mlx5e_update_tc_and_tx_queues_ctx, NULL, true); 3845 3846 if (!err && priv->mqprio_rl) { 3847 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3848 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3849 priv->mqprio_rl = NULL; 3850 } 3851 3852 priv->max_opened_tc = max_t(u8, priv->max_opened_tc, 3853 mlx5e_get_dcb_num_tc(&priv->channels.params)); 3854 return err; 3855 } 3856 3857 static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv, 3858 struct tc_mqprio_qopt_offload *mqprio) 3859 { 3860 struct net_device *netdev = priv->netdev; 3861 struct mlx5e_ptp *ptp_channel; 3862 int agg_count = 0; 3863 int i; 3864 3865 ptp_channel = priv->channels.ptp; 3866 if (ptp_channel && test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state)) { 3867 netdev_err(netdev, 3868 "Cannot activate MQPRIO mode channel since it conflicts with TX port TS\n"); 3869 return -EINVAL; 3870 } 3871 3872 if (mqprio->qopt.offset[0] != 0 || mqprio->qopt.num_tc < 1 || 3873 mqprio->qopt.num_tc > MLX5E_MAX_NUM_MQPRIO_CH_TC) 3874 return -EINVAL; 3875 3876 for (i = 0; i < mqprio->qopt.num_tc; i++) { 3877 if (!mqprio->qopt.count[i]) { 3878 netdev_err(netdev, "Zero size for queue-group (%d) is not supported\n", i); 3879 return -EINVAL; 3880 } 3881 if (mqprio->min_rate[i]) { 3882 netdev_err(netdev, "Min tx rate is not supported\n"); 3883 return -EINVAL; 3884 } 3885 3886 if (mqprio->max_rate[i]) { 3887 int err; 3888 3889 err = mlx5e_qos_bytes_rate_check(priv->mdev, mqprio->max_rate[i]); 3890 if (err) 3891 return err; 3892 } 3893 3894 if (mqprio->qopt.offset[i] != agg_count) { 3895 netdev_err(netdev, "Discontinuous queues config is not supported\n"); 3896 return -EINVAL; 3897 } 3898 agg_count += mqprio->qopt.count[i]; 3899 } 3900 3901 if (priv->channels.params.num_channels != agg_count) { 3902 netdev_err(netdev, "Num of queues (%d) does not match available (%d)\n", 3903 agg_count, priv->channels.params.num_channels); 3904 return -EINVAL; 3905 } 3906 3907 return 0; 3908 } 3909 3910 static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[]) 3911 { 3912 int tc; 3913 3914 for (tc = 0; tc < num_tc; tc++) 3915 if (max_rate[tc]) 3916 return true; 3917 return false; 3918 } 3919 3920 static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev, 3921 u8 num_tc, u64 max_rate[]) 3922 { 3923 struct mlx5e_mqprio_rl *rl; 3924 int err; 3925 3926 if (!mlx5e_mqprio_rate_limit(num_tc, max_rate)) 3927 return NULL; 3928 3929 rl = mlx5e_mqprio_rl_alloc(); 3930 if (!rl) 3931 return ERR_PTR(-ENOMEM); 3932 3933 err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate); 3934 if (err) { 3935 mlx5e_mqprio_rl_free(rl); 3936 return ERR_PTR(err); 3937 } 3938 3939 return rl; 3940 } 3941 3942 static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, 3943 struct tc_mqprio_qopt_offload *mqprio) 3944 { 3945 struct mlx5e_params new_params; 3946 struct mlx5e_mqprio_rl *rl; 3947 int err; 3948 3949 err = mlx5e_mqprio_channel_validate(priv, mqprio); 3950 if (err) 3951 return err; 3952 3953 rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate); 3954 if (IS_ERR(rl)) 3955 return PTR_ERR(rl); 3956 3957 new_params = priv->channels.params; 3958 mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl); 3959 3960 err = mlx5e_safe_switch_params(priv, &new_params, 3961 mlx5e_update_tc_and_tx_queues_ctx, NULL, true); 3962 if (err) { 3963 if (rl) { 3964 mlx5e_mqprio_rl_cleanup(rl); 3965 mlx5e_mqprio_rl_free(rl); 3966 } 3967 return err; 3968 } 3969 3970 if (priv->mqprio_rl) { 3971 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3972 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3973 } 3974 priv->mqprio_rl = rl; 3975 3976 return 0; 3977 } 3978 3979 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, 3980 struct tc_mqprio_qopt_offload *mqprio) 3981 { 3982 /* MQPRIO is another toplevel qdisc that can't be attached 3983 * simultaneously with the offloaded HTB. 3984 */ 3985 if (mlx5e_selq_is_htb_enabled(&priv->selq)) { 3986 NL_SET_ERR_MSG_MOD(mqprio->extack, 3987 "MQPRIO cannot be configured when HTB offload is enabled."); 3988 return -EOPNOTSUPP; 3989 } 3990 3991 switch (mqprio->mode) { 3992 case TC_MQPRIO_MODE_DCB: 3993 return mlx5e_setup_tc_mqprio_dcb(priv, &mqprio->qopt); 3994 case TC_MQPRIO_MODE_CHANNEL: 3995 return mlx5e_setup_tc_mqprio_channel(priv, mqprio); 3996 default: 3997 return -EOPNOTSUPP; 3998 } 3999 } 4000 4001 static LIST_HEAD(mlx5e_block_cb_list); 4002 4003 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, 4004 void *type_data) 4005 { 4006 struct mlx5e_priv *priv = netdev_priv(dev); 4007 bool tc_unbind = false; 4008 int err; 4009 4010 if (type == TC_SETUP_BLOCK && 4011 ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND) 4012 tc_unbind = true; 4013 4014 if (!netif_device_present(dev) && !tc_unbind) 4015 return -ENODEV; 4016 4017 switch (type) { 4018 case TC_SETUP_BLOCK: { 4019 struct flow_block_offload *f = type_data; 4020 4021 f->unlocked_driver_cb = true; 4022 return flow_block_cb_setup_simple(type_data, 4023 &mlx5e_block_cb_list, 4024 mlx5e_setup_tc_block_cb, 4025 priv, priv, true); 4026 } 4027 case TC_SETUP_QDISC_MQPRIO: 4028 mutex_lock(&priv->state_lock); 4029 err = mlx5e_setup_tc_mqprio(priv, type_data); 4030 mutex_unlock(&priv->state_lock); 4031 return err; 4032 case TC_SETUP_QDISC_HTB: 4033 mutex_lock(&priv->state_lock); 4034 err = mlx5e_htb_setup_tc(priv, type_data); 4035 mutex_unlock(&priv->state_lock); 4036 return err; 4037 default: 4038 return -EOPNOTSUPP; 4039 } 4040 } 4041 4042 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s) 4043 { 4044 u16 nch = mlx5e_stats_nch_read(priv); 4045 int i; 4046 4047 for (i = 0; i < nch; i++) { 4048 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; 4049 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq; 4050 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq; 4051 int j; 4052 4053 s->rx_packets += rq_stats->packets + xskrq_stats->packets; 4054 s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes; 4055 s->multicast += rq_stats->mcast_packets + xskrq_stats->mcast_packets; 4056 4057 for (j = 0; j < priv->max_opened_tc; j++) { 4058 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j]; 4059 4060 s->tx_packets += sq_stats->packets; 4061 s->tx_bytes += sq_stats->bytes; 4062 s->tx_dropped += sq_stats->dropped; 4063 } 4064 } 4065 if (priv->tx_ptp_opened) { 4066 for (i = 0; i < priv->max_opened_tc; i++) { 4067 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i]; 4068 4069 s->tx_packets += sq_stats->packets; 4070 s->tx_bytes += sq_stats->bytes; 4071 s->tx_dropped += sq_stats->dropped; 4072 } 4073 } 4074 if (priv->rx_ptp_opened) { 4075 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq; 4076 4077 s->rx_packets += rq_stats->packets; 4078 s->rx_bytes += rq_stats->bytes; 4079 s->multicast += rq_stats->mcast_packets; 4080 } 4081 4082 #ifdef CONFIG_MLX5_EN_PSP 4083 if (priv->psp) 4084 s->tx_dropped += atomic_read(&priv->psp->tx_drop); 4085 #endif 4086 } 4087 4088 void 4089 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 4090 { 4091 struct mlx5e_priv *priv = netdev_priv(dev); 4092 struct mlx5e_pport_stats *pstats = &priv->stats.pport; 4093 4094 if (!netif_device_present(dev)) 4095 return; 4096 4097 /* In switchdev mode, monitor counters doesn't monitor 4098 * rx/tx stats of 802_3. The update stats mechanism 4099 * should keep the 802_3 layout counters updated 4100 */ 4101 if (!mlx5e_monitor_counter_supported(priv) || 4102 mlx5e_is_uplink_rep(priv)) { 4103 /* update HW stats in background for next time */ 4104 mlx5e_queue_update_stats(priv); 4105 } 4106 4107 netdev_stats_to_stats64(stats, &dev->stats); 4108 4109 if (mlx5e_is_uplink_rep(priv)) { 4110 struct mlx5e_vport_stats *vstats = &priv->stats.vport; 4111 4112 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok); 4113 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok); 4114 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok); 4115 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok); 4116 4117 /* vport multicast also counts packets that are dropped due to steering 4118 * or rx out of buffer 4119 */ 4120 stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets); 4121 } else { 4122 mlx5e_fold_sw_stats64(priv, stats); 4123 } 4124 4125 stats->rx_missed_errors += priv->stats.qcnt.rx_out_of_buffer; 4126 stats->rx_dropped += PPORT_2863_GET(pstats, if_in_discards); 4127 4128 stats->rx_length_errors += 4129 PPORT_802_3_GET(pstats, a_in_range_length_errors) + 4130 PPORT_802_3_GET(pstats, a_out_of_range_length_field) + 4131 PPORT_802_3_GET(pstats, a_frame_too_long_errors) + 4132 VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small); 4133 stats->rx_crc_errors += 4134 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); 4135 stats->rx_frame_errors += PPORT_802_3_GET(pstats, a_alignment_errors); 4136 stats->tx_aborted_errors += PPORT_2863_GET(pstats, if_out_discards); 4137 stats->rx_errors += stats->rx_length_errors + stats->rx_crc_errors + 4138 stats->rx_frame_errors; 4139 stats->tx_errors += stats->tx_aborted_errors + stats->tx_carrier_errors; 4140 } 4141 4142 static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) 4143 { 4144 queue_work(priv->wq, &priv->set_rx_mode_work); 4145 } 4146 4147 static int mlx5e_set_rx_mode(struct net_device *dev, 4148 struct netdev_hw_addr_list *uc, 4149 struct netdev_hw_addr_list *mc) 4150 { 4151 struct mlx5e_priv *priv = netdev_priv(dev); 4152 4153 mlx5e_fs_set_rx_mode_work(priv->fs, dev, uc, mc); 4154 4155 return 0; 4156 } 4157 4158 static int mlx5e_set_mac(struct net_device *netdev, void *addr) 4159 { 4160 struct mlx5e_priv *priv = netdev_priv(netdev); 4161 struct sockaddr *saddr = addr; 4162 4163 if (!is_valid_ether_addr(saddr->sa_data)) 4164 return -EADDRNOTAVAIL; 4165 4166 netif_addr_lock_bh(netdev); 4167 eth_hw_addr_set(netdev, saddr->sa_data); 4168 netif_addr_unlock_bh(netdev); 4169 4170 mlx5e_nic_set_rx_mode(priv); 4171 4172 return 0; 4173 } 4174 4175 #define MLX5E_SET_FEATURE(features, feature, enable) \ 4176 do { \ 4177 if (enable) \ 4178 *features |= feature; \ 4179 else \ 4180 *features &= ~feature; \ 4181 } while (0) 4182 4183 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable); 4184 4185 static int set_feature_lro(struct net_device *netdev, bool enable) 4186 { 4187 struct mlx5e_priv *priv = netdev_priv(netdev); 4188 struct mlx5_core_dev *mdev = priv->mdev; 4189 struct mlx5e_params *cur_params; 4190 struct mlx5e_params new_params; 4191 bool reset = true; 4192 int err = 0; 4193 4194 mutex_lock(&priv->state_lock); 4195 4196 cur_params = &priv->channels.params; 4197 new_params = *cur_params; 4198 4199 if (enable) 4200 new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO; 4201 else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO) 4202 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 4203 else 4204 goto out; 4205 4206 if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO && 4207 new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) { 4208 if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 4209 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) == 4210 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL)) 4211 reset = false; 4212 } 4213 } 4214 4215 err = mlx5e_safe_switch_params(priv, &new_params, 4216 mlx5e_modify_tirs_packet_merge_ctx, NULL, reset); 4217 out: 4218 mutex_unlock(&priv->state_lock); 4219 return err; 4220 } 4221 4222 static int set_feature_hw_gro(struct net_device *netdev, bool enable) 4223 { 4224 struct mlx5e_priv *priv = netdev_priv(netdev); 4225 struct mlx5e_params new_params; 4226 bool reset = true; 4227 int err = 0; 4228 4229 mutex_lock(&priv->state_lock); 4230 new_params = priv->channels.params; 4231 4232 if (enable) { 4233 new_params.packet_merge.type = MLX5E_PACKET_MERGE_SHAMPO; 4234 } else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) { 4235 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 4236 } else { 4237 goto out; 4238 } 4239 4240 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 4241 out: 4242 mutex_unlock(&priv->state_lock); 4243 return err; 4244 } 4245 4246 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) 4247 { 4248 struct mlx5e_priv *priv = netdev_priv(netdev); 4249 4250 if (enable) 4251 mlx5e_enable_cvlan_filter(priv->fs, 4252 !!(priv->netdev->flags & IFF_PROMISC)); 4253 else 4254 mlx5e_disable_cvlan_filter(priv->fs, 4255 !!(priv->netdev->flags & IFF_PROMISC)); 4256 4257 return 0; 4258 } 4259 4260 static int set_feature_hw_tc(struct net_device *netdev, bool enable) 4261 { 4262 struct mlx5e_priv *priv = netdev_priv(netdev); 4263 int err = 0; 4264 4265 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 4266 int tc_flag = mlx5e_is_uplink_rep(priv) ? MLX5_TC_FLAG(ESW_OFFLOAD) : 4267 MLX5_TC_FLAG(NIC_OFFLOAD); 4268 if (!enable && mlx5e_tc_num_filters(priv, tc_flag)) { 4269 netdev_err(netdev, 4270 "Active offloaded tc filters, can't turn hw_tc_offload off\n"); 4271 return -EINVAL; 4272 } 4273 #endif 4274 4275 mutex_lock(&priv->state_lock); 4276 if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) { 4277 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n"); 4278 err = -EINVAL; 4279 } 4280 mutex_unlock(&priv->state_lock); 4281 4282 return err; 4283 } 4284 4285 static int set_feature_rx_all(struct net_device *netdev, bool enable) 4286 { 4287 struct mlx5e_priv *priv = netdev_priv(netdev); 4288 struct mlx5_core_dev *mdev = priv->mdev; 4289 4290 return mlx5_set_port_fcs(mdev, !enable); 4291 } 4292 4293 static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode) 4294 { 4295 return (struct dim_cq_moder) { 4296 .cq_period_mode = cq_period_mode, 4297 .pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS, 4298 .usec = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? 4299 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE : 4300 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC, 4301 }; 4302 } 4303 4304 bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode, 4305 bool dim_enabled) 4306 { 4307 bool reset_needed = cq_moder->cq_period_mode != cq_period_mode; 4308 4309 if (dim_enabled) 4310 *cq_moder = net_dim_get_def_rx_moderation(cq_period_mode); 4311 else 4312 *cq_moder = mlx5e_get_def_rx_moderation(cq_period_mode); 4313 4314 return reset_needed; 4315 } 4316 4317 bool mlx5e_reset_rx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode, 4318 bool dim_enabled, bool keep_dim_state) 4319 { 4320 bool reset = false; 4321 int i; 4322 4323 for (i = 0; i < chs->num; i++) { 4324 if (keep_dim_state) 4325 dim_enabled = !!chs->c[i]->rq.dim; 4326 4327 reset |= mlx5e_reset_rx_moderation(&chs->c[i]->rx_cq_moder, 4328 cq_period_mode, dim_enabled); 4329 } 4330 4331 return reset; 4332 } 4333 4334 static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable) 4335 { 4336 u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {}; 4337 bool supported, curr_state; 4338 int err; 4339 4340 if (!MLX5_CAP_GEN(mdev, ports_check)) 4341 return 0; 4342 4343 err = mlx5_query_ports_check(mdev, in, sizeof(in)); 4344 if (err) 4345 return err; 4346 4347 supported = MLX5_GET(pcmr_reg, in, rx_ts_over_crc_cap); 4348 curr_state = MLX5_GET(pcmr_reg, in, rx_ts_over_crc); 4349 4350 if (!supported || enable == curr_state) 4351 return 0; 4352 4353 MLX5_SET(pcmr_reg, in, local_port, 1); 4354 MLX5_SET(pcmr_reg, in, rx_ts_over_crc, enable); 4355 4356 return mlx5_set_ports_check(mdev, in, sizeof(in)); 4357 } 4358 4359 static int mlx5e_set_rx_port_ts_wrap(struct mlx5e_priv *priv, void *ctx) 4360 { 4361 struct mlx5_core_dev *mdev = priv->mdev; 4362 bool enable = *(bool *)ctx; 4363 4364 return mlx5e_set_rx_port_ts(mdev, enable); 4365 } 4366 4367 static int set_feature_rx_fcs(struct net_device *netdev, bool enable) 4368 { 4369 struct mlx5e_priv *priv = netdev_priv(netdev); 4370 struct mlx5e_channels *chs = &priv->channels; 4371 struct mlx5e_params new_params; 4372 int err; 4373 bool rx_ts_over_crc = !enable; 4374 4375 mutex_lock(&priv->state_lock); 4376 4377 new_params = chs->params; 4378 new_params.scatter_fcs_en = enable; 4379 err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_set_rx_port_ts_wrap, 4380 &rx_ts_over_crc, true); 4381 mutex_unlock(&priv->state_lock); 4382 return err; 4383 } 4384 4385 static int set_feature_rx_vlan(struct net_device *netdev, bool enable) 4386 { 4387 struct mlx5e_priv *priv = netdev_priv(netdev); 4388 int err = 0; 4389 4390 mutex_lock(&priv->state_lock); 4391 4392 mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable); 4393 priv->channels.params.vlan_strip_disable = !enable; 4394 4395 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4396 goto unlock; 4397 4398 err = mlx5e_modify_channels_vsd(&priv->channels, !enable); 4399 if (err) { 4400 mlx5e_fs_set_vlan_strip_disable(priv->fs, enable); 4401 priv->channels.params.vlan_strip_disable = enable; 4402 } 4403 unlock: 4404 mutex_unlock(&priv->state_lock); 4405 4406 return err; 4407 } 4408 4409 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 4410 { 4411 struct mlx5e_priv *priv = netdev_priv(dev); 4412 struct mlx5e_flow_steering *fs = priv->fs; 4413 4414 if (mlx5e_is_uplink_rep(priv)) 4415 return 0; /* no vlan table for uplink rep */ 4416 4417 return mlx5e_fs_vlan_rx_add_vid(fs, dev, proto, vid); 4418 } 4419 4420 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 4421 { 4422 struct mlx5e_priv *priv = netdev_priv(dev); 4423 struct mlx5e_flow_steering *fs = priv->fs; 4424 4425 if (mlx5e_is_uplink_rep(priv)) 4426 return 0; /* no vlan table for uplink rep */ 4427 4428 return mlx5e_fs_vlan_rx_kill_vid(fs, dev, proto, vid); 4429 } 4430 4431 #ifdef CONFIG_MLX5_EN_ARFS 4432 static int set_feature_arfs(struct net_device *netdev, bool enable) 4433 { 4434 struct mlx5e_priv *priv = netdev_priv(netdev); 4435 int err; 4436 4437 if (enable) 4438 err = mlx5e_arfs_enable(priv->fs); 4439 else 4440 err = mlx5e_arfs_disable(priv->fs); 4441 4442 return err; 4443 } 4444 #endif 4445 4446 static int mlx5e_handle_feature(struct net_device *netdev, 4447 netdev_features_t *features, 4448 netdev_features_t feature, 4449 mlx5e_feature_handler feature_handler) 4450 { 4451 netdev_features_t changes = *features ^ netdev->features; 4452 bool enable = !!(*features & feature); 4453 int err; 4454 4455 if (!(changes & feature)) 4456 return 0; 4457 4458 err = feature_handler(netdev, enable); 4459 if (err) { 4460 MLX5E_SET_FEATURE(features, feature, !enable); 4461 netdev_err(netdev, "%s feature %pNF failed, err %d\n", 4462 enable ? "Enable" : "Disable", &feature, err); 4463 return err; 4464 } 4465 4466 return 0; 4467 } 4468 4469 void mlx5e_set_xdp_feature(struct mlx5e_priv *priv) 4470 { 4471 struct mlx5e_params *params = &priv->channels.params; 4472 struct net_device *netdev = priv->netdev; 4473 xdp_features_t val = 0; 4474 4475 if (netdev->netdev_ops->ndo_bpf && 4476 params->packet_merge.type == MLX5E_PACKET_MERGE_NONE) 4477 val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 4478 NETDEV_XDP_ACT_XSK_ZEROCOPY | 4479 NETDEV_XDP_ACT_RX_SG; 4480 4481 if (netdev->netdev_ops->ndo_xdp_xmit && params->xdp_prog) 4482 val |= NETDEV_XDP_ACT_NDO_XMIT | 4483 NETDEV_XDP_ACT_NDO_XMIT_SG; 4484 4485 xdp_set_features_flag_locked(netdev, val); 4486 } 4487 4488 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) 4489 { 4490 netdev_features_t oper_features = features; 4491 int err = 0; 4492 4493 #define MLX5E_HANDLE_FEATURE(feature, handler) \ 4494 mlx5e_handle_feature(netdev, &oper_features, feature, handler) 4495 4496 if (features & (NETIF_F_GRO_HW | NETIF_F_LRO)) { 4497 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4498 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4499 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4500 } else { 4501 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4502 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4503 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4504 } 4505 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, 4506 set_feature_cvlan_filter); 4507 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc); 4508 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); 4509 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); 4510 #ifdef CONFIG_MLX5_EN_ARFS 4511 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); 4512 #endif 4513 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx); 4514 4515 if (err) { 4516 netdev->features = oper_features; 4517 return -EINVAL; 4518 } 4519 4520 return 0; 4521 } 4522 4523 static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev, 4524 netdev_features_t features) 4525 { 4526 features &= ~NETIF_F_HW_TLS_RX; 4527 if (netdev->features & NETIF_F_HW_TLS_RX) 4528 netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n"); 4529 4530 features &= ~NETIF_F_HW_TLS_TX; 4531 if (netdev->features & NETIF_F_HW_TLS_TX) 4532 netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n"); 4533 4534 features &= ~NETIF_F_NTUPLE; 4535 if (netdev->features & NETIF_F_NTUPLE) 4536 netdev_warn(netdev, "Disabling ntuple, not supported in switchdev mode\n"); 4537 4538 features &= ~NETIF_F_GRO_HW; 4539 if (netdev->features & NETIF_F_GRO_HW) 4540 netdev_warn(netdev, "Disabling HW_GRO, not supported in switchdev mode\n"); 4541 4542 features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4543 if (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER) 4544 netdev_warn(netdev, "Disabling HW_VLAN CTAG FILTERING, not supported in switchdev mode\n"); 4545 4546 features &= ~NETIF_F_HW_MACSEC; 4547 if (netdev->features & NETIF_F_HW_MACSEC) 4548 netdev_warn(netdev, "Disabling HW MACsec offload, not supported in switchdev mode\n"); 4549 4550 return features; 4551 } 4552 4553 static netdev_features_t mlx5e_fix_features(struct net_device *netdev, 4554 netdev_features_t features) 4555 { 4556 struct netdev_config *cfg = netdev->cfg_pending; 4557 struct mlx5e_priv *priv = netdev_priv(netdev); 4558 struct mlx5e_vlan_table *vlan; 4559 struct mlx5e_params *params; 4560 4561 if (!netif_device_present(netdev)) 4562 return features; 4563 4564 vlan = mlx5e_fs_get_vlan(priv->fs); 4565 mutex_lock(&priv->state_lock); 4566 params = &priv->channels.params; 4567 if (!vlan || 4568 !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) { 4569 /* HW strips the outer C-tag header, this is a problem 4570 * for S-tag traffic. 4571 */ 4572 features &= ~NETIF_F_HW_VLAN_CTAG_RX; 4573 if (!params->vlan_strip_disable) 4574 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n"); 4575 } 4576 4577 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 4578 if (features & NETIF_F_LRO) { 4579 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n"); 4580 features &= ~NETIF_F_LRO; 4581 } 4582 if (features & NETIF_F_GRO_HW) { 4583 netdev_warn(netdev, "Disabling HW-GRO, not supported in legacy RQ\n"); 4584 features &= ~NETIF_F_GRO_HW; 4585 } 4586 } 4587 4588 if (params->xdp_prog) { 4589 if (features & NETIF_F_LRO) { 4590 netdev_warn(netdev, "LRO is incompatible with XDP\n"); 4591 features &= ~NETIF_F_LRO; 4592 } 4593 if (features & NETIF_F_GRO_HW) { 4594 netdev_warn(netdev, "HW GRO is incompatible with XDP\n"); 4595 features &= ~NETIF_F_GRO_HW; 4596 } 4597 } 4598 4599 if (priv->xsk.refcnt) { 4600 if (features & NETIF_F_LRO) { 4601 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n", 4602 priv->xsk.refcnt); 4603 features &= ~NETIF_F_LRO; 4604 } 4605 if (features & NETIF_F_GRO_HW) { 4606 netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n", 4607 priv->xsk.refcnt); 4608 features &= ~NETIF_F_GRO_HW; 4609 } 4610 } 4611 4612 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 4613 features &= ~NETIF_F_RXHASH; 4614 if (netdev->features & NETIF_F_RXHASH) 4615 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n"); 4616 4617 if (features & NETIF_F_GRO_HW) { 4618 netdev_warn(netdev, "Disabling HW-GRO, not supported when CQE compress is active\n"); 4619 features &= ~NETIF_F_GRO_HW; 4620 } 4621 } 4622 4623 /* The header-data split ring param requires HW GRO to stay enabled. */ 4624 if (cfg && cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 4625 !(features & NETIF_F_GRO_HW)) { 4626 netdev_warn(netdev, "Keeping HW-GRO enabled, TCP header-data split depends on it\n"); 4627 features |= NETIF_F_GRO_HW; 4628 } 4629 4630 if (mlx5e_is_uplink_rep(priv)) { 4631 features = mlx5e_fix_uplink_rep_features(netdev, features); 4632 netdev->netns_immutable = true; 4633 } else { 4634 netdev->netns_immutable = false; 4635 } 4636 4637 mutex_unlock(&priv->state_lock); 4638 4639 return features; 4640 } 4641 4642 static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, 4643 struct mlx5e_channels *chs, 4644 struct mlx5e_params *new_params, 4645 struct mlx5_core_dev *mdev) 4646 { 4647 u16 ix; 4648 4649 for (ix = 0; ix < chs->params.num_channels; ix++) { 4650 struct xsk_buff_pool *xsk_pool = 4651 mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix); 4652 struct mlx5e_rq_opt_param rqo = {0}; 4653 struct mlx5e_xsk_param xsk; 4654 int max_xdp_mtu; 4655 4656 if (!xsk_pool) 4657 continue; 4658 4659 mlx5e_build_xsk_param(xsk_pool, &xsk); 4660 rqo.xsk = &xsk; 4661 max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &rqo); 4662 4663 /* Validate XSK params and XDP MTU in advance */ 4664 if (!mlx5e_validate_xsk_param(new_params, &rqo, mdev) || 4665 new_params->sw_mtu > max_xdp_mtu) { 4666 u32 hr = mlx5e_get_linear_rq_headroom(new_params, &rqo); 4667 int max_mtu_frame, max_mtu_page, max_mtu; 4668 4669 /* Two criteria must be met: 4670 * 1. HW MTU + all headrooms <= XSK frame size. 4671 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE. 4672 */ 4673 max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr); 4674 max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0)); 4675 max_mtu = min3(max_mtu_frame, max_mtu_page, max_xdp_mtu); 4676 4677 netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u or its redirection XDP program. Try MTU <= %d\n", 4678 new_params->sw_mtu, ix, max_mtu); 4679 return false; 4680 } 4681 } 4682 4683 return true; 4684 } 4685 4686 static bool mlx5e_params_validate_xdp(struct net_device *netdev, 4687 struct mlx5_core_dev *mdev, 4688 struct mlx5e_params *params) 4689 { 4690 bool is_linear; 4691 4692 /* No XSK params: AF_XDP can't be enabled yet at the point of setting 4693 * the XDP program. 4694 */ 4695 is_linear = params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC ? 4696 mlx5e_rx_is_linear_skb(mdev, params, NULL) : 4697 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL); 4698 4699 if (!is_linear) { 4700 if (!params->xdp_prog->aux->xdp_has_frags) { 4701 netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n", 4702 params->sw_mtu, 4703 mlx5e_xdp_max_mtu(params, NULL)); 4704 return false; 4705 } 4706 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && 4707 !mlx5e_verify_params_rx_mpwqe_strides(mdev, params, NULL)) { 4708 netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n", 4709 params->sw_mtu, 4710 mlx5e_xdp_max_mtu(params, NULL)); 4711 return false; 4712 } 4713 } 4714 4715 return true; 4716 } 4717 4718 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, 4719 mlx5e_fp_preactivate preactivate) 4720 { 4721 struct mlx5e_priv *priv = netdev_priv(netdev); 4722 struct mlx5e_params new_params; 4723 struct mlx5e_params *params; 4724 int err = 0; 4725 4726 mutex_lock(&priv->state_lock); 4727 4728 params = &priv->channels.params; 4729 4730 new_params = *params; 4731 new_params.sw_mtu = new_mtu; 4732 err = mlx5e_validate_params(priv->mdev, &new_params); 4733 if (err) 4734 goto out; 4735 4736 if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev, 4737 &new_params)) { 4738 err = -EINVAL; 4739 goto out; 4740 } 4741 4742 if (priv->xsk.refcnt && 4743 !mlx5e_xsk_validate_mtu(netdev, &priv->channels, 4744 &new_params, priv->mdev)) { 4745 err = -EINVAL; 4746 goto out; 4747 } 4748 4749 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, 4750 true); 4751 4752 out: 4753 WRITE_ONCE(netdev->mtu, params->sw_mtu); 4754 mutex_unlock(&priv->state_lock); 4755 4756 if (!err) 4757 netdev_update_features(netdev); 4758 4759 return err; 4760 } 4761 4762 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu) 4763 { 4764 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx); 4765 } 4766 4767 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx) 4768 { 4769 bool set = *(bool *)ctx; 4770 4771 return mlx5e_ptp_rx_manage_fs(priv, set); 4772 } 4773 4774 static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter) 4775 { 4776 bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4777 int err; 4778 4779 if (!rx_filter) 4780 /* Reset CQE compression to Admin default */ 4781 return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def, false); 4782 4783 if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 4784 return 0; 4785 4786 /* Disable CQE compression */ 4787 netdev_warn(priv->netdev, "Disabling RX cqe compression\n"); 4788 err = mlx5e_modify_rx_cqe_compression_locked(priv, false, true); 4789 if (err) 4790 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err); 4791 4792 return err; 4793 } 4794 4795 static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx) 4796 { 4797 struct mlx5e_params new_params; 4798 4799 if (ptp_rx == priv->channels.params.ptp_rx) 4800 return 0; 4801 4802 new_params = priv->channels.params; 4803 new_params.ptp_rx = ptp_rx; 4804 return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx, 4805 &new_params.ptp_rx, true); 4806 } 4807 4808 int mlx5e_hwtstamp_set(struct mlx5e_priv *priv, 4809 struct kernel_hwtstamp_config *config, 4810 struct netlink_ext_ack *extack) 4811 { 4812 bool rx_cqe_compress_def; 4813 bool ptp_rx; 4814 int err; 4815 4816 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) || 4817 (mlx5_clock_get_ptp_index(priv->mdev) == -1)) { 4818 NL_SET_ERR_MSG_MOD(extack, 4819 "Timestamps are not supported on this device"); 4820 return -EOPNOTSUPP; 4821 } 4822 4823 /* TX HW timestamp */ 4824 switch (config->tx_type) { 4825 case HWTSTAMP_TX_OFF: 4826 case HWTSTAMP_TX_ON: 4827 break; 4828 default: 4829 return -ERANGE; 4830 } 4831 4832 mutex_lock(&priv->state_lock); 4833 rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4834 4835 /* RX HW timestamp */ 4836 switch (config->rx_filter) { 4837 case HWTSTAMP_FILTER_NONE: 4838 ptp_rx = false; 4839 break; 4840 case HWTSTAMP_FILTER_ALL: 4841 case HWTSTAMP_FILTER_SOME: 4842 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 4843 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 4844 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 4845 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 4846 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 4847 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 4848 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 4849 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 4850 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 4851 case HWTSTAMP_FILTER_PTP_V2_EVENT: 4852 case HWTSTAMP_FILTER_PTP_V2_SYNC: 4853 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 4854 case HWTSTAMP_FILTER_NTP_ALL: 4855 config->rx_filter = HWTSTAMP_FILTER_ALL; 4856 /* ptp_rx is set if both HW TS is set and CQE 4857 * compression is set 4858 */ 4859 ptp_rx = rx_cqe_compress_def; 4860 break; 4861 default: 4862 err = -ERANGE; 4863 goto err_unlock; 4864 } 4865 4866 if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX)) 4867 err = mlx5e_hwstamp_config_no_ptp_rx(priv, 4868 config->rx_filter != HWTSTAMP_FILTER_NONE); 4869 else 4870 err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx); 4871 if (err) 4872 goto err_unlock; 4873 4874 priv->hwtstamp_config = *config; 4875 mutex_unlock(&priv->state_lock); 4876 4877 /* might need to fix some features */ 4878 netdev_update_features(priv->netdev); 4879 4880 return 0; 4881 err_unlock: 4882 mutex_unlock(&priv->state_lock); 4883 return err; 4884 } 4885 4886 static int mlx5e_hwtstamp_set_ndo(struct net_device *netdev, 4887 struct kernel_hwtstamp_config *config, 4888 struct netlink_ext_ack *extack) 4889 { 4890 struct mlx5e_priv *priv = netdev_priv(netdev); 4891 4892 return mlx5e_hwtstamp_set(priv, config, extack); 4893 } 4894 4895 int mlx5e_hwtstamp_get(struct mlx5e_priv *priv, 4896 struct kernel_hwtstamp_config *config) 4897 { 4898 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz)) 4899 return -EOPNOTSUPP; 4900 4901 *config = priv->hwtstamp_config; 4902 4903 return 0; 4904 } 4905 4906 static int mlx5e_hwtstamp_get_ndo(struct net_device *dev, 4907 struct kernel_hwtstamp_config *config) 4908 { 4909 struct mlx5e_priv *priv = netdev_priv(dev); 4910 4911 return mlx5e_hwtstamp_get(priv, config); 4912 } 4913 4914 #ifdef CONFIG_MLX5_ESWITCH 4915 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 4916 { 4917 struct mlx5e_priv *priv = netdev_priv(dev); 4918 struct mlx5_core_dev *mdev = priv->mdev; 4919 4920 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac); 4921 } 4922 4923 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 4924 __be16 vlan_proto) 4925 { 4926 struct mlx5e_priv *priv = netdev_priv(dev); 4927 struct mlx5_core_dev *mdev = priv->mdev; 4928 4929 if (vlan_proto != htons(ETH_P_8021Q)) 4930 return -EPROTONOSUPPORT; 4931 4932 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1, 4933 vlan, qos); 4934 } 4935 4936 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 4937 { 4938 struct mlx5e_priv *priv = netdev_priv(dev); 4939 struct mlx5_core_dev *mdev = priv->mdev; 4940 4941 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting); 4942 } 4943 4944 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting) 4945 { 4946 struct mlx5e_priv *priv = netdev_priv(dev); 4947 struct mlx5_core_dev *mdev = priv->mdev; 4948 4949 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting); 4950 } 4951 4952 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 4953 int max_tx_rate) 4954 { 4955 struct mlx5e_priv *priv = netdev_priv(dev); 4956 struct mlx5_core_dev *mdev = priv->mdev; 4957 4958 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1, 4959 max_tx_rate, min_tx_rate); 4960 } 4961 4962 static int mlx5_vport_link2ifla(u8 esw_link) 4963 { 4964 switch (esw_link) { 4965 case MLX5_VPORT_ADMIN_STATE_DOWN: 4966 return IFLA_VF_LINK_STATE_DISABLE; 4967 case MLX5_VPORT_ADMIN_STATE_UP: 4968 return IFLA_VF_LINK_STATE_ENABLE; 4969 } 4970 return IFLA_VF_LINK_STATE_AUTO; 4971 } 4972 4973 static int mlx5_ifla_link2vport(u8 ifla_link) 4974 { 4975 switch (ifla_link) { 4976 case IFLA_VF_LINK_STATE_DISABLE: 4977 return MLX5_VPORT_ADMIN_STATE_DOWN; 4978 case IFLA_VF_LINK_STATE_ENABLE: 4979 return MLX5_VPORT_ADMIN_STATE_UP; 4980 } 4981 return MLX5_VPORT_ADMIN_STATE_AUTO; 4982 } 4983 4984 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf, 4985 int link_state) 4986 { 4987 struct mlx5e_priv *priv = netdev_priv(dev); 4988 struct mlx5_core_dev *mdev = priv->mdev; 4989 4990 if (mlx5e_is_uplink_rep(priv)) 4991 return -EOPNOTSUPP; 4992 4993 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1, 4994 mlx5_ifla_link2vport(link_state)); 4995 } 4996 4997 int mlx5e_get_vf_config(struct net_device *dev, 4998 int vf, struct ifla_vf_info *ivi) 4999 { 5000 struct mlx5e_priv *priv = netdev_priv(dev); 5001 struct mlx5_core_dev *mdev = priv->mdev; 5002 int err; 5003 5004 if (!netif_device_present(dev)) 5005 return -EOPNOTSUPP; 5006 5007 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi); 5008 if (err) 5009 return err; 5010 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate); 5011 return 0; 5012 } 5013 5014 int mlx5e_get_vf_stats(struct net_device *dev, 5015 int vf, struct ifla_vf_stats *vf_stats) 5016 { 5017 struct mlx5e_priv *priv = netdev_priv(dev); 5018 struct mlx5_core_dev *mdev = priv->mdev; 5019 5020 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1, 5021 vf_stats); 5022 } 5023 5024 static bool 5025 mlx5e_has_offload_stats(const struct net_device *dev, int attr_id) 5026 { 5027 struct mlx5e_priv *priv = netdev_priv(dev); 5028 5029 if (!netif_device_present(dev)) 5030 return false; 5031 5032 if (!mlx5e_is_uplink_rep(priv)) 5033 return false; 5034 5035 return mlx5e_rep_has_offload_stats(dev, attr_id); 5036 } 5037 5038 static int 5039 mlx5e_get_offload_stats(int attr_id, const struct net_device *dev, 5040 void *sp) 5041 { 5042 struct mlx5e_priv *priv = netdev_priv(dev); 5043 5044 if (!mlx5e_is_uplink_rep(priv)) 5045 return -EOPNOTSUPP; 5046 5047 return mlx5e_rep_get_offload_stats(attr_id, dev, sp); 5048 } 5049 #endif 5050 5051 static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type) 5052 { 5053 switch (proto_type) { 5054 case IPPROTO_GRE: 5055 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre); 5056 case IPPROTO_IPIP: 5057 case IPPROTO_IPV6: 5058 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) || 5059 MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx)); 5060 default: 5061 return false; 5062 } 5063 } 5064 5065 static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev, 5066 struct sk_buff *skb) 5067 { 5068 switch (skb->inner_protocol) { 5069 case htons(ETH_P_IP): 5070 case htons(ETH_P_IPV6): 5071 case htons(ETH_P_TEB): 5072 return true; 5073 case htons(ETH_P_MPLS_UC): 5074 case htons(ETH_P_MPLS_MC): 5075 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre); 5076 } 5077 return false; 5078 } 5079 5080 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, 5081 struct sk_buff *skb, 5082 netdev_features_t features) 5083 { 5084 unsigned int offset = 0; 5085 struct udphdr *udph; 5086 u8 proto; 5087 u16 port; 5088 5089 switch (vlan_get_protocol(skb)) { 5090 case htons(ETH_P_IP): 5091 proto = ip_hdr(skb)->protocol; 5092 break; 5093 case htons(ETH_P_IPV6): 5094 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL); 5095 break; 5096 default: 5097 goto out; 5098 } 5099 5100 switch (proto) { 5101 case IPPROTO_GRE: 5102 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb)) 5103 return features; 5104 break; 5105 case IPPROTO_IPIP: 5106 case IPPROTO_IPV6: 5107 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP)) 5108 return features; 5109 break; 5110 case IPPROTO_UDP: 5111 udph = udp_hdr(skb); 5112 port = be16_to_cpu(udph->dest); 5113 5114 /* Verify if UDP port is being offloaded by HW */ 5115 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) 5116 return vxlan_features_check(skb, features); 5117 5118 #if IS_ENABLED(CONFIG_GENEVE) 5119 /* Support Geneve offload for default UDP port */ 5120 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) 5121 return features; 5122 #endif 5123 break; 5124 #ifdef CONFIG_MLX5_EN_IPSEC 5125 case IPPROTO_ESP: 5126 return mlx5e_ipsec_feature_check(skb, features); 5127 #endif 5128 } 5129 5130 out: 5131 /* Disable CSUM and GSO if skb cannot be offloaded by HW */ 5132 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 5133 } 5134 5135 netdev_features_t mlx5e_features_check(struct sk_buff *skb, 5136 struct net_device *netdev, 5137 netdev_features_t features) 5138 { 5139 struct mlx5e_priv *priv = netdev_priv(netdev); 5140 5141 features = vlan_features_check(skb, features); 5142 5143 /* Validate if the tunneled packet is being offloaded by HW */ 5144 if (skb->encapsulation && 5145 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK)) 5146 return mlx5e_tunnel_features_check(priv, skb, features); 5147 5148 return features; 5149 } 5150 5151 static void mlx5e_tx_timeout_work(struct work_struct *work) 5152 { 5153 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 5154 tx_timeout_work); 5155 struct net_device *netdev = priv->netdev; 5156 int i; 5157 5158 for (i = 0; i < netdev->real_num_tx_queues; i++) { 5159 struct netdev_queue *dev_queue = 5160 netdev_get_tx_queue(netdev, i); 5161 struct mlx5e_txqsq *sq = priv->txq2sq[i]; 5162 5163 if (!netif_xmit_timeout_ms(dev_queue)) 5164 continue; 5165 5166 if (mlx5e_reporter_tx_timeout(sq)) 5167 /* break if tried to reopened channels */ 5168 break; 5169 } 5170 } 5171 5172 static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) 5173 { 5174 struct mlx5e_priv *priv = netdev_priv(dev); 5175 5176 netdev_err(dev, "TX timeout detected\n"); 5177 queue_work(priv->wq, &priv->tx_timeout_work); 5178 } 5179 5180 static int mlx5e_xdp_allowed(struct net_device *netdev, struct mlx5_core_dev *mdev, 5181 struct mlx5e_params *params) 5182 { 5183 if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) { 5184 netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n"); 5185 return -EINVAL; 5186 } 5187 5188 if (!mlx5e_params_validate_xdp(netdev, mdev, params)) 5189 return -EINVAL; 5190 5191 return 0; 5192 } 5193 5194 static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog) 5195 { 5196 struct bpf_prog *old_prog; 5197 5198 old_prog = rcu_replace_pointer(rq->xdp_prog, prog, 5199 lockdep_is_held(&rq->priv->state_lock)); 5200 if (old_prog) 5201 bpf_prog_put(old_prog); 5202 } 5203 5204 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) 5205 { 5206 struct mlx5e_priv *priv = netdev_priv(netdev); 5207 struct mlx5e_params new_params; 5208 struct bpf_prog *old_prog; 5209 int err = 0; 5210 bool reset; 5211 int i; 5212 5213 mutex_lock(&priv->state_lock); 5214 5215 new_params = priv->channels.params; 5216 new_params.xdp_prog = prog; 5217 5218 if (prog) { 5219 err = mlx5e_xdp_allowed(netdev, priv->mdev, &new_params); 5220 if (err) 5221 goto unlock; 5222 } 5223 5224 /* no need for full reset when exchanging programs */ 5225 reset = (!priv->channels.params.xdp_prog || !prog); 5226 5227 old_prog = priv->channels.params.xdp_prog; 5228 5229 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 5230 if (err) 5231 goto unlock; 5232 5233 if (old_prog) 5234 bpf_prog_put(old_prog); 5235 5236 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset) 5237 goto unlock; 5238 5239 /* exchanging programs w/o reset, we update ref counts on behalf 5240 * of the channels RQs here. 5241 */ 5242 bpf_prog_add(prog, priv->channels.num); 5243 for (i = 0; i < priv->channels.num; i++) { 5244 struct mlx5e_channel *c = priv->channels.c[i]; 5245 5246 mlx5e_rq_replace_xdp_prog(&c->rq, prog); 5247 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) { 5248 bpf_prog_inc(prog); 5249 mlx5e_rq_replace_xdp_prog(&c->xskrq, prog); 5250 } 5251 } 5252 5253 unlock: 5254 mutex_unlock(&priv->state_lock); 5255 5256 /* Need to fix some features. */ 5257 if (!err) 5258 netdev_update_features(netdev); 5259 5260 return err; 5261 } 5262 5263 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) 5264 { 5265 switch (xdp->command) { 5266 case XDP_SETUP_PROG: 5267 return mlx5e_xdp_set(dev, xdp->prog); 5268 case XDP_SETUP_XSK_POOL: 5269 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool, 5270 xdp->xsk.queue_id); 5271 default: 5272 return -EINVAL; 5273 } 5274 } 5275 5276 #ifdef CONFIG_MLX5_ESWITCH 5277 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 5278 struct net_device *dev, u32 filter_mask, 5279 int nlflags) 5280 { 5281 struct mlx5e_priv *priv = netdev_priv(dev); 5282 struct mlx5_core_dev *mdev = priv->mdev; 5283 u8 mode, setting; 5284 5285 if (mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting)) 5286 return -EOPNOTSUPP; 5287 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB; 5288 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 5289 mode, 5290 0, 0, nlflags, filter_mask, NULL); 5291 } 5292 5293 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 5294 u16 flags, struct netlink_ext_ack *extack) 5295 { 5296 struct mlx5e_priv *priv = netdev_priv(dev); 5297 struct mlx5_core_dev *mdev = priv->mdev; 5298 struct nlattr *attr, *br_spec; 5299 u16 mode = BRIDGE_MODE_UNDEF; 5300 u8 setting; 5301 int rem; 5302 5303 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 5304 if (!br_spec) 5305 return -EINVAL; 5306 5307 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) { 5308 mode = nla_get_u16(attr); 5309 if (mode > BRIDGE_MODE_VEPA) 5310 return -EINVAL; 5311 5312 break; 5313 } 5314 5315 if (mode == BRIDGE_MODE_UNDEF) 5316 return -EINVAL; 5317 5318 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0; 5319 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting); 5320 } 5321 #endif 5322 5323 const struct net_device_ops mlx5e_netdev_ops = { 5324 .ndo_open = mlx5e_open, 5325 .ndo_stop = mlx5e_close, 5326 .ndo_start_xmit = mlx5e_xmit, 5327 .ndo_setup_tc = mlx5e_setup_tc, 5328 .ndo_select_queue = mlx5e_select_queue, 5329 .ndo_get_stats64 = mlx5e_get_stats, 5330 .ndo_set_rx_mode_async = mlx5e_set_rx_mode, 5331 .ndo_set_mac_address = mlx5e_set_mac, 5332 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid, 5333 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid, 5334 .ndo_set_features = mlx5e_set_features, 5335 .ndo_fix_features = mlx5e_fix_features, 5336 .ndo_change_mtu = mlx5e_change_nic_mtu, 5337 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, 5338 .ndo_features_check = mlx5e_features_check, 5339 .ndo_tx_timeout = mlx5e_tx_timeout, 5340 .ndo_bpf = mlx5e_xdp, 5341 .ndo_xdp_xmit = mlx5e_xdp_xmit, 5342 .ndo_xsk_wakeup = mlx5e_xsk_wakeup, 5343 .ndo_hwtstamp_get = mlx5e_hwtstamp_get_ndo, 5344 .ndo_hwtstamp_set = mlx5e_hwtstamp_set_ndo, 5345 #ifdef CONFIG_MLX5_EN_ARFS 5346 .ndo_rx_flow_steer = mlx5e_rx_flow_steer, 5347 #endif 5348 #ifdef CONFIG_MLX5_ESWITCH 5349 .ndo_bridge_setlink = mlx5e_bridge_setlink, 5350 .ndo_bridge_getlink = mlx5e_bridge_getlink, 5351 5352 /* SRIOV E-Switch NDOs */ 5353 .ndo_set_vf_mac = mlx5e_set_vf_mac, 5354 .ndo_set_vf_vlan = mlx5e_set_vf_vlan, 5355 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk, 5356 .ndo_set_vf_trust = mlx5e_set_vf_trust, 5357 .ndo_set_vf_rate = mlx5e_set_vf_rate, 5358 .ndo_get_vf_config = mlx5e_get_vf_config, 5359 .ndo_set_vf_link_state = mlx5e_set_vf_link_state, 5360 .ndo_get_vf_stats = mlx5e_get_vf_stats, 5361 .ndo_has_offload_stats = mlx5e_has_offload_stats, 5362 .ndo_get_offload_stats = mlx5e_get_offload_stats, 5363 #endif 5364 }; 5365 5366 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu) 5367 { 5368 struct mlx5e_params *params = &priv->channels.params; 5369 struct mlx5_core_dev *mdev = priv->mdev; 5370 5371 params->sw_mtu = mtu; 5372 params->hard_mtu = MLX5E_ETH_HARD_MTU; 5373 params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2, 5374 priv->max_nch); 5375 mlx5e_params_mqprio_reset(params); 5376 5377 /* SQ */ 5378 params->log_sq_size = is_kdump_kernel() ? 5379 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : 5380 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 5381 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 5382 5383 /* XDP SQ */ 5384 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 5385 5386 /* set CQE compression */ 5387 params->rx_cqe_compress_def = false; 5388 if (MLX5_CAP_GEN(mdev, cqe_compression) && 5389 MLX5_CAP_GEN(mdev, vport_group_manager)) 5390 params->rx_cqe_compress_def = slow_pci_heuristic(mdev); 5391 5392 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def); 5393 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); 5394 5395 /* RQ */ 5396 mlx5e_build_rq_params(mdev, params); 5397 5398 params->terminate_lkey_be = mlx5_core_get_terminate_scatter_list_mkey(mdev); 5399 5400 params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); 5401 5402 /* CQ moderation params */ 5403 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation) && 5404 MLX5_CAP_GEN(mdev, cq_period_mode_modify); 5405 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation) && 5406 MLX5_CAP_GEN(mdev, cq_period_mode_modify); 5407 params->rx_moder_use_cqe_mode = !!MLX5_CAP_GEN(mdev, cq_period_start_from_cqe); 5408 params->tx_moder_use_cqe_mode = false; 5409 mlx5e_reset_rx_moderation(¶ms->rx_cq_moderation, params->rx_moder_use_cqe_mode, 5410 params->rx_dim_enabled); 5411 mlx5e_reset_tx_moderation(¶ms->tx_cq_moderation, params->tx_moder_use_cqe_mode, 5412 params->tx_dim_enabled); 5413 5414 /* TX inline */ 5415 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 5416 5417 /* AF_XDP */ 5418 params->xsk = xsk; 5419 5420 /* Do not update netdev->features directly in here 5421 * on mlx5e_attach_netdev() we will call mlx5e_update_features() 5422 * To update netdev->features please modify mlx5e_fix_features() 5423 */ 5424 } 5425 5426 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) 5427 { 5428 struct mlx5e_priv *priv = netdev_priv(netdev); 5429 u8 addr[ETH_ALEN]; 5430 5431 mlx5_query_mac_address(priv->mdev, addr); 5432 if (is_zero_ether_addr(addr)) { 5433 eth_hw_addr_random(netdev); 5434 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr); 5435 return; 5436 } 5437 5438 eth_hw_addr_set(netdev, addr); 5439 } 5440 5441 static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table, 5442 unsigned int entry, struct udp_tunnel_info *ti) 5443 { 5444 struct mlx5e_priv *priv = netdev_priv(netdev); 5445 5446 return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port)); 5447 } 5448 5449 static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table, 5450 unsigned int entry, struct udp_tunnel_info *ti) 5451 { 5452 struct mlx5e_priv *priv = netdev_priv(netdev); 5453 5454 return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port)); 5455 } 5456 5457 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv) 5458 { 5459 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 5460 return; 5461 5462 priv->nic_info.set_port = mlx5e_vxlan_set_port; 5463 priv->nic_info.unset_port = mlx5e_vxlan_unset_port; 5464 priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN; 5465 priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN; 5466 /* Don't count the space hard-coded to the IANA port */ 5467 priv->nic_info.tables[0].n_entries = 5468 mlx5_vxlan_max_udp_ports(priv->mdev) - 1; 5469 5470 priv->netdev->udp_tunnel_nic_info = &priv->nic_info; 5471 } 5472 5473 static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev) 5474 { 5475 int tt; 5476 5477 for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) { 5478 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5_get_proto_by_tunnel_type(tt))) 5479 return true; 5480 } 5481 return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)); 5482 } 5483 5484 static void mlx5e_get_queue_stats_rx(struct net_device *dev, int i, 5485 struct netdev_queue_stats_rx *stats) 5486 { 5487 struct mlx5e_priv *priv = netdev_priv(dev); 5488 struct mlx5e_channel_stats *channel_stats; 5489 struct mlx5e_rq_stats *xskrq_stats; 5490 struct mlx5e_rq_stats *rq_stats; 5491 5492 if (mlx5e_is_uplink_rep(priv) || !mlx5e_stats_nch_read(priv)) 5493 return; 5494 5495 channel_stats = priv->channel_stats[i]; 5496 xskrq_stats = &channel_stats->xskrq; 5497 rq_stats = &channel_stats->rq; 5498 5499 stats->packets = rq_stats->packets + xskrq_stats->packets; 5500 stats->bytes = rq_stats->bytes + xskrq_stats->bytes; 5501 stats->alloc_fail = rq_stats->buff_alloc_err + 5502 xskrq_stats->buff_alloc_err; 5503 5504 stats->hw_gro_packets = rq_stats->gro_skbs + xskrq_stats->gro_skbs; 5505 stats->hw_gro_wire_packets = 5506 rq_stats->gro_packets + xskrq_stats->gro_packets; 5507 stats->hw_gro_wire_bytes = rq_stats->gro_bytes + xskrq_stats->gro_bytes; 5508 } 5509 5510 static void mlx5e_get_queue_stats_tx(struct net_device *dev, int i, 5511 struct netdev_queue_stats_tx *stats) 5512 { 5513 struct mlx5e_priv *priv = netdev_priv(dev); 5514 struct mlx5e_sq_stats *sq_stats; 5515 5516 if (!mlx5e_stats_nch_read(priv)) 5517 return; 5518 5519 /* no special case needed for ptp htb etc since txq2sq_stats is kept up 5520 * to date for active sq_stats, otherwise get_base_stats takes care of 5521 * inactive sqs. 5522 */ 5523 sq_stats = priv->txq2sq_stats[i]; 5524 stats->packets = sq_stats->packets; 5525 stats->bytes = sq_stats->bytes; 5526 5527 stats->hw_gso_packets = 5528 sq_stats->tso_packets + sq_stats->tso_inner_packets; 5529 stats->hw_gso_bytes = sq_stats->tso_bytes + sq_stats->tso_inner_bytes; 5530 5531 stats->csum_none = sq_stats->csum_none; 5532 5533 stats->stop = sq_stats->stopped; 5534 stats->wake = sq_stats->wake; 5535 } 5536 5537 static void mlx5e_get_base_stats(struct net_device *dev, 5538 struct netdev_queue_stats_rx *rx, 5539 struct netdev_queue_stats_tx *tx) 5540 { 5541 struct mlx5e_priv *priv = netdev_priv(dev); 5542 u16 nch = mlx5e_stats_nch_read(priv); 5543 struct mlx5e_ptp *ptp_channel; 5544 int i, tc; 5545 5546 if (!mlx5e_is_uplink_rep(priv)) { 5547 rx->packets = 0; 5548 rx->bytes = 0; 5549 rx->alloc_fail = 0; 5550 rx->hw_gro_packets = 0; 5551 rx->hw_gro_wire_packets = 0; 5552 rx->hw_gro_wire_bytes = 0; 5553 5554 for (i = priv->channels.params.num_channels; i < nch; i++) { 5555 struct netdev_queue_stats_rx rx_i = {0}; 5556 5557 mlx5e_get_queue_stats_rx(dev, i, &rx_i); 5558 5559 rx->packets += rx_i.packets; 5560 rx->bytes += rx_i.bytes; 5561 rx->alloc_fail += rx_i.alloc_fail; 5562 rx->hw_gro_packets += rx_i.hw_gro_packets; 5563 rx->hw_gro_wire_packets += rx_i.hw_gro_wire_packets; 5564 rx->hw_gro_wire_bytes += rx_i.hw_gro_wire_bytes; 5565 } 5566 5567 /* always report PTP RX stats from base as there is no 5568 * corresponding channel to report them under in 5569 * mlx5e_get_queue_stats_rx. 5570 */ 5571 if (priv->rx_ptp_opened) { 5572 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq; 5573 5574 rx->packets += rq_stats->packets; 5575 rx->bytes += rq_stats->bytes; 5576 rx->hw_gro_packets += rq_stats->gro_skbs; 5577 rx->hw_gro_wire_packets += rq_stats->gro_packets; 5578 rx->hw_gro_wire_bytes += rq_stats->gro_bytes; 5579 } 5580 } 5581 5582 tx->packets = 0; 5583 tx->bytes = 0; 5584 tx->hw_gso_packets = 0; 5585 tx->hw_gso_bytes = 0; 5586 tx->csum_none = 0; 5587 tx->stop = 0; 5588 tx->wake = 0; 5589 5590 for (i = 0; i < nch; i++) { 5591 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; 5592 5593 /* handle two cases: 5594 * 5595 * 1. channels which are active. In this case, 5596 * report only deactivated TCs on these channels. 5597 * 5598 * 2. channels which were deactivated 5599 * (i > priv->channels.params.num_channels) 5600 * must have all of their TCs [0 .. priv->max_opened_tc) 5601 * examined because deactivated channels will not be in the 5602 * range of [0..real_num_tx_queues) and will not have their 5603 * stats reported by mlx5e_get_queue_stats_tx. 5604 */ 5605 if (i < priv->channels.params.num_channels) 5606 tc = mlx5e_get_dcb_num_tc(&priv->channels.params); 5607 else 5608 tc = 0; 5609 5610 for (; tc < priv->max_opened_tc; tc++) { 5611 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[tc]; 5612 5613 tx->packets += sq_stats->packets; 5614 tx->bytes += sq_stats->bytes; 5615 tx->hw_gso_packets += sq_stats->tso_packets + 5616 sq_stats->tso_inner_packets; 5617 tx->hw_gso_bytes += sq_stats->tso_bytes + 5618 sq_stats->tso_inner_bytes; 5619 tx->csum_none += sq_stats->csum_none; 5620 tx->stop += sq_stats->stopped; 5621 tx->wake += sq_stats->wake; 5622 } 5623 } 5624 5625 /* if PTP TX was opened at some point and has since either: 5626 * - been shutdown and set to NULL, or 5627 * - simply disabled (bit unset) 5628 * 5629 * report stats directly from the ptp_stats structures as these queues 5630 * are now unavailable and there is no txq index to retrieve these 5631 * stats via calls to mlx5e_get_queue_stats_tx. 5632 */ 5633 ptp_channel = priv->channels.ptp; 5634 if (priv->tx_ptp_opened && (!ptp_channel || !test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state))) { 5635 for (tc = 0; tc < priv->max_opened_tc; tc++) { 5636 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[tc]; 5637 5638 tx->packets += sq_stats->packets; 5639 tx->bytes += sq_stats->bytes; 5640 tx->hw_gso_packets += sq_stats->tso_packets + 5641 sq_stats->tso_inner_packets; 5642 tx->hw_gso_bytes += sq_stats->tso_bytes + 5643 sq_stats->tso_inner_bytes; 5644 tx->csum_none += sq_stats->csum_none; 5645 tx->stop += sq_stats->stopped; 5646 tx->wake += sq_stats->wake; 5647 } 5648 } 5649 } 5650 5651 static const struct netdev_stat_ops mlx5e_stat_ops = { 5652 .get_queue_stats_rx = mlx5e_get_queue_stats_rx, 5653 .get_queue_stats_tx = mlx5e_get_queue_stats_tx, 5654 .get_base_stats = mlx5e_get_base_stats, 5655 }; 5656 5657 struct mlx5_qmgmt_data { 5658 struct mlx5e_channel *c; 5659 }; 5660 5661 static void mlx5e_queue_default_qcfg(struct net_device *dev, 5662 struct netdev_queue_config *qcfg) 5663 { 5664 qcfg->rx_page_size = PAGE_SIZE; 5665 } 5666 5667 static int mlx5e_queue_validate_qcfg(struct net_device *dev, 5668 struct netdev_queue_config *qcfg, 5669 struct netlink_ext_ack *extack) 5670 { 5671 struct mlx5e_priv *priv = netdev_priv(dev); 5672 struct mlx5_core_dev *mdev = priv->mdev; 5673 u32 max; 5674 5675 if (!is_power_of_2(qcfg->rx_page_size)) { 5676 netdev_err(priv->netdev, "rx_page_size not power of 2: %u", 5677 qcfg->rx_page_size); 5678 return -EINVAL; 5679 } 5680 5681 max = mlx5e_mpwrq_max_page_size(mdev); 5682 if (qcfg->rx_page_size < PAGE_SIZE || qcfg->rx_page_size > max) { 5683 netdev_err(priv->netdev, 5684 "Selected rx_page_size %u not in supported range [%lu, %u]\n", 5685 qcfg->rx_page_size, PAGE_SIZE, max); 5686 return -ERANGE; 5687 } 5688 5689 return 0; 5690 } 5691 5692 static bool mlx5e_queue_validate_page_size(struct net_device *dev, 5693 struct netdev_queue_config *qcfg, 5694 int queue_index) 5695 { 5696 if (qcfg->rx_page_size == PAGE_SIZE) 5697 return true; 5698 5699 if (!netif_rxq_has_unreadable_mp(dev, queue_index)) 5700 return false; 5701 5702 return true; 5703 } 5704 5705 static int mlx5e_queue_mem_alloc(struct net_device *dev, 5706 struct netdev_queue_config *qcfg, 5707 void *newq, int queue_index) 5708 { 5709 struct mlx5_qmgmt_data *new = (struct mlx5_qmgmt_data *)newq; 5710 struct mlx5e_priv *priv = netdev_priv(dev); 5711 struct mlx5e_channels *chs = &priv->channels; 5712 struct mlx5e_params params = chs->params; 5713 int err; 5714 5715 mutex_lock(&priv->state_lock); 5716 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 5717 err = -ENODEV; 5718 goto unlock; 5719 } 5720 5721 if (queue_index >= chs->num) { 5722 err = -ERANGE; 5723 goto unlock; 5724 } 5725 5726 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || 5727 chs->params.ptp_rx || 5728 chs->params.xdp_prog || 5729 priv->htb) { 5730 netdev_err(priv->netdev, 5731 "Cloning channels with Port/rx PTP, XDP or HTB is not supported\n"); 5732 err = -EOPNOTSUPP; 5733 goto unlock; 5734 } 5735 5736 if (!mlx5e_queue_validate_page_size(dev, qcfg, queue_index)) { 5737 netdev_err(priv->netdev, "High order pages are supported only in Zero-Copy mode\n"); 5738 err = -EINVAL; 5739 goto unlock; 5740 } 5741 5742 err = mlx5e_open_channel(priv, queue_index, ¶ms, qcfg, NULL, 5743 &new->c); 5744 unlock: 5745 mutex_unlock(&priv->state_lock); 5746 return err; 5747 } 5748 5749 static void mlx5e_queue_mem_free(struct net_device *dev, void *mem) 5750 { 5751 struct mlx5_qmgmt_data *data = (struct mlx5_qmgmt_data *)mem; 5752 5753 /* not supposed to happen since mlx5e_queue_start never fails 5754 * but this is how this should be implemented just in case 5755 */ 5756 if (data->c) 5757 mlx5e_close_channel(data->c); 5758 } 5759 5760 static int mlx5e_queue_stop(struct net_device *dev, void *oldq, int queue_index) 5761 { 5762 /* In mlx5 a txq cannot be simply stopped in isolation, only restarted. 5763 * mlx5e_queue_start does not fail, we stop the old queue there. 5764 * TODO: Improve this. 5765 */ 5766 return 0; 5767 } 5768 5769 static int mlx5e_queue_start(struct net_device *dev, 5770 struct netdev_queue_config *qcfg, 5771 void *newq, int queue_index) 5772 { 5773 struct mlx5_qmgmt_data *new = (struct mlx5_qmgmt_data *)newq; 5774 struct mlx5e_priv *priv = netdev_priv(dev); 5775 struct mlx5e_channel *old; 5776 5777 mutex_lock(&priv->state_lock); 5778 5779 /* stop and close the old */ 5780 old = priv->channels.c[queue_index]; 5781 mlx5e_deactivate_priv_channels(priv); 5782 /* close old before activating new, to avoid napi conflict */ 5783 mlx5e_close_channel(old); 5784 5785 /* start the new */ 5786 priv->channels.c[queue_index] = new->c; 5787 mlx5e_activate_priv_channels(priv); 5788 mutex_unlock(&priv->state_lock); 5789 return 0; 5790 } 5791 5792 static struct device *mlx5e_queue_get_dma_dev(struct net_device *dev, 5793 int queue_index) 5794 { 5795 struct mlx5e_priv *priv = netdev_priv(dev); 5796 struct mlx5e_channels *channels; 5797 struct device *pdev = NULL; 5798 struct mlx5e_channel *ch; 5799 5800 channels = &priv->channels; 5801 5802 mutex_lock(&priv->state_lock); 5803 5804 if (queue_index >= channels->num) 5805 goto out; 5806 5807 ch = channels->c[queue_index]; 5808 pdev = ch->pdev; 5809 out: 5810 mutex_unlock(&priv->state_lock); 5811 5812 return pdev; 5813 } 5814 5815 static const struct netdev_queue_mgmt_ops mlx5e_queue_mgmt_ops = { 5816 .ndo_queue_mem_size = sizeof(struct mlx5_qmgmt_data), 5817 .ndo_queue_mem_alloc = mlx5e_queue_mem_alloc, 5818 .ndo_queue_mem_free = mlx5e_queue_mem_free, 5819 .ndo_queue_start = mlx5e_queue_start, 5820 .ndo_queue_stop = mlx5e_queue_stop, 5821 .ndo_queue_get_dma_dev = mlx5e_queue_get_dma_dev, 5822 .ndo_default_qcfg = mlx5e_queue_default_qcfg, 5823 .ndo_validate_qcfg = mlx5e_queue_validate_qcfg, 5824 .supported_params = QCFG_RX_PAGE_SIZE, 5825 }; 5826 5827 static void mlx5e_build_nic_netdev(struct net_device *netdev) 5828 { 5829 struct mlx5e_priv *priv = netdev_priv(netdev); 5830 struct mlx5_core_dev *mdev = priv->mdev; 5831 bool fcs_supported; 5832 bool fcs_enabled; 5833 5834 SET_NETDEV_DEV(netdev, mdev->device); 5835 5836 netdev->netdev_ops = &mlx5e_netdev_ops; 5837 netdev->queue_mgmt_ops = &mlx5e_queue_mgmt_ops; 5838 netdev->xdp_metadata_ops = &mlx5e_xdp_metadata_ops; 5839 netdev->xsk_tx_metadata_ops = &mlx5e_xsk_tx_metadata_ops; 5840 netdev->request_ops_lock = true; 5841 netdev_lockdep_set_classes(netdev); 5842 5843 mlx5e_dcbnl_build_netdev(netdev); 5844 5845 netdev->watchdog_timeo = 15 * HZ; 5846 5847 netdev->stat_ops = &mlx5e_stat_ops; 5848 netdev->ethtool_ops = &mlx5e_ethtool_ops; 5849 5850 netdev->vlan_features |= NETIF_F_SG; 5851 netdev->vlan_features |= NETIF_F_HW_CSUM; 5852 netdev->vlan_features |= NETIF_F_GRO; 5853 netdev->vlan_features |= NETIF_F_TSO; 5854 netdev->vlan_features |= NETIF_F_TSO6; 5855 netdev->vlan_features |= NETIF_F_RXCSUM; 5856 netdev->vlan_features |= NETIF_F_RXHASH; 5857 netdev->vlan_features |= NETIF_F_GSO_PARTIAL; 5858 5859 netdev->mpls_features |= NETIF_F_SG; 5860 netdev->mpls_features |= NETIF_F_HW_CSUM; 5861 netdev->mpls_features |= NETIF_F_TSO; 5862 netdev->mpls_features |= NETIF_F_TSO6; 5863 5864 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX; 5865 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX; 5866 5867 /* Tunneled LRO is not supported in the driver, and the same RQs are 5868 * shared between inner and outer TIRs, so the driver can't disable LRO 5869 * for inner TIRs while having it enabled for outer TIRs. Due to this, 5870 * block LRO altogether if the firmware declares tunneled LRO support. 5871 */ 5872 if (!!MLX5_CAP_ETH(mdev, lro_cap) && 5873 !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) && 5874 !MLX5_CAP_ETH(mdev, tunnel_lro_gre) && 5875 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, 5876 MLX5E_MPWRQ_UMR_MODE_ALIGNED)) 5877 netdev->vlan_features |= NETIF_F_LRO; 5878 5879 if (mlx5e_hw_gro_supported(mdev) && 5880 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, 5881 MLX5E_MPWRQ_UMR_MODE_ALIGNED)) 5882 netdev->vlan_features |= NETIF_F_GRO_HW; 5883 5884 netdev->hw_features = netdev->vlan_features; 5885 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 5886 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 5887 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 5888 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 5889 5890 if (mlx5e_tunnel_any_tx_proto_supported(mdev)) { 5891 netdev->hw_enc_features |= NETIF_F_HW_CSUM; 5892 netdev->hw_enc_features |= NETIF_F_TSO; 5893 netdev->hw_enc_features |= NETIF_F_TSO6; 5894 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; 5895 } 5896 5897 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { 5898 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 5899 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5900 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 5901 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5902 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 5903 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL | 5904 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5905 } 5906 5907 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) { 5908 netdev->hw_features |= NETIF_F_GSO_GRE | 5909 NETIF_F_GSO_GRE_CSUM; 5910 netdev->hw_enc_features |= NETIF_F_GSO_GRE | 5911 NETIF_F_GSO_GRE_CSUM; 5912 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 5913 netdev->vlan_features |= NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM; 5914 } 5915 5916 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) { 5917 netdev->hw_features |= NETIF_F_GSO_IPXIP4 | 5918 NETIF_F_GSO_IPXIP6; 5919 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 | 5920 NETIF_F_GSO_IPXIP6; 5921 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 | 5922 NETIF_F_GSO_IPXIP6; 5923 } 5924 5925 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4; 5926 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 5927 netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4; 5928 5929 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled); 5930 5931 if (fcs_supported) 5932 netdev->hw_features |= NETIF_F_RXALL; 5933 5934 if (MLX5_CAP_ETH(mdev, scatter_fcs)) 5935 netdev->hw_features |= NETIF_F_RXFCS; 5936 5937 if (mlx5_qos_is_supported(mdev)) 5938 netdev->hw_features |= NETIF_F_HW_TC; 5939 5940 netdev->features = netdev->hw_features; 5941 5942 /* Defaults */ 5943 if (fcs_enabled) 5944 netdev->features &= ~NETIF_F_RXALL; 5945 netdev->features &= ~NETIF_F_LRO; 5946 netdev->features &= ~NETIF_F_GRO_HW; 5947 netdev->features &= ~NETIF_F_RXFCS; 5948 5949 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f) 5950 if (FT_CAP(flow_modify_en) && 5951 FT_CAP(modify_root) && 5952 FT_CAP(identified_miss_table_mode) && 5953 FT_CAP(flow_table_modify)) { 5954 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 5955 netdev->hw_features |= NETIF_F_HW_TC; 5956 #endif 5957 #if IS_ENABLED(CONFIG_MLX5_EN_ARFS) 5958 netdev->hw_features |= NETIF_F_NTUPLE; 5959 #elif IS_ENABLED(CONFIG_MLX5_EN_RXNFC) 5960 netdev->features |= NETIF_F_NTUPLE; 5961 #endif 5962 } 5963 5964 netdev->features |= NETIF_F_HIGHDMA; 5965 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 5966 5967 netdev->priv_flags |= IFF_UNICAST_FLT; 5968 5969 netdev->netmem_tx = NETMEM_TX_DMA; 5970 5971 netif_set_tso_max_size(netdev, GSO_MAX_SIZE); 5972 mlx5e_set_xdp_feature(priv); 5973 mlx5e_set_netdev_dev_addr(netdev); 5974 mlx5e_macsec_build_netdev(priv); 5975 mlx5e_ipsec_build_netdev(priv); 5976 mlx5e_ktls_build_netdev(priv); 5977 } 5978 5979 void mlx5e_create_q_counters(struct mlx5e_priv *priv) 5980 { 5981 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 5982 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 5983 struct mlx5_core_dev *mdev = priv->mdev; 5984 struct mlx5_core_dev *pos; 5985 int err, i; 5986 5987 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 5988 5989 mlx5_sd_for_each_dev(i, mdev, pos) { 5990 err = mlx5_cmd_exec_inout(pos, alloc_q_counter, in, out); 5991 if (!err) 5992 priv->q_counter[i] = 5993 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5994 } 5995 5996 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); 5997 if (!err) 5998 priv->drop_rq_q_counter = 5999 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 6000 } 6001 6002 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) 6003 { 6004 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 6005 struct mlx5_core_dev *pos; 6006 int i; 6007 6008 MLX5_SET(dealloc_q_counter_in, in, opcode, 6009 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 6010 mlx5_sd_for_each_dev(i, priv->mdev, pos) { 6011 if (priv->q_counter[i]) { 6012 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 6013 priv->q_counter[i]); 6014 mlx5_cmd_exec_in(pos, dealloc_q_counter, in); 6015 } 6016 } 6017 6018 if (priv->drop_rq_q_counter) { 6019 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 6020 priv->drop_rq_q_counter); 6021 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); 6022 } 6023 } 6024 6025 static int mlx5e_nic_init(struct mlx5_core_dev *mdev, 6026 struct net_device *netdev) 6027 { 6028 const bool take_rtnl = netdev->reg_state == NETREG_REGISTERED; 6029 struct mlx5e_priv *priv = netdev_priv(netdev); 6030 struct mlx5e_flow_steering *fs; 6031 int err; 6032 6033 mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu); 6034 mlx5e_vxlan_set_netdev_info(priv); 6035 6036 mlx5e_timestamp_init(priv); 6037 6038 priv->dfs_root = debugfs_create_dir("nic", 6039 mlx5_debugfs_get_dev_root(mdev)); 6040 6041 fs = mlx5e_fs_init(priv->profile, mdev, 6042 !test_bit(MLX5E_STATE_DESTROYING, &priv->state), 6043 priv->dfs_root); 6044 if (!fs) { 6045 err = -ENOMEM; 6046 mlx5_core_err(mdev, "FS initialization failed, %d\n", err); 6047 debugfs_remove_recursive(priv->dfs_root); 6048 return err; 6049 } 6050 priv->fs = fs; 6051 6052 err = mlx5e_psp_init(priv); 6053 if (err) 6054 mlx5_core_err(mdev, "PSP initialization failed, %d\n", err); 6055 6056 err = mlx5e_ktls_init(priv); 6057 if (err) 6058 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); 6059 6060 mlx5e_health_create_reporters(priv); 6061 6062 /* If netdev is already registered (e.g. move from uplink to nic profile), 6063 * RTNL lock must be held before triggering netdev notifiers. 6064 */ 6065 if (take_rtnl) 6066 rtnl_lock(); 6067 6068 /* update XDP supported features */ 6069 mlx5e_set_xdp_feature(priv); 6070 6071 if (take_rtnl) 6072 rtnl_unlock(); 6073 6074 return 0; 6075 } 6076 6077 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) 6078 { 6079 mlx5e_health_destroy_reporters(priv); 6080 mlx5e_ktls_cleanup(priv); 6081 mlx5e_psp_cleanup(priv); 6082 mlx5e_fs_cleanup(priv->fs); 6083 debugfs_remove_recursive(priv->dfs_root); 6084 priv->fs = NULL; 6085 } 6086 6087 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) 6088 { 6089 struct mlx5_core_dev *mdev = priv->mdev; 6090 enum mlx5e_rx_res_features features; 6091 int err; 6092 6093 mlx5e_create_q_counters(priv); 6094 6095 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 6096 if (err) { 6097 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 6098 goto err_destroy_q_counters; 6099 } 6100 6101 features = MLX5E_RX_RES_FEATURE_PTP; 6102 if (mlx5_tunnel_inner_ft_supported(mdev)) 6103 features |= MLX5E_RX_RES_FEATURE_INNER_FT; 6104 if (mlx5_get_sd(priv->mdev)) 6105 features |= MLX5E_RX_RES_FEATURE_MULTI_VHCA; 6106 6107 priv->rx_res = mlx5e_rx_res_create(priv->mdev, features, priv->max_nch, priv->drop_rq.rqn, 6108 &priv->channels.params.packet_merge, 6109 priv->channels.params.num_channels); 6110 if (IS_ERR(priv->rx_res)) { 6111 err = PTR_ERR(priv->rx_res); 6112 priv->rx_res = NULL; 6113 mlx5_core_err(mdev, "create rx resources failed, %d\n", err); 6114 goto err_close_drop_rq; 6115 } 6116 6117 err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile, 6118 priv->netdev); 6119 if (err) { 6120 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); 6121 goto err_destroy_rx_res; 6122 } 6123 6124 err = mlx5e_tc_nic_init(priv); 6125 if (err) 6126 goto err_destroy_flow_steering; 6127 6128 err = mlx5e_accel_init_rx(priv); 6129 if (err) 6130 goto err_tc_nic_cleanup; 6131 6132 #ifdef CONFIG_MLX5_EN_ARFS 6133 priv->netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(priv->mdev); 6134 #endif 6135 6136 return 0; 6137 6138 err_tc_nic_cleanup: 6139 mlx5e_tc_nic_cleanup(priv); 6140 err_destroy_flow_steering: 6141 mlx5e_destroy_flow_steering(priv->fs, mlx5e_fs_has_arfs(priv->netdev), 6142 priv->profile); 6143 err_destroy_rx_res: 6144 mlx5e_rx_res_destroy(priv->rx_res); 6145 priv->rx_res = NULL; 6146 err_close_drop_rq: 6147 mlx5e_close_drop_rq(&priv->drop_rq); 6148 err_destroy_q_counters: 6149 mlx5e_destroy_q_counters(priv); 6150 return err; 6151 } 6152 6153 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) 6154 { 6155 mlx5e_accel_cleanup_rx(priv); 6156 mlx5e_tc_nic_cleanup(priv); 6157 mlx5e_destroy_flow_steering(priv->fs, mlx5e_fs_has_arfs(priv->netdev), 6158 priv->profile); 6159 mlx5e_rx_res_destroy(priv->rx_res); 6160 priv->rx_res = NULL; 6161 mlx5e_close_drop_rq(&priv->drop_rq); 6162 mlx5e_destroy_q_counters(priv); 6163 } 6164 6165 static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv) 6166 { 6167 struct mlx5e_params *params; 6168 struct mlx5e_mqprio_rl *rl; 6169 6170 params = &priv->channels.params; 6171 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) 6172 return; 6173 6174 rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc, 6175 params->mqprio.channel.max_rate); 6176 if (IS_ERR(rl)) 6177 rl = NULL; 6178 priv->mqprio_rl = rl; 6179 mlx5e_mqprio_rl_update_params(params, rl); 6180 } 6181 6182 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) 6183 { 6184 int err; 6185 6186 err = mlx5e_accel_init_tx(priv); 6187 if (err) 6188 return err; 6189 6190 mlx5e_set_mqprio_rl(priv); 6191 mlx5e_dcbnl_initialize(priv); 6192 return 0; 6193 } 6194 6195 static int mlx5e_nic_enable(struct mlx5e_priv *priv) 6196 { 6197 struct net_device *netdev = priv->netdev; 6198 struct mlx5_core_dev *mdev = priv->mdev; 6199 int err; 6200 6201 mlx5e_fs_init_l2_addr(priv->fs, netdev); 6202 mlx5e_ipsec_init(priv); 6203 err = mlx5e_psp_register(priv); 6204 if (err) 6205 goto out_ipsec_cleanup; 6206 6207 err = mlx5e_macsec_init(priv); 6208 if (err) 6209 mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err); 6210 6211 /* Marking the link as currently not needed by the Driver */ 6212 if (!netif_running(netdev)) 6213 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN); 6214 6215 mlx5e_set_netdev_mtu_boundaries(priv); 6216 mlx5e_set_dev_port_mtu(priv); 6217 6218 mlx5_lag_add_netdev(mdev, netdev); 6219 6220 mlx5e_enable_async_events(priv); 6221 mlx5e_enable_blocking_events(priv); 6222 if (mlx5e_monitor_counter_supported(priv)) 6223 mlx5e_monitor_counter_init(priv); 6224 6225 mlx5e_pcie_cong_event_init(priv); 6226 mlx5e_hv_vhca_stats_create(priv); 6227 if (netdev->reg_state != NETREG_REGISTERED) 6228 return 0; 6229 mlx5e_dcbnl_init_app(priv); 6230 6231 mlx5e_nic_set_rx_mode(priv); 6232 6233 rtnl_lock(); 6234 netdev_lock(netdev); 6235 if (netif_running(netdev)) 6236 mlx5e_open(netdev); 6237 udp_tunnel_nic_reset_ntf(priv->netdev); 6238 netdev_unlock(netdev); 6239 netif_device_attach(netdev); 6240 rtnl_unlock(); 6241 6242 return 0; 6243 6244 out_ipsec_cleanup: 6245 mlx5e_ipsec_cleanup(priv); 6246 return err; 6247 } 6248 6249 static void mlx5e_nic_disable(struct mlx5e_priv *priv) 6250 { 6251 struct mlx5_core_dev *mdev = priv->mdev; 6252 6253 if (priv->netdev->reg_state == NETREG_REGISTERED) 6254 mlx5e_dcbnl_delete_app(priv); 6255 6256 rtnl_lock(); 6257 netdev_lock(priv->netdev); 6258 if (netif_running(priv->netdev)) 6259 mlx5e_close(priv->netdev); 6260 netif_device_detach(priv->netdev); 6261 if (priv->en_trap) { 6262 mlx5e_deactivate_trap(priv); 6263 mlx5e_close_trap(priv->en_trap); 6264 priv->en_trap = NULL; 6265 } 6266 netdev_unlock(priv->netdev); 6267 rtnl_unlock(); 6268 6269 mlx5e_nic_set_rx_mode(priv); 6270 6271 mlx5e_pcie_cong_event_cleanup(priv); 6272 mlx5e_hv_vhca_stats_destroy(priv); 6273 if (mlx5e_monitor_counter_supported(priv)) 6274 mlx5e_monitor_counter_cleanup(priv); 6275 6276 mlx5e_ipsec_disable_events(priv); 6277 mlx5e_disable_blocking_events(priv); 6278 mlx5e_disable_async_events(priv); 6279 mlx5_lag_remove_netdev(mdev, priv->netdev); 6280 mlx5_vxlan_reset_to_default(mdev->vxlan); 6281 mlx5e_macsec_cleanup(priv); 6282 mlx5e_psp_unregister(priv); 6283 mlx5e_ipsec_cleanup(priv); 6284 } 6285 6286 static int mlx5e_update_nic_rx(struct mlx5e_priv *priv) 6287 { 6288 return mlx5e_refresh_tirs(priv->mdev, false, false); 6289 } 6290 6291 static const struct mlx5e_profile mlx5e_nic_profile = { 6292 .init = mlx5e_nic_init, 6293 .cleanup = mlx5e_nic_cleanup, 6294 .init_rx = mlx5e_init_nic_rx, 6295 .cleanup_rx = mlx5e_cleanup_nic_rx, 6296 .init_tx = mlx5e_init_nic_tx, 6297 .cleanup_tx = mlx5e_cleanup_nic_tx, 6298 .enable = mlx5e_nic_enable, 6299 .disable = mlx5e_nic_disable, 6300 .update_rx = mlx5e_update_nic_rx, 6301 .update_stats = mlx5e_stats_update_ndo_stats, 6302 .update_carrier = mlx5e_update_carrier, 6303 .rx_handlers = &mlx5e_rx_handlers_nic, 6304 .max_tc = MLX5_MAX_NUM_TC, 6305 .stats_grps = mlx5e_nic_stats_grps, 6306 .stats_grps_num = mlx5e_nic_stats_grps_num, 6307 .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) | 6308 BIT(MLX5E_PROFILE_FEATURE_PTP_TX) | 6309 BIT(MLX5E_PROFILE_FEATURE_QOS_HTB) | 6310 BIT(MLX5E_PROFILE_FEATURE_FS_VLAN) | 6311 BIT(MLX5E_PROFILE_FEATURE_FS_TC), 6312 }; 6313 6314 static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev, 6315 const struct mlx5e_profile *profile) 6316 { 6317 int nch; 6318 6319 nch = mlx5e_get_max_num_channels(mdev); 6320 6321 if (profile->max_nch_limit) 6322 nch = min_t(int, nch, profile->max_nch_limit(mdev)); 6323 return nch; 6324 } 6325 6326 static unsigned int 6327 mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev, 6328 const struct mlx5e_profile *profile) 6329 6330 { 6331 unsigned int max_nch, tmp; 6332 6333 /* core resources */ 6334 max_nch = mlx5e_profile_max_num_channels(mdev, profile); 6335 6336 /* netdev rx queues */ 6337 max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues); 6338 6339 /* netdev tx queues */ 6340 tmp = netdev->num_tx_queues; 6341 if (mlx5_qos_is_supported(mdev)) 6342 tmp -= mlx5e_qos_max_leaf_nodes(mdev); 6343 if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn)) 6344 tmp -= profile->max_tc; 6345 tmp = tmp / profile->max_tc; 6346 max_nch = min_t(unsigned int, max_nch, tmp); 6347 6348 return max_nch; 6349 } 6350 6351 int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev) 6352 { 6353 /* Indirect TIRS: 2 sets of TTCs (inner + outer steering) 6354 * and 1 set of direct TIRS 6355 */ 6356 return 2 * MLX5E_NUM_INDIR_TIRS 6357 + mlx5e_profile_max_num_channels(mdev, &mlx5e_nic_profile); 6358 } 6359 6360 void mlx5e_set_rx_mode_work(struct work_struct *work) 6361 { 6362 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 6363 set_rx_mode_work); 6364 struct net_device *dev = priv->netdev; 6365 6366 netdev_lock_ops(dev); 6367 mlx5e_fs_set_rx_mode_work(priv->fs, dev, NULL, NULL); 6368 netdev_unlock_ops(dev); 6369 } 6370 6371 /* mlx5e generic netdev management API (move to en_common.c) */ 6372 int mlx5e_priv_init(struct mlx5e_priv *priv, 6373 const struct mlx5e_profile *profile, 6374 struct net_device *netdev, 6375 struct mlx5_core_dev *mdev) 6376 { 6377 int nch, num_txqs, node; 6378 int err; 6379 6380 num_txqs = netdev->num_tx_queues; 6381 nch = mlx5e_calc_max_nch(mdev, netdev, profile); 6382 node = dev_to_node(mlx5_core_dma_dev(mdev)); 6383 6384 /* priv init */ 6385 priv->mdev = mdev; 6386 priv->netdev = netdev; 6387 priv->max_nch = nch; 6388 priv->max_opened_tc = 1; 6389 6390 if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL)) 6391 return -ENOMEM; 6392 6393 mutex_init(&priv->state_lock); 6394 6395 err = mlx5e_selq_init(&priv->selq, &priv->state_lock); 6396 if (err) 6397 goto err_free_cpumask; 6398 6399 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); 6400 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); 6401 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); 6402 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); 6403 6404 priv->wq = create_singlethread_workqueue("mlx5e"); 6405 if (!priv->wq) 6406 goto err_free_selq; 6407 6408 priv->txq2sq = kcalloc_node(num_txqs, sizeof(*priv->txq2sq), GFP_KERNEL, node); 6409 if (!priv->txq2sq) 6410 goto err_destroy_workqueue; 6411 6412 priv->txq2sq_stats = kcalloc_node(num_txqs, sizeof(*priv->txq2sq_stats), GFP_KERNEL, node); 6413 if (!priv->txq2sq_stats) 6414 goto err_free_txq2sq; 6415 6416 priv->tx_rates = kcalloc_node(num_txqs, sizeof(*priv->tx_rates), GFP_KERNEL, node); 6417 if (!priv->tx_rates) 6418 goto err_free_txq2sq_stats; 6419 6420 priv->channel_stats = 6421 kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node); 6422 if (!priv->channel_stats) 6423 goto err_free_tx_rates; 6424 6425 return 0; 6426 6427 err_free_tx_rates: 6428 kfree(priv->tx_rates); 6429 err_free_txq2sq_stats: 6430 kfree(priv->txq2sq_stats); 6431 err_free_txq2sq: 6432 kfree(priv->txq2sq); 6433 err_destroy_workqueue: 6434 destroy_workqueue(priv->wq); 6435 err_free_selq: 6436 mlx5e_selq_cleanup(&priv->selq); 6437 err_free_cpumask: 6438 free_cpumask_var(priv->scratchpad.cpumask); 6439 return -ENOMEM; 6440 } 6441 6442 void mlx5e_priv_cleanup(struct mlx5e_priv *priv) 6443 { 6444 bool destroying = test_bit(MLX5E_STATE_DESTROYING, &priv->state); 6445 int i; 6446 6447 /* bail if change profile failed and also rollback failed */ 6448 if (!priv->mdev) 6449 return; 6450 6451 for (i = 0; i < priv->stats_nch; i++) 6452 kvfree(priv->channel_stats[i]); 6453 kfree(priv->channel_stats); 6454 kfree(priv->tx_rates); 6455 kfree(priv->txq2sq_stats); 6456 kfree(priv->txq2sq); 6457 destroy_workqueue(priv->wq); 6458 mlx5e_selq_cleanup(&priv->selq); 6459 free_cpumask_var(priv->scratchpad.cpumask); 6460 6461 for (i = 0; i < priv->htb_max_qos_sqs; i++) 6462 kfree(priv->htb_qos_sq_stats[i]); 6463 kvfree(priv->htb_qos_sq_stats); 6464 6465 if (priv->mqprio_rl) { 6466 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 6467 mlx5e_mqprio_rl_free(priv->mqprio_rl); 6468 } 6469 6470 memset(priv, 0, sizeof(*priv)); 6471 if (destroying) /* restore destroying bit, to allow unload */ 6472 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6473 } 6474 6475 static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev, 6476 const struct mlx5e_profile *profile) 6477 { 6478 unsigned int nch, ptp_txqs, qos_txqs; 6479 6480 nch = mlx5e_profile_max_num_channels(mdev, profile); 6481 6482 ptp_txqs = MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn) && 6483 mlx5e_profile_feature_cap(profile, PTP_TX) ? 6484 profile->max_tc : 0; 6485 6486 qos_txqs = mlx5_qos_is_supported(mdev) && 6487 mlx5e_profile_feature_cap(profile, QOS_HTB) ? 6488 mlx5e_qos_max_leaf_nodes(mdev) : 0; 6489 6490 return nch * profile->max_tc + ptp_txqs + qos_txqs; 6491 } 6492 6493 static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev, 6494 const struct mlx5e_profile *profile) 6495 { 6496 return mlx5e_profile_max_num_channels(mdev, profile); 6497 } 6498 6499 struct net_device * 6500 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile) 6501 { 6502 struct net_device *netdev; 6503 unsigned int txqs, rxqs; 6504 int err; 6505 6506 txqs = mlx5e_get_max_num_txqs(mdev, profile); 6507 rxqs = mlx5e_get_max_num_rxqs(mdev, profile); 6508 6509 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs); 6510 if (!netdev) { 6511 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n"); 6512 return NULL; 6513 } 6514 6515 err = mlx5e_priv_init(netdev_priv(netdev), profile, netdev, mdev); 6516 if (err) { 6517 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 6518 goto err_free_netdev; 6519 } 6520 6521 netif_carrier_off(netdev); 6522 netif_tx_disable(netdev); 6523 dev_net_set(netdev, mlx5_core_net(mdev)); 6524 6525 return netdev; 6526 6527 err_free_netdev: 6528 free_netdev(netdev); 6529 6530 return NULL; 6531 } 6532 6533 static void mlx5e_update_features(struct net_device *netdev) 6534 { 6535 if (netdev->reg_state != NETREG_REGISTERED) 6536 return; /* features will be updated on netdev registration */ 6537 6538 rtnl_lock(); 6539 netdev_lock(netdev); 6540 netdev_update_features(netdev); 6541 netdev_unlock(netdev); 6542 rtnl_unlock(); 6543 } 6544 6545 static void mlx5e_reset_channels(struct net_device *netdev) 6546 { 6547 netdev_reset_tc(netdev); 6548 } 6549 6550 int mlx5e_attach_netdev(struct mlx5e_priv *priv) 6551 { 6552 const bool need_lock = priv->netdev->reg_state == NETREG_REGISTERED; 6553 const struct mlx5e_profile *profile = priv->profile; 6554 int max_nch; 6555 int err; 6556 6557 clear_bit(MLX5E_STATE_DESTROYING, &priv->state); 6558 if (priv->fs) 6559 mlx5e_fs_set_rx_mode_active(priv->fs, 6560 !test_bit(MLX5E_STATE_DESTROYING, 6561 &priv->state)); 6562 6563 /* Validate the max_wqe_size_sq capability. */ 6564 if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) { 6565 mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %u\n", 6566 mlx5e_get_max_sq_wqebbs(priv->mdev), (unsigned int)MLX5E_MAX_TX_WQEBBS); 6567 return -EIO; 6568 } 6569 6570 /* max number of channels may have changed */ 6571 max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile); 6572 6573 /* Locking is required by ethtool_rxfh_indir_lost() (sends 6574 * ETHTOOL_MSG_RSS_NTF) and by netif_set_real_num_*_queues in case 6575 * the netdev has been registered by this point (if this function 6576 * was called in the reload or resume flow). 6577 */ 6578 if (need_lock) { 6579 rtnl_lock(); 6580 netdev_lock(priv->netdev); 6581 } 6582 6583 if (priv->channels.params.num_channels > max_nch) { 6584 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch); 6585 /* Reducing the number of channels - RXFH has to be reset, and 6586 * mlx5e_num_channels_changed below will build the RQT. 6587 */ 6588 ethtool_rxfh_indir_lost(priv->netdev); 6589 priv->channels.params.num_channels = max_nch; 6590 if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) { 6591 mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n"); 6592 mlx5e_params_mqprio_reset(&priv->channels.params); 6593 } 6594 } 6595 if (max_nch != priv->max_nch) { 6596 mlx5_core_warn(priv->mdev, 6597 "MLX5E: Updating max number of channels from %u to %u\n", 6598 priv->max_nch, max_nch); 6599 priv->max_nch = max_nch; 6600 } 6601 6602 /* 1. Set the real number of queues in the kernel the first time. 6603 * 2. Set our default XPS cpumask. 6604 * 3. Build the RQT. 6605 */ 6606 err = mlx5e_num_channels_changed(priv); 6607 if (need_lock) { 6608 netdev_unlock(priv->netdev); 6609 rtnl_unlock(); 6610 } 6611 if (err) 6612 goto out; 6613 6614 err = profile->init_tx(priv); 6615 if (err) 6616 goto out; 6617 6618 err = profile->init_rx(priv); 6619 if (err) 6620 goto err_cleanup_tx; 6621 6622 if (profile->enable) { 6623 err = profile->enable(priv); 6624 if (err) 6625 goto err_cleanup_rx; 6626 } 6627 6628 mlx5e_update_features(priv->netdev); 6629 6630 return 0; 6631 6632 err_cleanup_rx: 6633 profile->cleanup_rx(priv); 6634 err_cleanup_tx: 6635 profile->cleanup_tx(priv); 6636 6637 out: 6638 mlx5e_reset_channels(priv->netdev); 6639 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6640 if (priv->fs) 6641 mlx5e_fs_set_rx_mode_active(priv->fs, 6642 !test_bit(MLX5E_STATE_DESTROYING, 6643 &priv->state)); 6644 cancel_work_sync(&priv->update_stats_work); 6645 return err; 6646 } 6647 6648 void mlx5e_detach_netdev(struct mlx5e_priv *priv) 6649 { 6650 const struct mlx5e_profile *profile = priv->profile; 6651 6652 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6653 if (priv->fs) 6654 mlx5e_fs_set_rx_mode_active(priv->fs, 6655 !test_bit(MLX5E_STATE_DESTROYING, 6656 &priv->state)); 6657 6658 if (profile->disable) 6659 profile->disable(priv); 6660 flush_workqueue(priv->wq); 6661 6662 profile->cleanup_rx(priv); 6663 profile->cleanup_tx(priv); 6664 mlx5e_reset_channels(priv->netdev); 6665 cancel_work_sync(&priv->update_stats_work); 6666 } 6667 6668 static int 6669 mlx5e_netdev_init_profile(struct net_device *netdev, struct mlx5_core_dev *mdev, 6670 const struct mlx5e_profile *new_profile, void *new_ppriv) 6671 { 6672 struct mlx5e_priv *priv = netdev_priv(netdev); 6673 int err; 6674 6675 err = mlx5e_priv_init(priv, new_profile, netdev, mdev); 6676 if (err) { 6677 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 6678 return err; 6679 } 6680 netif_carrier_off(netdev); 6681 priv->profile = new_profile; 6682 priv->ppriv = new_ppriv; 6683 err = new_profile->init(priv->mdev, priv->netdev); 6684 if (err) 6685 goto priv_cleanup; 6686 6687 return 0; 6688 6689 priv_cleanup: 6690 mlx5e_priv_cleanup(priv); 6691 return err; 6692 } 6693 6694 static int 6695 mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev, 6696 const struct mlx5e_profile *new_profile, void *new_ppriv) 6697 { 6698 struct mlx5e_priv *priv = netdev_priv(netdev); 6699 int err; 6700 6701 err = mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv); 6702 if (err) 6703 return err; 6704 6705 err = mlx5e_attach_netdev(priv); 6706 if (err) 6707 goto profile_cleanup; 6708 return err; 6709 6710 profile_cleanup: 6711 new_profile->cleanup(priv); 6712 mlx5e_priv_cleanup(priv); 6713 return err; 6714 } 6715 6716 int mlx5e_netdev_change_profile(struct net_device *netdev, 6717 struct mlx5_core_dev *mdev, 6718 const struct mlx5e_profile *new_profile, 6719 void *new_ppriv) 6720 { 6721 struct mlx5e_priv *priv = netdev_priv(netdev); 6722 const struct mlx5e_profile *orig_profile; 6723 int err, rollback_err; 6724 void *orig_ppriv; 6725 6726 orig_profile = priv->profile; 6727 orig_ppriv = priv->ppriv; 6728 6729 /* NULL could happen if previous change_profile failed to rollback */ 6730 if (priv->profile) { 6731 WARN_ON_ONCE(priv->mdev != mdev); 6732 /* cleanup old profile */ 6733 mlx5e_detach_netdev(priv); 6734 priv->profile->cleanup(priv); 6735 mlx5e_priv_cleanup(priv); 6736 } 6737 /* priv members are not valid from this point ... */ 6738 6739 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 6740 mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv); 6741 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6742 return -EIO; 6743 } 6744 6745 err = mlx5e_netdev_attach_profile(netdev, mdev, new_profile, new_ppriv); 6746 if (err) { /* roll back to original profile */ 6747 netdev_warn(netdev, "%s: new profile init failed, %d\n", __func__, err); 6748 goto rollback; 6749 } 6750 6751 return 0; 6752 6753 rollback: 6754 if (!orig_profile) { 6755 netdev_warn(netdev, "no original profile to rollback to\n"); 6756 priv->profile = NULL; 6757 return err; 6758 } 6759 6760 rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv); 6761 if (rollback_err) { 6762 netdev_err(netdev, "failed to rollback to orig profile, %d\n", 6763 rollback_err); 6764 priv->profile = NULL; 6765 } 6766 return err; 6767 } 6768 6769 void mlx5e_netdev_attach_nic_profile(struct net_device *netdev, 6770 struct mlx5_core_dev *mdev) 6771 { 6772 mlx5e_netdev_change_profile(netdev, mdev, &mlx5e_nic_profile, NULL); 6773 } 6774 6775 void mlx5e_destroy_netdev(struct net_device *netdev) 6776 { 6777 struct mlx5e_priv *priv = netdev_priv(netdev); 6778 6779 if (priv->profile) 6780 mlx5e_priv_cleanup(priv); 6781 free_netdev(netdev); 6782 } 6783 6784 static int _mlx5e_resume(struct auxiliary_device *adev) 6785 { 6786 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6787 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6788 struct mlx5e_priv *priv = netdev_priv(mlx5e_dev->netdev); 6789 struct net_device *netdev = mlx5e_dev->netdev; 6790 struct mlx5_core_dev *mdev = edev->mdev; 6791 struct mlx5_core_dev *pos, *to; 6792 int err, i; 6793 6794 if (netif_device_present(netdev)) 6795 return 0; 6796 6797 mlx5_sd_for_each_dev(i, mdev, pos) { 6798 err = mlx5e_create_mdev_resources(pos, true); 6799 if (err) 6800 goto err_destroy_mdev_res; 6801 } 6802 6803 err = mlx5e_attach_netdev(priv); 6804 if (err) 6805 goto err_destroy_mdev_res; 6806 6807 return 0; 6808 6809 err_destroy_mdev_res: 6810 to = pos; 6811 mlx5_sd_for_each_dev_to(i, mdev, to, pos) 6812 mlx5e_destroy_mdev_resources(pos); 6813 return err; 6814 } 6815 6816 static int mlx5e_resume(struct auxiliary_device *adev) 6817 { 6818 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6819 struct mlx5_core_dev *mdev = edev->mdev; 6820 struct auxiliary_device *actual_adev; 6821 int err; 6822 6823 err = mlx5_sd_init(mdev); 6824 if (err) 6825 return err; 6826 6827 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6828 if (actual_adev) { 6829 err = _mlx5e_resume(actual_adev); 6830 mlx5_sd_put_adev(actual_adev, adev); 6831 } 6832 return err; 6833 } 6834 6835 static int _mlx5e_suspend(struct auxiliary_device *adev, bool pre_netdev_reg) 6836 { 6837 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6838 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6839 struct mlx5e_priv *priv = netdev_priv(mlx5e_dev->netdev); 6840 struct net_device *netdev = mlx5e_dev->netdev; 6841 struct mlx5_core_dev *mdev = edev->mdev; 6842 struct mlx5_core_dev *pos; 6843 int i; 6844 6845 if (!pre_netdev_reg && !netif_device_present(netdev)) { 6846 if (test_bit(MLX5E_STATE_DESTROYING, &priv->state)) 6847 mlx5_sd_for_each_dev(i, mdev, pos) 6848 mlx5e_destroy_mdev_resources(pos); 6849 return -ENODEV; 6850 } 6851 6852 mlx5e_detach_netdev(priv); 6853 mlx5_sd_for_each_dev(i, mdev, pos) 6854 mlx5e_destroy_mdev_resources(pos); 6855 6856 return 0; 6857 } 6858 6859 static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) 6860 { 6861 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6862 struct mlx5_core_dev *mdev = edev->mdev; 6863 struct auxiliary_device *actual_adev; 6864 int err = 0; 6865 6866 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6867 if (actual_adev) 6868 err = _mlx5e_suspend(actual_adev, false); 6869 6870 mlx5_sd_cleanup(mdev); 6871 if (actual_adev) 6872 mlx5_sd_put_adev(actual_adev, adev); 6873 return err; 6874 } 6875 6876 static int _mlx5e_probe(struct auxiliary_device *adev) 6877 { 6878 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6879 const struct mlx5e_profile *profile = &mlx5e_nic_profile; 6880 struct mlx5_core_dev *mdev = edev->mdev; 6881 struct mlx5e_dev *mlx5e_dev; 6882 struct net_device *netdev; 6883 struct mlx5e_priv *priv; 6884 int err; 6885 6886 mlx5e_dev = mlx5e_create_devlink(&adev->dev, mdev); 6887 if (IS_ERR(mlx5e_dev)) 6888 return PTR_ERR(mlx5e_dev); 6889 auxiliary_set_drvdata(adev, mlx5e_dev); 6890 6891 err = mlx5e_devlink_port_register(mlx5e_dev, mdev); 6892 if (err) { 6893 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err); 6894 goto err_devlink_unregister; 6895 } 6896 6897 netdev = mlx5e_create_netdev(mdev, profile); 6898 if (!netdev) { 6899 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n"); 6900 err = -ENOMEM; 6901 goto err_devlink_port_unregister; 6902 } 6903 SET_NETDEV_DEVLINK_PORT(netdev, &mlx5e_dev->dl_port); 6904 mlx5e_dev->netdev = netdev; 6905 6906 mlx5e_build_nic_netdev(netdev); 6907 6908 priv = netdev_priv(netdev); 6909 6910 priv->profile = profile; 6911 priv->ppriv = NULL; 6912 6913 err = profile->init(mdev, netdev); 6914 if (err) { 6915 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err); 6916 goto err_destroy_netdev; 6917 } 6918 6919 err = _mlx5e_resume(adev); 6920 if (err) { 6921 mlx5_core_err(mdev, "_mlx5e_resume failed, %d\n", err); 6922 goto err_profile_cleanup; 6923 } 6924 6925 err = register_netdev(netdev); 6926 if (err) { 6927 mlx5_core_err(mdev, "register_netdev failed, %d\n", err); 6928 goto err_resume; 6929 } 6930 6931 /* mlx5e_fix_features() returns early when the device is not present 6932 * to avoid dereferencing cleared priv during profile changes. 6933 * This also causes it to be a no-op during register_netdev(), where 6934 * the device is not yet present. 6935 * Trigger an additional features update that will actually work. 6936 */ 6937 mlx5e_update_features(netdev); 6938 6939 mlx5e_dcbnl_init_app(priv); 6940 mlx5_core_uplink_netdev_set(mdev, netdev); 6941 mlx5e_params_print_info(mdev, &priv->channels.params); 6942 return 0; 6943 6944 err_resume: 6945 _mlx5e_suspend(adev, true); 6946 err_profile_cleanup: 6947 profile->cleanup(priv); 6948 err_destroy_netdev: 6949 mlx5e_destroy_netdev(netdev); 6950 err_devlink_port_unregister: 6951 mlx5e_devlink_port_unregister(mlx5e_dev); 6952 err_devlink_unregister: 6953 mlx5e_destroy_devlink(mlx5e_dev); 6954 return err; 6955 } 6956 6957 static int mlx5e_probe(struct auxiliary_device *adev, 6958 const struct auxiliary_device_id *id) 6959 { 6960 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6961 struct mlx5_core_dev *mdev = edev->mdev; 6962 struct auxiliary_device *actual_adev; 6963 int err; 6964 6965 err = mlx5_sd_init(mdev); 6966 if (err) 6967 return err; 6968 6969 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6970 if (actual_adev) { 6971 err = _mlx5e_probe(actual_adev); 6972 if (err) 6973 goto sd_cleanup; 6974 mlx5_sd_put_adev(actual_adev, adev); 6975 } 6976 return 0; 6977 6978 sd_cleanup: 6979 mlx5_sd_cleanup(mdev); 6980 if (actual_adev) 6981 mlx5_sd_put_adev(actual_adev, adev); 6982 return err; 6983 } 6984 6985 static void _mlx5e_remove(struct auxiliary_device *adev) 6986 { 6987 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6988 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6989 struct net_device *netdev = mlx5e_dev->netdev; 6990 struct mlx5e_priv *priv = netdev_priv(netdev); 6991 struct mlx5_core_dev *mdev = edev->mdev; 6992 6993 mlx5_eswitch_safe_aux_devs_remove(mdev); 6994 mlx5_core_uplink_netdev_set(mdev, NULL); 6995 6996 if (priv->profile) 6997 mlx5e_dcbnl_delete_app(priv); 6998 /* When unload driver, the netdev is in registered state 6999 * if it's from legacy mode. If from switchdev mode, it 7000 * is already unregistered before changing to NIC profile. 7001 */ 7002 if (netdev->reg_state == NETREG_REGISTERED) { 7003 unregister_netdev(netdev); 7004 _mlx5e_suspend(adev, false); 7005 } else { 7006 struct mlx5_core_dev *pos; 7007 int i; 7008 7009 if (test_bit(MLX5E_STATE_DESTROYING, &priv->state)) 7010 mlx5_sd_for_each_dev(i, mdev, pos) 7011 mlx5e_destroy_mdev_resources(pos); 7012 else 7013 _mlx5e_suspend(adev, true); 7014 } 7015 /* Avoid cleanup if profile rollback failed. */ 7016 if (priv->profile) 7017 priv->profile->cleanup(priv); 7018 mlx5e_destroy_netdev(netdev); 7019 mlx5e_devlink_port_unregister(mlx5e_dev); 7020 mlx5e_destroy_devlink(mlx5e_dev); 7021 } 7022 7023 static void mlx5e_remove(struct auxiliary_device *adev) 7024 { 7025 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 7026 struct mlx5_core_dev *mdev = edev->mdev; 7027 struct auxiliary_device *actual_adev; 7028 7029 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 7030 if (actual_adev) 7031 _mlx5e_remove(actual_adev); 7032 7033 mlx5_sd_cleanup(mdev); 7034 if (actual_adev) 7035 mlx5_sd_put_adev(actual_adev, adev); 7036 } 7037 7038 static const struct auxiliary_device_id mlx5e_id_table[] = { 7039 { .name = MLX5_ADEV_NAME ".eth", }, 7040 {}, 7041 }; 7042 7043 MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table); 7044 7045 static struct auxiliary_driver mlx5e_driver = { 7046 .name = "eth", 7047 .probe = mlx5e_probe, 7048 .remove = mlx5e_remove, 7049 .suspend = mlx5e_suspend, 7050 .resume = mlx5e_resume, 7051 .id_table = mlx5e_id_table, 7052 }; 7053 7054 int mlx5e_init(void) 7055 { 7056 int ret; 7057 7058 mlx5e_build_ptys2ethtool_map(); 7059 ret = auxiliary_driver_register(&mlx5e_driver); 7060 if (ret) 7061 return ret; 7062 7063 ret = mlx5e_rep_init(); 7064 if (ret) 7065 auxiliary_driver_unregister(&mlx5e_driver); 7066 return ret; 7067 } 7068 7069 void mlx5e_cleanup(void) 7070 { 7071 mlx5e_rep_cleanup(); 7072 auxiliary_driver_unregister(&mlx5e_driver); 7073 } 7074