1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/dim.h> 34 #include <net/tc_act/tc_gact.h> 35 #include <linux/mlx5/fs.h> 36 #include <net/vxlan.h> 37 #include <net/geneve.h> 38 #include <linux/bpf.h> 39 #include <linux/debugfs.h> 40 #include <linux/if_bridge.h> 41 #include <linux/filter.h> 42 #include <net/netdev_lock.h> 43 #include <net/netdev_queues.h> 44 #include <net/netdev_rx_queue.h> 45 #include <net/page_pool/types.h> 46 #include <net/pkt_sched.h> 47 #include <net/xdp_sock_drv.h> 48 #include "eswitch.h" 49 #include "en.h" 50 #include "en/dim.h" 51 #include "en/txrx.h" 52 #include "en_tc.h" 53 #include "en_rep.h" 54 #include "en_accel/ipsec.h" 55 #include "en_accel/psp.h" 56 #include "en_accel/macsec.h" 57 #include "en_accel/en_accel.h" 58 #include "en_accel/ktls.h" 59 #include "lib/vxlan.h" 60 #include "lib/clock.h" 61 #include "en/port.h" 62 #include "en/xdp.h" 63 #include "lib/eq.h" 64 #include "en/monitor_stats.h" 65 #include "en/health.h" 66 #include "en/params.h" 67 #include "en/xsk/pool.h" 68 #include "en/xsk/setup.h" 69 #include "en/xsk/rx.h" 70 #include "en/xsk/tx.h" 71 #include "en/hv_vhca_stats.h" 72 #include "en/devlink.h" 73 #include "lib/mlx5.h" 74 #include "en/ptp.h" 75 #include "en/htb.h" 76 #include "qos.h" 77 #include "en/trap.h" 78 #include "lib/devcom.h" 79 #include "lib/sd.h" 80 #include "en/pcie_cong_event.h" 81 82 static bool mlx5e_hw_gro_supported(struct mlx5_core_dev *mdev) 83 { 84 if (!MLX5_CAP_GEN(mdev, shampo) || 85 !MLX5_CAP_SHAMPO(mdev, shampo_header_split_data_merge)) 86 return false; 87 88 /* Our HW-GRO implementation relies on "KSM Mkey" for 89 * SHAMPO headers buffer mapping 90 */ 91 if (!MLX5_CAP_GEN(mdev, fixed_buffer_size)) 92 return false; 93 94 if (!MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer_valid)) 95 return false; 96 97 if (MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer) > 98 MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE) 99 return false; 100 101 return true; 102 } 103 104 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift, 105 enum mlx5e_mpwrq_umr_mode umr_mode) 106 { 107 u16 umr_wqebbs, max_wqebbs; 108 bool striding_rq_umr; 109 110 striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) && 111 MLX5_CAP_ETH(mdev, reg_umr_sq); 112 if (!striding_rq_umr) 113 return false; 114 115 umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode); 116 max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 117 /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is 118 * calculated from mlx5e_get_max_sq_aligned_wqebbs. 119 */ 120 if (WARN_ON(umr_wqebbs > max_wqebbs)) 121 return false; 122 123 return true; 124 } 125 126 void mlx5e_update_carrier(struct mlx5e_priv *priv) 127 { 128 struct mlx5_core_dev *mdev = priv->mdev; 129 u8 port_state; 130 bool up; 131 132 port_state = mlx5_query_vport_state(mdev, 133 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 134 0); 135 136 up = port_state == VPORT_STATE_UP; 137 if (up == netif_carrier_ok(priv->netdev)) 138 netif_carrier_event(priv->netdev); 139 if (up) { 140 netdev_info(priv->netdev, "Link up\n"); 141 netif_carrier_on(priv->netdev); 142 } else { 143 netdev_info(priv->netdev, "Link down\n"); 144 netif_carrier_off(priv->netdev); 145 } 146 } 147 148 static void mlx5e_update_carrier_work(struct work_struct *work) 149 { 150 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 151 update_carrier_work); 152 153 mutex_lock(&priv->state_lock); 154 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 155 if (priv->profile->update_carrier) 156 priv->profile->update_carrier(priv); 157 mutex_unlock(&priv->state_lock); 158 } 159 160 static void mlx5e_update_stats_work(struct work_struct *work) 161 { 162 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 163 update_stats_work); 164 165 mutex_lock(&priv->state_lock); 166 priv->profile->update_stats(priv); 167 mutex_unlock(&priv->state_lock); 168 } 169 170 void mlx5e_queue_update_stats(struct mlx5e_priv *priv) 171 { 172 if (!priv->profile->update_stats) 173 return; 174 175 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state))) 176 return; 177 178 queue_work(priv->wq, &priv->update_stats_work); 179 } 180 181 static int async_event(struct notifier_block *nb, unsigned long event, void *data) 182 { 183 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 184 struct mlx5_eqe *eqe = data; 185 186 if (event != MLX5_EVENT_TYPE_PORT_CHANGE) 187 return NOTIFY_DONE; 188 189 switch (eqe->sub_type) { 190 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 191 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 192 queue_work(priv->wq, &priv->update_carrier_work); 193 break; 194 default: 195 return NOTIFY_DONE; 196 } 197 198 return NOTIFY_OK; 199 } 200 201 static void mlx5e_enable_async_events(struct mlx5e_priv *priv) 202 { 203 priv->events_nb.notifier_call = async_event; 204 mlx5_notifier_register(priv->mdev, &priv->events_nb); 205 } 206 207 static void mlx5e_disable_async_events(struct mlx5e_priv *priv) 208 { 209 mlx5_notifier_unregister(priv->mdev, &priv->events_nb); 210 } 211 212 static int mlx5e_devcom_event_mpv(int event, void *my_data, void *event_data) 213 { 214 struct mlx5e_priv *slave_priv = my_data; 215 216 switch (event) { 217 case MPV_DEVCOM_MASTER_UP: 218 mlx5_devcom_comp_set_ready(slave_priv->devcom, true); 219 break; 220 case MPV_DEVCOM_MASTER_DOWN: 221 /* no need for comp set ready false since we unregister after 222 * and it hurts cleanup flow. 223 */ 224 break; 225 case MPV_DEVCOM_IPSEC_MASTER_UP: 226 case MPV_DEVCOM_IPSEC_MASTER_DOWN: 227 mlx5e_ipsec_handle_mpv_event(event, my_data, event_data); 228 break; 229 } 230 231 return 0; 232 } 233 234 static int mlx5e_devcom_init_mpv(struct mlx5e_priv *priv, u64 *data) 235 { 236 struct mlx5_devcom_match_attr attr = { 237 .key.val = *data, 238 }; 239 240 priv->devcom = mlx5_devcom_register_component(priv->mdev->priv.devc, 241 MLX5_DEVCOM_MPV, 242 &attr, 243 mlx5e_devcom_event_mpv, 244 priv); 245 if (!priv->devcom) 246 return -EINVAL; 247 248 if (mlx5_core_is_mp_master(priv->mdev)) { 249 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_UP, 250 MPV_DEVCOM_MASTER_UP, priv); 251 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_UP); 252 } 253 254 return 0; 255 } 256 257 static void mlx5e_devcom_cleanup_mpv(struct mlx5e_priv *priv) 258 { 259 if (!priv->devcom) 260 return; 261 262 if (mlx5_core_is_mp_master(priv->mdev)) { 263 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_DOWN, 264 MPV_DEVCOM_MASTER_DOWN, priv); 265 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_DOWN); 266 } 267 268 mlx5_devcom_unregister_component(priv->devcom); 269 priv->devcom = NULL; 270 } 271 272 static int blocking_event(struct notifier_block *nb, unsigned long event, void *data) 273 { 274 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb); 275 struct mlx5_devlink_trap_event_ctx *trap_event_ctx = data; 276 int err; 277 278 switch (event) { 279 case MLX5_DRIVER_EVENT_TYPE_TRAP: 280 err = mlx5e_handle_trap_event(priv, trap_event_ctx->trap); 281 if (err) { 282 trap_event_ctx->err = err; 283 return NOTIFY_BAD; 284 } 285 break; 286 case MLX5_DRIVER_EVENT_AFFILIATION_DONE: 287 if (mlx5e_devcom_init_mpv(priv, data)) 288 return NOTIFY_BAD; 289 break; 290 case MLX5_DRIVER_EVENT_AFFILIATION_REMOVED: 291 mlx5e_devcom_cleanup_mpv(priv); 292 break; 293 default: 294 return NOTIFY_DONE; 295 } 296 return NOTIFY_OK; 297 } 298 299 static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv) 300 { 301 priv->blocking_events_nb.notifier_call = blocking_event; 302 mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb); 303 } 304 305 static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv) 306 { 307 mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb); 308 } 309 310 static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode) 311 { 312 u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode); 313 u32 sz; 314 315 sz = ALIGN(entries * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT); 316 317 return sz / MLX5_OCTWORD; 318 } 319 320 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, 321 struct mlx5e_icosq *sq, 322 struct mlx5e_umr_wqe *wqe) 323 { 324 struct mlx5_wqe_ctrl_seg *cseg = &wqe->hdr.ctrl; 325 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->hdr.uctrl; 326 u16 octowords; 327 u8 ds_cnt; 328 329 ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift, 330 rq->mpwqe.umr_mode), 331 MLX5_SEND_WQE_DS); 332 333 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | 334 ds_cnt); 335 cseg->umr_mkey = rq->mpwqe.umr_mkey_be; 336 337 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 338 octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode); 339 ucseg->xlt_octowords = cpu_to_be16(octowords); 340 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 341 } 342 343 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) 344 { 345 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 346 size_t alloc_size; 347 348 alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, 349 alloc_units.frag_pages, 350 rq->mpwqe.pages_per_wqe)); 351 352 rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node); 353 if (!rq->mpwqe.info) 354 return -ENOMEM; 355 356 /* For deferred page release (release right before alloc), make sure 357 * that on first round release is not called. 358 */ 359 for (int i = 0; i < wq_sz; i++) { 360 struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, i); 361 362 bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe); 363 } 364 365 mlx5e_build_umr_wqe(rq, rq->icosq, 366 container_of(&rq->mpwqe.umr_wqe, 367 struct mlx5e_umr_wqe, hdr)); 368 369 return 0; 370 } 371 372 static int mlx5e_rq_alloc_mpwqe_linear_info(struct mlx5e_rq *rq, int node, 373 struct mlx5e_params *params, 374 struct mlx5e_rq_opt_param *rqo) 375 { 376 struct mlx5_core_dev *mdev = rq->mdev; 377 struct mlx5e_mpw_linear_info *li; 378 u32 linear_frag_count; 379 380 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo) || 381 !params->xdp_prog) 382 return 0; 383 384 li = kvzalloc_node(sizeof(*li), GFP_KERNEL, node); 385 if (!li) 386 return -ENOMEM; 387 388 linear_frag_count = 389 BIT(rq->mpwqe.page_shift - MLX5E_XDP_LOG_MAX_LINEAR_SZ); 390 if (linear_frag_count > U16_MAX) { 391 netdev_warn(rq->netdev, 392 "rq %d: linear_frag_count (%u) larger than expected (%u), page_shift: %u, log_max_linear_sz: %u\n", 393 rq->ix, linear_frag_count, U16_MAX, 394 rq->mpwqe.page_shift, MLX5E_XDP_LOG_MAX_LINEAR_SZ); 395 kvfree(li); 396 return -EINVAL; 397 } 398 399 li->max_frags = linear_frag_count; 400 rq->mpwqe.linear_info = li; 401 402 /* Set to max to force allocation on first run. */ 403 li->frag_page.frags = li->max_frags; 404 405 return 0; 406 } 407 408 static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode) 409 { 410 switch (umr_mode) { 411 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 412 return MLX5_MKC_ACCESS_MODE_MTT; 413 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 414 return MLX5_MKC_ACCESS_MODE_KSM; 415 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 416 return MLX5_MKC_ACCESS_MODE_KLMS; 417 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 418 return MLX5_MKC_ACCESS_MODE_KSM; 419 } 420 WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode); 421 return 0; 422 } 423 424 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, 425 u32 npages, u8 page_shift, u32 *umr_mkey, 426 dma_addr_t filler_addr, 427 enum mlx5e_mpwrq_umr_mode umr_mode, 428 u32 xsk_chunk_size) 429 { 430 struct mlx5_mtt *mtt; 431 struct mlx5_ksm *ksm; 432 struct mlx5_klm *klm; 433 u32 octwords; 434 int inlen; 435 void *mkc; 436 u32 *in; 437 int err; 438 int i; 439 440 if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED || 441 umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) && 442 !MLX5_CAP_GEN(mdev, fixed_buffer_size)) { 443 mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n"); 444 return -EINVAL; 445 } 446 447 octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode); 448 449 inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), 450 MLX5_OCTWORD, octwords); 451 if (inlen < 0) 452 return inlen; 453 454 in = kvzalloc(inlen, GFP_KERNEL); 455 if (!in) 456 return -ENOMEM; 457 458 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 459 460 MLX5_SET(mkc, mkc, free, 1); 461 MLX5_SET(mkc, mkc, umr_en, 1); 462 MLX5_SET(mkc, mkc, lw, 1); 463 MLX5_SET(mkc, mkc, lr, 1); 464 MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode)); 465 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 466 MLX5_SET(mkc, mkc, qpn, 0xffffff); 467 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 468 MLX5_SET64(mkc, mkc, len, npages << page_shift); 469 MLX5_SET(mkc, mkc, translations_octword_size, octwords); 470 if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) 471 MLX5_SET(mkc, mkc, log_page_size, page_shift - 2); 472 else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED) 473 MLX5_SET(mkc, mkc, log_page_size, page_shift); 474 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords); 475 476 /* Initialize the mkey with all MTTs pointing to a default 477 * page (filler_addr). When the channels are activated, UMR 478 * WQEs will redirect the RX WQEs to the actual memory from 479 * the RQ's pool, while the gaps (wqe_overflow) remain mapped 480 * to the default page. 481 */ 482 switch (umr_mode) { 483 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED: 484 klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 485 for (i = 0; i < npages; i++) { 486 klm[i << 1] = (struct mlx5_klm) { 487 .va = cpu_to_be64(filler_addr), 488 .bcount = cpu_to_be32(xsk_chunk_size), 489 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 490 }; 491 klm[(i << 1) + 1] = (struct mlx5_klm) { 492 .va = cpu_to_be64(filler_addr), 493 .bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size), 494 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 495 }; 496 } 497 break; 498 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED: 499 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 500 for (i = 0; i < npages; i++) 501 ksm[i] = (struct mlx5_ksm) { 502 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 503 .va = cpu_to_be64(filler_addr), 504 }; 505 break; 506 case MLX5E_MPWRQ_UMR_MODE_ALIGNED: 507 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 508 for (i = 0; i < npages; i++) 509 mtt[i] = (struct mlx5_mtt) { 510 .ptag = cpu_to_be64(filler_addr), 511 }; 512 break; 513 case MLX5E_MPWRQ_UMR_MODE_TRIPLE: 514 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 515 for (i = 0; i < npages * 4; i++) { 516 ksm[i] = (struct mlx5_ksm) { 517 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey), 518 .va = cpu_to_be64(filler_addr), 519 }; 520 } 521 break; 522 } 523 524 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 525 526 kvfree(in); 527 return err; 528 } 529 530 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) 531 { 532 u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0; 533 u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 534 u32 num_entries, max_num_entries; 535 u32 umr_mkey; 536 int err; 537 538 max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode); 539 540 /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */ 541 if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe, 542 &num_entries) || 543 num_entries > max_num_entries)) 544 mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n", 545 __func__, wq_size, rq->mpwqe.mtts_per_wqe, 546 max_num_entries); 547 548 err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift, 549 &umr_mkey, rq->wqe_overflow.addr, 550 rq->mpwqe.umr_mode, xsk_chunk_size); 551 rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey); 552 return err; 553 } 554 555 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) 556 { 557 struct mlx5e_wqe_frag_info next_frag = {}; 558 struct mlx5e_wqe_frag_info *prev = NULL; 559 int i; 560 561 WARN_ON(rq->xsk_pool); 562 563 next_frag.frag_page = &rq->wqe.alloc_units->frag_pages[0]; 564 565 /* Skip first release due to deferred release. */ 566 next_frag.flags = BIT(MLX5E_WQE_FRAG_SKIP_RELEASE); 567 568 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 569 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; 570 struct mlx5e_wqe_frag_info *frag = 571 &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; 572 int f; 573 574 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { 575 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { 576 /* Pages are assigned at runtime. */ 577 next_frag.frag_page++; 578 next_frag.offset = 0; 579 if (prev) 580 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE); 581 } 582 *frag = next_frag; 583 584 /* prepare next */ 585 next_frag.offset += frag_info[f].frag_stride; 586 prev = frag; 587 } 588 } 589 590 if (prev) 591 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE); 592 } 593 594 static void mlx5e_init_xsk_buffs(struct mlx5e_rq *rq) 595 { 596 int i; 597 598 /* Assumptions used by XSK batched allocator. */ 599 WARN_ON(rq->wqe.info.num_frags != 1); 600 WARN_ON(rq->wqe.info.log_num_frags != 0); 601 WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE); 602 603 /* Considering the above assumptions a fragment maps to a single 604 * xsk_buff. 605 */ 606 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 607 rq->wqe.frags[i].xskp = &rq->wqe.alloc_units->xsk_buffs[i]; 608 609 /* Skip first release due to deferred release as WQES are 610 * not allocated yet. 611 */ 612 rq->wqe.frags[i].flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE); 613 } 614 } 615 616 static int mlx5e_init_wqe_alloc_info(struct mlx5e_rq *rq, int node) 617 { 618 int wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 619 int len = wq_sz << rq->wqe.info.log_num_frags; 620 struct mlx5e_wqe_frag_info *frags; 621 union mlx5e_alloc_units *aus; 622 int aus_sz; 623 624 if (rq->xsk_pool) 625 aus_sz = sizeof(*aus->xsk_buffs); 626 else 627 aus_sz = sizeof(*aus->frag_pages); 628 629 aus = kvzalloc_node(array_size(len, aus_sz), GFP_KERNEL, node); 630 if (!aus) 631 return -ENOMEM; 632 633 frags = kvzalloc_node(array_size(len, sizeof(*frags)), GFP_KERNEL, node); 634 if (!frags) { 635 kvfree(aus); 636 return -ENOMEM; 637 } 638 639 rq->wqe.alloc_units = aus; 640 rq->wqe.frags = frags; 641 642 if (rq->xsk_pool) 643 mlx5e_init_xsk_buffs(rq); 644 else 645 mlx5e_init_frags_partition(rq); 646 647 return 0; 648 } 649 650 static void mlx5e_free_wqe_alloc_info(struct mlx5e_rq *rq) 651 { 652 kvfree(rq->wqe.frags); 653 kvfree(rq->wqe.alloc_units); 654 } 655 656 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work) 657 { 658 struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work); 659 660 mlx5e_reporter_rq_cqe_err(rq); 661 } 662 663 static void mlx5e_rq_timeout_work(struct work_struct *timeout_work) 664 { 665 struct mlx5e_rq *rq = container_of(timeout_work, 666 struct mlx5e_rq, 667 rx_timeout_work); 668 669 mlx5e_reporter_rx_timeout(rq); 670 } 671 672 static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 673 { 674 /* xsk can have page_shift < PAGE_SHIFT */ 675 u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0); 676 u32 page_size = BIT(PAGE_SHIFT + page_order); 677 678 rq->wqe_overflow.page = alloc_pages(GFP_KERNEL, page_order); 679 if (!rq->wqe_overflow.page) 680 return -ENOMEM; 681 682 rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0, 683 page_size, rq->buff.map_dir); 684 if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) { 685 __free_pages(rq->wqe_overflow.page, page_order); 686 return -ENOMEM; 687 } 688 return 0; 689 } 690 691 static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq) 692 { 693 u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0); 694 u32 page_size = BIT(PAGE_SHIFT + page_order); 695 696 dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, page_size, 697 rq->buff.map_dir); 698 __free_pages(rq->wqe_overflow.page, page_order); 699 } 700 701 static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 702 u32 xdp_frag_size, struct mlx5e_rq *rq) 703 { 704 struct mlx5_core_dev *mdev = c->mdev; 705 int err; 706 707 rq->wq_type = params->rq_wq_type; 708 rq->pdev = c->pdev; 709 rq->netdev = c->netdev; 710 rq->priv = c->priv; 711 rq->hwtstamp_config = &c->priv->hwtstamp_config; 712 rq->clock = mdev->clock; 713 rq->icosq = &c->icosq; 714 rq->ix = c->ix; 715 rq->channel = c; 716 rq->mdev = mdev; 717 rq->hw_mtu = 718 MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN * !params->scatter_fcs_en; 719 rq->xdpsq = &c->rq_xdpsq; 720 rq->stats = &c->priv->channel_stats[c->ix]->rq; 721 rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev); 722 err = mlx5e_rq_set_handlers(rq, params, NULL); 723 if (err) 724 return err; 725 726 return __xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id, 727 xdp_frag_size); 728 } 729 730 static void mlx5e_release_rq_hd_pages(struct mlx5e_rq *rq, 731 struct mlx5e_shampo_hd *shampo) 732 733 { 734 for (int i = 0; i < shampo->nentries; i++) { 735 struct mlx5e_dma_info *info = &shampo->hd_buf_pages[i]; 736 737 if (!info->page) 738 continue; 739 740 dma_unmap_page(rq->pdev, info->addr, PAGE_SIZE, 741 rq->buff.map_dir); 742 __free_page(info->page); 743 } 744 } 745 746 static int mlx5e_alloc_rq_hd_pages(struct mlx5e_rq *rq, int node, 747 struct mlx5e_shampo_hd *shampo) 748 { 749 int err, i; 750 751 for (i = 0; i < shampo->nentries; i++) { 752 struct page *page = alloc_pages_node(node, GFP_KERNEL, 0); 753 dma_addr_t addr; 754 755 if (!page) { 756 err = -ENOMEM; 757 goto err_free_pages; 758 } 759 760 addr = dma_map_page(rq->pdev, page, 0, PAGE_SIZE, 761 rq->buff.map_dir); 762 err = dma_mapping_error(rq->pdev, addr); 763 if (err) { 764 __free_page(page); 765 goto err_free_pages; 766 } 767 768 shampo->hd_buf_pages[i].page = page; 769 shampo->hd_buf_pages[i].addr = addr; 770 } 771 772 return 0; 773 774 err_free_pages: 775 mlx5e_release_rq_hd_pages(rq, shampo); 776 777 return err; 778 } 779 780 static int mlx5e_create_rq_hd_mkey(struct mlx5_core_dev *mdev, 781 struct mlx5e_shampo_hd *shampo) 782 { 783 enum mlx5e_mpwrq_umr_mode umr_mode = MLX5E_MPWRQ_UMR_MODE_ALIGNED; 784 struct mlx5_mtt *mtt; 785 void *mkc, *in; 786 int inlen, err; 787 u32 octwords; 788 789 octwords = mlx5e_mpwrq_umr_octowords(shampo->nentries, umr_mode); 790 inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in), 791 MLX5_OCTWORD, octwords); 792 if (inlen < 0) 793 return inlen; 794 795 in = kvzalloc(inlen, GFP_KERNEL); 796 if (!in) 797 return -ENOMEM; 798 799 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 800 801 MLX5_SET(mkc, mkc, lw, 1); 802 MLX5_SET(mkc, mkc, lr, 1); 803 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 804 mlx5e_mkey_set_relaxed_ordering(mdev, mkc); 805 MLX5_SET(mkc, mkc, qpn, 0xffffff); 806 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn); 807 MLX5_SET64(mkc, mkc, len, shampo->hd_buf_size); 808 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 809 MLX5_SET(mkc, mkc, translations_octword_size, octwords); 810 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 811 octwords); 812 813 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 814 for (int i = 0; i < shampo->nentries; i++) 815 mtt[i].ptag = cpu_to_be64(shampo->hd_buf_pages[i].addr); 816 817 err = mlx5_core_create_mkey(mdev, &shampo->mkey, in, inlen); 818 819 kvfree(in); 820 return err; 821 } 822 823 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, 824 struct mlx5e_params *params, 825 struct mlx5e_rq_param *rq_param, 826 struct mlx5e_rq *rq, 827 int node) 828 { 829 struct mlx5e_shampo_hd *shampo; 830 int nentries, err, shampo_sz; 831 u32 hd_per_wq, hd_buf_size; 832 833 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 834 return 0; 835 836 hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rq_param); 837 hd_buf_size = hd_per_wq * BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE); 838 nentries = hd_buf_size / PAGE_SIZE; 839 if (!nentries) { 840 mlx5_core_err(mdev, "SHAMPO header buffer size %u < %lu\n", 841 hd_buf_size, PAGE_SIZE); 842 return -EINVAL; 843 } 844 845 shampo_sz = struct_size(shampo, hd_buf_pages, nentries); 846 shampo = kvzalloc_node(shampo_sz, GFP_KERNEL, node); 847 if (!shampo) 848 return -ENOMEM; 849 850 shampo->hd_per_wq = hd_per_wq; 851 shampo->hd_buf_size = hd_buf_size; 852 shampo->nentries = nentries; 853 err = mlx5e_alloc_rq_hd_pages(rq, node, shampo); 854 if (err) 855 goto err_free; 856 857 err = mlx5e_create_rq_hd_mkey(mdev, shampo); 858 if (err) 859 goto err_release_pages; 860 861 /* gro only data structures */ 862 rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node); 863 if (!rq->hw_gro_data) { 864 err = -ENOMEM; 865 goto err_destroy_mkey; 866 } 867 868 rq->mpwqe.shampo = shampo; 869 870 return 0; 871 872 err_destroy_mkey: 873 mlx5_core_destroy_mkey(mdev, shampo->mkey); 874 err_release_pages: 875 mlx5e_release_rq_hd_pages(rq, shampo); 876 err_free: 877 kvfree(shampo); 878 879 return err; 880 } 881 882 static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) 883 { 884 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo; 885 886 if (!shampo) 887 return; 888 889 kvfree(rq->hw_gro_data); 890 mlx5_core_destroy_mkey(rq->mdev, shampo->mkey); 891 mlx5e_release_rq_hd_pages(rq, shampo); 892 kvfree(shampo); 893 } 894 895 static int mlx5e_alloc_rq(struct mlx5e_params *params, 896 struct mlx5e_rq_param *rq_param, 897 struct mlx5e_rq_opt_param *rqo, 898 int node, struct mlx5e_rq *rq) 899 { 900 void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); 901 struct mlx5_core_dev *mdev = rq->mdev; 902 u32 pool_order = 0; 903 u32 pool_size; 904 int wq_sz; 905 int err; 906 int i; 907 908 rq_param->wq.db_numa_node = node; 909 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work); 910 INIT_WORK(&rq->rx_timeout_work, mlx5e_rq_timeout_work); 911 912 if (params->xdp_prog) 913 bpf_prog_inc(params->xdp_prog); 914 RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog); 915 916 rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 917 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, rqo); 918 pool_size = 1 << params->log_rq_mtu_frames; 919 920 rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); 921 922 switch (rq->wq_type) { 923 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 924 err = mlx5_wq_ll_create(mdev, &rq_param->wq, rqc_wq, 925 &rq->mpwqe.wq, &rq->wq_ctrl); 926 if (err) 927 goto err_rq_xdp_prog; 928 929 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; 930 931 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 932 933 rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); 934 err = mlx5e_alloc_mpwqe_rq_drop_page(rq); 935 if (err) 936 goto err_rq_wq_destroy; 937 938 rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); 939 rq->mpwqe.pages_per_wqe = 940 mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift, 941 rq->mpwqe.umr_mode); 942 rq->mpwqe.umr_wqebbs = 943 mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift, 944 rq->mpwqe.umr_mode); 945 rq->mpwqe.mtts_per_wqe = 946 mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift, 947 rq->mpwqe.umr_mode); 948 949 pool_size = rq->mpwqe.pages_per_wqe << 950 mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo); 951 pool_order = rq->mpwqe.page_shift - PAGE_SHIFT; 952 953 rq->mpwqe.log_stride_sz = 954 mlx5e_mpwqe_get_log_stride_size(mdev, params, 955 rqo); 956 rq->mpwqe.num_strides = 957 BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo)); 958 rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz); 959 960 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz); 961 962 err = mlx5e_create_rq_umr_mkey(mdev, rq); 963 if (err) 964 goto err_rq_drop_page; 965 966 err = mlx5e_rq_alloc_mpwqe_info(rq, node); 967 if (err) 968 goto err_rq_mkey; 969 970 err = mlx5e_rq_alloc_mpwqe_linear_info(rq, node, params, rqo); 971 if (err) 972 goto err_free_mpwqe_info; 973 974 err = mlx5_rq_shampo_alloc(mdev, params, rq_param, rq, node); 975 if (err) 976 goto err_free_mpwqe_linear_info; 977 978 break; 979 default: /* MLX5_WQ_TYPE_CYCLIC */ 980 err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, 981 &rq->wqe.wq, &rq->wq_ctrl); 982 if (err) 983 goto err_rq_xdp_prog; 984 985 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; 986 987 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 988 989 rq->wqe.info = rq_param->frags_info; 990 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride; 991 992 err = mlx5e_init_wqe_alloc_info(rq, node); 993 if (err) 994 goto err_rq_wq_destroy; 995 } 996 997 if (mlx5e_rqo_xsk_param(rqo)) { 998 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 999 MEM_TYPE_XSK_BUFF_POOL, NULL); 1000 if (err) 1001 goto err_free_by_rq_type; 1002 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq); 1003 } else { 1004 /* Create a page_pool and register it with rxq */ 1005 struct page_pool_params pp_params = { 0 }; 1006 1007 if (WARN_ON(BIT(PAGE_SHIFT + pool_order) / 64 > 1008 MLX5E_PAGECNT_BIAS_MAX)) { 1009 err = -E2BIG; 1010 goto err_free_by_rq_type; 1011 } 1012 1013 pp_params.order = pool_order; 1014 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; 1015 pp_params.pool_size = pool_size; 1016 pp_params.nid = node; 1017 pp_params.dev = rq->pdev; 1018 pp_params.napi = rq->cq.napi; 1019 pp_params.netdev = rq->netdev; 1020 pp_params.dma_dir = rq->buff.map_dir; 1021 pp_params.max_len = BIT(PAGE_SHIFT + pool_order); 1022 pp_params.queue_idx = rq->ix; 1023 1024 /* Shampo header data split allow for unreadable netmem */ 1025 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) 1026 pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; 1027 1028 /* page_pool can be used even when there is no rq->xdp_prog, 1029 * given page_pool does not handle DMA mapping there is no 1030 * required state to clear. And page_pool gracefully handle 1031 * elevated refcnt. 1032 */ 1033 rq->page_pool = page_pool_create(&pp_params); 1034 if (IS_ERR(rq->page_pool)) { 1035 err = PTR_ERR(rq->page_pool); 1036 rq->page_pool = NULL; 1037 goto err_free_by_rq_type; 1038 } 1039 if (!rq->hd_page_pool) 1040 rq->hd_page_pool = rq->page_pool; 1041 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) { 1042 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 1043 MEM_TYPE_PAGE_POOL, rq->page_pool); 1044 if (err) 1045 goto err_destroy_page_pool; 1046 } 1047 } 1048 1049 for (i = 0; i < wq_sz; i++) { 1050 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 1051 struct mlx5e_rx_wqe_ll *wqe = 1052 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i); 1053 u32 byte_count = 1054 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz; 1055 u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) << 1056 rq->mpwqe.page_shift; 1057 u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ? 1058 0 : rq->buff.headroom; 1059 1060 wqe->data[0].addr = cpu_to_be64(dma_offset + headroom); 1061 wqe->data[0].byte_count = cpu_to_be32(byte_count); 1062 wqe->data[0].lkey = rq->mpwqe.umr_mkey_be; 1063 } else { 1064 struct mlx5e_rx_wqe_cyc *wqe = 1065 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); 1066 int f; 1067 1068 for (f = 0; f < rq->wqe.info.num_frags; f++) { 1069 u32 frag_size = rq->wqe.info.arr[f].frag_size | 1070 MLX5_HW_START_PADDING; 1071 1072 wqe->data[f].byte_count = cpu_to_be32(frag_size); 1073 wqe->data[f].lkey = rq->mkey_be; 1074 } 1075 /* check if num_frags is not a pow of two */ 1076 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { 1077 wqe->data[f].byte_count = 0; 1078 wqe->data[f].lkey = params->terminate_lkey_be; 1079 wqe->data[f].addr = 0; 1080 } 1081 } 1082 } 1083 1084 return 0; 1085 1086 err_destroy_page_pool: 1087 page_pool_destroy(rq->page_pool); 1088 err_free_by_rq_type: 1089 switch (rq->wq_type) { 1090 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 1091 mlx5e_rq_free_shampo(rq); 1092 err_free_mpwqe_linear_info: 1093 kvfree(rq->mpwqe.linear_info); 1094 err_free_mpwqe_info: 1095 kvfree(rq->mpwqe.info); 1096 err_rq_mkey: 1097 mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 1098 err_rq_drop_page: 1099 mlx5e_free_mpwqe_rq_drop_page(rq); 1100 break; 1101 default: /* MLX5_WQ_TYPE_CYCLIC */ 1102 mlx5e_free_wqe_alloc_info(rq); 1103 } 1104 err_rq_wq_destroy: 1105 mlx5_wq_destroy(&rq->wq_ctrl); 1106 err_rq_xdp_prog: 1107 if (params->xdp_prog) 1108 bpf_prog_put(params->xdp_prog); 1109 1110 return err; 1111 } 1112 1113 static void mlx5e_free_rq(struct mlx5e_rq *rq) 1114 { 1115 kvfree(rq->dim); 1116 page_pool_destroy(rq->page_pool); 1117 1118 switch (rq->wq_type) { 1119 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 1120 mlx5e_rq_free_shampo(rq); 1121 kvfree(rq->mpwqe.linear_info); 1122 kvfree(rq->mpwqe.info); 1123 mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); 1124 mlx5e_free_mpwqe_rq_drop_page(rq); 1125 break; 1126 default: /* MLX5_WQ_TYPE_CYCLIC */ 1127 mlx5e_free_wqe_alloc_info(rq); 1128 } 1129 1130 mlx5_wq_destroy(&rq->wq_ctrl); 1131 1132 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) { 1133 struct bpf_prog *old_prog; 1134 1135 old_prog = rcu_dereference_protected(rq->xdp_prog, 1136 lockdep_is_held(&rq->priv->state_lock)); 1137 if (old_prog) 1138 bpf_prog_put(old_prog); 1139 } 1140 xdp_rxq_info_unreg(&rq->xdp_rxq); 1141 } 1142 1143 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param, 1144 u16 q_counter) 1145 { 1146 struct mlx5_core_dev *mdev = rq->mdev; 1147 u8 ts_format; 1148 void *in; 1149 void *rqc; 1150 void *wq; 1151 int inlen; 1152 int err; 1153 1154 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + 1155 sizeof(u64) * rq->wq_ctrl.buf.npages; 1156 in = kvzalloc(inlen, GFP_KERNEL); 1157 if (!in) 1158 return -ENOMEM; 1159 1160 ts_format = mlx5_is_real_time_rq(mdev) ? 1161 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 1162 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 1163 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 1164 wq = MLX5_ADDR_OF(rqc, rqc, wq); 1165 1166 memcpy(rqc, rq_param->rqc, sizeof(rq_param->rqc)); 1167 1168 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); 1169 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 1170 MLX5_SET(rqc, rqc, ts_format, ts_format); 1171 MLX5_SET(rqc, rqc, counter_set_id, q_counter); 1172 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - 1173 MLX5_ADAPTER_PAGE_SHIFT); 1174 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); 1175 1176 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) { 1177 MLX5_SET(wq, wq, log_headers_buffer_entry_num, 1178 order_base_2(rq->mpwqe.shampo->hd_per_wq)); 1179 MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey); 1180 } 1181 1182 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, 1183 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1184 1185 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); 1186 1187 kvfree(in); 1188 1189 return err; 1190 } 1191 1192 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) 1193 { 1194 struct mlx5_core_dev *mdev = rq->mdev; 1195 1196 void *in; 1197 void *rqc; 1198 int inlen; 1199 int err; 1200 1201 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1202 in = kvzalloc(inlen, GFP_KERNEL); 1203 if (!in) 1204 return -ENOMEM; 1205 1206 if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY) 1207 mlx5e_rqwq_reset(rq); 1208 1209 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1210 1211 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 1212 MLX5_SET(rqc, rqc, state, next_state); 1213 1214 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1215 1216 kvfree(in); 1217 1218 return err; 1219 } 1220 1221 static void mlx5e_flush_rq_cq(struct mlx5e_rq *rq) 1222 { 1223 struct mlx5_cqwq *cqwq = &rq->cq.wq; 1224 struct mlx5_cqe64 *cqe; 1225 1226 if (test_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state)) { 1227 while ((cqe = mlx5_cqwq_get_cqe_enhanced_comp(cqwq))) 1228 mlx5_cqwq_pop(cqwq); 1229 } else { 1230 while ((cqe = mlx5_cqwq_get_cqe(cqwq))) 1231 mlx5_cqwq_pop(cqwq); 1232 } 1233 1234 mlx5_cqwq_update_db_record(cqwq); 1235 } 1236 1237 int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state) 1238 { 1239 struct net_device *dev = rq->netdev; 1240 int err; 1241 1242 err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST); 1243 if (err) { 1244 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn); 1245 return err; 1246 } 1247 1248 mlx5e_free_rx_descs(rq); 1249 mlx5e_flush_rq_cq(rq); 1250 1251 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1252 if (err) { 1253 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn); 1254 return err; 1255 } 1256 1257 return 0; 1258 } 1259 1260 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) 1261 { 1262 struct mlx5_core_dev *mdev = rq->mdev; 1263 void *in; 1264 void *rqc; 1265 int inlen; 1266 int err; 1267 1268 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 1269 in = kvzalloc(inlen, GFP_KERNEL); 1270 if (!in) 1271 return -ENOMEM; 1272 1273 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 1274 1275 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 1276 MLX5_SET64(modify_rq_in, in, modify_bitmask, 1277 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD); 1278 MLX5_SET(rqc, rqc, vsd, vsd); 1279 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 1280 1281 err = mlx5_core_modify_rq(mdev, rq->rqn, in); 1282 1283 kvfree(in); 1284 1285 return err; 1286 } 1287 1288 void mlx5e_destroy_rq(struct mlx5e_rq *rq) 1289 { 1290 mlx5_core_destroy_rq(rq->mdev, rq->rqn); 1291 } 1292 1293 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time) 1294 { 1295 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time); 1296 1297 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq)); 1298 1299 do { 1300 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes) 1301 return 0; 1302 1303 msleep(20); 1304 } while (time_before(jiffies, exp_time)); 1305 1306 netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n", 1307 rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes); 1308 1309 queue_work(rq->priv->wq, &rq->rx_timeout_work); 1310 1311 return -ETIMEDOUT; 1312 } 1313 1314 void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq) 1315 { 1316 struct mlx5_wq_ll *wq; 1317 u16 head; 1318 int i; 1319 1320 if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) 1321 return; 1322 1323 wq = &rq->mpwqe.wq; 1324 head = wq->head; 1325 1326 /* Release WQEs that are in missing state: they have been 1327 * popped from the list after completion but were not freed 1328 * due to deferred release. 1329 * Also free the linked-list reserved entry, hence the "+ 1". 1330 */ 1331 for (i = 0; i < mlx5_wq_ll_missing(wq) + 1; i++) { 1332 rq->dealloc_wqe(rq, head); 1333 head = mlx5_wq_ll_get_wqe_next_ix(wq, head); 1334 } 1335 1336 rq->mpwqe.actual_wq_head = wq->head; 1337 rq->mpwqe.umr_in_progress = 0; 1338 rq->mpwqe.umr_completed = 0; 1339 } 1340 1341 void mlx5e_free_rx_descs(struct mlx5e_rq *rq) 1342 { 1343 __be16 wqe_ix_be; 1344 u16 wqe_ix; 1345 1346 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 1347 struct mlx5_wq_ll *wq = &rq->mpwqe.wq; 1348 1349 mlx5e_free_rx_missing_descs(rq); 1350 1351 while (!mlx5_wq_ll_is_empty(wq)) { 1352 struct mlx5e_rx_wqe_ll *wqe; 1353 1354 wqe_ix_be = *wq->tail_next; 1355 wqe_ix = be16_to_cpu(wqe_ix_be); 1356 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix); 1357 rq->dealloc_wqe(rq, wqe_ix); 1358 mlx5_wq_ll_pop(wq, wqe_ix_be, 1359 &wqe->next.next_wqe_index); 1360 } 1361 1362 mlx5e_mpwqe_dealloc_linear_page(rq); 1363 } else { 1364 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 1365 u16 missing = mlx5_wq_cyc_missing(wq); 1366 u16 head = mlx5_wq_cyc_get_head(wq); 1367 1368 while (!mlx5_wq_cyc_is_empty(wq)) { 1369 wqe_ix = mlx5_wq_cyc_get_tail(wq); 1370 rq->dealloc_wqe(rq, wqe_ix); 1371 mlx5_wq_cyc_pop(wq); 1372 } 1373 /* Missing slots might also contain unreleased pages due to 1374 * deferred release. 1375 */ 1376 while (missing--) { 1377 wqe_ix = mlx5_wq_cyc_ctr2ix(wq, head++); 1378 rq->dealloc_wqe(rq, wqe_ix); 1379 } 1380 } 1381 1382 } 1383 1384 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param, 1385 struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter, 1386 struct mlx5e_rq *rq) 1387 { 1388 struct mlx5_core_dev *mdev = rq->mdev; 1389 int err; 1390 1391 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) 1392 __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state); 1393 1394 err = mlx5e_alloc_rq(params, rq_param, rqo, node, rq); 1395 if (err) 1396 return err; 1397 1398 err = mlx5e_create_rq(rq, rq_param, q_counter); 1399 if (err) 1400 goto err_free_rq; 1401 1402 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 1403 if (err) 1404 goto err_destroy_rq; 1405 1406 if (MLX5_CAP_ETH(mdev, cqe_checksum_full)) 1407 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state); 1408 1409 if (rq->channel && !params->rx_dim_enabled) { 1410 rq->channel->rx_cq_moder = params->rx_cq_moderation; 1411 } else if (rq->channel) { 1412 u8 cq_period_mode; 1413 1414 cq_period_mode = params->rx_moder_use_cqe_mode ? 1415 DIM_CQ_PERIOD_MODE_START_FROM_CQE : 1416 DIM_CQ_PERIOD_MODE_START_FROM_EQE; 1417 mlx5e_reset_rx_moderation(&rq->channel->rx_cq_moder, cq_period_mode, 1418 params->rx_dim_enabled); 1419 1420 err = mlx5e_dim_rx_change(rq, params->rx_dim_enabled); 1421 if (err) 1422 goto err_destroy_rq; 1423 } 1424 1425 /* We disable csum_complete when XDP is enabled since 1426 * XDP programs might manipulate packets which will render 1427 * skb->checksum incorrect. 1428 */ 1429 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog) 1430 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state); 1431 1432 /* For CQE compression on striding RQ, use stride index provided by 1433 * HW if capability is supported. 1434 */ 1435 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) && 1436 MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) 1437 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state); 1438 1439 /* For enhanced CQE compression packet processing. decompress 1440 * session according to the enhanced layout. 1441 */ 1442 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) && 1443 MLX5_CAP_GEN(mdev, enhanced_cqe_compression)) 1444 __set_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state); 1445 1446 return 0; 1447 1448 err_destroy_rq: 1449 mlx5e_destroy_rq(rq); 1450 err_free_rq: 1451 mlx5e_free_rq(rq); 1452 1453 return err; 1454 } 1455 1456 void mlx5e_activate_rq(struct mlx5e_rq *rq) 1457 { 1458 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1459 } 1460 1461 void mlx5e_deactivate_rq(struct mlx5e_rq *rq) 1462 { 1463 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 1464 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */ 1465 } 1466 1467 void mlx5e_close_rq(struct mlx5e_rq *rq) 1468 { 1469 if (rq->dim) 1470 cancel_work_sync(&rq->dim->work); 1471 cancel_work_sync(&rq->recover_work); 1472 cancel_work_sync(&rq->rx_timeout_work); 1473 mlx5e_destroy_rq(rq); 1474 mlx5e_free_rx_descs(rq); 1475 mlx5e_free_rq(rq); 1476 } 1477 1478 u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev, 1479 struct mlx5e_priv *priv, 1480 const struct mlx5e_profile *profile, 1481 u8 lag_port, u8 tc) 1482 { 1483 if (profile->get_tisn) 1484 return profile->get_tisn(mdev, priv, lag_port, tc); 1485 1486 return mdev->mlx5e_res.hw_objs.tisn[lag_port][tc]; 1487 } 1488 1489 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) 1490 { 1491 kvfree(sq->db.xdpi_fifo.xi); 1492 kvfree(sq->db.wqe_info); 1493 } 1494 1495 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa) 1496 { 1497 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 1498 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1499 int entries; 1500 size_t size; 1501 1502 /* upper bound for maximum num of entries of all xmit_modes. */ 1503 entries = roundup_pow_of_two(wq_sz * MLX5_SEND_WQEBB_NUM_DS * 1504 MLX5E_XDP_FIFO_ENTRIES2DS_MAX_RATIO); 1505 1506 size = array_size(sizeof(*xdpi_fifo->xi), entries); 1507 xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa); 1508 if (!xdpi_fifo->xi) 1509 return -ENOMEM; 1510 1511 xdpi_fifo->pc = &sq->xdpi_fifo_pc; 1512 xdpi_fifo->cc = &sq->xdpi_fifo_cc; 1513 xdpi_fifo->mask = entries - 1; 1514 1515 return 0; 1516 } 1517 1518 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) 1519 { 1520 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1521 size_t size; 1522 int err; 1523 1524 size = array_size(sizeof(*sq->db.wqe_info), wq_sz); 1525 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1526 if (!sq->db.wqe_info) 1527 return -ENOMEM; 1528 1529 err = mlx5e_alloc_xdpsq_fifo(sq, numa); 1530 if (err) { 1531 mlx5e_free_xdpsq_db(sq); 1532 return err; 1533 } 1534 1535 return 0; 1536 } 1537 1538 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 1539 struct mlx5e_params *params, 1540 struct xsk_buff_pool *xsk_pool, 1541 struct mlx5e_sq_param *param, 1542 struct mlx5e_xdpsq *sq, 1543 bool is_redirect) 1544 { 1545 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1546 struct mlx5_core_dev *mdev = c->mdev; 1547 struct mlx5_wq_cyc *wq = &sq->wq; 1548 int err; 1549 1550 sq->pdev = c->pdev; 1551 sq->mkey_be = c->mkey_be; 1552 sq->channel = c; 1553 sq->uar_map = c->bfreg->map; 1554 sq->min_inline_mode = params->tx_min_inline_mode; 1555 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN; 1556 sq->xsk_pool = xsk_pool; 1557 1558 sq->stats = sq->xsk_pool ? 1559 &c->priv->channel_stats[c->ix]->xsksq : 1560 is_redirect ? 1561 &c->priv->channel_stats[c->ix]->xdpsq : 1562 &c->priv->channel_stats[c->ix]->rq_xdpsq; 1563 sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) : 1564 mlx5e_stop_room_for_max_wqe(mdev); 1565 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1566 1567 param->wq.db_numa_node = cpu_to_node(c->cpu); 1568 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1569 if (err) 1570 return err; 1571 wq->db = &wq->db[MLX5_SND_DBR]; 1572 1573 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu)); 1574 if (err) 1575 goto err_sq_wq_destroy; 1576 1577 return 0; 1578 1579 err_sq_wq_destroy: 1580 mlx5_wq_destroy(&sq->wq_ctrl); 1581 1582 return err; 1583 } 1584 1585 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq) 1586 { 1587 mlx5e_free_xdpsq_db(sq); 1588 mlx5_wq_destroy(&sq->wq_ctrl); 1589 } 1590 1591 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) 1592 { 1593 kvfree(sq->db.wqe_info); 1594 } 1595 1596 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa) 1597 { 1598 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1599 size_t size; 1600 1601 size = array_size(wq_sz, sizeof(*sq->db.wqe_info)); 1602 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa); 1603 if (!sq->db.wqe_info) 1604 return -ENOMEM; 1605 1606 return 0; 1607 } 1608 1609 static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work) 1610 { 1611 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1612 recover_work); 1613 1614 mlx5e_reporter_icosq_cqe_err(sq); 1615 } 1616 1617 static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work) 1618 { 1619 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1620 recover_work); 1621 1622 /* Not implemented yet. */ 1623 1624 netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n"); 1625 } 1626 1627 static int mlx5e_alloc_icosq(struct mlx5e_channel *c, 1628 struct mlx5e_sq_param *param, 1629 struct mlx5e_icosq *sq, 1630 work_func_t recover_work_func) 1631 { 1632 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1633 struct mlx5_core_dev *mdev = c->mdev; 1634 struct mlx5_wq_cyc *wq = &sq->wq; 1635 int err; 1636 1637 sq->channel = c; 1638 sq->uar_map = c->bfreg->map; 1639 sq->reserved_room = param->stop_room; 1640 1641 param->wq.db_numa_node = cpu_to_node(c->cpu); 1642 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1643 if (err) 1644 return err; 1645 wq->db = &wq->db[MLX5_SND_DBR]; 1646 1647 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu)); 1648 if (err) 1649 goto err_sq_wq_destroy; 1650 1651 INIT_WORK(&sq->recover_work, recover_work_func); 1652 1653 return 0; 1654 1655 err_sq_wq_destroy: 1656 mlx5_wq_destroy(&sq->wq_ctrl); 1657 1658 return err; 1659 } 1660 1661 static void mlx5e_free_icosq(struct mlx5e_icosq *sq) 1662 { 1663 mlx5e_free_icosq_db(sq); 1664 mlx5_wq_destroy(&sq->wq_ctrl); 1665 } 1666 1667 void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq) 1668 { 1669 kvfree(sq->db.wqe_info); 1670 kvfree(sq->db.skb_fifo.fifo); 1671 kvfree(sq->db.dma_fifo); 1672 } 1673 1674 int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa) 1675 { 1676 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1677 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 1678 1679 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz, 1680 sizeof(*sq->db.dma_fifo)), 1681 GFP_KERNEL, numa); 1682 sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz, 1683 sizeof(*sq->db.skb_fifo.fifo)), 1684 GFP_KERNEL, numa); 1685 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz, 1686 sizeof(*sq->db.wqe_info)), 1687 GFP_KERNEL, numa); 1688 if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) { 1689 mlx5e_free_txqsq_db(sq); 1690 return -ENOMEM; 1691 } 1692 1693 sq->dma_fifo_mask = df_sz - 1; 1694 1695 sq->db.skb_fifo.pc = &sq->skb_fifo_pc; 1696 sq->db.skb_fifo.cc = &sq->skb_fifo_cc; 1697 sq->db.skb_fifo.mask = df_sz - 1; 1698 1699 return 0; 1700 } 1701 1702 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 1703 int txq_ix, 1704 struct mlx5e_params *params, 1705 struct mlx5e_sq_param *param, 1706 struct mlx5e_txqsq *sq, 1707 int tc) 1708 { 1709 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1710 struct mlx5_core_dev *mdev = c->mdev; 1711 struct mlx5_wq_cyc *wq = &sq->wq; 1712 int err; 1713 1714 sq->pdev = c->pdev; 1715 sq->clock = mdev->clock; 1716 sq->mkey_be = c->mkey_be; 1717 sq->netdev = c->netdev; 1718 sq->mdev = c->mdev; 1719 sq->channel = c; 1720 sq->priv = c->priv; 1721 sq->ch_ix = c->ix; 1722 sq->txq_ix = txq_ix; 1723 sq->uar_map = c->bfreg->map; 1724 sq->min_inline_mode = params->tx_min_inline_mode; 1725 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 1726 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev); 1727 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); 1728 if (mlx5_ipsec_device_caps(c->priv->mdev)) 1729 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state); 1730 if (param->is_mpw) 1731 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state); 1732 sq->stop_room = param->stop_room; 1733 sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev); 1734 1735 param->wq.db_numa_node = cpu_to_node(c->cpu); 1736 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1737 if (err) 1738 return err; 1739 wq->db = &wq->db[MLX5_SND_DBR]; 1740 1741 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu)); 1742 if (err) 1743 goto err_sq_wq_destroy; 1744 1745 return 0; 1746 1747 err_sq_wq_destroy: 1748 mlx5_wq_destroy(&sq->wq_ctrl); 1749 1750 return err; 1751 } 1752 1753 void mlx5e_free_txqsq(struct mlx5e_txqsq *sq) 1754 { 1755 kvfree(sq->dim); 1756 mlx5e_free_txqsq_db(sq); 1757 mlx5_wq_destroy(&sq->wq_ctrl); 1758 } 1759 1760 static int mlx5e_create_sq(struct mlx5_core_dev *mdev, 1761 struct mlx5e_sq_param *param, 1762 struct mlx5e_create_sq_param *csp, 1763 u32 *sqn) 1764 { 1765 u8 ts_format; 1766 void *in; 1767 void *sqc; 1768 void *wq; 1769 int inlen; 1770 int err; 1771 1772 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + 1773 sizeof(u64) * csp->wq_ctrl->buf.npages; 1774 in = kvzalloc(inlen, GFP_KERNEL); 1775 if (!in) 1776 return -ENOMEM; 1777 1778 ts_format = mlx5_is_real_time_sq(mdev) ? 1779 MLX5_TIMESTAMP_FORMAT_REAL_TIME : 1780 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING; 1781 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 1782 wq = MLX5_ADDR_OF(sqc, sqc, wq); 1783 1784 memcpy(sqc, param->sqc, sizeof(param->sqc)); 1785 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz); 1786 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn); 1787 MLX5_SET(sqc, sqc, cqn, csp->cqn); 1788 MLX5_SET(sqc, sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn); 1789 MLX5_SET(sqc, sqc, ts_format, ts_format); 1790 1791 1792 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) 1793 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode); 1794 1795 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 1796 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 1797 1798 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1799 MLX5_SET(wq, wq, uar_page, csp->uar_page); 1800 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift - 1801 MLX5_ADAPTER_PAGE_SHIFT); 1802 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma); 1803 1804 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf, 1805 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1806 1807 err = mlx5_core_create_sq(mdev, in, inlen, sqn); 1808 1809 kvfree(in); 1810 1811 return err; 1812 } 1813 1814 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, 1815 struct mlx5e_modify_sq_param *p) 1816 { 1817 u64 bitmask = 0; 1818 void *in; 1819 void *sqc; 1820 int inlen; 1821 int err; 1822 1823 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 1824 in = kvzalloc(inlen, GFP_KERNEL); 1825 if (!in) 1826 return -ENOMEM; 1827 1828 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 1829 1830 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state); 1831 MLX5_SET(sqc, sqc, state, p->next_state); 1832 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) { 1833 bitmask |= 1; 1834 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); 1835 } 1836 if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) { 1837 bitmask |= 1 << 2; 1838 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id); 1839 } 1840 MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask); 1841 1842 err = mlx5_core_modify_sq(mdev, sqn, in); 1843 1844 kvfree(in); 1845 1846 return err; 1847 } 1848 1849 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) 1850 { 1851 mlx5_core_destroy_sq(mdev, sqn); 1852 } 1853 1854 int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, 1855 struct mlx5e_sq_param *param, 1856 struct mlx5e_create_sq_param *csp, 1857 u16 qos_queue_group_id, 1858 u32 *sqn) 1859 { 1860 struct mlx5e_modify_sq_param msp = {0}; 1861 int err; 1862 1863 err = mlx5e_create_sq(mdev, param, csp, sqn); 1864 if (err) 1865 return err; 1866 1867 msp.curr_state = MLX5_SQC_STATE_RST; 1868 msp.next_state = MLX5_SQC_STATE_RDY; 1869 if (qos_queue_group_id) { 1870 msp.qos_update = true; 1871 msp.qos_queue_group_id = qos_queue_group_id; 1872 } 1873 err = mlx5e_modify_sq(mdev, *sqn, &msp); 1874 if (err) 1875 mlx5e_destroy_sq(mdev, *sqn); 1876 1877 return err; 1878 } 1879 1880 static int mlx5e_set_sq_maxrate(struct net_device *dev, 1881 struct mlx5e_txqsq *sq, u32 rate); 1882 1883 int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix, 1884 struct mlx5e_params *params, struct mlx5e_sq_param *param, 1885 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id, 1886 struct mlx5e_sq_stats *sq_stats) 1887 { 1888 struct mlx5e_create_sq_param csp = {}; 1889 u32 tx_rate; 1890 int err; 1891 1892 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc); 1893 if (err) 1894 return err; 1895 1896 sq->stats = sq_stats; 1897 1898 csp.tisn = tisn; 1899 csp.tis_lst_sz = 1; 1900 csp.cqn = sq->cq.mcq.cqn; 1901 csp.wq_ctrl = &sq->wq_ctrl; 1902 csp.min_inline_mode = sq->min_inline_mode; 1903 csp.uar_page = c->bfreg->index; 1904 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn); 1905 if (err) 1906 goto err_free_txqsq; 1907 1908 tx_rate = c->priv->tx_rates[sq->txq_ix]; 1909 if (tx_rate) 1910 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate); 1911 1912 if (sq->channel && !params->tx_dim_enabled) { 1913 sq->channel->tx_cq_moder = params->tx_cq_moderation; 1914 } else if (sq->channel) { 1915 u8 cq_period_mode; 1916 1917 cq_period_mode = params->tx_moder_use_cqe_mode ? 1918 DIM_CQ_PERIOD_MODE_START_FROM_CQE : 1919 DIM_CQ_PERIOD_MODE_START_FROM_EQE; 1920 mlx5e_reset_tx_moderation(&sq->channel->tx_cq_moder, 1921 cq_period_mode, 1922 params->tx_dim_enabled); 1923 1924 err = mlx5e_dim_tx_change(sq, params->tx_dim_enabled); 1925 if (err) 1926 goto err_destroy_sq; 1927 } 1928 1929 return 0; 1930 1931 err_destroy_sq: 1932 mlx5e_destroy_sq(c->mdev, sq->sqn); 1933 err_free_txqsq: 1934 mlx5e_free_txqsq(sq); 1935 1936 return err; 1937 } 1938 1939 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) 1940 { 1941 sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix); 1942 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1943 netdev_tx_reset_queue(sq->txq); 1944 netif_tx_start_queue(sq->txq); 1945 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, sq->cq.napi); 1946 } 1947 1948 void mlx5e_tx_disable_queue(struct netdev_queue *txq) 1949 { 1950 __netif_tx_lock_bh(txq); 1951 netif_tx_stop_queue(txq); 1952 __netif_tx_unlock_bh(txq); 1953 } 1954 1955 void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) 1956 { 1957 struct mlx5_wq_cyc *wq = &sq->wq; 1958 1959 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, NULL); 1960 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1961 synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */ 1962 1963 mlx5e_tx_disable_queue(sq->txq); 1964 1965 /* last doorbell out, godspeed .. */ 1966 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) { 1967 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1968 struct mlx5e_tx_wqe *nop; 1969 1970 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) { 1971 .num_wqebbs = 1, 1972 }; 1973 1974 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc); 1975 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl); 1976 } 1977 } 1978 1979 void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) 1980 { 1981 struct mlx5_core_dev *mdev = sq->mdev; 1982 struct mlx5_rate_limit rl = {0}; 1983 1984 if (sq->dim) 1985 cancel_work_sync(&sq->dim->work); 1986 cancel_work_sync(&sq->recover_work); 1987 mlx5e_destroy_sq(mdev, sq->sqn); 1988 if (sq->rate_limit) { 1989 rl.rate = sq->rate_limit; 1990 mlx5_rl_remove_rate(mdev, &rl); 1991 } 1992 mlx5e_free_txqsq_descs(sq); 1993 mlx5e_free_txqsq(sq); 1994 } 1995 1996 void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) 1997 { 1998 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, 1999 recover_work); 2000 2001 mlx5e_reporter_tx_err_cqe(sq); 2002 } 2003 2004 static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) 2005 { 2006 return (struct dim_cq_moder) { 2007 .cq_period_mode = cq_period_mode, 2008 .pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS, 2009 .usec = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? 2010 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE : 2011 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC, 2012 }; 2013 } 2014 2015 bool mlx5e_reset_tx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode, 2016 bool dim_enabled) 2017 { 2018 bool reset_needed = cq_moder->cq_period_mode != cq_period_mode; 2019 2020 if (dim_enabled) 2021 *cq_moder = net_dim_get_def_tx_moderation(cq_period_mode); 2022 else 2023 *cq_moder = mlx5e_get_def_tx_moderation(cq_period_mode); 2024 2025 return reset_needed; 2026 } 2027 2028 bool mlx5e_reset_tx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode, 2029 bool dim_enabled, bool keep_dim_state) 2030 { 2031 bool reset = false; 2032 int i, tc; 2033 2034 for (i = 0; i < chs->num; i++) { 2035 for (tc = 0; tc < mlx5e_get_dcb_num_tc(&chs->params); tc++) { 2036 if (keep_dim_state) 2037 dim_enabled = !!chs->c[i]->sq[tc].dim; 2038 2039 reset |= mlx5e_reset_tx_moderation(&chs->c[i]->tx_cq_moder, 2040 cq_period_mode, dim_enabled); 2041 } 2042 } 2043 2044 return reset; 2045 } 2046 2047 static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, 2048 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq, 2049 work_func_t recover_work_func) 2050 { 2051 struct mlx5e_create_sq_param csp = {}; 2052 int err; 2053 2054 err = mlx5e_alloc_icosq(c, param, sq, recover_work_func); 2055 if (err) 2056 return err; 2057 2058 csp.cqn = sq->cq.mcq.cqn; 2059 csp.wq_ctrl = &sq->wq_ctrl; 2060 csp.min_inline_mode = params->tx_min_inline_mode; 2061 csp.uar_page = c->bfreg->index; 2062 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 2063 if (err) 2064 goto err_free_icosq; 2065 2066 spin_lock_init(&sq->lock); 2067 2068 if (param->is_tls) { 2069 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list(); 2070 if (IS_ERR(sq->ktls_resync)) { 2071 err = PTR_ERR(sq->ktls_resync); 2072 goto err_destroy_icosq; 2073 } 2074 } 2075 return 0; 2076 2077 err_destroy_icosq: 2078 mlx5e_destroy_sq(c->mdev, sq->sqn); 2079 err_free_icosq: 2080 mlx5e_free_icosq(sq); 2081 2082 return err; 2083 } 2084 2085 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq) 2086 { 2087 set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 2088 } 2089 2090 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq) 2091 { 2092 clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 2093 synchronize_net(); /* Sync with NAPI. */ 2094 } 2095 2096 static void mlx5e_close_icosq(struct mlx5e_icosq *sq) 2097 { 2098 struct mlx5e_channel *c = sq->channel; 2099 2100 if (sq->ktls_resync) 2101 mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync); 2102 mlx5e_destroy_sq(c->mdev, sq->sqn); 2103 mlx5e_free_icosq_descs(sq); 2104 mlx5e_free_icosq(sq); 2105 } 2106 2107 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, 2108 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool, 2109 struct mlx5e_xdpsq *sq, bool is_redirect) 2110 { 2111 struct mlx5e_create_sq_param csp = {}; 2112 int err; 2113 2114 err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect); 2115 if (err) 2116 return err; 2117 2118 csp.tis_lst_sz = 1; 2119 csp.tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, 2120 c->lag_port, 0); /* tc = 0 */ 2121 csp.cqn = sq->cq.mcq.cqn; 2122 csp.wq_ctrl = &sq->wq_ctrl; 2123 csp.min_inline_mode = sq->min_inline_mode; 2124 csp.uar_page = c->bfreg->index; 2125 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 2126 2127 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn); 2128 if (err) 2129 goto err_free_xdpsq; 2130 2131 mlx5e_set_xmit_fp(sq, param->is_mpw); 2132 2133 return 0; 2134 2135 err_free_xdpsq: 2136 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 2137 mlx5e_free_xdpsq(sq); 2138 2139 return err; 2140 } 2141 2142 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq) 2143 { 2144 struct mlx5e_channel *c = sq->channel; 2145 2146 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 2147 synchronize_net(); /* Sync with NAPI. */ 2148 2149 mlx5e_destroy_sq(c->mdev, sq->sqn); 2150 mlx5e_free_xdpsq_descs(sq); 2151 mlx5e_free_xdpsq(sq); 2152 } 2153 2154 static struct mlx5e_xdpsq *mlx5e_open_xdpredirect_sq(struct mlx5e_channel *c, 2155 struct mlx5e_params *params, 2156 struct mlx5e_channel_param *cparam, 2157 struct mlx5e_create_cq_param *ccp) 2158 { 2159 struct mlx5e_xdpsq *xdpsq; 2160 int err; 2161 2162 xdpsq = kvzalloc_node(sizeof(*xdpsq), GFP_KERNEL, cpu_to_node(c->cpu)); 2163 if (!xdpsq) 2164 return ERR_PTR(-ENOMEM); 2165 2166 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, 2167 &cparam->xdp_sq.cqp, ccp, &xdpsq->cq); 2168 if (err) 2169 goto err_free_xdpsq; 2170 2171 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, xdpsq, true); 2172 if (err) 2173 goto err_close_xdpsq_cq; 2174 2175 return xdpsq; 2176 2177 err_close_xdpsq_cq: 2178 mlx5e_close_cq(&xdpsq->cq); 2179 err_free_xdpsq: 2180 kvfree(xdpsq); 2181 2182 return ERR_PTR(err); 2183 } 2184 2185 static void mlx5e_close_xdpredirect_sq(struct mlx5e_xdpsq *xdpsq) 2186 { 2187 mlx5e_close_xdpsq(xdpsq); 2188 mlx5e_close_cq(&xdpsq->cq); 2189 kvfree(xdpsq); 2190 } 2191 2192 static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, 2193 struct net_device *netdev, 2194 struct workqueue_struct *workqueue, 2195 struct mlx5_uars_page *uar, 2196 struct mlx5e_cq_param *param, 2197 struct mlx5e_cq *cq) 2198 { 2199 struct mlx5_core_cq *mcq = &cq->mcq; 2200 int err; 2201 u32 i; 2202 2203 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, 2204 &cq->wq_ctrl); 2205 if (err) 2206 return err; 2207 2208 mcq->cqe_sz = 64; 2209 mcq->set_ci_db = cq->wq_ctrl.db.db; 2210 mcq->arm_db = cq->wq_ctrl.db.db + 1; 2211 *mcq->set_ci_db = 0; 2212 mcq->vector = param->eq_ix; 2213 mcq->comp = mlx5e_completion_event; 2214 mcq->event = mlx5e_cq_error_event; 2215 2216 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 2217 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 2218 2219 cqe->op_own = 0xf1; 2220 cqe->validity_iteration_count = 0xff; 2221 } 2222 2223 cq->mdev = mdev; 2224 cq->netdev = netdev; 2225 cq->workqueue = workqueue; 2226 cq->uar = uar; 2227 2228 return 0; 2229 } 2230 2231 static int mlx5e_alloc_cq(struct mlx5_core_dev *mdev, 2232 struct mlx5e_cq_param *param, 2233 struct mlx5e_create_cq_param *ccp, 2234 struct mlx5e_cq *cq) 2235 { 2236 int err; 2237 2238 param->wq.buf_numa_node = ccp->node; 2239 param->wq.db_numa_node = ccp->node; 2240 param->eq_ix = ccp->ix; 2241 2242 err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq, 2243 ccp->uar, param, cq); 2244 2245 cq->napi = ccp->napi; 2246 cq->ch_stats = ccp->ch_stats; 2247 2248 return err; 2249 } 2250 2251 static void mlx5e_free_cq(struct mlx5e_cq *cq) 2252 { 2253 mlx5_wq_destroy(&cq->wq_ctrl); 2254 } 2255 2256 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 2257 { 2258 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 2259 struct mlx5_core_dev *mdev = cq->mdev; 2260 struct mlx5_core_cq *mcq = &cq->mcq; 2261 2262 void *in; 2263 void *cqc; 2264 int inlen; 2265 int eqn; 2266 int err; 2267 2268 err = mlx5_comp_eqn_get(mdev, param->eq_ix, &eqn); 2269 if (err) 2270 return err; 2271 2272 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 2273 sizeof(u64) * cq->wq_ctrl.buf.npages; 2274 in = kvzalloc(inlen, GFP_KERNEL); 2275 if (!in) 2276 return -ENOMEM; 2277 2278 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 2279 2280 memcpy(cqc, param->cqc, sizeof(param->cqc)); 2281 2282 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, 2283 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); 2284 2285 MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(param->cq_period_mode)); 2286 2287 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); 2288 MLX5_SET(cqc, cqc, uar_page, cq->uar->index); 2289 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 2290 MLX5_ADAPTER_PAGE_SHIFT); 2291 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 2292 2293 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); 2294 2295 kvfree(in); 2296 2297 if (err) 2298 return err; 2299 2300 mlx5e_cq_arm(cq); 2301 2302 return 0; 2303 } 2304 2305 static void mlx5e_destroy_cq(struct mlx5e_cq *cq) 2306 { 2307 mlx5_core_destroy_cq(cq->mdev, &cq->mcq); 2308 } 2309 2310 int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder, 2311 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp, 2312 struct mlx5e_cq *cq) 2313 { 2314 int err; 2315 2316 err = mlx5e_alloc_cq(mdev, param, ccp, cq); 2317 if (err) 2318 return err; 2319 2320 err = mlx5e_create_cq(cq, param); 2321 if (err) 2322 goto err_free_cq; 2323 2324 if (MLX5_CAP_GEN(mdev, cq_moderation) && 2325 MLX5_CAP_GEN(mdev, cq_period_mode_modify)) 2326 mlx5e_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts, 2327 mlx5e_cq_period_mode(moder.cq_period_mode)); 2328 return 0; 2329 2330 err_free_cq: 2331 mlx5e_free_cq(cq); 2332 2333 return err; 2334 } 2335 2336 void mlx5e_close_cq(struct mlx5e_cq *cq) 2337 { 2338 mlx5e_destroy_cq(cq); 2339 mlx5e_free_cq(cq); 2340 } 2341 2342 int mlx5e_modify_cq_period_mode(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 2343 u8 cq_period_mode) 2344 { 2345 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {}; 2346 void *cqc; 2347 2348 MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 2349 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 2350 MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(cq_period_mode)); 2351 MLX5_SET(modify_cq_in, in, 2352 modify_field_select_resize_field_select.modify_field_select.modify_field_select, 2353 MLX5_CQ_MODIFY_PERIOD_MODE); 2354 2355 return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 2356 } 2357 2358 int mlx5e_modify_cq_moderation(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 2359 u16 cq_period, u16 cq_max_count, u8 cq_period_mode) 2360 { 2361 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {}; 2362 void *cqc; 2363 2364 MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 2365 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 2366 MLX5_SET(cqc, cqc, cq_period, cq_period); 2367 MLX5_SET(cqc, cqc, cq_max_count, cq_max_count); 2368 MLX5_SET(cqc, cqc, cq_period_mode, cq_period_mode); 2369 MLX5_SET(modify_cq_in, in, 2370 modify_field_select_resize_field_select.modify_field_select.modify_field_select, 2371 MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT | MLX5_CQ_MODIFY_PERIOD_MODE); 2372 2373 return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 2374 } 2375 2376 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, 2377 struct mlx5e_params *params, 2378 struct mlx5e_create_cq_param *ccp, 2379 struct mlx5e_channel_param *cparam) 2380 { 2381 int err; 2382 int tc; 2383 2384 for (tc = 0; tc < c->num_tc; tc++) { 2385 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->txq_sq.cqp, 2386 ccp, &c->sq[tc].cq); 2387 if (err) 2388 goto err_close_tx_cqs; 2389 } 2390 2391 return 0; 2392 2393 err_close_tx_cqs: 2394 for (tc--; tc >= 0; tc--) 2395 mlx5e_close_cq(&c->sq[tc].cq); 2396 2397 return err; 2398 } 2399 2400 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c) 2401 { 2402 int tc; 2403 2404 for (tc = 0; tc < c->num_tc; tc++) 2405 mlx5e_close_cq(&c->sq[tc].cq); 2406 } 2407 2408 static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq) 2409 { 2410 int tc; 2411 2412 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 2413 if (txq - tc_to_txq[tc].offset < tc_to_txq[tc].count) 2414 return tc; 2415 2416 WARN(1, "Unexpected TCs configuration. No match found for txq %u", txq); 2417 return -ENOENT; 2418 } 2419 2420 static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, 2421 u32 *hw_id) 2422 { 2423 int tc; 2424 2425 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) { 2426 *hw_id = 0; 2427 return 0; 2428 } 2429 2430 tc = mlx5e_mqprio_txq_to_tc(params->mqprio.tc_to_txq, txq_ix); 2431 if (tc < 0) 2432 return tc; 2433 2434 if (tc >= params->mqprio.num_tc) { 2435 WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u", 2436 tc, params->mqprio.num_tc); 2437 return -EINVAL; 2438 } 2439 2440 *hw_id = params->mqprio.channel.hw_id[tc]; 2441 return 0; 2442 } 2443 2444 static int mlx5e_open_sqs(struct mlx5e_channel *c, 2445 struct mlx5e_params *params, 2446 struct mlx5e_channel_param *cparam) 2447 { 2448 int err, tc; 2449 2450 for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) { 2451 int txq_ix = c->ix + tc * params->num_channels; 2452 u32 qos_queue_group_id; 2453 u32 tisn; 2454 2455 tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile, 2456 c->lag_port, tc); 2457 err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id); 2458 if (err) 2459 goto err_close_sqs; 2460 2461 err = mlx5e_open_txqsq(c, tisn, txq_ix, 2462 params, &cparam->txq_sq, &c->sq[tc], tc, 2463 qos_queue_group_id, 2464 &c->priv->channel_stats[c->ix]->sq[tc]); 2465 if (err) 2466 goto err_close_sqs; 2467 } 2468 2469 return 0; 2470 2471 err_close_sqs: 2472 for (tc--; tc >= 0; tc--) 2473 mlx5e_close_txqsq(&c->sq[tc]); 2474 2475 return err; 2476 } 2477 2478 static void mlx5e_close_sqs(struct mlx5e_channel *c) 2479 { 2480 int tc; 2481 2482 for (tc = 0; tc < c->num_tc; tc++) 2483 mlx5e_close_txqsq(&c->sq[tc]); 2484 } 2485 2486 static int mlx5e_set_sq_maxrate(struct net_device *dev, 2487 struct mlx5e_txqsq *sq, u32 rate) 2488 { 2489 struct mlx5e_priv *priv = netdev_priv(dev); 2490 struct mlx5_core_dev *mdev = priv->mdev; 2491 struct mlx5e_modify_sq_param msp = {0}; 2492 struct mlx5_rate_limit rl = {0}; 2493 u16 rl_index = 0; 2494 int err; 2495 2496 if (rate == sq->rate_limit) 2497 /* nothing to do */ 2498 return 0; 2499 2500 if (sq->rate_limit) { 2501 rl.rate = sq->rate_limit; 2502 /* remove current rl index to free space to next ones */ 2503 mlx5_rl_remove_rate(mdev, &rl); 2504 } 2505 2506 sq->rate_limit = 0; 2507 2508 if (rate) { 2509 rl.rate = rate; 2510 err = mlx5_rl_add_rate(mdev, &rl_index, &rl); 2511 if (err) { 2512 netdev_err(dev, "Failed configuring rate %u: %d\n", 2513 rate, err); 2514 return err; 2515 } 2516 } 2517 2518 msp.curr_state = MLX5_SQC_STATE_RDY; 2519 msp.next_state = MLX5_SQC_STATE_RDY; 2520 msp.rl_index = rl_index; 2521 msp.rl_update = true; 2522 err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 2523 if (err) { 2524 netdev_err(dev, "Failed configuring rate %u: %d\n", 2525 rate, err); 2526 /* remove the rate from the table */ 2527 if (rate) 2528 mlx5_rl_remove_rate(mdev, &rl); 2529 return err; 2530 } 2531 2532 sq->rate_limit = rate; 2533 return 0; 2534 } 2535 2536 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 2537 { 2538 struct mlx5e_priv *priv = netdev_priv(dev); 2539 struct mlx5_core_dev *mdev = priv->mdev; 2540 struct mlx5e_txqsq *sq = priv->txq2sq[index]; 2541 int err = 0; 2542 2543 if (!mlx5_rl_is_supported(mdev)) { 2544 netdev_err(dev, "Rate limiting is not supported on this device\n"); 2545 return -EINVAL; 2546 } 2547 2548 /* rate is given in Mb/sec, HW config is in Kb/sec */ 2549 rate = rate << 10; 2550 2551 /* Check whether rate in valid range, 0 is always valid */ 2552 if (rate && !mlx5_rl_is_in_range(mdev, rate)) { 2553 netdev_err(dev, "TX rate %u, is not in range\n", rate); 2554 return -ERANGE; 2555 } 2556 2557 mutex_lock(&priv->state_lock); 2558 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 2559 err = mlx5e_set_sq_maxrate(dev, sq, rate); 2560 if (!err) 2561 priv->tx_rates[index] = rate; 2562 mutex_unlock(&priv->state_lock); 2563 2564 return err; 2565 } 2566 2567 static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, 2568 struct mlx5e_params *params, 2569 struct mlx5e_rq_param *rq_param, 2570 struct mlx5e_rq_opt_param *rqo) 2571 { 2572 u16 q_counter = c->priv->q_counter[c->sd_ix]; 2573 int err; 2574 2575 err = mlx5e_init_rxq_rq(c, params, rq_param->xdp_frag_size, &c->rq); 2576 if (err) 2577 return err; 2578 2579 return mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu), 2580 q_counter, &c->rq); 2581 } 2582 2583 static struct mlx5e_icosq * 2584 mlx5e_open_async_icosq(struct mlx5e_channel *c, 2585 struct mlx5e_params *params, 2586 struct mlx5e_channel_param *cparam, 2587 struct mlx5e_create_cq_param *ccp) 2588 { 2589 struct dim_cq_moder icocq_moder = {0, 0}; 2590 struct mlx5e_icosq *async_icosq; 2591 int err; 2592 2593 async_icosq = kvzalloc_node(sizeof(*async_icosq), GFP_KERNEL, 2594 cpu_to_node(c->cpu)); 2595 if (!async_icosq) 2596 return ERR_PTR(-ENOMEM); 2597 2598 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->async_icosq.cqp, ccp, 2599 &async_icosq->cq); 2600 if (err) 2601 goto err_free_async_icosq; 2602 2603 err = mlx5e_open_icosq(c, params, &cparam->async_icosq, async_icosq, 2604 mlx5e_async_icosq_err_cqe_work); 2605 if (err) 2606 goto err_close_async_icosq_cq; 2607 2608 return async_icosq; 2609 2610 err_close_async_icosq_cq: 2611 mlx5e_close_cq(&async_icosq->cq); 2612 err_free_async_icosq: 2613 kvfree(async_icosq); 2614 return ERR_PTR(err); 2615 } 2616 2617 static void mlx5e_close_async_icosq(struct mlx5e_icosq *async_icosq) 2618 { 2619 mlx5e_close_icosq(async_icosq); 2620 mlx5e_close_cq(&async_icosq->cq); 2621 kvfree(async_icosq); 2622 } 2623 2624 static int mlx5e_open_queues(struct mlx5e_channel *c, 2625 struct mlx5e_params *params, 2626 struct mlx5e_channel_param *cparam, 2627 bool async_icosq_needed) 2628 { 2629 const struct net_device_ops *netdev_ops = c->netdev->netdev_ops; 2630 struct dim_cq_moder icocq_moder = {0, 0}; 2631 struct mlx5e_create_cq_param ccp; 2632 int err; 2633 2634 mlx5e_build_create_cq_param(&ccp, c); 2635 2636 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->icosq.cqp, &ccp, 2637 &c->icosq.cq); 2638 if (err) 2639 return err; 2640 2641 err = mlx5e_open_tx_cqs(c, params, &ccp, cparam); 2642 if (err) 2643 goto err_close_icosq_cq; 2644 2645 if (netdev_ops->ndo_xdp_xmit && c->xdp) { 2646 c->xdpsq = mlx5e_open_xdpredirect_sq(c, params, cparam, &ccp); 2647 if (IS_ERR(c->xdpsq)) { 2648 err = PTR_ERR(c->xdpsq); 2649 goto err_close_tx_cqs; 2650 } 2651 } 2652 2653 err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, 2654 &c->rq.cq); 2655 if (err) 2656 goto err_close_xdpredirect_sq; 2657 2658 err = c->xdp ? mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, 2659 &ccp, &c->rq_xdpsq.cq) : 0; 2660 if (err) 2661 goto err_close_rx_cq; 2662 2663 if (async_icosq_needed) { 2664 c->async_icosq = mlx5e_open_async_icosq(c, params, cparam, 2665 &ccp); 2666 if (IS_ERR(c->async_icosq)) { 2667 err = PTR_ERR(c->async_icosq); 2668 goto err_close_rq_xdpsq_cq; 2669 } 2670 } 2671 2672 mutex_init(&c->icosq_recovery_lock); 2673 2674 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq, 2675 mlx5e_icosq_err_cqe_work); 2676 if (err) 2677 goto err_close_async_icosq; 2678 2679 err = mlx5e_open_sqs(c, params, cparam); 2680 if (err) 2681 goto err_close_icosq; 2682 2683 err = mlx5e_open_rxq_rq(c, params, &cparam->rq, &cparam->rq_opt); 2684 if (err) 2685 goto err_close_sqs; 2686 2687 if (c->xdp) { 2688 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, 2689 &c->rq_xdpsq, false); 2690 if (err) 2691 goto err_close_rq; 2692 } 2693 2694 return 0; 2695 2696 err_close_rq: 2697 mlx5e_close_rq(&c->rq); 2698 2699 err_close_sqs: 2700 mlx5e_close_sqs(c); 2701 2702 err_close_icosq: 2703 mlx5e_close_icosq(&c->icosq); 2704 2705 err_close_async_icosq: 2706 if (c->async_icosq) 2707 mlx5e_close_async_icosq(c->async_icosq); 2708 2709 err_close_rq_xdpsq_cq: 2710 if (c->xdp) 2711 mlx5e_close_cq(&c->rq_xdpsq.cq); 2712 2713 err_close_rx_cq: 2714 mlx5e_close_cq(&c->rq.cq); 2715 2716 err_close_xdpredirect_sq: 2717 if (c->xdpsq) 2718 mlx5e_close_xdpredirect_sq(c->xdpsq); 2719 2720 err_close_tx_cqs: 2721 mlx5e_close_tx_cqs(c); 2722 2723 err_close_icosq_cq: 2724 mlx5e_close_cq(&c->icosq.cq); 2725 2726 return err; 2727 } 2728 2729 static void mlx5e_close_queues(struct mlx5e_channel *c) 2730 { 2731 if (c->xdp) 2732 mlx5e_close_xdpsq(&c->rq_xdpsq); 2733 /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */ 2734 cancel_work_sync(&c->icosq.recover_work); 2735 mlx5e_close_rq(&c->rq); 2736 mlx5e_close_sqs(c); 2737 mlx5e_close_icosq(&c->icosq); 2738 mutex_destroy(&c->icosq_recovery_lock); 2739 if (c->async_icosq) 2740 mlx5e_close_async_icosq(c->async_icosq); 2741 if (c->xdp) 2742 mlx5e_close_cq(&c->rq_xdpsq.cq); 2743 mlx5e_close_cq(&c->rq.cq); 2744 if (c->xdpsq) 2745 mlx5e_close_xdpredirect_sq(c->xdpsq); 2746 mlx5e_close_tx_cqs(c); 2747 mlx5e_close_cq(&c->icosq.cq); 2748 } 2749 2750 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) 2751 { 2752 u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id); 2753 2754 return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev); 2755 } 2756 2757 static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu) 2758 { 2759 if (ix > priv->stats_nch) { 2760 netdev_warn(priv->netdev, "Unexpected channel stats index %d > %d\n", ix, 2761 priv->stats_nch); 2762 return -EINVAL; 2763 } 2764 2765 if (priv->channel_stats[ix]) 2766 return 0; 2767 2768 /* Asymmetric dynamic memory allocation. 2769 * Freed in mlx5e_priv_arrays_free, not on channel closure. 2770 */ 2771 netdev_dbg(priv->netdev, "Creating channel stats %d\n", ix); 2772 priv->channel_stats[ix] = kvzalloc_node(sizeof(**priv->channel_stats), 2773 GFP_KERNEL, cpu_to_node(cpu)); 2774 if (!priv->channel_stats[ix]) 2775 return -ENOMEM; 2776 priv->stats_nch++; 2777 2778 return 0; 2779 } 2780 2781 void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c) 2782 { 2783 struct mlx5e_icosq *sq = &c->icosq; 2784 bool locked; 2785 2786 set_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &sq->state); 2787 synchronize_net(); 2788 2789 locked = mlx5e_icosq_sync_lock(sq); 2790 mlx5e_trigger_irq(sq); 2791 mlx5e_icosq_sync_unlock(sq, locked); 2792 2793 clear_bit(MLX5E_SQ_STATE_LOCK_NEEDED, &sq->state); 2794 } 2795 2796 void mlx5e_trigger_napi_async_icosq(struct mlx5e_channel *c) 2797 { 2798 struct mlx5e_icosq *sq = c->async_icosq; 2799 2800 spin_lock_bh(&sq->lock); 2801 mlx5e_trigger_irq(sq); 2802 spin_unlock_bh(&sq->lock); 2803 } 2804 2805 void mlx5e_trigger_napi_sched(struct napi_struct *napi) 2806 { 2807 local_bh_disable(); 2808 napi_schedule(napi); 2809 local_bh_enable(); 2810 } 2811 2812 static void mlx5e_channel_pick_doorbell(struct mlx5e_channel *c) 2813 { 2814 struct mlx5e_hw_objs *hw_objs = &c->mdev->mlx5e_res.hw_objs; 2815 2816 /* No dedicated Ethernet doorbells, use the global one. */ 2817 if (hw_objs->num_bfregs == 0) { 2818 c->bfreg = &c->mdev->priv.bfreg; 2819 return; 2820 } 2821 2822 /* Round-robin between doorbells. */ 2823 c->bfreg = hw_objs->bfregs + c->vec_ix % hw_objs->num_bfregs; 2824 } 2825 2826 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 2827 struct mlx5e_params *params, 2828 struct netdev_queue_config *qcfg, 2829 struct xsk_buff_pool *xsk_pool, 2830 struct mlx5e_channel **cp) 2831 { 2832 struct net_device *netdev = priv->netdev; 2833 struct mlx5e_channel_param *cparam; 2834 struct mlx5_core_dev *mdev; 2835 struct mlx5e_xsk_param xsk; 2836 bool async_icosq_needed; 2837 struct mlx5e_channel *c; 2838 unsigned int irq; 2839 int vec_ix; 2840 int cpu; 2841 int err; 2842 2843 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); 2844 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); 2845 cpu = mlx5_comp_vector_get_cpu(mdev, vec_ix); 2846 2847 err = mlx5_comp_irqn_get(mdev, vec_ix, &irq); 2848 if (err) 2849 return err; 2850 2851 err = mlx5e_channel_stats_alloc(priv, ix, cpu); 2852 if (err) 2853 return err; 2854 2855 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu)); 2856 cparam = kvzalloc_obj(*cparam); 2857 if (!c || !cparam) { 2858 err = -ENOMEM; 2859 goto err_free; 2860 } 2861 2862 err = mlx5e_build_channel_param(mdev, params, qcfg, cparam); 2863 if (err) 2864 goto err_free; 2865 2866 c->priv = priv; 2867 c->mdev = mdev; 2868 c->ix = ix; 2869 c->vec_ix = vec_ix; 2870 c->sd_ix = mlx5_sd_ch_ix_get_dev_ix(mdev, ix); 2871 c->cpu = cpu; 2872 c->pdev = mlx5_core_dma_dev(mdev); 2873 c->netdev = priv->netdev; 2874 c->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); 2875 c->num_tc = mlx5e_get_dcb_num_tc(params); 2876 c->xdp = !!params->xdp_prog; 2877 c->stats = &priv->channel_stats[ix]->ch; 2878 c->aff_mask = irq_get_effective_affinity_mask(irq); 2879 c->lag_port = mlx5e_enumerate_lag_port(mdev, ix); 2880 2881 mlx5e_channel_pick_doorbell(c); 2882 2883 netif_napi_add_config_locked(netdev, &c->napi, mlx5e_napi_poll, ix); 2884 netif_napi_set_irq_locked(&c->napi, irq); 2885 2886 async_icosq_needed = !!params->xdp_prog || priv->ktls_rx_was_enabled; 2887 err = mlx5e_open_queues(c, params, cparam, async_icosq_needed); 2888 if (unlikely(err)) 2889 goto err_napi_del; 2890 2891 if (xsk_pool) { 2892 mlx5e_build_xsk_param(xsk_pool, &xsk); 2893 mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam); 2894 err = mlx5e_open_xsk(priv, params, cparam, xsk_pool, c); 2895 if (unlikely(err)) 2896 goto err_close_queues; 2897 } 2898 2899 *cp = c; 2900 2901 kvfree(cparam); 2902 return 0; 2903 2904 err_close_queues: 2905 mlx5e_close_queues(c); 2906 2907 err_napi_del: 2908 netif_napi_del_locked(&c->napi); 2909 2910 err_free: 2911 kvfree(cparam); 2912 kvfree(c); 2913 2914 return err; 2915 } 2916 2917 static void mlx5e_activate_channel(struct mlx5e_channel *c) 2918 { 2919 int tc; 2920 2921 napi_enable_locked(&c->napi); 2922 2923 for (tc = 0; tc < c->num_tc; tc++) 2924 mlx5e_activate_txqsq(&c->sq[tc]); 2925 mlx5e_activate_icosq(&c->icosq); 2926 if (c->async_icosq) 2927 mlx5e_activate_icosq(c->async_icosq); 2928 2929 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2930 mlx5e_activate_xsk(c); 2931 else 2932 mlx5e_activate_rq(&c->rq); 2933 2934 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, &c->napi); 2935 } 2936 2937 static void mlx5e_deactivate_channel(struct mlx5e_channel *c) 2938 { 2939 int tc; 2940 2941 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, NULL); 2942 2943 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2944 mlx5e_deactivate_xsk(c); 2945 else 2946 mlx5e_deactivate_rq(&c->rq); 2947 2948 if (c->async_icosq) 2949 mlx5e_deactivate_icosq(c->async_icosq); 2950 mlx5e_deactivate_icosq(&c->icosq); 2951 for (tc = 0; tc < c->num_tc; tc++) 2952 mlx5e_deactivate_txqsq(&c->sq[tc]); 2953 mlx5e_qos_deactivate_queues(c); 2954 2955 napi_disable_locked(&c->napi); 2956 } 2957 2958 static void mlx5e_close_channel(struct mlx5e_channel *c) 2959 { 2960 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2961 mlx5e_close_xsk(c); 2962 mlx5e_close_queues(c); 2963 mlx5e_qos_close_queues(c); 2964 netif_napi_del_locked(&c->napi); 2965 2966 kvfree(c); 2967 } 2968 2969 int mlx5e_open_channels(struct mlx5e_priv *priv, 2970 struct mlx5e_channels *chs) 2971 { 2972 int err = -ENOMEM; 2973 int i; 2974 2975 chs->num = chs->params.num_channels; 2976 2977 chs->c = kzalloc_objs(struct mlx5e_channel *, chs->num); 2978 if (!chs->c) 2979 goto err_out; 2980 2981 for (i = 0; i < chs->num; i++) { 2982 struct xsk_buff_pool *xsk_pool = NULL; 2983 2984 if (chs->params.xdp_prog) 2985 xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i); 2986 2987 err = mlx5e_open_channel(priv, i, &chs->params, NULL, 2988 xsk_pool, &chs->c[i]); 2989 if (err) 2990 goto err_close_channels; 2991 } 2992 2993 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) { 2994 err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp); 2995 if (err) 2996 goto err_close_channels; 2997 } 2998 2999 if (priv->htb) { 3000 err = mlx5e_qos_open_queues(priv, chs); 3001 if (err) 3002 goto err_close_ptp; 3003 } 3004 3005 mlx5e_health_channels_update(priv); 3006 return 0; 3007 3008 err_close_ptp: 3009 if (chs->ptp) 3010 mlx5e_ptp_close(chs->ptp); 3011 3012 err_close_channels: 3013 for (i--; i >= 0; i--) 3014 mlx5e_close_channel(chs->c[i]); 3015 3016 kfree(chs->c); 3017 err_out: 3018 chs->num = 0; 3019 return err; 3020 } 3021 3022 static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs) 3023 { 3024 int i; 3025 3026 for (i = 0; i < chs->num; i++) 3027 mlx5e_activate_channel(chs->c[i]); 3028 3029 if (priv->htb) 3030 mlx5e_qos_activate_queues(priv); 3031 3032 for (i = 0; i < chs->num; i++) 3033 mlx5e_trigger_napi_icosq(chs->c[i]); 3034 3035 if (chs->ptp) 3036 mlx5e_ptp_activate_channel(chs->ptp); 3037 } 3038 3039 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) 3040 { 3041 int err = 0; 3042 int i; 3043 3044 for (i = 0; i < chs->num; i++) { 3045 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; 3046 struct mlx5e_channel *c = chs->c[i]; 3047 3048 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 3049 continue; 3050 3051 err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout); 3052 3053 /* Don't wait on the XSK RQ, because the newer xdpsock sample 3054 * doesn't provide any Fill Ring entries at the setup stage. 3055 */ 3056 } 3057 3058 return err ? -ETIMEDOUT : 0; 3059 } 3060 3061 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs) 3062 { 3063 int i; 3064 3065 if (chs->ptp) 3066 mlx5e_ptp_deactivate_channel(chs->ptp); 3067 3068 for (i = 0; i < chs->num; i++) 3069 mlx5e_deactivate_channel(chs->c[i]); 3070 } 3071 3072 void mlx5e_close_channels(struct mlx5e_channels *chs) 3073 { 3074 int i; 3075 3076 ASSERT_RTNL(); 3077 if (chs->ptp) { 3078 mlx5e_ptp_close(chs->ptp); 3079 chs->ptp = NULL; 3080 } 3081 for (i = 0; i < chs->num; i++) 3082 mlx5e_close_channel(chs->c[i]); 3083 3084 kfree(chs->c); 3085 chs->num = 0; 3086 } 3087 3088 static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv) 3089 { 3090 struct mlx5e_rx_res *res = priv->rx_res; 3091 3092 return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge); 3093 } 3094 3095 static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge); 3096 3097 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev, 3098 struct mlx5e_params *params, u16 mtu) 3099 { 3100 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu); 3101 int err; 3102 3103 err = mlx5_set_port_mtu(mdev, hw_mtu, 1); 3104 if (err) 3105 return err; 3106 3107 /* Update vport context MTU */ 3108 mlx5_modify_nic_vport_mtu(mdev, hw_mtu); 3109 return 0; 3110 } 3111 3112 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev, 3113 struct mlx5e_params *params, u16 *mtu) 3114 { 3115 u16 hw_mtu = 0; 3116 int err; 3117 3118 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 3119 if (err || !hw_mtu) /* fallback to port oper mtu */ 3120 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1); 3121 3122 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu); 3123 } 3124 3125 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) 3126 { 3127 struct mlx5e_params *params = &priv->channels.params; 3128 struct net_device *netdev = priv->netdev; 3129 struct mlx5_core_dev *mdev = priv->mdev; 3130 u16 mtu; 3131 int err; 3132 3133 err = mlx5e_set_mtu(mdev, params, params->sw_mtu); 3134 if (err) 3135 return err; 3136 3137 mlx5e_query_mtu(mdev, params, &mtu); 3138 if (mtu != params->sw_mtu) 3139 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", 3140 __func__, mtu, params->sw_mtu); 3141 3142 params->sw_mtu = mtu; 3143 return 0; 3144 } 3145 3146 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu); 3147 3148 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv) 3149 { 3150 struct mlx5e_params *params = &priv->channels.params; 3151 struct net_device *netdev = priv->netdev; 3152 struct mlx5_core_dev *mdev = priv->mdev; 3153 u16 max_mtu; 3154 3155 /* MTU range: 68 - hw-specific max */ 3156 netdev->min_mtu = ETH_MIN_MTU; 3157 3158 mlx5_query_port_max_mtu(mdev, &max_mtu, 1); 3159 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu), 3160 ETH_MAX_MTU); 3161 } 3162 3163 static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc, 3164 struct netdev_tc_txq *tc_to_txq) 3165 { 3166 int tc, err; 3167 3168 netdev_reset_tc(netdev); 3169 3170 if (ntc == 1) 3171 return 0; 3172 3173 err = netdev_set_num_tc(netdev, ntc); 3174 if (err) { 3175 netdev_WARN(netdev, "netdev_set_num_tc failed (%d), ntc = %d\n", err, ntc); 3176 return err; 3177 } 3178 3179 for (tc = 0; tc < ntc; tc++) { 3180 u16 count, offset; 3181 3182 count = tc_to_txq[tc].count; 3183 offset = tc_to_txq[tc].offset; 3184 netdev_set_tc_queue(netdev, tc, count, offset); 3185 } 3186 3187 return 0; 3188 } 3189 3190 int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv) 3191 { 3192 int nch, ntc, num_txqs, err; 3193 int qos_queues = 0; 3194 3195 if (priv->htb) 3196 qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb); 3197 3198 nch = priv->channels.params.num_channels; 3199 ntc = mlx5e_get_dcb_num_tc(&priv->channels.params); 3200 num_txqs = nch * ntc + qos_queues; 3201 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS)) 3202 num_txqs += ntc; 3203 3204 netdev_dbg(priv->netdev, "Setting num_txqs %d\n", num_txqs); 3205 err = netif_set_real_num_tx_queues(priv->netdev, num_txqs); 3206 if (err) 3207 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err); 3208 3209 return err; 3210 } 3211 3212 static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv, 3213 struct mlx5e_params *params) 3214 { 3215 int ix; 3216 3217 for (ix = 0; ix < params->num_channels; ix++) { 3218 int num_comp_vectors, irq, vec_ix; 3219 struct mlx5_core_dev *mdev; 3220 3221 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix); 3222 num_comp_vectors = mlx5_comp_vectors_max(mdev); 3223 cpumask_clear(priv->scratchpad.cpumask); 3224 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix); 3225 3226 for (irq = vec_ix; irq < num_comp_vectors; irq += params->num_channels) { 3227 int cpu = mlx5_comp_vector_get_cpu(mdev, irq); 3228 3229 cpumask_set_cpu(cpu, priv->scratchpad.cpumask); 3230 } 3231 3232 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix); 3233 } 3234 } 3235 3236 static int mlx5e_update_tc_and_tx_queues(struct mlx5e_priv *priv) 3237 { 3238 struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq; 3239 struct net_device *netdev = priv->netdev; 3240 int old_num_txqs, old_ntc; 3241 int nch, ntc; 3242 int err; 3243 int i; 3244 3245 old_num_txqs = netdev->real_num_tx_queues; 3246 old_ntc = netdev->num_tc ? : 1; 3247 for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++) 3248 old_tc_to_txq[i] = netdev->tc_to_txq[i]; 3249 3250 nch = priv->channels.params.num_channels; 3251 ntc = priv->channels.params.mqprio.num_tc; 3252 tc_to_txq = priv->channels.params.mqprio.tc_to_txq; 3253 3254 err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq); 3255 if (err) 3256 goto err_out; 3257 err = mlx5e_update_tx_netdev_queues(priv); 3258 if (err) 3259 goto err_tcs; 3260 mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params); 3261 3262 return 0; 3263 3264 err_tcs: 3265 WARN_ON_ONCE(mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc, 3266 old_tc_to_txq)); 3267 err_out: 3268 return err; 3269 } 3270 3271 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_tc_and_tx_queues); 3272 3273 static int mlx5e_num_channels_changed(struct mlx5e_priv *priv) 3274 { 3275 u16 count = priv->channels.params.num_channels; 3276 struct net_device *netdev = priv->netdev; 3277 int old_num_rxqs; 3278 int err; 3279 3280 old_num_rxqs = netdev->real_num_rx_queues; 3281 err = netif_set_real_num_rx_queues(netdev, count); 3282 if (err) { 3283 netdev_warn(netdev, "%s: netif_set_real_num_rx_queues failed, %d\n", 3284 __func__, err); 3285 return err; 3286 } 3287 err = mlx5e_update_tc_and_tx_queues(priv); 3288 if (err) { 3289 /* mlx5e_update_tc_and_tx_queues can fail if channels or TCs number increases. 3290 * Since channel number changed, it increased. That means, the call to 3291 * netif_set_real_num_rx_queues below should not fail, because it 3292 * decreases the number of RX queues. 3293 */ 3294 WARN_ON_ONCE(netif_set_real_num_rx_queues(netdev, old_num_rxqs)); 3295 return err; 3296 } 3297 3298 /* This function may be called on attach, before priv->rx_res is created. */ 3299 if (priv->rx_res) 3300 mlx5e_rx_res_rss_update_num_channels(priv->rx_res, count, 3301 netdev); 3302 3303 return 0; 3304 } 3305 3306 MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed); 3307 3308 static void mlx5e_build_txq_maps(struct mlx5e_priv *priv) 3309 { 3310 int i, ch, tc, num_tc; 3311 3312 ch = priv->channels.num; 3313 num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params); 3314 3315 for (i = 0; i < ch; i++) { 3316 for (tc = 0; tc < num_tc; tc++) { 3317 struct mlx5e_channel *c = priv->channels.c[i]; 3318 struct mlx5e_txqsq *sq = &c->sq[tc]; 3319 3320 priv->txq2sq[sq->txq_ix] = sq; 3321 priv->txq2sq_stats[sq->txq_ix] = sq->stats; 3322 } 3323 } 3324 3325 if (!priv->channels.ptp) 3326 goto out; 3327 3328 if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state)) 3329 goto out; 3330 3331 for (tc = 0; tc < num_tc; tc++) { 3332 struct mlx5e_ptp *c = priv->channels.ptp; 3333 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq; 3334 3335 priv->txq2sq[sq->txq_ix] = sq; 3336 priv->txq2sq_stats[sq->txq_ix] = sq->stats; 3337 } 3338 3339 out: 3340 /* Make the change to txq2sq visible before the queue is started. 3341 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE, 3342 * which pairs with this barrier. 3343 */ 3344 smp_wmb(); 3345 } 3346 3347 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) 3348 { 3349 mlx5e_build_txq_maps(priv); 3350 mlx5e_activate_channels(priv, &priv->channels); 3351 mlx5e_xdp_tx_enable(priv); 3352 3353 /* dev_watchdog() wants all TX queues to be started when the carrier is 3354 * OK, including the ones in range real_num_tx_queues..num_tx_queues-1. 3355 * Make it happy to avoid TX timeout false alarms. 3356 */ 3357 netif_tx_start_all_queues(priv->netdev); 3358 3359 if (mlx5e_is_vport_rep(priv)) 3360 mlx5e_rep_activate_channels(priv); 3361 3362 set_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state); 3363 3364 mlx5e_wait_channels_min_rx_wqes(&priv->channels); 3365 3366 if (priv->rx_res) 3367 mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels); 3368 } 3369 3370 static void mlx5e_cancel_tx_timeout_work(struct mlx5e_priv *priv) 3371 { 3372 WARN_ON_ONCE(test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state)); 3373 if (current_work() != &priv->tx_timeout_work) 3374 cancel_work_sync(&priv->tx_timeout_work); 3375 } 3376 3377 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) 3378 { 3379 if (priv->rx_res) 3380 mlx5e_rx_res_channels_deactivate(priv->rx_res); 3381 3382 clear_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state); 3383 mlx5e_cancel_tx_timeout_work(priv); 3384 3385 if (mlx5e_is_vport_rep(priv)) 3386 mlx5e_rep_deactivate_channels(priv); 3387 3388 /* The results of ndo_select_queue are unreliable, while netdev config 3389 * is being changed (real_num_tx_queues, num_tc). Stop all queues to 3390 * prevent ndo_start_xmit from being called, so that it can assume that 3391 * the selected queue is always valid. 3392 */ 3393 netif_tx_disable(priv->netdev); 3394 3395 mlx5e_xdp_tx_disable(priv); 3396 mlx5e_deactivate_channels(&priv->channels); 3397 } 3398 3399 static int mlx5e_switch_priv_params(struct mlx5e_priv *priv, 3400 struct mlx5e_params *new_params, 3401 mlx5e_fp_preactivate preactivate, 3402 void *context) 3403 { 3404 struct mlx5e_params old_params; 3405 3406 old_params = priv->channels.params; 3407 priv->channels.params = *new_params; 3408 3409 if (preactivate) { 3410 int err; 3411 3412 err = preactivate(priv, context); 3413 if (err) { 3414 priv->channels.params = old_params; 3415 return err; 3416 } 3417 } 3418 3419 mlx5e_set_xdp_feature(priv); 3420 return 0; 3421 } 3422 3423 static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv, 3424 struct mlx5e_channels *old_chs, 3425 struct mlx5e_channels *new_chs, 3426 mlx5e_fp_preactivate preactivate, 3427 void *context) 3428 { 3429 struct net_device *netdev = priv->netdev; 3430 int carrier_ok; 3431 int err = 0; 3432 3433 carrier_ok = netif_carrier_ok(netdev); 3434 netif_carrier_off(netdev); 3435 3436 mlx5e_deactivate_priv_channels(priv); 3437 3438 priv->channels = *new_chs; 3439 3440 /* New channels are ready to roll, call the preactivate hook if needed 3441 * to modify HW settings or update kernel parameters. 3442 */ 3443 if (preactivate) { 3444 err = preactivate(priv, context); 3445 if (err) { 3446 priv->channels = *old_chs; 3447 goto out; 3448 } 3449 } 3450 3451 mlx5e_set_xdp_feature(priv); 3452 if (!MLX5_CAP_GEN(priv->mdev, tis_tir_td_order)) 3453 mlx5e_close_channels(old_chs); 3454 priv->profile->update_rx(priv); 3455 3456 mlx5e_selq_apply(&priv->selq); 3457 out: 3458 mlx5e_activate_priv_channels(priv); 3459 3460 /* return carrier back if needed */ 3461 if (carrier_ok) 3462 netif_carrier_on(netdev); 3463 3464 return err; 3465 } 3466 3467 int mlx5e_safe_switch_params(struct mlx5e_priv *priv, 3468 struct mlx5e_params *params, 3469 mlx5e_fp_preactivate preactivate, 3470 void *context, bool reset) 3471 { 3472 struct mlx5e_channels *old_chs, *new_chs; 3473 int err; 3474 3475 reset &= test_bit(MLX5E_STATE_OPENED, &priv->state); 3476 if (!reset) 3477 return mlx5e_switch_priv_params(priv, params, preactivate, context); 3478 3479 old_chs = kzalloc_obj(*old_chs); 3480 new_chs = kzalloc_obj(*new_chs); 3481 if (!old_chs || !new_chs) { 3482 err = -ENOMEM; 3483 goto err_free_chs; 3484 } 3485 3486 new_chs->params = *params; 3487 3488 mlx5e_selq_prepare_params(&priv->selq, &new_chs->params); 3489 3490 err = mlx5e_open_channels(priv, new_chs); 3491 if (err) 3492 goto err_cancel_selq; 3493 3494 *old_chs = priv->channels; 3495 3496 err = mlx5e_switch_priv_channels(priv, old_chs, new_chs, 3497 preactivate, context); 3498 if (err) 3499 goto err_close; 3500 3501 if (MLX5_CAP_GEN(priv->mdev, tis_tir_td_order)) 3502 mlx5e_close_channels(old_chs); 3503 3504 kfree(new_chs); 3505 kfree(old_chs); 3506 return 0; 3507 3508 err_close: 3509 mlx5e_close_channels(new_chs); 3510 3511 err_cancel_selq: 3512 mlx5e_selq_cancel(&priv->selq); 3513 err_free_chs: 3514 kfree(new_chs); 3515 kfree(old_chs); 3516 return err; 3517 } 3518 3519 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv) 3520 { 3521 return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true); 3522 } 3523 3524 void mlx5e_timestamp_init(struct mlx5e_priv *priv) 3525 { 3526 priv->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; 3527 priv->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; 3528 } 3529 3530 static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev, 3531 enum mlx5_port_status state) 3532 { 3533 struct mlx5_eswitch *esw = mdev->priv.eswitch; 3534 int vport_admin_state; 3535 3536 mlx5_set_port_admin_status(mdev, state); 3537 3538 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS || 3539 !MLX5_CAP_GEN(mdev, uplink_follow)) 3540 return; 3541 3542 if (state == MLX5_PORT_UP) 3543 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO; 3544 else 3545 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3546 3547 mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state); 3548 } 3549 3550 int mlx5e_open_locked(struct net_device *netdev) 3551 { 3552 struct mlx5e_priv *priv = netdev_priv(netdev); 3553 int err; 3554 3555 mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params); 3556 3557 set_bit(MLX5E_STATE_OPENED, &priv->state); 3558 3559 err = mlx5e_open_channels(priv, &priv->channels); 3560 if (err) 3561 goto err_clear_state_opened_flag; 3562 3563 err = priv->profile->update_rx(priv); 3564 if (err) 3565 goto err_close_channels; 3566 3567 mlx5e_selq_apply(&priv->selq); 3568 mlx5e_activate_priv_channels(priv); 3569 mlx5e_apply_traps(priv, true); 3570 if (priv->profile->update_carrier) 3571 priv->profile->update_carrier(priv); 3572 3573 mlx5e_queue_update_stats(priv); 3574 return 0; 3575 3576 err_close_channels: 3577 mlx5e_close_channels(&priv->channels); 3578 err_clear_state_opened_flag: 3579 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3580 mlx5e_selq_cancel(&priv->selq); 3581 return err; 3582 } 3583 3584 int mlx5e_open(struct net_device *netdev) 3585 { 3586 struct mlx5e_priv *priv = netdev_priv(netdev); 3587 int err; 3588 3589 mutex_lock(&priv->state_lock); 3590 err = mlx5e_open_locked(netdev); 3591 if (!err) 3592 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP); 3593 mutex_unlock(&priv->state_lock); 3594 3595 return err; 3596 } 3597 3598 int mlx5e_close_locked(struct net_device *netdev) 3599 { 3600 struct mlx5e_priv *priv = netdev_priv(netdev); 3601 3602 /* May already be CLOSED in case a previous configuration operation 3603 * (e.g RX/TX queue size change) that involves close&open failed. 3604 */ 3605 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3606 return 0; 3607 3608 mlx5e_apply_traps(priv, false); 3609 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3610 3611 netif_carrier_off(priv->netdev); 3612 mlx5e_deactivate_priv_channels(priv); 3613 mlx5e_close_channels(&priv->channels); 3614 3615 return 0; 3616 } 3617 3618 int mlx5e_close(struct net_device *netdev) 3619 { 3620 struct mlx5e_priv *priv = netdev_priv(netdev); 3621 int err; 3622 3623 if (!netif_device_present(netdev)) 3624 return -ENODEV; 3625 3626 mutex_lock(&priv->state_lock); 3627 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN); 3628 err = mlx5e_close_locked(netdev); 3629 mutex_unlock(&priv->state_lock); 3630 3631 return err; 3632 } 3633 3634 static void mlx5e_free_drop_rq(struct mlx5e_rq *rq) 3635 { 3636 mlx5_wq_destroy(&rq->wq_ctrl); 3637 } 3638 3639 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, 3640 struct mlx5e_rq *rq, 3641 struct mlx5e_rq_param *rq_param) 3642 { 3643 void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); 3644 int err; 3645 3646 rq_param->wq.db_numa_node = rq_param->wq.buf_numa_node; 3647 3648 err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, &rq->wqe.wq, 3649 &rq->wq_ctrl); 3650 if (err) 3651 return err; 3652 3653 /* Mark as unused given "Drop-RQ" packets never reach XDP */ 3654 xdp_rxq_info_unused(&rq->xdp_rxq); 3655 3656 rq->mdev = mdev; 3657 3658 return 0; 3659 } 3660 3661 static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv, 3662 struct mlx5e_cq *cq, 3663 struct mlx5e_cq_param *param) 3664 { 3665 struct mlx5_core_dev *mdev = priv->mdev; 3666 3667 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3668 param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); 3669 3670 return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq, 3671 mdev->priv.bfreg.up, param, cq); 3672 } 3673 3674 int mlx5e_open_drop_rq(struct mlx5e_priv *priv, 3675 struct mlx5e_rq *drop_rq) 3676 { 3677 struct mlx5_core_dev *mdev = priv->mdev; 3678 struct mlx5e_cq_param cq_param = {}; 3679 struct mlx5e_rq_param rq_param = {}; 3680 struct mlx5e_cq *cq = &drop_rq->cq; 3681 int err; 3682 3683 mlx5e_build_drop_rq_param(mdev, &rq_param); 3684 3685 err = mlx5e_alloc_drop_cq(priv, cq, &cq_param); 3686 if (err) 3687 return err; 3688 3689 err = mlx5e_create_cq(cq, &cq_param); 3690 if (err) 3691 goto err_free_cq; 3692 3693 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param); 3694 if (err) 3695 goto err_destroy_cq; 3696 3697 err = mlx5e_create_rq(drop_rq, &rq_param, priv->drop_rq_q_counter); 3698 if (err) 3699 goto err_free_rq; 3700 3701 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 3702 if (err) 3703 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err); 3704 3705 return 0; 3706 3707 err_free_rq: 3708 mlx5e_free_drop_rq(drop_rq); 3709 3710 err_destroy_cq: 3711 mlx5e_destroy_cq(cq); 3712 3713 err_free_cq: 3714 mlx5e_free_cq(cq); 3715 3716 return err; 3717 } 3718 3719 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) 3720 { 3721 mlx5e_destroy_rq(drop_rq); 3722 mlx5e_free_drop_rq(drop_rq); 3723 mlx5e_destroy_cq(&drop_rq->cq); 3724 mlx5e_free_cq(&drop_rq->cq); 3725 } 3726 3727 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) 3728 { 3729 if (priv->mqprio_rl) { 3730 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3731 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3732 priv->mqprio_rl = NULL; 3733 } 3734 mlx5e_accel_cleanup_tx(priv); 3735 } 3736 3737 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) 3738 { 3739 int err; 3740 int i; 3741 3742 for (i = 0; i < chs->num; i++) { 3743 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd); 3744 if (err) 3745 return err; 3746 } 3747 if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state)) 3748 return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd); 3749 3750 return 0; 3751 } 3752 3753 static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3754 int ntc, int nch) 3755 { 3756 int tc; 3757 3758 memset(tc_to_txq, 0, sizeof(*tc_to_txq) * TC_MAX_QUEUE); 3759 3760 /* Map netdev TCs to offset 0. 3761 * We have our own UP to TXQ mapping for DCB mode of QoS 3762 */ 3763 for (tc = 0; tc < ntc; tc++) { 3764 tc_to_txq[tc] = (struct netdev_tc_txq) { 3765 .count = nch, 3766 .offset = 0, 3767 }; 3768 } 3769 } 3770 3771 static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq, 3772 struct tc_mqprio_qopt *qopt) 3773 { 3774 int tc; 3775 3776 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3777 tc_to_txq[tc] = (struct netdev_tc_txq) { 3778 .count = qopt->count[tc], 3779 .offset = qopt->offset[tc], 3780 }; 3781 } 3782 } 3783 3784 static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc) 3785 { 3786 params->mqprio.mode = TC_MQPRIO_MODE_DCB; 3787 params->mqprio.num_tc = num_tc; 3788 mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc, 3789 params->num_channels); 3790 } 3791 3792 static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params, 3793 struct mlx5e_mqprio_rl *rl) 3794 { 3795 int tc; 3796 3797 for (tc = 0; tc < TC_MAX_QUEUE; tc++) { 3798 u32 hw_id = 0; 3799 3800 if (rl) 3801 mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id); 3802 params->mqprio.channel.hw_id[tc] = hw_id; 3803 } 3804 } 3805 3806 static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params, 3807 struct tc_mqprio_qopt_offload *mqprio, 3808 struct mlx5e_mqprio_rl *rl) 3809 { 3810 int tc; 3811 3812 params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL; 3813 params->mqprio.num_tc = mqprio->qopt.num_tc; 3814 3815 for (tc = 0; tc < TC_MAX_QUEUE; tc++) 3816 params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc]; 3817 3818 mlx5e_mqprio_rl_update_params(params, rl); 3819 mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt); 3820 } 3821 3822 static void mlx5e_params_mqprio_reset(struct mlx5e_params *params) 3823 { 3824 mlx5e_params_mqprio_dcb_set(params, 1); 3825 } 3826 3827 static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, 3828 struct tc_mqprio_qopt *mqprio) 3829 { 3830 struct mlx5e_params new_params; 3831 u8 tc = mqprio->num_tc; 3832 int err; 3833 3834 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 3835 3836 if (tc && tc != MLX5_MAX_NUM_TC) 3837 return -EINVAL; 3838 3839 new_params = priv->channels.params; 3840 mlx5e_params_mqprio_dcb_set(&new_params, tc ? tc : 1); 3841 3842 err = mlx5e_safe_switch_params(priv, &new_params, 3843 mlx5e_update_tc_and_tx_queues_ctx, NULL, true); 3844 3845 if (!err && priv->mqprio_rl) { 3846 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3847 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3848 priv->mqprio_rl = NULL; 3849 } 3850 3851 priv->max_opened_tc = max_t(u8, priv->max_opened_tc, 3852 mlx5e_get_dcb_num_tc(&priv->channels.params)); 3853 return err; 3854 } 3855 3856 static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv, 3857 struct tc_mqprio_qopt_offload *mqprio) 3858 { 3859 struct net_device *netdev = priv->netdev; 3860 struct mlx5e_ptp *ptp_channel; 3861 int agg_count = 0; 3862 int i; 3863 3864 ptp_channel = priv->channels.ptp; 3865 if (ptp_channel && test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state)) { 3866 netdev_err(netdev, 3867 "Cannot activate MQPRIO mode channel since it conflicts with TX port TS\n"); 3868 return -EINVAL; 3869 } 3870 3871 if (mqprio->qopt.offset[0] != 0 || mqprio->qopt.num_tc < 1 || 3872 mqprio->qopt.num_tc > MLX5E_MAX_NUM_MQPRIO_CH_TC) 3873 return -EINVAL; 3874 3875 for (i = 0; i < mqprio->qopt.num_tc; i++) { 3876 if (!mqprio->qopt.count[i]) { 3877 netdev_err(netdev, "Zero size for queue-group (%d) is not supported\n", i); 3878 return -EINVAL; 3879 } 3880 if (mqprio->min_rate[i]) { 3881 netdev_err(netdev, "Min tx rate is not supported\n"); 3882 return -EINVAL; 3883 } 3884 3885 if (mqprio->max_rate[i]) { 3886 int err; 3887 3888 err = mlx5e_qos_bytes_rate_check(priv->mdev, mqprio->max_rate[i]); 3889 if (err) 3890 return err; 3891 } 3892 3893 if (mqprio->qopt.offset[i] != agg_count) { 3894 netdev_err(netdev, "Discontinuous queues config is not supported\n"); 3895 return -EINVAL; 3896 } 3897 agg_count += mqprio->qopt.count[i]; 3898 } 3899 3900 if (priv->channels.params.num_channels != agg_count) { 3901 netdev_err(netdev, "Num of queues (%d) does not match available (%d)\n", 3902 agg_count, priv->channels.params.num_channels); 3903 return -EINVAL; 3904 } 3905 3906 return 0; 3907 } 3908 3909 static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[]) 3910 { 3911 int tc; 3912 3913 for (tc = 0; tc < num_tc; tc++) 3914 if (max_rate[tc]) 3915 return true; 3916 return false; 3917 } 3918 3919 static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev, 3920 u8 num_tc, u64 max_rate[]) 3921 { 3922 struct mlx5e_mqprio_rl *rl; 3923 int err; 3924 3925 if (!mlx5e_mqprio_rate_limit(num_tc, max_rate)) 3926 return NULL; 3927 3928 rl = mlx5e_mqprio_rl_alloc(); 3929 if (!rl) 3930 return ERR_PTR(-ENOMEM); 3931 3932 err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate); 3933 if (err) { 3934 mlx5e_mqprio_rl_free(rl); 3935 return ERR_PTR(err); 3936 } 3937 3938 return rl; 3939 } 3940 3941 static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, 3942 struct tc_mqprio_qopt_offload *mqprio) 3943 { 3944 struct mlx5e_params new_params; 3945 struct mlx5e_mqprio_rl *rl; 3946 int err; 3947 3948 err = mlx5e_mqprio_channel_validate(priv, mqprio); 3949 if (err) 3950 return err; 3951 3952 rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate); 3953 if (IS_ERR(rl)) 3954 return PTR_ERR(rl); 3955 3956 new_params = priv->channels.params; 3957 mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl); 3958 3959 err = mlx5e_safe_switch_params(priv, &new_params, 3960 mlx5e_update_tc_and_tx_queues_ctx, NULL, true); 3961 if (err) { 3962 if (rl) { 3963 mlx5e_mqprio_rl_cleanup(rl); 3964 mlx5e_mqprio_rl_free(rl); 3965 } 3966 return err; 3967 } 3968 3969 if (priv->mqprio_rl) { 3970 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 3971 mlx5e_mqprio_rl_free(priv->mqprio_rl); 3972 } 3973 priv->mqprio_rl = rl; 3974 3975 return 0; 3976 } 3977 3978 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, 3979 struct tc_mqprio_qopt_offload *mqprio) 3980 { 3981 /* MQPRIO is another toplevel qdisc that can't be attached 3982 * simultaneously with the offloaded HTB. 3983 */ 3984 if (mlx5e_selq_is_htb_enabled(&priv->selq)) { 3985 NL_SET_ERR_MSG_MOD(mqprio->extack, 3986 "MQPRIO cannot be configured when HTB offload is enabled."); 3987 return -EOPNOTSUPP; 3988 } 3989 3990 switch (mqprio->mode) { 3991 case TC_MQPRIO_MODE_DCB: 3992 return mlx5e_setup_tc_mqprio_dcb(priv, &mqprio->qopt); 3993 case TC_MQPRIO_MODE_CHANNEL: 3994 return mlx5e_setup_tc_mqprio_channel(priv, mqprio); 3995 default: 3996 return -EOPNOTSUPP; 3997 } 3998 } 3999 4000 static LIST_HEAD(mlx5e_block_cb_list); 4001 4002 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, 4003 void *type_data) 4004 { 4005 struct mlx5e_priv *priv = netdev_priv(dev); 4006 bool tc_unbind = false; 4007 int err; 4008 4009 if (type == TC_SETUP_BLOCK && 4010 ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND) 4011 tc_unbind = true; 4012 4013 if (!netif_device_present(dev) && !tc_unbind) 4014 return -ENODEV; 4015 4016 switch (type) { 4017 case TC_SETUP_BLOCK: { 4018 struct flow_block_offload *f = type_data; 4019 4020 f->unlocked_driver_cb = true; 4021 return flow_block_cb_setup_simple(type_data, 4022 &mlx5e_block_cb_list, 4023 mlx5e_setup_tc_block_cb, 4024 priv, priv, true); 4025 } 4026 case TC_SETUP_QDISC_MQPRIO: 4027 mutex_lock(&priv->state_lock); 4028 err = mlx5e_setup_tc_mqprio(priv, type_data); 4029 mutex_unlock(&priv->state_lock); 4030 return err; 4031 case TC_SETUP_QDISC_HTB: 4032 mutex_lock(&priv->state_lock); 4033 err = mlx5e_htb_setup_tc(priv, type_data); 4034 mutex_unlock(&priv->state_lock); 4035 return err; 4036 default: 4037 return -EOPNOTSUPP; 4038 } 4039 } 4040 4041 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s) 4042 { 4043 int i; 4044 4045 for (i = 0; i < priv->stats_nch; i++) { 4046 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; 4047 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq; 4048 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq; 4049 int j; 4050 4051 s->rx_packets += rq_stats->packets + xskrq_stats->packets; 4052 s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes; 4053 s->multicast += rq_stats->mcast_packets + xskrq_stats->mcast_packets; 4054 4055 for (j = 0; j < priv->max_opened_tc; j++) { 4056 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j]; 4057 4058 s->tx_packets += sq_stats->packets; 4059 s->tx_bytes += sq_stats->bytes; 4060 s->tx_dropped += sq_stats->dropped; 4061 } 4062 } 4063 if (priv->tx_ptp_opened) { 4064 for (i = 0; i < priv->max_opened_tc; i++) { 4065 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i]; 4066 4067 s->tx_packets += sq_stats->packets; 4068 s->tx_bytes += sq_stats->bytes; 4069 s->tx_dropped += sq_stats->dropped; 4070 } 4071 } 4072 if (priv->rx_ptp_opened) { 4073 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq; 4074 4075 s->rx_packets += rq_stats->packets; 4076 s->rx_bytes += rq_stats->bytes; 4077 s->multicast += rq_stats->mcast_packets; 4078 } 4079 4080 #ifdef CONFIG_MLX5_EN_PSP 4081 if (priv->psp) 4082 s->tx_dropped += atomic_read(&priv->psp->tx_drop); 4083 #endif 4084 } 4085 4086 void 4087 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 4088 { 4089 struct mlx5e_priv *priv = netdev_priv(dev); 4090 struct mlx5e_pport_stats *pstats = &priv->stats.pport; 4091 4092 if (!netif_device_present(dev)) 4093 return; 4094 4095 /* In switchdev mode, monitor counters doesn't monitor 4096 * rx/tx stats of 802_3. The update stats mechanism 4097 * should keep the 802_3 layout counters updated 4098 */ 4099 if (!mlx5e_monitor_counter_supported(priv) || 4100 mlx5e_is_uplink_rep(priv)) { 4101 /* update HW stats in background for next time */ 4102 mlx5e_queue_update_stats(priv); 4103 } 4104 4105 netdev_stats_to_stats64(stats, &dev->stats); 4106 4107 if (mlx5e_is_uplink_rep(priv)) { 4108 struct mlx5e_vport_stats *vstats = &priv->stats.vport; 4109 4110 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok); 4111 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok); 4112 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok); 4113 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok); 4114 4115 /* vport multicast also counts packets that are dropped due to steering 4116 * or rx out of buffer 4117 */ 4118 stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets); 4119 } else { 4120 mlx5e_fold_sw_stats64(priv, stats); 4121 } 4122 4123 stats->rx_missed_errors += priv->stats.qcnt.rx_out_of_buffer; 4124 stats->rx_dropped += PPORT_2863_GET(pstats, if_in_discards); 4125 4126 stats->rx_length_errors += 4127 PPORT_802_3_GET(pstats, a_in_range_length_errors) + 4128 PPORT_802_3_GET(pstats, a_out_of_range_length_field) + 4129 PPORT_802_3_GET(pstats, a_frame_too_long_errors) + 4130 VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small); 4131 stats->rx_crc_errors += 4132 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); 4133 stats->rx_frame_errors += PPORT_802_3_GET(pstats, a_alignment_errors); 4134 stats->tx_aborted_errors += PPORT_2863_GET(pstats, if_out_discards); 4135 stats->rx_errors += stats->rx_length_errors + stats->rx_crc_errors + 4136 stats->rx_frame_errors; 4137 stats->tx_errors += stats->tx_aborted_errors + stats->tx_carrier_errors; 4138 } 4139 4140 static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) 4141 { 4142 queue_work(priv->wq, &priv->set_rx_mode_work); 4143 } 4144 4145 static int mlx5e_set_rx_mode(struct net_device *dev, 4146 struct netdev_hw_addr_list *uc, 4147 struct netdev_hw_addr_list *mc) 4148 { 4149 struct mlx5e_priv *priv = netdev_priv(dev); 4150 4151 mlx5e_fs_set_rx_mode_work(priv->fs, dev, uc, mc); 4152 4153 return 0; 4154 } 4155 4156 static int mlx5e_set_mac(struct net_device *netdev, void *addr) 4157 { 4158 struct mlx5e_priv *priv = netdev_priv(netdev); 4159 struct sockaddr *saddr = addr; 4160 4161 if (!is_valid_ether_addr(saddr->sa_data)) 4162 return -EADDRNOTAVAIL; 4163 4164 netif_addr_lock_bh(netdev); 4165 eth_hw_addr_set(netdev, saddr->sa_data); 4166 netif_addr_unlock_bh(netdev); 4167 4168 mlx5e_nic_set_rx_mode(priv); 4169 4170 return 0; 4171 } 4172 4173 #define MLX5E_SET_FEATURE(features, feature, enable) \ 4174 do { \ 4175 if (enable) \ 4176 *features |= feature; \ 4177 else \ 4178 *features &= ~feature; \ 4179 } while (0) 4180 4181 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable); 4182 4183 static int set_feature_lro(struct net_device *netdev, bool enable) 4184 { 4185 struct mlx5e_priv *priv = netdev_priv(netdev); 4186 struct mlx5_core_dev *mdev = priv->mdev; 4187 struct mlx5e_params *cur_params; 4188 struct mlx5e_params new_params; 4189 bool reset = true; 4190 int err = 0; 4191 4192 mutex_lock(&priv->state_lock); 4193 4194 cur_params = &priv->channels.params; 4195 new_params = *cur_params; 4196 4197 if (enable) 4198 new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO; 4199 else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO) 4200 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 4201 else 4202 goto out; 4203 4204 if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO && 4205 new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) { 4206 if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 4207 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) == 4208 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL)) 4209 reset = false; 4210 } 4211 } 4212 4213 err = mlx5e_safe_switch_params(priv, &new_params, 4214 mlx5e_modify_tirs_packet_merge_ctx, NULL, reset); 4215 out: 4216 mutex_unlock(&priv->state_lock); 4217 return err; 4218 } 4219 4220 static int set_feature_hw_gro(struct net_device *netdev, bool enable) 4221 { 4222 struct mlx5e_priv *priv = netdev_priv(netdev); 4223 struct mlx5e_params new_params; 4224 bool reset = true; 4225 int err = 0; 4226 4227 mutex_lock(&priv->state_lock); 4228 new_params = priv->channels.params; 4229 4230 if (enable) { 4231 new_params.packet_merge.type = MLX5E_PACKET_MERGE_SHAMPO; 4232 } else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) { 4233 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE; 4234 } else { 4235 goto out; 4236 } 4237 4238 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 4239 out: 4240 mutex_unlock(&priv->state_lock); 4241 return err; 4242 } 4243 4244 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) 4245 { 4246 struct mlx5e_priv *priv = netdev_priv(netdev); 4247 4248 if (enable) 4249 mlx5e_enable_cvlan_filter(priv->fs, 4250 !!(priv->netdev->flags & IFF_PROMISC)); 4251 else 4252 mlx5e_disable_cvlan_filter(priv->fs, 4253 !!(priv->netdev->flags & IFF_PROMISC)); 4254 4255 return 0; 4256 } 4257 4258 static int set_feature_hw_tc(struct net_device *netdev, bool enable) 4259 { 4260 struct mlx5e_priv *priv = netdev_priv(netdev); 4261 int err = 0; 4262 4263 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 4264 int tc_flag = mlx5e_is_uplink_rep(priv) ? MLX5_TC_FLAG(ESW_OFFLOAD) : 4265 MLX5_TC_FLAG(NIC_OFFLOAD); 4266 if (!enable && mlx5e_tc_num_filters(priv, tc_flag)) { 4267 netdev_err(netdev, 4268 "Active offloaded tc filters, can't turn hw_tc_offload off\n"); 4269 return -EINVAL; 4270 } 4271 #endif 4272 4273 mutex_lock(&priv->state_lock); 4274 if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) { 4275 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n"); 4276 err = -EINVAL; 4277 } 4278 mutex_unlock(&priv->state_lock); 4279 4280 return err; 4281 } 4282 4283 static int set_feature_rx_all(struct net_device *netdev, bool enable) 4284 { 4285 struct mlx5e_priv *priv = netdev_priv(netdev); 4286 struct mlx5_core_dev *mdev = priv->mdev; 4287 4288 return mlx5_set_port_fcs(mdev, !enable); 4289 } 4290 4291 static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode) 4292 { 4293 return (struct dim_cq_moder) { 4294 .cq_period_mode = cq_period_mode, 4295 .pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS, 4296 .usec = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ? 4297 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE : 4298 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC, 4299 }; 4300 } 4301 4302 bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode, 4303 bool dim_enabled) 4304 { 4305 bool reset_needed = cq_moder->cq_period_mode != cq_period_mode; 4306 4307 if (dim_enabled) 4308 *cq_moder = net_dim_get_def_rx_moderation(cq_period_mode); 4309 else 4310 *cq_moder = mlx5e_get_def_rx_moderation(cq_period_mode); 4311 4312 return reset_needed; 4313 } 4314 4315 bool mlx5e_reset_rx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode, 4316 bool dim_enabled, bool keep_dim_state) 4317 { 4318 bool reset = false; 4319 int i; 4320 4321 for (i = 0; i < chs->num; i++) { 4322 if (keep_dim_state) 4323 dim_enabled = !!chs->c[i]->rq.dim; 4324 4325 reset |= mlx5e_reset_rx_moderation(&chs->c[i]->rx_cq_moder, 4326 cq_period_mode, dim_enabled); 4327 } 4328 4329 return reset; 4330 } 4331 4332 static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable) 4333 { 4334 u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {}; 4335 bool supported, curr_state; 4336 int err; 4337 4338 if (!MLX5_CAP_GEN(mdev, ports_check)) 4339 return 0; 4340 4341 err = mlx5_query_ports_check(mdev, in, sizeof(in)); 4342 if (err) 4343 return err; 4344 4345 supported = MLX5_GET(pcmr_reg, in, rx_ts_over_crc_cap); 4346 curr_state = MLX5_GET(pcmr_reg, in, rx_ts_over_crc); 4347 4348 if (!supported || enable == curr_state) 4349 return 0; 4350 4351 MLX5_SET(pcmr_reg, in, local_port, 1); 4352 MLX5_SET(pcmr_reg, in, rx_ts_over_crc, enable); 4353 4354 return mlx5_set_ports_check(mdev, in, sizeof(in)); 4355 } 4356 4357 static int mlx5e_set_rx_port_ts_wrap(struct mlx5e_priv *priv, void *ctx) 4358 { 4359 struct mlx5_core_dev *mdev = priv->mdev; 4360 bool enable = *(bool *)ctx; 4361 4362 return mlx5e_set_rx_port_ts(mdev, enable); 4363 } 4364 4365 static int set_feature_rx_fcs(struct net_device *netdev, bool enable) 4366 { 4367 struct mlx5e_priv *priv = netdev_priv(netdev); 4368 struct mlx5e_channels *chs = &priv->channels; 4369 struct mlx5e_params new_params; 4370 int err; 4371 bool rx_ts_over_crc = !enable; 4372 4373 mutex_lock(&priv->state_lock); 4374 4375 new_params = chs->params; 4376 new_params.scatter_fcs_en = enable; 4377 err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_set_rx_port_ts_wrap, 4378 &rx_ts_over_crc, true); 4379 mutex_unlock(&priv->state_lock); 4380 return err; 4381 } 4382 4383 static int set_feature_rx_vlan(struct net_device *netdev, bool enable) 4384 { 4385 struct mlx5e_priv *priv = netdev_priv(netdev); 4386 int err = 0; 4387 4388 mutex_lock(&priv->state_lock); 4389 4390 mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable); 4391 priv->channels.params.vlan_strip_disable = !enable; 4392 4393 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4394 goto unlock; 4395 4396 err = mlx5e_modify_channels_vsd(&priv->channels, !enable); 4397 if (err) { 4398 mlx5e_fs_set_vlan_strip_disable(priv->fs, enable); 4399 priv->channels.params.vlan_strip_disable = enable; 4400 } 4401 unlock: 4402 mutex_unlock(&priv->state_lock); 4403 4404 return err; 4405 } 4406 4407 int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) 4408 { 4409 struct mlx5e_priv *priv = netdev_priv(dev); 4410 struct mlx5e_flow_steering *fs = priv->fs; 4411 4412 if (mlx5e_is_uplink_rep(priv)) 4413 return 0; /* no vlan table for uplink rep */ 4414 4415 return mlx5e_fs_vlan_rx_add_vid(fs, dev, proto, vid); 4416 } 4417 4418 int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) 4419 { 4420 struct mlx5e_priv *priv = netdev_priv(dev); 4421 struct mlx5e_flow_steering *fs = priv->fs; 4422 4423 if (mlx5e_is_uplink_rep(priv)) 4424 return 0; /* no vlan table for uplink rep */ 4425 4426 return mlx5e_fs_vlan_rx_kill_vid(fs, dev, proto, vid); 4427 } 4428 4429 #ifdef CONFIG_MLX5_EN_ARFS 4430 static int set_feature_arfs(struct net_device *netdev, bool enable) 4431 { 4432 struct mlx5e_priv *priv = netdev_priv(netdev); 4433 int err; 4434 4435 if (enable) 4436 err = mlx5e_arfs_enable(priv->fs); 4437 else 4438 err = mlx5e_arfs_disable(priv->fs); 4439 4440 return err; 4441 } 4442 #endif 4443 4444 static int mlx5e_handle_feature(struct net_device *netdev, 4445 netdev_features_t *features, 4446 netdev_features_t feature, 4447 mlx5e_feature_handler feature_handler) 4448 { 4449 netdev_features_t changes = *features ^ netdev->features; 4450 bool enable = !!(*features & feature); 4451 int err; 4452 4453 if (!(changes & feature)) 4454 return 0; 4455 4456 err = feature_handler(netdev, enable); 4457 if (err) { 4458 MLX5E_SET_FEATURE(features, feature, !enable); 4459 netdev_err(netdev, "%s feature %pNF failed, err %d\n", 4460 enable ? "Enable" : "Disable", &feature, err); 4461 return err; 4462 } 4463 4464 return 0; 4465 } 4466 4467 void mlx5e_set_xdp_feature(struct mlx5e_priv *priv) 4468 { 4469 struct mlx5e_params *params = &priv->channels.params; 4470 struct net_device *netdev = priv->netdev; 4471 xdp_features_t val = 0; 4472 4473 if (netdev->netdev_ops->ndo_bpf && 4474 params->packet_merge.type == MLX5E_PACKET_MERGE_NONE) 4475 val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 4476 NETDEV_XDP_ACT_XSK_ZEROCOPY | 4477 NETDEV_XDP_ACT_RX_SG; 4478 4479 if (netdev->netdev_ops->ndo_xdp_xmit && params->xdp_prog) 4480 val |= NETDEV_XDP_ACT_NDO_XMIT | 4481 NETDEV_XDP_ACT_NDO_XMIT_SG; 4482 4483 xdp_set_features_flag_locked(netdev, val); 4484 } 4485 4486 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) 4487 { 4488 netdev_features_t oper_features = features; 4489 int err = 0; 4490 4491 #define MLX5E_HANDLE_FEATURE(feature, handler) \ 4492 mlx5e_handle_feature(netdev, &oper_features, feature, handler) 4493 4494 if (features & (NETIF_F_GRO_HW | NETIF_F_LRO)) { 4495 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4496 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4497 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4498 } else { 4499 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 4500 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro); 4501 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 4502 } 4503 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, 4504 set_feature_cvlan_filter); 4505 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc); 4506 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); 4507 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); 4508 #ifdef CONFIG_MLX5_EN_ARFS 4509 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); 4510 #endif 4511 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx); 4512 4513 if (err) { 4514 netdev->features = oper_features; 4515 return -EINVAL; 4516 } 4517 4518 return 0; 4519 } 4520 4521 static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev, 4522 netdev_features_t features) 4523 { 4524 features &= ~NETIF_F_HW_TLS_RX; 4525 if (netdev->features & NETIF_F_HW_TLS_RX) 4526 netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n"); 4527 4528 features &= ~NETIF_F_HW_TLS_TX; 4529 if (netdev->features & NETIF_F_HW_TLS_TX) 4530 netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n"); 4531 4532 features &= ~NETIF_F_NTUPLE; 4533 if (netdev->features & NETIF_F_NTUPLE) 4534 netdev_warn(netdev, "Disabling ntuple, not supported in switchdev mode\n"); 4535 4536 features &= ~NETIF_F_GRO_HW; 4537 if (netdev->features & NETIF_F_GRO_HW) 4538 netdev_warn(netdev, "Disabling HW_GRO, not supported in switchdev mode\n"); 4539 4540 features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4541 if (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER) 4542 netdev_warn(netdev, "Disabling HW_VLAN CTAG FILTERING, not supported in switchdev mode\n"); 4543 4544 features &= ~NETIF_F_HW_MACSEC; 4545 if (netdev->features & NETIF_F_HW_MACSEC) 4546 netdev_warn(netdev, "Disabling HW MACsec offload, not supported in switchdev mode\n"); 4547 4548 return features; 4549 } 4550 4551 static netdev_features_t mlx5e_fix_features(struct net_device *netdev, 4552 netdev_features_t features) 4553 { 4554 struct netdev_config *cfg = netdev->cfg_pending; 4555 struct mlx5e_priv *priv = netdev_priv(netdev); 4556 struct mlx5e_vlan_table *vlan; 4557 struct mlx5e_params *params; 4558 4559 if (!netif_device_present(netdev)) 4560 return features; 4561 4562 vlan = mlx5e_fs_get_vlan(priv->fs); 4563 mutex_lock(&priv->state_lock); 4564 params = &priv->channels.params; 4565 if (!vlan || 4566 !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) { 4567 /* HW strips the outer C-tag header, this is a problem 4568 * for S-tag traffic. 4569 */ 4570 features &= ~NETIF_F_HW_VLAN_CTAG_RX; 4571 if (!params->vlan_strip_disable) 4572 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n"); 4573 } 4574 4575 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 4576 if (features & NETIF_F_LRO) { 4577 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n"); 4578 features &= ~NETIF_F_LRO; 4579 } 4580 if (features & NETIF_F_GRO_HW) { 4581 netdev_warn(netdev, "Disabling HW-GRO, not supported in legacy RQ\n"); 4582 features &= ~NETIF_F_GRO_HW; 4583 } 4584 } 4585 4586 if (params->xdp_prog) { 4587 if (features & NETIF_F_LRO) { 4588 netdev_warn(netdev, "LRO is incompatible with XDP\n"); 4589 features &= ~NETIF_F_LRO; 4590 } 4591 if (features & NETIF_F_GRO_HW) { 4592 netdev_warn(netdev, "HW GRO is incompatible with XDP\n"); 4593 features &= ~NETIF_F_GRO_HW; 4594 } 4595 } 4596 4597 if (priv->xsk.refcnt) { 4598 if (features & NETIF_F_LRO) { 4599 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n", 4600 priv->xsk.refcnt); 4601 features &= ~NETIF_F_LRO; 4602 } 4603 if (features & NETIF_F_GRO_HW) { 4604 netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n", 4605 priv->xsk.refcnt); 4606 features &= ~NETIF_F_GRO_HW; 4607 } 4608 } 4609 4610 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 4611 features &= ~NETIF_F_RXHASH; 4612 if (netdev->features & NETIF_F_RXHASH) 4613 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n"); 4614 4615 if (features & NETIF_F_GRO_HW) { 4616 netdev_warn(netdev, "Disabling HW-GRO, not supported when CQE compress is active\n"); 4617 features &= ~NETIF_F_GRO_HW; 4618 } 4619 } 4620 4621 /* The header-data split ring param requires HW GRO to stay enabled. */ 4622 if (cfg && cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 4623 !(features & NETIF_F_GRO_HW)) { 4624 netdev_warn(netdev, "Keeping HW-GRO enabled, TCP header-data split depends on it\n"); 4625 features |= NETIF_F_GRO_HW; 4626 } 4627 4628 if (mlx5e_is_uplink_rep(priv)) { 4629 features = mlx5e_fix_uplink_rep_features(netdev, features); 4630 netdev->netns_immutable = true; 4631 } else { 4632 netdev->netns_immutable = false; 4633 } 4634 4635 mutex_unlock(&priv->state_lock); 4636 4637 return features; 4638 } 4639 4640 static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, 4641 struct mlx5e_channels *chs, 4642 struct mlx5e_params *new_params, 4643 struct mlx5_core_dev *mdev) 4644 { 4645 u16 ix; 4646 4647 for (ix = 0; ix < chs->params.num_channels; ix++) { 4648 struct xsk_buff_pool *xsk_pool = 4649 mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix); 4650 struct mlx5e_rq_opt_param rqo = {0}; 4651 struct mlx5e_xsk_param xsk; 4652 int max_xdp_mtu; 4653 4654 if (!xsk_pool) 4655 continue; 4656 4657 mlx5e_build_xsk_param(xsk_pool, &xsk); 4658 rqo.xsk = &xsk; 4659 max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &rqo); 4660 4661 /* Validate XSK params and XDP MTU in advance */ 4662 if (!mlx5e_validate_xsk_param(new_params, &rqo, mdev) || 4663 new_params->sw_mtu > max_xdp_mtu) { 4664 u32 hr = mlx5e_get_linear_rq_headroom(new_params, &rqo); 4665 int max_mtu_frame, max_mtu_page, max_mtu; 4666 4667 /* Two criteria must be met: 4668 * 1. HW MTU + all headrooms <= XSK frame size. 4669 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE. 4670 */ 4671 max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr); 4672 max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0)); 4673 max_mtu = min3(max_mtu_frame, max_mtu_page, max_xdp_mtu); 4674 4675 netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u or its redirection XDP program. Try MTU <= %d\n", 4676 new_params->sw_mtu, ix, max_mtu); 4677 return false; 4678 } 4679 } 4680 4681 return true; 4682 } 4683 4684 static bool mlx5e_params_validate_xdp(struct net_device *netdev, 4685 struct mlx5_core_dev *mdev, 4686 struct mlx5e_params *params) 4687 { 4688 bool is_linear; 4689 4690 /* No XSK params: AF_XDP can't be enabled yet at the point of setting 4691 * the XDP program. 4692 */ 4693 is_linear = params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC ? 4694 mlx5e_rx_is_linear_skb(mdev, params, NULL) : 4695 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL); 4696 4697 if (!is_linear) { 4698 if (!params->xdp_prog->aux->xdp_has_frags) { 4699 netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n", 4700 params->sw_mtu, 4701 mlx5e_xdp_max_mtu(params, NULL)); 4702 return false; 4703 } 4704 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && 4705 !mlx5e_verify_params_rx_mpwqe_strides(mdev, params, NULL)) { 4706 netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n", 4707 params->sw_mtu, 4708 mlx5e_xdp_max_mtu(params, NULL)); 4709 return false; 4710 } 4711 } 4712 4713 return true; 4714 } 4715 4716 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, 4717 mlx5e_fp_preactivate preactivate) 4718 { 4719 struct mlx5e_priv *priv = netdev_priv(netdev); 4720 struct mlx5e_params new_params; 4721 struct mlx5e_params *params; 4722 int err = 0; 4723 4724 mutex_lock(&priv->state_lock); 4725 4726 params = &priv->channels.params; 4727 4728 new_params = *params; 4729 new_params.sw_mtu = new_mtu; 4730 err = mlx5e_validate_params(priv->mdev, &new_params); 4731 if (err) 4732 goto out; 4733 4734 if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev, 4735 &new_params)) { 4736 err = -EINVAL; 4737 goto out; 4738 } 4739 4740 if (priv->xsk.refcnt && 4741 !mlx5e_xsk_validate_mtu(netdev, &priv->channels, 4742 &new_params, priv->mdev)) { 4743 err = -EINVAL; 4744 goto out; 4745 } 4746 4747 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, 4748 true); 4749 4750 out: 4751 WRITE_ONCE(netdev->mtu, params->sw_mtu); 4752 mutex_unlock(&priv->state_lock); 4753 4754 if (!err) 4755 netdev_update_features(netdev); 4756 4757 return err; 4758 } 4759 4760 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu) 4761 { 4762 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx); 4763 } 4764 4765 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx) 4766 { 4767 bool set = *(bool *)ctx; 4768 4769 return mlx5e_ptp_rx_manage_fs(priv, set); 4770 } 4771 4772 static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter) 4773 { 4774 bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4775 int err; 4776 4777 if (!rx_filter) 4778 /* Reset CQE compression to Admin default */ 4779 return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def, false); 4780 4781 if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 4782 return 0; 4783 4784 /* Disable CQE compression */ 4785 netdev_warn(priv->netdev, "Disabling RX cqe compression\n"); 4786 err = mlx5e_modify_rx_cqe_compression_locked(priv, false, true); 4787 if (err) 4788 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err); 4789 4790 return err; 4791 } 4792 4793 static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx) 4794 { 4795 struct mlx5e_params new_params; 4796 4797 if (ptp_rx == priv->channels.params.ptp_rx) 4798 return 0; 4799 4800 new_params = priv->channels.params; 4801 new_params.ptp_rx = ptp_rx; 4802 return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx, 4803 &new_params.ptp_rx, true); 4804 } 4805 4806 int mlx5e_hwtstamp_set(struct mlx5e_priv *priv, 4807 struct kernel_hwtstamp_config *config, 4808 struct netlink_ext_ack *extack) 4809 { 4810 bool rx_cqe_compress_def; 4811 bool ptp_rx; 4812 int err; 4813 4814 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) || 4815 (mlx5_clock_get_ptp_index(priv->mdev) == -1)) { 4816 NL_SET_ERR_MSG_MOD(extack, 4817 "Timestamps are not supported on this device"); 4818 return -EOPNOTSUPP; 4819 } 4820 4821 /* TX HW timestamp */ 4822 switch (config->tx_type) { 4823 case HWTSTAMP_TX_OFF: 4824 case HWTSTAMP_TX_ON: 4825 break; 4826 default: 4827 return -ERANGE; 4828 } 4829 4830 mutex_lock(&priv->state_lock); 4831 rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def; 4832 4833 /* RX HW timestamp */ 4834 switch (config->rx_filter) { 4835 case HWTSTAMP_FILTER_NONE: 4836 ptp_rx = false; 4837 break; 4838 case HWTSTAMP_FILTER_ALL: 4839 case HWTSTAMP_FILTER_SOME: 4840 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 4841 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 4842 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 4843 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 4844 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 4845 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 4846 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 4847 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 4848 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 4849 case HWTSTAMP_FILTER_PTP_V2_EVENT: 4850 case HWTSTAMP_FILTER_PTP_V2_SYNC: 4851 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 4852 case HWTSTAMP_FILTER_NTP_ALL: 4853 config->rx_filter = HWTSTAMP_FILTER_ALL; 4854 /* ptp_rx is set if both HW TS is set and CQE 4855 * compression is set 4856 */ 4857 ptp_rx = rx_cqe_compress_def; 4858 break; 4859 default: 4860 err = -ERANGE; 4861 goto err_unlock; 4862 } 4863 4864 if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX)) 4865 err = mlx5e_hwstamp_config_no_ptp_rx(priv, 4866 config->rx_filter != HWTSTAMP_FILTER_NONE); 4867 else 4868 err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx); 4869 if (err) 4870 goto err_unlock; 4871 4872 priv->hwtstamp_config = *config; 4873 mutex_unlock(&priv->state_lock); 4874 4875 /* might need to fix some features */ 4876 netdev_update_features(priv->netdev); 4877 4878 return 0; 4879 err_unlock: 4880 mutex_unlock(&priv->state_lock); 4881 return err; 4882 } 4883 4884 static int mlx5e_hwtstamp_set_ndo(struct net_device *netdev, 4885 struct kernel_hwtstamp_config *config, 4886 struct netlink_ext_ack *extack) 4887 { 4888 struct mlx5e_priv *priv = netdev_priv(netdev); 4889 4890 return mlx5e_hwtstamp_set(priv, config, extack); 4891 } 4892 4893 int mlx5e_hwtstamp_get(struct mlx5e_priv *priv, 4894 struct kernel_hwtstamp_config *config) 4895 { 4896 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz)) 4897 return -EOPNOTSUPP; 4898 4899 *config = priv->hwtstamp_config; 4900 4901 return 0; 4902 } 4903 4904 static int mlx5e_hwtstamp_get_ndo(struct net_device *dev, 4905 struct kernel_hwtstamp_config *config) 4906 { 4907 struct mlx5e_priv *priv = netdev_priv(dev); 4908 4909 return mlx5e_hwtstamp_get(priv, config); 4910 } 4911 4912 #ifdef CONFIG_MLX5_ESWITCH 4913 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 4914 { 4915 struct mlx5e_priv *priv = netdev_priv(dev); 4916 struct mlx5_core_dev *mdev = priv->mdev; 4917 4918 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac); 4919 } 4920 4921 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 4922 __be16 vlan_proto) 4923 { 4924 struct mlx5e_priv *priv = netdev_priv(dev); 4925 struct mlx5_core_dev *mdev = priv->mdev; 4926 4927 if (vlan_proto != htons(ETH_P_8021Q)) 4928 return -EPROTONOSUPPORT; 4929 4930 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1, 4931 vlan, qos); 4932 } 4933 4934 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 4935 { 4936 struct mlx5e_priv *priv = netdev_priv(dev); 4937 struct mlx5_core_dev *mdev = priv->mdev; 4938 4939 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting); 4940 } 4941 4942 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting) 4943 { 4944 struct mlx5e_priv *priv = netdev_priv(dev); 4945 struct mlx5_core_dev *mdev = priv->mdev; 4946 4947 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting); 4948 } 4949 4950 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 4951 int max_tx_rate) 4952 { 4953 struct mlx5e_priv *priv = netdev_priv(dev); 4954 struct mlx5_core_dev *mdev = priv->mdev; 4955 4956 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1, 4957 max_tx_rate, min_tx_rate); 4958 } 4959 4960 static int mlx5_vport_link2ifla(u8 esw_link) 4961 { 4962 switch (esw_link) { 4963 case MLX5_VPORT_ADMIN_STATE_DOWN: 4964 return IFLA_VF_LINK_STATE_DISABLE; 4965 case MLX5_VPORT_ADMIN_STATE_UP: 4966 return IFLA_VF_LINK_STATE_ENABLE; 4967 } 4968 return IFLA_VF_LINK_STATE_AUTO; 4969 } 4970 4971 static int mlx5_ifla_link2vport(u8 ifla_link) 4972 { 4973 switch (ifla_link) { 4974 case IFLA_VF_LINK_STATE_DISABLE: 4975 return MLX5_VPORT_ADMIN_STATE_DOWN; 4976 case IFLA_VF_LINK_STATE_ENABLE: 4977 return MLX5_VPORT_ADMIN_STATE_UP; 4978 } 4979 return MLX5_VPORT_ADMIN_STATE_AUTO; 4980 } 4981 4982 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf, 4983 int link_state) 4984 { 4985 struct mlx5e_priv *priv = netdev_priv(dev); 4986 struct mlx5_core_dev *mdev = priv->mdev; 4987 4988 if (mlx5e_is_uplink_rep(priv)) 4989 return -EOPNOTSUPP; 4990 4991 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1, 4992 mlx5_ifla_link2vport(link_state)); 4993 } 4994 4995 int mlx5e_get_vf_config(struct net_device *dev, 4996 int vf, struct ifla_vf_info *ivi) 4997 { 4998 struct mlx5e_priv *priv = netdev_priv(dev); 4999 struct mlx5_core_dev *mdev = priv->mdev; 5000 int err; 5001 5002 if (!netif_device_present(dev)) 5003 return -EOPNOTSUPP; 5004 5005 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi); 5006 if (err) 5007 return err; 5008 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate); 5009 return 0; 5010 } 5011 5012 int mlx5e_get_vf_stats(struct net_device *dev, 5013 int vf, struct ifla_vf_stats *vf_stats) 5014 { 5015 struct mlx5e_priv *priv = netdev_priv(dev); 5016 struct mlx5_core_dev *mdev = priv->mdev; 5017 5018 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1, 5019 vf_stats); 5020 } 5021 5022 static bool 5023 mlx5e_has_offload_stats(const struct net_device *dev, int attr_id) 5024 { 5025 struct mlx5e_priv *priv = netdev_priv(dev); 5026 5027 if (!netif_device_present(dev)) 5028 return false; 5029 5030 if (!mlx5e_is_uplink_rep(priv)) 5031 return false; 5032 5033 return mlx5e_rep_has_offload_stats(dev, attr_id); 5034 } 5035 5036 static int 5037 mlx5e_get_offload_stats(int attr_id, const struct net_device *dev, 5038 void *sp) 5039 { 5040 struct mlx5e_priv *priv = netdev_priv(dev); 5041 5042 if (!mlx5e_is_uplink_rep(priv)) 5043 return -EOPNOTSUPP; 5044 5045 return mlx5e_rep_get_offload_stats(attr_id, dev, sp); 5046 } 5047 #endif 5048 5049 static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type) 5050 { 5051 switch (proto_type) { 5052 case IPPROTO_GRE: 5053 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre); 5054 case IPPROTO_IPIP: 5055 case IPPROTO_IPV6: 5056 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) || 5057 MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx)); 5058 default: 5059 return false; 5060 } 5061 } 5062 5063 static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev, 5064 struct sk_buff *skb) 5065 { 5066 switch (skb->inner_protocol) { 5067 case htons(ETH_P_IP): 5068 case htons(ETH_P_IPV6): 5069 case htons(ETH_P_TEB): 5070 return true; 5071 case htons(ETH_P_MPLS_UC): 5072 case htons(ETH_P_MPLS_MC): 5073 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre); 5074 } 5075 return false; 5076 } 5077 5078 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, 5079 struct sk_buff *skb, 5080 netdev_features_t features) 5081 { 5082 unsigned int offset = 0; 5083 struct udphdr *udph; 5084 u8 proto; 5085 u16 port; 5086 5087 switch (vlan_get_protocol(skb)) { 5088 case htons(ETH_P_IP): 5089 proto = ip_hdr(skb)->protocol; 5090 break; 5091 case htons(ETH_P_IPV6): 5092 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL); 5093 break; 5094 default: 5095 goto out; 5096 } 5097 5098 switch (proto) { 5099 case IPPROTO_GRE: 5100 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb)) 5101 return features; 5102 break; 5103 case IPPROTO_IPIP: 5104 case IPPROTO_IPV6: 5105 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP)) 5106 return features; 5107 break; 5108 case IPPROTO_UDP: 5109 udph = udp_hdr(skb); 5110 port = be16_to_cpu(udph->dest); 5111 5112 /* Verify if UDP port is being offloaded by HW */ 5113 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) 5114 return vxlan_features_check(skb, features); 5115 5116 #if IS_ENABLED(CONFIG_GENEVE) 5117 /* Support Geneve offload for default UDP port */ 5118 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) 5119 return features; 5120 #endif 5121 break; 5122 #ifdef CONFIG_MLX5_EN_IPSEC 5123 case IPPROTO_ESP: 5124 return mlx5e_ipsec_feature_check(skb, features); 5125 #endif 5126 } 5127 5128 out: 5129 /* Disable CSUM and GSO if skb cannot be offloaded by HW */ 5130 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 5131 } 5132 5133 netdev_features_t mlx5e_features_check(struct sk_buff *skb, 5134 struct net_device *netdev, 5135 netdev_features_t features) 5136 { 5137 struct mlx5e_priv *priv = netdev_priv(netdev); 5138 5139 features = vlan_features_check(skb, features); 5140 5141 /* Validate if the tunneled packet is being offloaded by HW */ 5142 if (skb->encapsulation && 5143 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK)) 5144 return mlx5e_tunnel_features_check(priv, skb, features); 5145 5146 return features; 5147 } 5148 5149 static void mlx5e_tx_timeout_work(struct work_struct *work) 5150 { 5151 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 5152 tx_timeout_work); 5153 struct net_device *netdev = priv->netdev; 5154 int i; 5155 5156 for (i = 0; i < netdev->real_num_tx_queues; i++) { 5157 struct netdev_queue *dev_queue = 5158 netdev_get_tx_queue(netdev, i); 5159 struct mlx5e_txqsq *sq = priv->txq2sq[i]; 5160 5161 if (!netif_xmit_timeout_ms(dev_queue)) 5162 continue; 5163 5164 if (mlx5e_reporter_tx_timeout(sq)) 5165 /* break if tried to reopened channels */ 5166 break; 5167 } 5168 } 5169 5170 static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue) 5171 { 5172 struct mlx5e_priv *priv = netdev_priv(dev); 5173 5174 netdev_err(dev, "TX timeout detected\n"); 5175 queue_work(priv->wq, &priv->tx_timeout_work); 5176 } 5177 5178 static int mlx5e_xdp_allowed(struct net_device *netdev, struct mlx5_core_dev *mdev, 5179 struct mlx5e_params *params) 5180 { 5181 if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) { 5182 netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n"); 5183 return -EINVAL; 5184 } 5185 5186 if (!mlx5e_params_validate_xdp(netdev, mdev, params)) 5187 return -EINVAL; 5188 5189 return 0; 5190 } 5191 5192 static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog) 5193 { 5194 struct bpf_prog *old_prog; 5195 5196 old_prog = rcu_replace_pointer(rq->xdp_prog, prog, 5197 lockdep_is_held(&rq->priv->state_lock)); 5198 if (old_prog) 5199 bpf_prog_put(old_prog); 5200 } 5201 5202 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) 5203 { 5204 struct mlx5e_priv *priv = netdev_priv(netdev); 5205 struct mlx5e_params new_params; 5206 struct bpf_prog *old_prog; 5207 int err = 0; 5208 bool reset; 5209 int i; 5210 5211 mutex_lock(&priv->state_lock); 5212 5213 new_params = priv->channels.params; 5214 new_params.xdp_prog = prog; 5215 5216 if (prog) { 5217 err = mlx5e_xdp_allowed(netdev, priv->mdev, &new_params); 5218 if (err) 5219 goto unlock; 5220 } 5221 5222 /* no need for full reset when exchanging programs */ 5223 reset = (!priv->channels.params.xdp_prog || !prog); 5224 5225 old_prog = priv->channels.params.xdp_prog; 5226 5227 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset); 5228 if (err) 5229 goto unlock; 5230 5231 if (old_prog) 5232 bpf_prog_put(old_prog); 5233 5234 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset) 5235 goto unlock; 5236 5237 /* exchanging programs w/o reset, we update ref counts on behalf 5238 * of the channels RQs here. 5239 */ 5240 bpf_prog_add(prog, priv->channels.num); 5241 for (i = 0; i < priv->channels.num; i++) { 5242 struct mlx5e_channel *c = priv->channels.c[i]; 5243 5244 mlx5e_rq_replace_xdp_prog(&c->rq, prog); 5245 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) { 5246 bpf_prog_inc(prog); 5247 mlx5e_rq_replace_xdp_prog(&c->xskrq, prog); 5248 } 5249 } 5250 5251 unlock: 5252 mutex_unlock(&priv->state_lock); 5253 5254 /* Need to fix some features. */ 5255 if (!err) 5256 netdev_update_features(netdev); 5257 5258 return err; 5259 } 5260 5261 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) 5262 { 5263 switch (xdp->command) { 5264 case XDP_SETUP_PROG: 5265 return mlx5e_xdp_set(dev, xdp->prog); 5266 case XDP_SETUP_XSK_POOL: 5267 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool, 5268 xdp->xsk.queue_id); 5269 default: 5270 return -EINVAL; 5271 } 5272 } 5273 5274 #ifdef CONFIG_MLX5_ESWITCH 5275 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 5276 struct net_device *dev, u32 filter_mask, 5277 int nlflags) 5278 { 5279 struct mlx5e_priv *priv = netdev_priv(dev); 5280 struct mlx5_core_dev *mdev = priv->mdev; 5281 u8 mode, setting; 5282 5283 if (mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting)) 5284 return -EOPNOTSUPP; 5285 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB; 5286 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 5287 mode, 5288 0, 0, nlflags, filter_mask, NULL); 5289 } 5290 5291 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 5292 u16 flags, struct netlink_ext_ack *extack) 5293 { 5294 struct mlx5e_priv *priv = netdev_priv(dev); 5295 struct mlx5_core_dev *mdev = priv->mdev; 5296 struct nlattr *attr, *br_spec; 5297 u16 mode = BRIDGE_MODE_UNDEF; 5298 u8 setting; 5299 int rem; 5300 5301 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 5302 if (!br_spec) 5303 return -EINVAL; 5304 5305 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) { 5306 mode = nla_get_u16(attr); 5307 if (mode > BRIDGE_MODE_VEPA) 5308 return -EINVAL; 5309 5310 break; 5311 } 5312 5313 if (mode == BRIDGE_MODE_UNDEF) 5314 return -EINVAL; 5315 5316 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0; 5317 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting); 5318 } 5319 #endif 5320 5321 const struct net_device_ops mlx5e_netdev_ops = { 5322 .ndo_open = mlx5e_open, 5323 .ndo_stop = mlx5e_close, 5324 .ndo_start_xmit = mlx5e_xmit, 5325 .ndo_setup_tc = mlx5e_setup_tc, 5326 .ndo_select_queue = mlx5e_select_queue, 5327 .ndo_get_stats64 = mlx5e_get_stats, 5328 .ndo_set_rx_mode_async = mlx5e_set_rx_mode, 5329 .ndo_set_mac_address = mlx5e_set_mac, 5330 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid, 5331 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid, 5332 .ndo_set_features = mlx5e_set_features, 5333 .ndo_fix_features = mlx5e_fix_features, 5334 .ndo_change_mtu = mlx5e_change_nic_mtu, 5335 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, 5336 .ndo_features_check = mlx5e_features_check, 5337 .ndo_tx_timeout = mlx5e_tx_timeout, 5338 .ndo_bpf = mlx5e_xdp, 5339 .ndo_xdp_xmit = mlx5e_xdp_xmit, 5340 .ndo_xsk_wakeup = mlx5e_xsk_wakeup, 5341 .ndo_hwtstamp_get = mlx5e_hwtstamp_get_ndo, 5342 .ndo_hwtstamp_set = mlx5e_hwtstamp_set_ndo, 5343 #ifdef CONFIG_MLX5_EN_ARFS 5344 .ndo_rx_flow_steer = mlx5e_rx_flow_steer, 5345 #endif 5346 #ifdef CONFIG_MLX5_ESWITCH 5347 .ndo_bridge_setlink = mlx5e_bridge_setlink, 5348 .ndo_bridge_getlink = mlx5e_bridge_getlink, 5349 5350 /* SRIOV E-Switch NDOs */ 5351 .ndo_set_vf_mac = mlx5e_set_vf_mac, 5352 .ndo_set_vf_vlan = mlx5e_set_vf_vlan, 5353 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk, 5354 .ndo_set_vf_trust = mlx5e_set_vf_trust, 5355 .ndo_set_vf_rate = mlx5e_set_vf_rate, 5356 .ndo_get_vf_config = mlx5e_get_vf_config, 5357 .ndo_set_vf_link_state = mlx5e_set_vf_link_state, 5358 .ndo_get_vf_stats = mlx5e_get_vf_stats, 5359 .ndo_has_offload_stats = mlx5e_has_offload_stats, 5360 .ndo_get_offload_stats = mlx5e_get_offload_stats, 5361 #endif 5362 }; 5363 5364 void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu) 5365 { 5366 struct mlx5e_params *params = &priv->channels.params; 5367 struct mlx5_core_dev *mdev = priv->mdev; 5368 5369 params->sw_mtu = mtu; 5370 params->hard_mtu = MLX5E_ETH_HARD_MTU; 5371 params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2, 5372 priv->max_nch); 5373 mlx5e_params_mqprio_reset(params); 5374 5375 /* SQ */ 5376 params->log_sq_size = is_kdump_kernel() ? 5377 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : 5378 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 5379 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 5380 5381 /* XDP SQ */ 5382 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev)); 5383 5384 /* set CQE compression */ 5385 params->rx_cqe_compress_def = false; 5386 if (MLX5_CAP_GEN(mdev, cqe_compression) && 5387 MLX5_CAP_GEN(mdev, vport_group_manager)) 5388 params->rx_cqe_compress_def = slow_pci_heuristic(mdev); 5389 5390 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def); 5391 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); 5392 5393 /* RQ */ 5394 mlx5e_build_rq_params(mdev, params); 5395 5396 params->terminate_lkey_be = mlx5_core_get_terminate_scatter_list_mkey(mdev); 5397 5398 params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); 5399 5400 /* CQ moderation params */ 5401 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation) && 5402 MLX5_CAP_GEN(mdev, cq_period_mode_modify); 5403 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation) && 5404 MLX5_CAP_GEN(mdev, cq_period_mode_modify); 5405 params->rx_moder_use_cqe_mode = !!MLX5_CAP_GEN(mdev, cq_period_start_from_cqe); 5406 params->tx_moder_use_cqe_mode = false; 5407 mlx5e_reset_rx_moderation(¶ms->rx_cq_moderation, params->rx_moder_use_cqe_mode, 5408 params->rx_dim_enabled); 5409 mlx5e_reset_tx_moderation(¶ms->tx_cq_moderation, params->tx_moder_use_cqe_mode, 5410 params->tx_dim_enabled); 5411 5412 /* TX inline */ 5413 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 5414 5415 /* AF_XDP */ 5416 params->xsk = xsk; 5417 5418 /* Do not update netdev->features directly in here 5419 * on mlx5e_attach_netdev() we will call mlx5e_update_features() 5420 * To update netdev->features please modify mlx5e_fix_features() 5421 */ 5422 } 5423 5424 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) 5425 { 5426 struct mlx5e_priv *priv = netdev_priv(netdev); 5427 u8 addr[ETH_ALEN]; 5428 5429 mlx5_query_mac_address(priv->mdev, addr); 5430 if (is_zero_ether_addr(addr) && 5431 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) { 5432 eth_hw_addr_random(netdev); 5433 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr); 5434 return; 5435 } 5436 5437 eth_hw_addr_set(netdev, addr); 5438 } 5439 5440 static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table, 5441 unsigned int entry, struct udp_tunnel_info *ti) 5442 { 5443 struct mlx5e_priv *priv = netdev_priv(netdev); 5444 5445 return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port)); 5446 } 5447 5448 static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table, 5449 unsigned int entry, struct udp_tunnel_info *ti) 5450 { 5451 struct mlx5e_priv *priv = netdev_priv(netdev); 5452 5453 return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port)); 5454 } 5455 5456 void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv) 5457 { 5458 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 5459 return; 5460 5461 priv->nic_info.set_port = mlx5e_vxlan_set_port; 5462 priv->nic_info.unset_port = mlx5e_vxlan_unset_port; 5463 priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN; 5464 priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN; 5465 /* Don't count the space hard-coded to the IANA port */ 5466 priv->nic_info.tables[0].n_entries = 5467 mlx5_vxlan_max_udp_ports(priv->mdev) - 1; 5468 5469 priv->netdev->udp_tunnel_nic_info = &priv->nic_info; 5470 } 5471 5472 static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev) 5473 { 5474 int tt; 5475 5476 for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) { 5477 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5_get_proto_by_tunnel_type(tt))) 5478 return true; 5479 } 5480 return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)); 5481 } 5482 5483 static void mlx5e_get_queue_stats_rx(struct net_device *dev, int i, 5484 struct netdev_queue_stats_rx *stats) 5485 { 5486 struct mlx5e_priv *priv = netdev_priv(dev); 5487 struct mlx5e_channel_stats *channel_stats; 5488 struct mlx5e_rq_stats *xskrq_stats; 5489 struct mlx5e_rq_stats *rq_stats; 5490 5491 if (mlx5e_is_uplink_rep(priv) || !priv->stats_nch) 5492 return; 5493 5494 channel_stats = priv->channel_stats[i]; 5495 xskrq_stats = &channel_stats->xskrq; 5496 rq_stats = &channel_stats->rq; 5497 5498 stats->packets = rq_stats->packets + xskrq_stats->packets; 5499 stats->bytes = rq_stats->bytes + xskrq_stats->bytes; 5500 stats->alloc_fail = rq_stats->buff_alloc_err + 5501 xskrq_stats->buff_alloc_err; 5502 5503 stats->hw_gro_packets = rq_stats->gro_skbs + xskrq_stats->gro_skbs; 5504 stats->hw_gro_wire_packets = 5505 rq_stats->gro_packets + xskrq_stats->gro_packets; 5506 stats->hw_gro_wire_bytes = rq_stats->gro_bytes + xskrq_stats->gro_bytes; 5507 } 5508 5509 static void mlx5e_get_queue_stats_tx(struct net_device *dev, int i, 5510 struct netdev_queue_stats_tx *stats) 5511 { 5512 struct mlx5e_priv *priv = netdev_priv(dev); 5513 struct mlx5e_sq_stats *sq_stats; 5514 5515 if (!priv->stats_nch) 5516 return; 5517 5518 /* no special case needed for ptp htb etc since txq2sq_stats is kept up 5519 * to date for active sq_stats, otherwise get_base_stats takes care of 5520 * inactive sqs. 5521 */ 5522 sq_stats = priv->txq2sq_stats[i]; 5523 stats->packets = sq_stats->packets; 5524 stats->bytes = sq_stats->bytes; 5525 5526 stats->hw_gso_packets = 5527 sq_stats->tso_packets + sq_stats->tso_inner_packets; 5528 stats->hw_gso_bytes = sq_stats->tso_bytes + sq_stats->tso_inner_bytes; 5529 5530 stats->csum_none = sq_stats->csum_none; 5531 5532 stats->stop = sq_stats->stopped; 5533 stats->wake = sq_stats->wake; 5534 } 5535 5536 static void mlx5e_get_base_stats(struct net_device *dev, 5537 struct netdev_queue_stats_rx *rx, 5538 struct netdev_queue_stats_tx *tx) 5539 { 5540 struct mlx5e_priv *priv = netdev_priv(dev); 5541 struct mlx5e_ptp *ptp_channel; 5542 int i, tc; 5543 5544 if (!mlx5e_is_uplink_rep(priv)) { 5545 rx->packets = 0; 5546 rx->bytes = 0; 5547 rx->alloc_fail = 0; 5548 rx->hw_gro_packets = 0; 5549 rx->hw_gro_wire_packets = 0; 5550 rx->hw_gro_wire_bytes = 0; 5551 5552 for (i = priv->channels.params.num_channels; i < priv->stats_nch; i++) { 5553 struct netdev_queue_stats_rx rx_i = {0}; 5554 5555 mlx5e_get_queue_stats_rx(dev, i, &rx_i); 5556 5557 rx->packets += rx_i.packets; 5558 rx->bytes += rx_i.bytes; 5559 rx->alloc_fail += rx_i.alloc_fail; 5560 rx->hw_gro_packets += rx_i.hw_gro_packets; 5561 rx->hw_gro_wire_packets += rx_i.hw_gro_wire_packets; 5562 rx->hw_gro_wire_bytes += rx_i.hw_gro_wire_bytes; 5563 } 5564 5565 /* always report PTP RX stats from base as there is no 5566 * corresponding channel to report them under in 5567 * mlx5e_get_queue_stats_rx. 5568 */ 5569 if (priv->rx_ptp_opened) { 5570 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq; 5571 5572 rx->packets += rq_stats->packets; 5573 rx->bytes += rq_stats->bytes; 5574 rx->hw_gro_packets += rq_stats->gro_skbs; 5575 rx->hw_gro_wire_packets += rq_stats->gro_packets; 5576 rx->hw_gro_wire_bytes += rq_stats->gro_bytes; 5577 } 5578 } 5579 5580 tx->packets = 0; 5581 tx->bytes = 0; 5582 tx->hw_gso_packets = 0; 5583 tx->hw_gso_bytes = 0; 5584 tx->csum_none = 0; 5585 tx->stop = 0; 5586 tx->wake = 0; 5587 5588 for (i = 0; i < priv->stats_nch; i++) { 5589 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i]; 5590 5591 /* handle two cases: 5592 * 5593 * 1. channels which are active. In this case, 5594 * report only deactivated TCs on these channels. 5595 * 5596 * 2. channels which were deactivated 5597 * (i > priv->channels.params.num_channels) 5598 * must have all of their TCs [0 .. priv->max_opened_tc) 5599 * examined because deactivated channels will not be in the 5600 * range of [0..real_num_tx_queues) and will not have their 5601 * stats reported by mlx5e_get_queue_stats_tx. 5602 */ 5603 if (i < priv->channels.params.num_channels) 5604 tc = mlx5e_get_dcb_num_tc(&priv->channels.params); 5605 else 5606 tc = 0; 5607 5608 for (; tc < priv->max_opened_tc; tc++) { 5609 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[tc]; 5610 5611 tx->packets += sq_stats->packets; 5612 tx->bytes += sq_stats->bytes; 5613 tx->hw_gso_packets += sq_stats->tso_packets + 5614 sq_stats->tso_inner_packets; 5615 tx->hw_gso_bytes += sq_stats->tso_bytes + 5616 sq_stats->tso_inner_bytes; 5617 tx->csum_none += sq_stats->csum_none; 5618 tx->stop += sq_stats->stopped; 5619 tx->wake += sq_stats->wake; 5620 } 5621 } 5622 5623 /* if PTP TX was opened at some point and has since either: 5624 * - been shutdown and set to NULL, or 5625 * - simply disabled (bit unset) 5626 * 5627 * report stats directly from the ptp_stats structures as these queues 5628 * are now unavailable and there is no txq index to retrieve these 5629 * stats via calls to mlx5e_get_queue_stats_tx. 5630 */ 5631 ptp_channel = priv->channels.ptp; 5632 if (priv->tx_ptp_opened && (!ptp_channel || !test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state))) { 5633 for (tc = 0; tc < priv->max_opened_tc; tc++) { 5634 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[tc]; 5635 5636 tx->packets += sq_stats->packets; 5637 tx->bytes += sq_stats->bytes; 5638 tx->hw_gso_packets += sq_stats->tso_packets + 5639 sq_stats->tso_inner_packets; 5640 tx->hw_gso_bytes += sq_stats->tso_bytes + 5641 sq_stats->tso_inner_bytes; 5642 tx->csum_none += sq_stats->csum_none; 5643 tx->stop += sq_stats->stopped; 5644 tx->wake += sq_stats->wake; 5645 } 5646 } 5647 } 5648 5649 static const struct netdev_stat_ops mlx5e_stat_ops = { 5650 .get_queue_stats_rx = mlx5e_get_queue_stats_rx, 5651 .get_queue_stats_tx = mlx5e_get_queue_stats_tx, 5652 .get_base_stats = mlx5e_get_base_stats, 5653 }; 5654 5655 struct mlx5_qmgmt_data { 5656 struct mlx5e_channel *c; 5657 }; 5658 5659 static void mlx5e_queue_default_qcfg(struct net_device *dev, 5660 struct netdev_queue_config *qcfg) 5661 { 5662 qcfg->rx_page_size = PAGE_SIZE; 5663 } 5664 5665 static int mlx5e_queue_validate_qcfg(struct net_device *dev, 5666 struct netdev_queue_config *qcfg, 5667 struct netlink_ext_ack *extack) 5668 { 5669 struct mlx5e_priv *priv = netdev_priv(dev); 5670 struct mlx5_core_dev *mdev = priv->mdev; 5671 u32 max; 5672 5673 if (!is_power_of_2(qcfg->rx_page_size)) { 5674 netdev_err(priv->netdev, "rx_page_size not power of 2: %u", 5675 qcfg->rx_page_size); 5676 return -EINVAL; 5677 } 5678 5679 max = mlx5e_mpwrq_max_page_size(mdev); 5680 if (qcfg->rx_page_size < PAGE_SIZE || qcfg->rx_page_size > max) { 5681 netdev_err(priv->netdev, 5682 "Selected rx_page_size %u not in supported range [%lu, %u]\n", 5683 qcfg->rx_page_size, PAGE_SIZE, max); 5684 return -ERANGE; 5685 } 5686 5687 return 0; 5688 } 5689 5690 static bool mlx5e_queue_validate_page_size(struct net_device *dev, 5691 struct netdev_queue_config *qcfg, 5692 int queue_index) 5693 { 5694 if (qcfg->rx_page_size == PAGE_SIZE) 5695 return true; 5696 5697 if (!netif_rxq_has_unreadable_mp(dev, queue_index)) 5698 return false; 5699 5700 return true; 5701 } 5702 5703 static int mlx5e_queue_mem_alloc(struct net_device *dev, 5704 struct netdev_queue_config *qcfg, 5705 void *newq, int queue_index) 5706 { 5707 struct mlx5_qmgmt_data *new = (struct mlx5_qmgmt_data *)newq; 5708 struct mlx5e_priv *priv = netdev_priv(dev); 5709 struct mlx5e_channels *chs = &priv->channels; 5710 struct mlx5e_params params = chs->params; 5711 int err; 5712 5713 mutex_lock(&priv->state_lock); 5714 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 5715 err = -ENODEV; 5716 goto unlock; 5717 } 5718 5719 if (queue_index >= chs->num) { 5720 err = -ERANGE; 5721 goto unlock; 5722 } 5723 5724 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || 5725 chs->params.ptp_rx || 5726 chs->params.xdp_prog || 5727 priv->htb) { 5728 netdev_err(priv->netdev, 5729 "Cloning channels with Port/rx PTP, XDP or HTB is not supported\n"); 5730 err = -EOPNOTSUPP; 5731 goto unlock; 5732 } 5733 5734 if (!mlx5e_queue_validate_page_size(dev, qcfg, queue_index)) { 5735 netdev_err(priv->netdev, "High order pages are supported only in Zero-Copy mode\n"); 5736 err = -EINVAL; 5737 goto unlock; 5738 } 5739 5740 err = mlx5e_open_channel(priv, queue_index, ¶ms, qcfg, NULL, 5741 &new->c); 5742 unlock: 5743 mutex_unlock(&priv->state_lock); 5744 return err; 5745 } 5746 5747 static void mlx5e_queue_mem_free(struct net_device *dev, void *mem) 5748 { 5749 struct mlx5_qmgmt_data *data = (struct mlx5_qmgmt_data *)mem; 5750 5751 /* not supposed to happen since mlx5e_queue_start never fails 5752 * but this is how this should be implemented just in case 5753 */ 5754 if (data->c) 5755 mlx5e_close_channel(data->c); 5756 } 5757 5758 static int mlx5e_queue_stop(struct net_device *dev, void *oldq, int queue_index) 5759 { 5760 /* In mlx5 a txq cannot be simply stopped in isolation, only restarted. 5761 * mlx5e_queue_start does not fail, we stop the old queue there. 5762 * TODO: Improve this. 5763 */ 5764 return 0; 5765 } 5766 5767 static int mlx5e_queue_start(struct net_device *dev, 5768 struct netdev_queue_config *qcfg, 5769 void *newq, int queue_index) 5770 { 5771 struct mlx5_qmgmt_data *new = (struct mlx5_qmgmt_data *)newq; 5772 struct mlx5e_priv *priv = netdev_priv(dev); 5773 struct mlx5e_channel *old; 5774 5775 mutex_lock(&priv->state_lock); 5776 5777 /* stop and close the old */ 5778 old = priv->channels.c[queue_index]; 5779 mlx5e_deactivate_priv_channels(priv); 5780 /* close old before activating new, to avoid napi conflict */ 5781 mlx5e_close_channel(old); 5782 5783 /* start the new */ 5784 priv->channels.c[queue_index] = new->c; 5785 mlx5e_activate_priv_channels(priv); 5786 mutex_unlock(&priv->state_lock); 5787 return 0; 5788 } 5789 5790 static struct device *mlx5e_queue_get_dma_dev(struct net_device *dev, 5791 int queue_index) 5792 { 5793 struct mlx5e_priv *priv = netdev_priv(dev); 5794 struct mlx5e_channels *channels; 5795 struct device *pdev = NULL; 5796 struct mlx5e_channel *ch; 5797 5798 channels = &priv->channels; 5799 5800 mutex_lock(&priv->state_lock); 5801 5802 if (queue_index >= channels->num) 5803 goto out; 5804 5805 ch = channels->c[queue_index]; 5806 pdev = ch->pdev; 5807 out: 5808 mutex_unlock(&priv->state_lock); 5809 5810 return pdev; 5811 } 5812 5813 static const struct netdev_queue_mgmt_ops mlx5e_queue_mgmt_ops = { 5814 .ndo_queue_mem_size = sizeof(struct mlx5_qmgmt_data), 5815 .ndo_queue_mem_alloc = mlx5e_queue_mem_alloc, 5816 .ndo_queue_mem_free = mlx5e_queue_mem_free, 5817 .ndo_queue_start = mlx5e_queue_start, 5818 .ndo_queue_stop = mlx5e_queue_stop, 5819 .ndo_queue_get_dma_dev = mlx5e_queue_get_dma_dev, 5820 .ndo_default_qcfg = mlx5e_queue_default_qcfg, 5821 .ndo_validate_qcfg = mlx5e_queue_validate_qcfg, 5822 .supported_params = QCFG_RX_PAGE_SIZE, 5823 }; 5824 5825 static void mlx5e_build_nic_netdev(struct net_device *netdev) 5826 { 5827 struct mlx5e_priv *priv = netdev_priv(netdev); 5828 struct mlx5_core_dev *mdev = priv->mdev; 5829 bool fcs_supported; 5830 bool fcs_enabled; 5831 5832 SET_NETDEV_DEV(netdev, mdev->device); 5833 5834 netdev->netdev_ops = &mlx5e_netdev_ops; 5835 netdev->queue_mgmt_ops = &mlx5e_queue_mgmt_ops; 5836 netdev->xdp_metadata_ops = &mlx5e_xdp_metadata_ops; 5837 netdev->xsk_tx_metadata_ops = &mlx5e_xsk_tx_metadata_ops; 5838 netdev->request_ops_lock = true; 5839 netdev_lockdep_set_classes(netdev); 5840 5841 mlx5e_dcbnl_build_netdev(netdev); 5842 5843 netdev->watchdog_timeo = 15 * HZ; 5844 5845 netdev->stat_ops = &mlx5e_stat_ops; 5846 netdev->ethtool_ops = &mlx5e_ethtool_ops; 5847 5848 netdev->vlan_features |= NETIF_F_SG; 5849 netdev->vlan_features |= NETIF_F_HW_CSUM; 5850 netdev->vlan_features |= NETIF_F_HW_MACSEC; 5851 netdev->vlan_features |= NETIF_F_GRO; 5852 netdev->vlan_features |= NETIF_F_TSO; 5853 netdev->vlan_features |= NETIF_F_TSO6; 5854 netdev->vlan_features |= NETIF_F_RXCSUM; 5855 netdev->vlan_features |= NETIF_F_RXHASH; 5856 netdev->vlan_features |= NETIF_F_GSO_PARTIAL; 5857 5858 netdev->mpls_features |= NETIF_F_SG; 5859 netdev->mpls_features |= NETIF_F_HW_CSUM; 5860 netdev->mpls_features |= NETIF_F_TSO; 5861 netdev->mpls_features |= NETIF_F_TSO6; 5862 5863 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX; 5864 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX; 5865 5866 /* Tunneled LRO is not supported in the driver, and the same RQs are 5867 * shared between inner and outer TIRs, so the driver can't disable LRO 5868 * for inner TIRs while having it enabled for outer TIRs. Due to this, 5869 * block LRO altogether if the firmware declares tunneled LRO support. 5870 */ 5871 if (!!MLX5_CAP_ETH(mdev, lro_cap) && 5872 !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) && 5873 !MLX5_CAP_ETH(mdev, tunnel_lro_gre) && 5874 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, 5875 MLX5E_MPWRQ_UMR_MODE_ALIGNED)) 5876 netdev->vlan_features |= NETIF_F_LRO; 5877 5878 if (mlx5e_hw_gro_supported(mdev) && 5879 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT, 5880 MLX5E_MPWRQ_UMR_MODE_ALIGNED)) 5881 netdev->vlan_features |= NETIF_F_GRO_HW; 5882 5883 netdev->hw_features = netdev->vlan_features; 5884 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 5885 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 5886 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 5887 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 5888 5889 if (mlx5e_tunnel_any_tx_proto_supported(mdev)) { 5890 netdev->hw_enc_features |= NETIF_F_HW_CSUM; 5891 netdev->hw_enc_features |= NETIF_F_TSO; 5892 netdev->hw_enc_features |= NETIF_F_TSO6; 5893 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; 5894 } 5895 5896 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { 5897 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 5898 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5899 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 5900 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5901 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 5902 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL | 5903 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5904 } 5905 5906 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) { 5907 netdev->hw_features |= NETIF_F_GSO_GRE | 5908 NETIF_F_GSO_GRE_CSUM; 5909 netdev->hw_enc_features |= NETIF_F_GSO_GRE | 5910 NETIF_F_GSO_GRE_CSUM; 5911 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 5912 netdev->vlan_features |= NETIF_F_GSO_GRE | NETIF_F_GSO_GRE_CSUM; 5913 } 5914 5915 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) { 5916 netdev->hw_features |= NETIF_F_GSO_IPXIP4 | 5917 NETIF_F_GSO_IPXIP6; 5918 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 | 5919 NETIF_F_GSO_IPXIP6; 5920 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 | 5921 NETIF_F_GSO_IPXIP6; 5922 } 5923 5924 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4; 5925 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 5926 netdev->hw_enc_features |= NETIF_F_GSO_UDP_L4; 5927 5928 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled); 5929 5930 if (fcs_supported) 5931 netdev->hw_features |= NETIF_F_RXALL; 5932 5933 if (MLX5_CAP_ETH(mdev, scatter_fcs)) 5934 netdev->hw_features |= NETIF_F_RXFCS; 5935 5936 if (mlx5_qos_is_supported(mdev)) 5937 netdev->hw_features |= NETIF_F_HW_TC; 5938 5939 netdev->features = netdev->hw_features; 5940 5941 /* Defaults */ 5942 if (fcs_enabled) 5943 netdev->features &= ~NETIF_F_RXALL; 5944 netdev->features &= ~NETIF_F_LRO; 5945 netdev->features &= ~NETIF_F_GRO_HW; 5946 netdev->features &= ~NETIF_F_RXFCS; 5947 5948 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f) 5949 if (FT_CAP(flow_modify_en) && 5950 FT_CAP(modify_root) && 5951 FT_CAP(identified_miss_table_mode) && 5952 FT_CAP(flow_table_modify)) { 5953 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 5954 netdev->hw_features |= NETIF_F_HW_TC; 5955 #endif 5956 #if IS_ENABLED(CONFIG_MLX5_EN_ARFS) 5957 netdev->hw_features |= NETIF_F_NTUPLE; 5958 #elif IS_ENABLED(CONFIG_MLX5_EN_RXNFC) 5959 netdev->features |= NETIF_F_NTUPLE; 5960 #endif 5961 } 5962 5963 netdev->features |= NETIF_F_HIGHDMA; 5964 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 5965 5966 netdev->priv_flags |= IFF_UNICAST_FLT; 5967 5968 netdev->netmem_tx = NETMEM_TX_DMA; 5969 5970 netif_set_tso_max_size(netdev, GSO_MAX_SIZE); 5971 mlx5e_set_xdp_feature(priv); 5972 mlx5e_set_netdev_dev_addr(netdev); 5973 mlx5e_macsec_build_netdev(priv); 5974 mlx5e_ipsec_build_netdev(priv); 5975 mlx5e_ktls_build_netdev(priv); 5976 } 5977 5978 void mlx5e_create_q_counters(struct mlx5e_priv *priv) 5979 { 5980 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 5981 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 5982 struct mlx5_core_dev *mdev = priv->mdev; 5983 struct mlx5_core_dev *pos; 5984 int err, i; 5985 5986 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 5987 5988 mlx5_sd_for_each_dev(i, mdev, pos) { 5989 err = mlx5_cmd_exec_inout(pos, alloc_q_counter, in, out); 5990 if (!err) 5991 priv->q_counter[i] = 5992 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5993 } 5994 5995 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out); 5996 if (!err) 5997 priv->drop_rq_q_counter = 5998 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 5999 } 6000 6001 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) 6002 { 6003 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 6004 struct mlx5_core_dev *pos; 6005 int i; 6006 6007 MLX5_SET(dealloc_q_counter_in, in, opcode, 6008 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 6009 mlx5_sd_for_each_dev(i, priv->mdev, pos) { 6010 if (priv->q_counter[i]) { 6011 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 6012 priv->q_counter[i]); 6013 mlx5_cmd_exec_in(pos, dealloc_q_counter, in); 6014 } 6015 } 6016 6017 if (priv->drop_rq_q_counter) { 6018 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 6019 priv->drop_rq_q_counter); 6020 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in); 6021 } 6022 } 6023 6024 static int mlx5e_nic_init(struct mlx5_core_dev *mdev, 6025 struct net_device *netdev) 6026 { 6027 const bool take_rtnl = netdev->reg_state == NETREG_REGISTERED; 6028 struct mlx5e_priv *priv = netdev_priv(netdev); 6029 struct mlx5e_flow_steering *fs; 6030 int err; 6031 6032 mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu); 6033 mlx5e_vxlan_set_netdev_info(priv); 6034 6035 mlx5e_timestamp_init(priv); 6036 6037 priv->dfs_root = debugfs_create_dir("nic", 6038 mlx5_debugfs_get_dev_root(mdev)); 6039 6040 fs = mlx5e_fs_init(priv->profile, mdev, 6041 !test_bit(MLX5E_STATE_DESTROYING, &priv->state), 6042 priv->dfs_root); 6043 if (!fs) { 6044 err = -ENOMEM; 6045 mlx5_core_err(mdev, "FS initialization failed, %d\n", err); 6046 debugfs_remove_recursive(priv->dfs_root); 6047 return err; 6048 } 6049 priv->fs = fs; 6050 6051 err = mlx5e_psp_init(priv); 6052 if (err) 6053 mlx5_core_err(mdev, "PSP initialization failed, %d\n", err); 6054 6055 err = mlx5e_ktls_init(priv); 6056 if (err) 6057 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); 6058 6059 mlx5e_health_create_reporters(priv); 6060 6061 /* If netdev is already registered (e.g. move from uplink to nic profile), 6062 * RTNL lock must be held before triggering netdev notifiers. 6063 */ 6064 if (take_rtnl) 6065 rtnl_lock(); 6066 6067 /* update XDP supported features */ 6068 mlx5e_set_xdp_feature(priv); 6069 6070 if (take_rtnl) 6071 rtnl_unlock(); 6072 6073 return 0; 6074 } 6075 6076 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) 6077 { 6078 mlx5e_health_destroy_reporters(priv); 6079 mlx5e_ktls_cleanup(priv); 6080 mlx5e_psp_cleanup(priv); 6081 mlx5e_fs_cleanup(priv->fs); 6082 debugfs_remove_recursive(priv->dfs_root); 6083 priv->fs = NULL; 6084 } 6085 6086 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) 6087 { 6088 struct mlx5_core_dev *mdev = priv->mdev; 6089 enum mlx5e_rx_res_features features; 6090 int err; 6091 6092 mlx5e_create_q_counters(priv); 6093 6094 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 6095 if (err) { 6096 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 6097 goto err_destroy_q_counters; 6098 } 6099 6100 features = MLX5E_RX_RES_FEATURE_PTP; 6101 if (mlx5_tunnel_inner_ft_supported(mdev)) 6102 features |= MLX5E_RX_RES_FEATURE_INNER_FT; 6103 if (mlx5_get_sd(priv->mdev)) 6104 features |= MLX5E_RX_RES_FEATURE_MULTI_VHCA; 6105 6106 priv->rx_res = mlx5e_rx_res_create(priv->mdev, features, priv->max_nch, priv->drop_rq.rqn, 6107 &priv->channels.params.packet_merge, 6108 priv->channels.params.num_channels); 6109 if (IS_ERR(priv->rx_res)) { 6110 err = PTR_ERR(priv->rx_res); 6111 priv->rx_res = NULL; 6112 mlx5_core_err(mdev, "create rx resources failed, %d\n", err); 6113 goto err_close_drop_rq; 6114 } 6115 6116 err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile, 6117 priv->netdev); 6118 if (err) { 6119 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); 6120 goto err_destroy_rx_res; 6121 } 6122 6123 err = mlx5e_tc_nic_init(priv); 6124 if (err) 6125 goto err_destroy_flow_steering; 6126 6127 err = mlx5e_accel_init_rx(priv); 6128 if (err) 6129 goto err_tc_nic_cleanup; 6130 6131 #ifdef CONFIG_MLX5_EN_ARFS 6132 priv->netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(priv->mdev); 6133 #endif 6134 6135 return 0; 6136 6137 err_tc_nic_cleanup: 6138 mlx5e_tc_nic_cleanup(priv); 6139 err_destroy_flow_steering: 6140 mlx5e_destroy_flow_steering(priv->fs, mlx5e_fs_has_arfs(priv->netdev), 6141 priv->profile); 6142 err_destroy_rx_res: 6143 mlx5e_rx_res_destroy(priv->rx_res); 6144 priv->rx_res = NULL; 6145 err_close_drop_rq: 6146 mlx5e_close_drop_rq(&priv->drop_rq); 6147 err_destroy_q_counters: 6148 mlx5e_destroy_q_counters(priv); 6149 return err; 6150 } 6151 6152 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) 6153 { 6154 mlx5e_accel_cleanup_rx(priv); 6155 mlx5e_tc_nic_cleanup(priv); 6156 mlx5e_destroy_flow_steering(priv->fs, mlx5e_fs_has_arfs(priv->netdev), 6157 priv->profile); 6158 mlx5e_rx_res_destroy(priv->rx_res); 6159 priv->rx_res = NULL; 6160 mlx5e_close_drop_rq(&priv->drop_rq); 6161 mlx5e_destroy_q_counters(priv); 6162 } 6163 6164 static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv) 6165 { 6166 struct mlx5e_params *params; 6167 struct mlx5e_mqprio_rl *rl; 6168 6169 params = &priv->channels.params; 6170 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) 6171 return; 6172 6173 rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc, 6174 params->mqprio.channel.max_rate); 6175 if (IS_ERR(rl)) 6176 rl = NULL; 6177 priv->mqprio_rl = rl; 6178 mlx5e_mqprio_rl_update_params(params, rl); 6179 } 6180 6181 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) 6182 { 6183 int err; 6184 6185 err = mlx5e_accel_init_tx(priv); 6186 if (err) 6187 return err; 6188 6189 mlx5e_set_mqprio_rl(priv); 6190 mlx5e_dcbnl_initialize(priv); 6191 return 0; 6192 } 6193 6194 static void mlx5e_nic_enable(struct mlx5e_priv *priv) 6195 { 6196 struct net_device *netdev = priv->netdev; 6197 struct mlx5_core_dev *mdev = priv->mdev; 6198 int err; 6199 6200 mlx5e_fs_init_l2_addr(priv->fs, netdev); 6201 mlx5e_ipsec_init(priv); 6202 mlx5e_psp_register(priv); 6203 6204 err = mlx5e_macsec_init(priv); 6205 if (err) 6206 mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err); 6207 6208 /* Marking the link as currently not needed by the Driver */ 6209 if (!netif_running(netdev)) 6210 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN); 6211 6212 mlx5e_set_netdev_mtu_boundaries(priv); 6213 mlx5e_set_dev_port_mtu(priv); 6214 6215 mlx5_lag_add_netdev(mdev, netdev); 6216 6217 mlx5e_enable_async_events(priv); 6218 mlx5e_enable_blocking_events(priv); 6219 if (mlx5e_monitor_counter_supported(priv)) 6220 mlx5e_monitor_counter_init(priv); 6221 6222 mlx5e_pcie_cong_event_init(priv); 6223 mlx5e_hv_vhca_stats_create(priv); 6224 if (netdev->reg_state != NETREG_REGISTERED) 6225 return; 6226 mlx5e_dcbnl_init_app(priv); 6227 6228 mlx5e_nic_set_rx_mode(priv); 6229 6230 rtnl_lock(); 6231 netdev_lock(netdev); 6232 if (netif_running(netdev)) 6233 mlx5e_open(netdev); 6234 udp_tunnel_nic_reset_ntf(priv->netdev); 6235 netdev_unlock(netdev); 6236 netif_device_attach(netdev); 6237 rtnl_unlock(); 6238 } 6239 6240 static void mlx5e_nic_disable(struct mlx5e_priv *priv) 6241 { 6242 struct mlx5_core_dev *mdev = priv->mdev; 6243 6244 if (priv->netdev->reg_state == NETREG_REGISTERED) 6245 mlx5e_dcbnl_delete_app(priv); 6246 6247 rtnl_lock(); 6248 netdev_lock(priv->netdev); 6249 if (netif_running(priv->netdev)) 6250 mlx5e_close(priv->netdev); 6251 netif_device_detach(priv->netdev); 6252 if (priv->en_trap) { 6253 mlx5e_deactivate_trap(priv); 6254 mlx5e_close_trap(priv->en_trap); 6255 priv->en_trap = NULL; 6256 } 6257 netdev_unlock(priv->netdev); 6258 rtnl_unlock(); 6259 6260 mlx5e_nic_set_rx_mode(priv); 6261 6262 mlx5e_pcie_cong_event_cleanup(priv); 6263 mlx5e_hv_vhca_stats_destroy(priv); 6264 if (mlx5e_monitor_counter_supported(priv)) 6265 mlx5e_monitor_counter_cleanup(priv); 6266 6267 mlx5e_ipsec_disable_events(priv); 6268 mlx5e_disable_blocking_events(priv); 6269 mlx5e_disable_async_events(priv); 6270 mlx5_lag_remove_netdev(mdev, priv->netdev); 6271 mlx5_vxlan_reset_to_default(mdev->vxlan); 6272 mlx5e_macsec_cleanup(priv); 6273 mlx5e_psp_unregister(priv); 6274 mlx5e_ipsec_cleanup(priv); 6275 } 6276 6277 static int mlx5e_update_nic_rx(struct mlx5e_priv *priv) 6278 { 6279 return mlx5e_refresh_tirs(priv->mdev, false, false); 6280 } 6281 6282 static const struct mlx5e_profile mlx5e_nic_profile = { 6283 .init = mlx5e_nic_init, 6284 .cleanup = mlx5e_nic_cleanup, 6285 .init_rx = mlx5e_init_nic_rx, 6286 .cleanup_rx = mlx5e_cleanup_nic_rx, 6287 .init_tx = mlx5e_init_nic_tx, 6288 .cleanup_tx = mlx5e_cleanup_nic_tx, 6289 .enable = mlx5e_nic_enable, 6290 .disable = mlx5e_nic_disable, 6291 .update_rx = mlx5e_update_nic_rx, 6292 .update_stats = mlx5e_stats_update_ndo_stats, 6293 .update_carrier = mlx5e_update_carrier, 6294 .rx_handlers = &mlx5e_rx_handlers_nic, 6295 .max_tc = MLX5_MAX_NUM_TC, 6296 .stats_grps = mlx5e_nic_stats_grps, 6297 .stats_grps_num = mlx5e_nic_stats_grps_num, 6298 .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) | 6299 BIT(MLX5E_PROFILE_FEATURE_PTP_TX) | 6300 BIT(MLX5E_PROFILE_FEATURE_QOS_HTB) | 6301 BIT(MLX5E_PROFILE_FEATURE_FS_VLAN) | 6302 BIT(MLX5E_PROFILE_FEATURE_FS_TC), 6303 }; 6304 6305 static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev, 6306 const struct mlx5e_profile *profile) 6307 { 6308 int nch; 6309 6310 nch = mlx5e_get_max_num_channels(mdev); 6311 6312 if (profile->max_nch_limit) 6313 nch = min_t(int, nch, profile->max_nch_limit(mdev)); 6314 return nch; 6315 } 6316 6317 static unsigned int 6318 mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev, 6319 const struct mlx5e_profile *profile) 6320 6321 { 6322 unsigned int max_nch, tmp; 6323 6324 /* core resources */ 6325 max_nch = mlx5e_profile_max_num_channels(mdev, profile); 6326 6327 /* netdev rx queues */ 6328 max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues); 6329 6330 /* netdev tx queues */ 6331 tmp = netdev->num_tx_queues; 6332 if (mlx5_qos_is_supported(mdev)) 6333 tmp -= mlx5e_qos_max_leaf_nodes(mdev); 6334 if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn)) 6335 tmp -= profile->max_tc; 6336 tmp = tmp / profile->max_tc; 6337 max_nch = min_t(unsigned int, max_nch, tmp); 6338 6339 return max_nch; 6340 } 6341 6342 int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev) 6343 { 6344 /* Indirect TIRS: 2 sets of TTCs (inner + outer steering) 6345 * and 1 set of direct TIRS 6346 */ 6347 return 2 * MLX5E_NUM_INDIR_TIRS 6348 + mlx5e_profile_max_num_channels(mdev, &mlx5e_nic_profile); 6349 } 6350 6351 void mlx5e_set_rx_mode_work(struct work_struct *work) 6352 { 6353 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 6354 set_rx_mode_work); 6355 struct net_device *dev = priv->netdev; 6356 6357 netdev_lock_ops(dev); 6358 mlx5e_fs_set_rx_mode_work(priv->fs, dev, NULL, NULL); 6359 netdev_unlock_ops(dev); 6360 } 6361 6362 /* mlx5e generic netdev management API (move to en_common.c) */ 6363 int mlx5e_priv_init(struct mlx5e_priv *priv, 6364 const struct mlx5e_profile *profile, 6365 struct net_device *netdev, 6366 struct mlx5_core_dev *mdev) 6367 { 6368 int nch, num_txqs, node; 6369 int err; 6370 6371 num_txqs = netdev->num_tx_queues; 6372 nch = mlx5e_calc_max_nch(mdev, netdev, profile); 6373 node = dev_to_node(mlx5_core_dma_dev(mdev)); 6374 6375 /* priv init */ 6376 priv->mdev = mdev; 6377 priv->netdev = netdev; 6378 priv->max_nch = nch; 6379 priv->max_opened_tc = 1; 6380 6381 if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL)) 6382 return -ENOMEM; 6383 6384 mutex_init(&priv->state_lock); 6385 6386 err = mlx5e_selq_init(&priv->selq, &priv->state_lock); 6387 if (err) 6388 goto err_free_cpumask; 6389 6390 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); 6391 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); 6392 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); 6393 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); 6394 6395 priv->wq = create_singlethread_workqueue("mlx5e"); 6396 if (!priv->wq) 6397 goto err_free_selq; 6398 6399 priv->txq2sq = kcalloc_node(num_txqs, sizeof(*priv->txq2sq), GFP_KERNEL, node); 6400 if (!priv->txq2sq) 6401 goto err_destroy_workqueue; 6402 6403 priv->txq2sq_stats = kcalloc_node(num_txqs, sizeof(*priv->txq2sq_stats), GFP_KERNEL, node); 6404 if (!priv->txq2sq_stats) 6405 goto err_free_txq2sq; 6406 6407 priv->tx_rates = kcalloc_node(num_txqs, sizeof(*priv->tx_rates), GFP_KERNEL, node); 6408 if (!priv->tx_rates) 6409 goto err_free_txq2sq_stats; 6410 6411 priv->channel_stats = 6412 kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node); 6413 if (!priv->channel_stats) 6414 goto err_free_tx_rates; 6415 6416 priv->fec_ranges = kzalloc_objs(*priv->fec_ranges, ETHTOOL_FEC_HIST_MAX); 6417 if (!priv->fec_ranges) 6418 goto err_free_channel_stats; 6419 6420 return 0; 6421 6422 err_free_channel_stats: 6423 kfree(priv->channel_stats); 6424 err_free_tx_rates: 6425 kfree(priv->tx_rates); 6426 err_free_txq2sq_stats: 6427 kfree(priv->txq2sq_stats); 6428 err_free_txq2sq: 6429 kfree(priv->txq2sq); 6430 err_destroy_workqueue: 6431 destroy_workqueue(priv->wq); 6432 err_free_selq: 6433 mlx5e_selq_cleanup(&priv->selq); 6434 err_free_cpumask: 6435 free_cpumask_var(priv->scratchpad.cpumask); 6436 return -ENOMEM; 6437 } 6438 6439 void mlx5e_priv_cleanup(struct mlx5e_priv *priv) 6440 { 6441 bool destroying = test_bit(MLX5E_STATE_DESTROYING, &priv->state); 6442 int i; 6443 6444 /* bail if change profile failed and also rollback failed */ 6445 if (!priv->mdev) 6446 return; 6447 6448 kfree(priv->fec_ranges); 6449 for (i = 0; i < priv->stats_nch; i++) 6450 kvfree(priv->channel_stats[i]); 6451 kfree(priv->channel_stats); 6452 kfree(priv->tx_rates); 6453 kfree(priv->txq2sq_stats); 6454 kfree(priv->txq2sq); 6455 destroy_workqueue(priv->wq); 6456 mlx5e_selq_cleanup(&priv->selq); 6457 free_cpumask_var(priv->scratchpad.cpumask); 6458 6459 for (i = 0; i < priv->htb_max_qos_sqs; i++) 6460 kfree(priv->htb_qos_sq_stats[i]); 6461 kvfree(priv->htb_qos_sq_stats); 6462 6463 if (priv->mqprio_rl) { 6464 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); 6465 mlx5e_mqprio_rl_free(priv->mqprio_rl); 6466 } 6467 6468 memset(priv, 0, sizeof(*priv)); 6469 if (destroying) /* restore destroying bit, to allow unload */ 6470 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6471 } 6472 6473 static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev, 6474 const struct mlx5e_profile *profile) 6475 { 6476 unsigned int nch, ptp_txqs, qos_txqs; 6477 6478 nch = mlx5e_profile_max_num_channels(mdev, profile); 6479 6480 ptp_txqs = MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn) && 6481 mlx5e_profile_feature_cap(profile, PTP_TX) ? 6482 profile->max_tc : 0; 6483 6484 qos_txqs = mlx5_qos_is_supported(mdev) && 6485 mlx5e_profile_feature_cap(profile, QOS_HTB) ? 6486 mlx5e_qos_max_leaf_nodes(mdev) : 0; 6487 6488 return nch * profile->max_tc + ptp_txqs + qos_txqs; 6489 } 6490 6491 static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev, 6492 const struct mlx5e_profile *profile) 6493 { 6494 return mlx5e_profile_max_num_channels(mdev, profile); 6495 } 6496 6497 struct net_device * 6498 mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile) 6499 { 6500 struct net_device *netdev; 6501 unsigned int txqs, rxqs; 6502 int err; 6503 6504 txqs = mlx5e_get_max_num_txqs(mdev, profile); 6505 rxqs = mlx5e_get_max_num_rxqs(mdev, profile); 6506 6507 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs); 6508 if (!netdev) { 6509 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n"); 6510 return NULL; 6511 } 6512 6513 err = mlx5e_priv_init(netdev_priv(netdev), profile, netdev, mdev); 6514 if (err) { 6515 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 6516 goto err_free_netdev; 6517 } 6518 6519 netif_carrier_off(netdev); 6520 netif_tx_disable(netdev); 6521 dev_net_set(netdev, mlx5_core_net(mdev)); 6522 6523 return netdev; 6524 6525 err_free_netdev: 6526 free_netdev(netdev); 6527 6528 return NULL; 6529 } 6530 6531 static void mlx5e_update_features(struct net_device *netdev) 6532 { 6533 if (netdev->reg_state != NETREG_REGISTERED) 6534 return; /* features will be updated on netdev registration */ 6535 6536 rtnl_lock(); 6537 netdev_lock(netdev); 6538 netdev_update_features(netdev); 6539 netdev_unlock(netdev); 6540 rtnl_unlock(); 6541 } 6542 6543 static void mlx5e_reset_channels(struct net_device *netdev) 6544 { 6545 netdev_reset_tc(netdev); 6546 } 6547 6548 int mlx5e_attach_netdev(struct mlx5e_priv *priv) 6549 { 6550 const bool need_lock = priv->netdev->reg_state == NETREG_REGISTERED; 6551 const struct mlx5e_profile *profile = priv->profile; 6552 int max_nch; 6553 int err; 6554 6555 clear_bit(MLX5E_STATE_DESTROYING, &priv->state); 6556 if (priv->fs) 6557 mlx5e_fs_set_state_destroy(priv->fs, 6558 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 6559 6560 /* Validate the max_wqe_size_sq capability. */ 6561 if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) { 6562 mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %u\n", 6563 mlx5e_get_max_sq_wqebbs(priv->mdev), (unsigned int)MLX5E_MAX_TX_WQEBBS); 6564 return -EIO; 6565 } 6566 6567 /* max number of channels may have changed */ 6568 max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile); 6569 6570 /* Locking is required by ethtool_rxfh_indir_lost() (sends 6571 * ETHTOOL_MSG_RSS_NTF) and by netif_set_real_num_*_queues in case 6572 * the netdev has been registered by this point (if this function 6573 * was called in the reload or resume flow). 6574 */ 6575 if (need_lock) { 6576 rtnl_lock(); 6577 netdev_lock(priv->netdev); 6578 } 6579 6580 if (priv->channels.params.num_channels > max_nch) { 6581 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch); 6582 /* Reducing the number of channels - RXFH has to be reset, and 6583 * mlx5e_num_channels_changed below will build the RQT. 6584 */ 6585 ethtool_rxfh_indir_lost(priv->netdev); 6586 priv->channels.params.num_channels = max_nch; 6587 if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) { 6588 mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n"); 6589 mlx5e_params_mqprio_reset(&priv->channels.params); 6590 } 6591 } 6592 if (max_nch != priv->max_nch) { 6593 mlx5_core_warn(priv->mdev, 6594 "MLX5E: Updating max number of channels from %u to %u\n", 6595 priv->max_nch, max_nch); 6596 priv->max_nch = max_nch; 6597 } 6598 6599 /* 1. Set the real number of queues in the kernel the first time. 6600 * 2. Set our default XPS cpumask. 6601 * 3. Build the RQT. 6602 */ 6603 err = mlx5e_num_channels_changed(priv); 6604 if (need_lock) { 6605 netdev_unlock(priv->netdev); 6606 rtnl_unlock(); 6607 } 6608 if (err) 6609 goto out; 6610 6611 err = profile->init_tx(priv); 6612 if (err) 6613 goto out; 6614 6615 err = profile->init_rx(priv); 6616 if (err) 6617 goto err_cleanup_tx; 6618 6619 if (profile->enable) 6620 profile->enable(priv); 6621 6622 mlx5e_update_features(priv->netdev); 6623 6624 return 0; 6625 6626 err_cleanup_tx: 6627 profile->cleanup_tx(priv); 6628 6629 out: 6630 mlx5e_reset_channels(priv->netdev); 6631 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6632 if (priv->fs) 6633 mlx5e_fs_set_state_destroy(priv->fs, 6634 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 6635 cancel_work_sync(&priv->update_stats_work); 6636 return err; 6637 } 6638 6639 void mlx5e_detach_netdev(struct mlx5e_priv *priv) 6640 { 6641 const struct mlx5e_profile *profile = priv->profile; 6642 6643 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6644 if (priv->fs) 6645 mlx5e_fs_set_state_destroy(priv->fs, 6646 !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); 6647 6648 if (profile->disable) 6649 profile->disable(priv); 6650 flush_workqueue(priv->wq); 6651 6652 profile->cleanup_rx(priv); 6653 profile->cleanup_tx(priv); 6654 mlx5e_reset_channels(priv->netdev); 6655 cancel_work_sync(&priv->update_stats_work); 6656 } 6657 6658 static int 6659 mlx5e_netdev_init_profile(struct net_device *netdev, struct mlx5_core_dev *mdev, 6660 const struct mlx5e_profile *new_profile, void *new_ppriv) 6661 { 6662 struct mlx5e_priv *priv = netdev_priv(netdev); 6663 int err; 6664 6665 err = mlx5e_priv_init(priv, new_profile, netdev, mdev); 6666 if (err) { 6667 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err); 6668 return err; 6669 } 6670 netif_carrier_off(netdev); 6671 priv->profile = new_profile; 6672 priv->ppriv = new_ppriv; 6673 err = new_profile->init(priv->mdev, priv->netdev); 6674 if (err) 6675 goto priv_cleanup; 6676 6677 return 0; 6678 6679 priv_cleanup: 6680 mlx5e_priv_cleanup(priv); 6681 return err; 6682 } 6683 6684 static int 6685 mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev, 6686 const struct mlx5e_profile *new_profile, void *new_ppriv) 6687 { 6688 struct mlx5e_priv *priv = netdev_priv(netdev); 6689 int err; 6690 6691 err = mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv); 6692 if (err) 6693 return err; 6694 6695 err = mlx5e_attach_netdev(priv); 6696 if (err) 6697 goto profile_cleanup; 6698 return err; 6699 6700 profile_cleanup: 6701 new_profile->cleanup(priv); 6702 mlx5e_priv_cleanup(priv); 6703 return err; 6704 } 6705 6706 int mlx5e_netdev_change_profile(struct net_device *netdev, 6707 struct mlx5_core_dev *mdev, 6708 const struct mlx5e_profile *new_profile, 6709 void *new_ppriv) 6710 { 6711 struct mlx5e_priv *priv = netdev_priv(netdev); 6712 const struct mlx5e_profile *orig_profile; 6713 int err, rollback_err; 6714 void *orig_ppriv; 6715 6716 orig_profile = priv->profile; 6717 orig_ppriv = priv->ppriv; 6718 6719 /* NULL could happen if previous change_profile failed to rollback */ 6720 if (priv->profile) { 6721 WARN_ON_ONCE(priv->mdev != mdev); 6722 /* cleanup old profile */ 6723 mlx5e_detach_netdev(priv); 6724 priv->profile->cleanup(priv); 6725 mlx5e_priv_cleanup(priv); 6726 } 6727 /* priv members are not valid from this point ... */ 6728 6729 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) { 6730 mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv); 6731 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 6732 return -EIO; 6733 } 6734 6735 err = mlx5e_netdev_attach_profile(netdev, mdev, new_profile, new_ppriv); 6736 if (err) { /* roll back to original profile */ 6737 netdev_warn(netdev, "%s: new profile init failed, %d\n", __func__, err); 6738 goto rollback; 6739 } 6740 6741 return 0; 6742 6743 rollback: 6744 if (!orig_profile) { 6745 netdev_warn(netdev, "no original profile to rollback to\n"); 6746 priv->profile = NULL; 6747 return err; 6748 } 6749 6750 rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv); 6751 if (rollback_err) { 6752 netdev_err(netdev, "failed to rollback to orig profile, %d\n", 6753 rollback_err); 6754 priv->profile = NULL; 6755 } 6756 return err; 6757 } 6758 6759 void mlx5e_netdev_attach_nic_profile(struct net_device *netdev, 6760 struct mlx5_core_dev *mdev) 6761 { 6762 mlx5e_netdev_change_profile(netdev, mdev, &mlx5e_nic_profile, NULL); 6763 } 6764 6765 void mlx5e_destroy_netdev(struct net_device *netdev) 6766 { 6767 struct mlx5e_priv *priv = netdev_priv(netdev); 6768 6769 if (priv->profile) 6770 mlx5e_priv_cleanup(priv); 6771 free_netdev(netdev); 6772 } 6773 6774 static int _mlx5e_resume(struct auxiliary_device *adev) 6775 { 6776 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6777 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6778 struct mlx5e_priv *priv = netdev_priv(mlx5e_dev->netdev); 6779 struct net_device *netdev = mlx5e_dev->netdev; 6780 struct mlx5_core_dev *mdev = edev->mdev; 6781 struct mlx5_core_dev *pos, *to; 6782 int err, i; 6783 6784 if (netif_device_present(netdev)) 6785 return 0; 6786 6787 mlx5_sd_for_each_dev(i, mdev, pos) { 6788 err = mlx5e_create_mdev_resources(pos, true); 6789 if (err) 6790 goto err_destroy_mdev_res; 6791 } 6792 6793 err = mlx5e_attach_netdev(priv); 6794 if (err) 6795 goto err_destroy_mdev_res; 6796 6797 return 0; 6798 6799 err_destroy_mdev_res: 6800 to = pos; 6801 mlx5_sd_for_each_dev_to(i, mdev, to, pos) 6802 mlx5e_destroy_mdev_resources(pos); 6803 return err; 6804 } 6805 6806 static int mlx5e_resume(struct auxiliary_device *adev) 6807 { 6808 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6809 struct mlx5_core_dev *mdev = edev->mdev; 6810 struct auxiliary_device *actual_adev; 6811 int err; 6812 6813 err = mlx5_sd_init(mdev); 6814 if (err) 6815 return err; 6816 6817 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6818 if (actual_adev) { 6819 err = _mlx5e_resume(actual_adev); 6820 mlx5_sd_put_adev(actual_adev, adev); 6821 } 6822 return err; 6823 } 6824 6825 static int _mlx5e_suspend(struct auxiliary_device *adev, bool pre_netdev_reg) 6826 { 6827 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6828 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6829 struct mlx5e_priv *priv = netdev_priv(mlx5e_dev->netdev); 6830 struct net_device *netdev = mlx5e_dev->netdev; 6831 struct mlx5_core_dev *mdev = edev->mdev; 6832 struct mlx5_core_dev *pos; 6833 int i; 6834 6835 if (!pre_netdev_reg && !netif_device_present(netdev)) { 6836 if (test_bit(MLX5E_STATE_DESTROYING, &priv->state)) 6837 mlx5_sd_for_each_dev(i, mdev, pos) 6838 mlx5e_destroy_mdev_resources(pos); 6839 return -ENODEV; 6840 } 6841 6842 mlx5e_detach_netdev(priv); 6843 mlx5_sd_for_each_dev(i, mdev, pos) 6844 mlx5e_destroy_mdev_resources(pos); 6845 6846 return 0; 6847 } 6848 6849 static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state) 6850 { 6851 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6852 struct mlx5_core_dev *mdev = edev->mdev; 6853 struct auxiliary_device *actual_adev; 6854 int err = 0; 6855 6856 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6857 if (actual_adev) 6858 err = _mlx5e_suspend(actual_adev, false); 6859 6860 mlx5_sd_cleanup(mdev); 6861 if (actual_adev) 6862 mlx5_sd_put_adev(actual_adev, adev); 6863 return err; 6864 } 6865 6866 static int _mlx5e_probe(struct auxiliary_device *adev) 6867 { 6868 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6869 const struct mlx5e_profile *profile = &mlx5e_nic_profile; 6870 struct mlx5_core_dev *mdev = edev->mdev; 6871 struct mlx5e_dev *mlx5e_dev; 6872 struct net_device *netdev; 6873 struct mlx5e_priv *priv; 6874 int err; 6875 6876 mlx5e_dev = mlx5e_create_devlink(&adev->dev, mdev); 6877 if (IS_ERR(mlx5e_dev)) 6878 return PTR_ERR(mlx5e_dev); 6879 auxiliary_set_drvdata(adev, mlx5e_dev); 6880 6881 err = mlx5e_devlink_port_register(mlx5e_dev, mdev); 6882 if (err) { 6883 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err); 6884 goto err_devlink_unregister; 6885 } 6886 6887 netdev = mlx5e_create_netdev(mdev, profile); 6888 if (!netdev) { 6889 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n"); 6890 err = -ENOMEM; 6891 goto err_devlink_port_unregister; 6892 } 6893 SET_NETDEV_DEVLINK_PORT(netdev, &mlx5e_dev->dl_port); 6894 mlx5e_dev->netdev = netdev; 6895 6896 mlx5e_build_nic_netdev(netdev); 6897 6898 priv = netdev_priv(netdev); 6899 6900 priv->profile = profile; 6901 priv->ppriv = NULL; 6902 6903 err = profile->init(mdev, netdev); 6904 if (err) { 6905 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err); 6906 goto err_destroy_netdev; 6907 } 6908 6909 err = _mlx5e_resume(adev); 6910 if (err) { 6911 mlx5_core_err(mdev, "_mlx5e_resume failed, %d\n", err); 6912 goto err_profile_cleanup; 6913 } 6914 6915 err = register_netdev(netdev); 6916 if (err) { 6917 mlx5_core_err(mdev, "register_netdev failed, %d\n", err); 6918 goto err_resume; 6919 } 6920 6921 /* mlx5e_fix_features() returns early when the device is not present 6922 * to avoid dereferencing cleared priv during profile changes. 6923 * This also causes it to be a no-op during register_netdev(), where 6924 * the device is not yet present. 6925 * Trigger an additional features update that will actually work. 6926 */ 6927 mlx5e_update_features(netdev); 6928 6929 mlx5e_dcbnl_init_app(priv); 6930 mlx5_core_uplink_netdev_set(mdev, netdev); 6931 mlx5e_params_print_info(mdev, &priv->channels.params); 6932 return 0; 6933 6934 err_resume: 6935 _mlx5e_suspend(adev, true); 6936 err_profile_cleanup: 6937 profile->cleanup(priv); 6938 err_destroy_netdev: 6939 mlx5e_destroy_netdev(netdev); 6940 err_devlink_port_unregister: 6941 mlx5e_devlink_port_unregister(mlx5e_dev); 6942 err_devlink_unregister: 6943 mlx5e_destroy_devlink(mlx5e_dev); 6944 return err; 6945 } 6946 6947 static int mlx5e_probe(struct auxiliary_device *adev, 6948 const struct auxiliary_device_id *id) 6949 { 6950 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6951 struct mlx5_core_dev *mdev = edev->mdev; 6952 struct auxiliary_device *actual_adev; 6953 int err; 6954 6955 err = mlx5_sd_init(mdev); 6956 if (err) 6957 return err; 6958 6959 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 6960 if (actual_adev) { 6961 err = _mlx5e_probe(actual_adev); 6962 if (err) 6963 goto sd_cleanup; 6964 mlx5_sd_put_adev(actual_adev, adev); 6965 } 6966 return 0; 6967 6968 sd_cleanup: 6969 mlx5_sd_cleanup(mdev); 6970 if (actual_adev) 6971 mlx5_sd_put_adev(actual_adev, adev); 6972 return err; 6973 } 6974 6975 static void _mlx5e_remove(struct auxiliary_device *adev) 6976 { 6977 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 6978 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev); 6979 struct net_device *netdev = mlx5e_dev->netdev; 6980 struct mlx5e_priv *priv = netdev_priv(netdev); 6981 struct mlx5_core_dev *mdev = edev->mdev; 6982 6983 mlx5_eswitch_safe_aux_devs_remove(mdev); 6984 mlx5_core_uplink_netdev_set(mdev, NULL); 6985 6986 if (priv->profile) 6987 mlx5e_dcbnl_delete_app(priv); 6988 /* When unload driver, the netdev is in registered state 6989 * if it's from legacy mode. If from switchdev mode, it 6990 * is already unregistered before changing to NIC profile. 6991 */ 6992 if (netdev->reg_state == NETREG_REGISTERED) { 6993 unregister_netdev(netdev); 6994 _mlx5e_suspend(adev, false); 6995 } else { 6996 struct mlx5_core_dev *pos; 6997 int i; 6998 6999 if (test_bit(MLX5E_STATE_DESTROYING, &priv->state)) 7000 mlx5_sd_for_each_dev(i, mdev, pos) 7001 mlx5e_destroy_mdev_resources(pos); 7002 else 7003 _mlx5e_suspend(adev, true); 7004 } 7005 /* Avoid cleanup if profile rollback failed. */ 7006 if (priv->profile) 7007 priv->profile->cleanup(priv); 7008 mlx5e_destroy_netdev(netdev); 7009 mlx5e_devlink_port_unregister(mlx5e_dev); 7010 mlx5e_destroy_devlink(mlx5e_dev); 7011 } 7012 7013 static void mlx5e_remove(struct auxiliary_device *adev) 7014 { 7015 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 7016 struct mlx5_core_dev *mdev = edev->mdev; 7017 struct auxiliary_device *actual_adev; 7018 7019 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx); 7020 if (actual_adev) 7021 _mlx5e_remove(actual_adev); 7022 7023 mlx5_sd_cleanup(mdev); 7024 if (actual_adev) 7025 mlx5_sd_put_adev(actual_adev, adev); 7026 } 7027 7028 static const struct auxiliary_device_id mlx5e_id_table[] = { 7029 { .name = MLX5_ADEV_NAME ".eth", }, 7030 {}, 7031 }; 7032 7033 MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table); 7034 7035 static struct auxiliary_driver mlx5e_driver = { 7036 .name = "eth", 7037 .probe = mlx5e_probe, 7038 .remove = mlx5e_remove, 7039 .suspend = mlx5e_suspend, 7040 .resume = mlx5e_resume, 7041 .id_table = mlx5e_id_table, 7042 }; 7043 7044 int mlx5e_init(void) 7045 { 7046 int ret; 7047 7048 mlx5e_build_ptys2ethtool_map(); 7049 ret = auxiliary_driver_register(&mlx5e_driver); 7050 if (ret) 7051 return ret; 7052 7053 ret = mlx5e_rep_init(); 7054 if (ret) 7055 auxiliary_driver_unregister(&mlx5e_driver); 7056 return ret; 7057 } 7058 7059 void mlx5e_cleanup(void) 7060 { 7061 mlx5e_rep_cleanup(); 7062 auxiliary_driver_unregister(&mlx5e_driver); 7063 } 7064