1 /* 2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <linux/bpf.h> 35 #include <linux/etherdevice.h> 36 #include <linux/filter.h> 37 #include <linux/tcp.h> 38 #include <linux/if_vlan.h> 39 #include <linux/delay.h> 40 #include <linux/slab.h> 41 #include <linux/hash.h> 42 #include <net/ip.h> 43 #include <net/vxlan.h> 44 #include <net/devlink.h> 45 #include <net/rps.h> 46 #include <net/netdev_queues.h> 47 48 #include <linux/mlx4/driver.h> 49 #include <linux/mlx4/device.h> 50 #include <linux/mlx4/cmd.h> 51 #include <linux/mlx4/cq.h> 52 53 #include "mlx4_en.h" 54 #include "en_port.h" 55 56 #define MLX4_EN_MAX_XDP_MTU ((int)(PAGE_SIZE - ETH_HLEN - (2 * VLAN_HLEN) - \ 57 XDP_PACKET_HEADROOM - \ 58 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))) 59 60 int mlx4_en_setup_tc(struct net_device *dev, u8 up) 61 { 62 struct mlx4_en_priv *priv = netdev_priv(dev); 63 int i; 64 unsigned int offset = 0; 65 66 if (up && up != MLX4_EN_NUM_UP_HIGH) 67 return -EINVAL; 68 69 netdev_set_num_tc(dev, up); 70 netif_set_real_num_tx_queues(dev, priv->tx_ring_num[TX]); 71 /* Partition Tx queues evenly amongst UP's */ 72 for (i = 0; i < up; i++) { 73 netdev_set_tc_queue(dev, i, priv->num_tx_rings_p_up, offset); 74 offset += priv->num_tx_rings_p_up; 75 } 76 77 #ifdef CONFIG_MLX4_EN_DCB 78 if (!mlx4_is_slave(priv->mdev->dev)) { 79 if (up) { 80 if (priv->dcbx_cap) 81 priv->flags |= MLX4_EN_FLAG_DCB_ENABLED; 82 } else { 83 priv->flags &= ~MLX4_EN_FLAG_DCB_ENABLED; 84 priv->cee_config.pfc_state = false; 85 } 86 } 87 #endif /* CONFIG_MLX4_EN_DCB */ 88 89 return 0; 90 } 91 92 int mlx4_en_alloc_tx_queue_per_tc(struct net_device *dev, u8 tc) 93 { 94 struct mlx4_en_priv *priv = netdev_priv(dev); 95 struct mlx4_en_dev *mdev = priv->mdev; 96 struct mlx4_en_port_profile new_prof; 97 struct mlx4_en_priv *tmp; 98 int total_count; 99 int port_up = 0; 100 int err = 0; 101 102 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); 103 if (!tmp) 104 return -ENOMEM; 105 106 mutex_lock(&mdev->state_lock); 107 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); 108 new_prof.num_up = (tc == 0) ? MLX4_EN_NUM_UP_LOW : 109 MLX4_EN_NUM_UP_HIGH; 110 new_prof.tx_ring_num[TX] = new_prof.num_tx_rings_p_up * 111 new_prof.num_up; 112 total_count = new_prof.tx_ring_num[TX] + new_prof.tx_ring_num[TX_XDP]; 113 if (total_count > MAX_TX_RINGS) { 114 err = -EINVAL; 115 en_err(priv, 116 "Total number of TX and XDP rings (%d) exceeds the maximum supported (%d)\n", 117 total_count, MAX_TX_RINGS); 118 goto out; 119 } 120 err = mlx4_en_try_alloc_resources(priv, tmp, &new_prof, true); 121 if (err) 122 goto out; 123 124 if (priv->port_up) { 125 port_up = 1; 126 mlx4_en_stop_port(dev, 1); 127 } 128 129 mlx4_en_safe_replace_resources(priv, tmp); 130 if (port_up) { 131 err = mlx4_en_start_port(dev); 132 if (err) { 133 en_err(priv, "Failed starting port for setup TC\n"); 134 goto out; 135 } 136 } 137 138 err = mlx4_en_setup_tc(dev, tc); 139 out: 140 mutex_unlock(&mdev->state_lock); 141 kfree(tmp); 142 return err; 143 } 144 145 static int __mlx4_en_setup_tc(struct net_device *dev, enum tc_setup_type type, 146 void *type_data) 147 { 148 struct tc_mqprio_qopt *mqprio = type_data; 149 150 if (type != TC_SETUP_QDISC_MQPRIO) 151 return -EOPNOTSUPP; 152 153 if (mqprio->num_tc && mqprio->num_tc != MLX4_EN_NUM_UP_HIGH) 154 return -EINVAL; 155 156 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 157 158 return mlx4_en_alloc_tx_queue_per_tc(dev, mqprio->num_tc); 159 } 160 161 #ifdef CONFIG_RFS_ACCEL 162 163 struct mlx4_en_filter { 164 struct list_head next; 165 struct work_struct work; 166 167 u8 ip_proto; 168 __be32 src_ip; 169 __be32 dst_ip; 170 __be16 src_port; 171 __be16 dst_port; 172 173 int rxq_index; 174 struct mlx4_en_priv *priv; 175 u32 flow_id; /* RFS infrastructure id */ 176 int id; /* mlx4_en driver id */ 177 u64 reg_id; /* Flow steering API id */ 178 u8 activated; /* Used to prevent expiry before filter 179 * is attached 180 */ 181 struct hlist_node filter_chain; 182 }; 183 184 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv); 185 186 static enum mlx4_net_trans_rule_id mlx4_ip_proto_to_trans_rule_id(u8 ip_proto) 187 { 188 switch (ip_proto) { 189 case IPPROTO_UDP: 190 return MLX4_NET_TRANS_RULE_ID_UDP; 191 case IPPROTO_TCP: 192 return MLX4_NET_TRANS_RULE_ID_TCP; 193 default: 194 return MLX4_NET_TRANS_RULE_NUM; 195 } 196 }; 197 198 /* Must not acquire state_lock, as its corresponding work_sync 199 * is done under it. 200 */ 201 static void mlx4_en_filter_work(struct work_struct *work) 202 { 203 struct mlx4_en_filter *filter = container_of(work, 204 struct mlx4_en_filter, 205 work); 206 struct mlx4_en_priv *priv = filter->priv; 207 struct mlx4_spec_list spec_tcp_udp = { 208 .id = mlx4_ip_proto_to_trans_rule_id(filter->ip_proto), 209 { 210 .tcp_udp = { 211 .dst_port = filter->dst_port, 212 .dst_port_msk = (__force __be16)-1, 213 .src_port = filter->src_port, 214 .src_port_msk = (__force __be16)-1, 215 }, 216 }, 217 }; 218 struct mlx4_spec_list spec_ip = { 219 .id = MLX4_NET_TRANS_RULE_ID_IPV4, 220 { 221 .ipv4 = { 222 .dst_ip = filter->dst_ip, 223 .dst_ip_msk = (__force __be32)-1, 224 .src_ip = filter->src_ip, 225 .src_ip_msk = (__force __be32)-1, 226 }, 227 }, 228 }; 229 struct mlx4_spec_list spec_eth = { 230 .id = MLX4_NET_TRANS_RULE_ID_ETH, 231 }; 232 struct mlx4_net_trans_rule rule = { 233 .list = LIST_HEAD_INIT(rule.list), 234 .queue_mode = MLX4_NET_TRANS_Q_LIFO, 235 .exclusive = 1, 236 .allow_loopback = 1, 237 .promisc_mode = MLX4_FS_REGULAR, 238 .port = priv->port, 239 .priority = MLX4_DOMAIN_RFS, 240 }; 241 int rc; 242 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); 243 244 if (spec_tcp_udp.id >= MLX4_NET_TRANS_RULE_NUM) { 245 en_warn(priv, "RFS: ignoring unsupported ip protocol (%d)\n", 246 filter->ip_proto); 247 goto ignore; 248 } 249 list_add_tail(&spec_eth.list, &rule.list); 250 list_add_tail(&spec_ip.list, &rule.list); 251 list_add_tail(&spec_tcp_udp.list, &rule.list); 252 253 rule.qpn = priv->rss_map.qps[filter->rxq_index].qpn; 254 memcpy(spec_eth.eth.dst_mac, priv->dev->dev_addr, ETH_ALEN); 255 memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); 256 257 filter->activated = 0; 258 259 if (filter->reg_id) { 260 rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); 261 if (rc && rc != -ENOENT) 262 en_err(priv, "Error detaching flow. rc = %d\n", rc); 263 } 264 265 rc = mlx4_flow_attach(priv->mdev->dev, &rule, &filter->reg_id); 266 if (rc) 267 en_err(priv, "Error attaching flow. err = %d\n", rc); 268 269 ignore: 270 mlx4_en_filter_rfs_expire(priv); 271 272 filter->activated = 1; 273 } 274 275 static inline struct hlist_head * 276 filter_hash_bucket(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, 277 __be16 src_port, __be16 dst_port) 278 { 279 unsigned long l; 280 int bucket_idx; 281 282 l = (__force unsigned long)src_port | 283 ((__force unsigned long)dst_port << 2); 284 l ^= (__force unsigned long)(src_ip ^ dst_ip); 285 286 bucket_idx = hash_long(l, MLX4_EN_FILTER_HASH_SHIFT); 287 288 return &priv->filter_hash[bucket_idx]; 289 } 290 291 static struct mlx4_en_filter * 292 mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip, 293 __be32 dst_ip, u8 ip_proto, __be16 src_port, 294 __be16 dst_port, u32 flow_id) 295 { 296 struct mlx4_en_filter *filter; 297 298 filter = kzalloc(sizeof(struct mlx4_en_filter), GFP_ATOMIC); 299 if (!filter) 300 return NULL; 301 302 filter->priv = priv; 303 filter->rxq_index = rxq_index; 304 INIT_WORK(&filter->work, mlx4_en_filter_work); 305 306 filter->src_ip = src_ip; 307 filter->dst_ip = dst_ip; 308 filter->ip_proto = ip_proto; 309 filter->src_port = src_port; 310 filter->dst_port = dst_port; 311 312 filter->flow_id = flow_id; 313 314 filter->id = priv->last_filter_id++ % RPS_NO_FILTER; 315 316 list_add_tail(&filter->next, &priv->filters); 317 hlist_add_head(&filter->filter_chain, 318 filter_hash_bucket(priv, src_ip, dst_ip, src_port, 319 dst_port)); 320 321 return filter; 322 } 323 324 static void mlx4_en_filter_free(struct mlx4_en_filter *filter) 325 { 326 struct mlx4_en_priv *priv = filter->priv; 327 int rc; 328 329 list_del(&filter->next); 330 331 rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); 332 if (rc && rc != -ENOENT) 333 en_err(priv, "Error detaching flow. rc = %d\n", rc); 334 335 kfree(filter); 336 } 337 338 static inline struct mlx4_en_filter * 339 mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, 340 u8 ip_proto, __be16 src_port, __be16 dst_port) 341 { 342 struct mlx4_en_filter *filter; 343 struct mlx4_en_filter *ret = NULL; 344 345 hlist_for_each_entry(filter, 346 filter_hash_bucket(priv, src_ip, dst_ip, 347 src_port, dst_port), 348 filter_chain) { 349 if (filter->src_ip == src_ip && 350 filter->dst_ip == dst_ip && 351 filter->ip_proto == ip_proto && 352 filter->src_port == src_port && 353 filter->dst_port == dst_port) { 354 ret = filter; 355 break; 356 } 357 } 358 359 return ret; 360 } 361 362 static int 363 mlx4_en_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, 364 u16 rxq_index, u32 flow_id) 365 { 366 struct mlx4_en_priv *priv = netdev_priv(net_dev); 367 struct mlx4_en_filter *filter; 368 const struct iphdr *ip; 369 const __be16 *ports; 370 u8 ip_proto; 371 __be32 src_ip; 372 __be32 dst_ip; 373 __be16 src_port; 374 __be16 dst_port; 375 int nhoff = skb_network_offset(skb); 376 int ret = 0; 377 378 if (skb->encapsulation) 379 return -EPROTONOSUPPORT; 380 381 if (skb->protocol != htons(ETH_P_IP)) 382 return -EPROTONOSUPPORT; 383 384 ip = (const struct iphdr *)(skb->data + nhoff); 385 if (ip_is_fragment(ip)) 386 return -EPROTONOSUPPORT; 387 388 if ((ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP)) 389 return -EPROTONOSUPPORT; 390 ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); 391 392 ip_proto = ip->protocol; 393 src_ip = ip->saddr; 394 dst_ip = ip->daddr; 395 src_port = ports[0]; 396 dst_port = ports[1]; 397 398 spin_lock_bh(&priv->filters_lock); 399 filter = mlx4_en_filter_find(priv, src_ip, dst_ip, ip_proto, 400 src_port, dst_port); 401 if (filter) { 402 if (filter->rxq_index == rxq_index) 403 goto out; 404 405 filter->rxq_index = rxq_index; 406 } else { 407 filter = mlx4_en_filter_alloc(priv, rxq_index, 408 src_ip, dst_ip, ip_proto, 409 src_port, dst_port, flow_id); 410 if (!filter) { 411 ret = -ENOMEM; 412 goto err; 413 } 414 } 415 416 queue_work(priv->mdev->workqueue, &filter->work); 417 418 out: 419 ret = filter->id; 420 err: 421 spin_unlock_bh(&priv->filters_lock); 422 423 return ret; 424 } 425 426 void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv) 427 { 428 struct mlx4_en_filter *filter, *tmp; 429 LIST_HEAD(del_list); 430 431 spin_lock_bh(&priv->filters_lock); 432 list_for_each_entry_safe(filter, tmp, &priv->filters, next) { 433 list_move(&filter->next, &del_list); 434 hlist_del(&filter->filter_chain); 435 } 436 spin_unlock_bh(&priv->filters_lock); 437 438 list_for_each_entry_safe(filter, tmp, &del_list, next) { 439 cancel_work_sync(&filter->work); 440 mlx4_en_filter_free(filter); 441 } 442 } 443 444 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv) 445 { 446 struct mlx4_en_filter *filter = NULL, *tmp, *last_filter = NULL; 447 LIST_HEAD(del_list); 448 int i = 0; 449 450 spin_lock_bh(&priv->filters_lock); 451 list_for_each_entry_safe(filter, tmp, &priv->filters, next) { 452 if (i > MLX4_EN_FILTER_EXPIRY_QUOTA) 453 break; 454 455 if (filter->activated && 456 !work_pending(&filter->work) && 457 rps_may_expire_flow(priv->dev, 458 filter->rxq_index, filter->flow_id, 459 filter->id)) { 460 list_move(&filter->next, &del_list); 461 hlist_del(&filter->filter_chain); 462 } else 463 last_filter = filter; 464 465 i++; 466 } 467 468 if (last_filter && (&last_filter->next != priv->filters.next)) 469 list_move(&priv->filters, &last_filter->next); 470 471 spin_unlock_bh(&priv->filters_lock); 472 473 list_for_each_entry_safe(filter, tmp, &del_list, next) 474 mlx4_en_filter_free(filter); 475 } 476 #endif 477 478 static int mlx4_en_vlan_rx_add_vid(struct net_device *dev, 479 __be16 proto, u16 vid) 480 { 481 struct mlx4_en_priv *priv = netdev_priv(dev); 482 struct mlx4_en_dev *mdev = priv->mdev; 483 int err; 484 int idx; 485 486 en_dbg(HW, priv, "adding VLAN:%d\n", vid); 487 488 set_bit(vid, priv->active_vlans); 489 490 /* Add VID to port VLAN filter */ 491 mutex_lock(&mdev->state_lock); 492 if (mdev->device_up && priv->port_up) { 493 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); 494 if (err) { 495 en_err(priv, "Failed configuring VLAN filter\n"); 496 goto out; 497 } 498 } 499 err = mlx4_register_vlan(mdev->dev, priv->port, vid, &idx); 500 if (err) 501 en_dbg(HW, priv, "Failed adding vlan %d\n", vid); 502 503 out: 504 mutex_unlock(&mdev->state_lock); 505 return err; 506 } 507 508 static int mlx4_en_vlan_rx_kill_vid(struct net_device *dev, 509 __be16 proto, u16 vid) 510 { 511 struct mlx4_en_priv *priv = netdev_priv(dev); 512 struct mlx4_en_dev *mdev = priv->mdev; 513 int err = 0; 514 515 en_dbg(HW, priv, "Killing VID:%d\n", vid); 516 517 clear_bit(vid, priv->active_vlans); 518 519 /* Remove VID from port VLAN filter */ 520 mutex_lock(&mdev->state_lock); 521 mlx4_unregister_vlan(mdev->dev, priv->port, vid); 522 523 if (mdev->device_up && priv->port_up) { 524 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); 525 if (err) 526 en_err(priv, "Failed configuring VLAN filter\n"); 527 } 528 mutex_unlock(&mdev->state_lock); 529 530 return err; 531 } 532 533 static void mlx4_en_u64_to_mac(struct net_device *dev, u64 src_mac) 534 { 535 u8 addr[ETH_ALEN]; 536 537 u64_to_ether_addr(src_mac, addr); 538 eth_hw_addr_set(dev, addr); 539 } 540 541 542 static int mlx4_en_tunnel_steer_add(struct mlx4_en_priv *priv, 543 const unsigned char *addr, 544 int qpn, u64 *reg_id) 545 { 546 int err; 547 548 if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN || 549 priv->mdev->dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC) 550 return 0; /* do nothing */ 551 552 err = mlx4_tunnel_steer_add(priv->mdev->dev, addr, priv->port, qpn, 553 MLX4_DOMAIN_NIC, reg_id); 554 if (err) { 555 en_err(priv, "failed to add vxlan steering rule, err %d\n", err); 556 return err; 557 } 558 en_dbg(DRV, priv, "added vxlan steering rule, mac %pM reg_id %llx\n", addr, *reg_id); 559 return 0; 560 } 561 562 563 static int mlx4_en_uc_steer_add(struct mlx4_en_priv *priv, 564 const unsigned char *mac, int *qpn, u64 *reg_id) 565 { 566 struct mlx4_en_dev *mdev = priv->mdev; 567 struct mlx4_dev *dev = mdev->dev; 568 int err; 569 570 switch (dev->caps.steering_mode) { 571 case MLX4_STEERING_MODE_B0: { 572 struct mlx4_qp qp; 573 u8 gid[16] = {0}; 574 575 qp.qpn = *qpn; 576 memcpy(&gid[10], mac, ETH_ALEN); 577 gid[5] = priv->port; 578 579 err = mlx4_unicast_attach(dev, &qp, gid, 0, MLX4_PROT_ETH); 580 break; 581 } 582 case MLX4_STEERING_MODE_DEVICE_MANAGED: { 583 struct mlx4_spec_list spec_eth = { {NULL} }; 584 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); 585 586 struct mlx4_net_trans_rule rule = { 587 .queue_mode = MLX4_NET_TRANS_Q_FIFO, 588 .exclusive = 0, 589 .allow_loopback = 1, 590 .promisc_mode = MLX4_FS_REGULAR, 591 .priority = MLX4_DOMAIN_NIC, 592 }; 593 594 rule.port = priv->port; 595 rule.qpn = *qpn; 596 INIT_LIST_HEAD(&rule.list); 597 598 spec_eth.id = MLX4_NET_TRANS_RULE_ID_ETH; 599 memcpy(spec_eth.eth.dst_mac, mac, ETH_ALEN); 600 memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); 601 list_add_tail(&spec_eth.list, &rule.list); 602 603 err = mlx4_flow_attach(dev, &rule, reg_id); 604 break; 605 } 606 default: 607 return -EINVAL; 608 } 609 if (err) 610 en_warn(priv, "Failed Attaching Unicast\n"); 611 612 return err; 613 } 614 615 static void mlx4_en_uc_steer_release(struct mlx4_en_priv *priv, 616 const unsigned char *mac, 617 int qpn, u64 reg_id) 618 { 619 struct mlx4_en_dev *mdev = priv->mdev; 620 struct mlx4_dev *dev = mdev->dev; 621 622 switch (dev->caps.steering_mode) { 623 case MLX4_STEERING_MODE_B0: { 624 struct mlx4_qp qp; 625 u8 gid[16] = {0}; 626 627 qp.qpn = qpn; 628 memcpy(&gid[10], mac, ETH_ALEN); 629 gid[5] = priv->port; 630 631 mlx4_unicast_detach(dev, &qp, gid, MLX4_PROT_ETH); 632 break; 633 } 634 case MLX4_STEERING_MODE_DEVICE_MANAGED: { 635 mlx4_flow_detach(dev, reg_id); 636 break; 637 } 638 default: 639 en_err(priv, "Invalid steering mode.\n"); 640 } 641 } 642 643 static int mlx4_en_get_qp(struct mlx4_en_priv *priv) 644 { 645 struct mlx4_en_dev *mdev = priv->mdev; 646 struct mlx4_dev *dev = mdev->dev; 647 int index = 0; 648 int err = 0; 649 int *qpn = &priv->base_qpn; 650 u64 mac = ether_addr_to_u64(priv->dev->dev_addr); 651 652 en_dbg(DRV, priv, "Registering MAC: %pM for adding\n", 653 priv->dev->dev_addr); 654 index = mlx4_register_mac(dev, priv->port, mac); 655 if (index < 0) { 656 err = index; 657 en_err(priv, "Failed adding MAC: %pM\n", 658 priv->dev->dev_addr); 659 return err; 660 } 661 662 en_info(priv, "Steering Mode %d\n", dev->caps.steering_mode); 663 664 if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { 665 int base_qpn = mlx4_get_base_qpn(dev, priv->port); 666 *qpn = base_qpn + index; 667 return 0; 668 } 669 670 err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP, 671 MLX4_RES_USAGE_DRIVER); 672 en_dbg(DRV, priv, "Reserved qp %d\n", *qpn); 673 if (err) { 674 en_err(priv, "Failed to reserve qp for mac registration\n"); 675 mlx4_unregister_mac(dev, priv->port, mac); 676 return err; 677 } 678 679 return 0; 680 } 681 682 static void mlx4_en_put_qp(struct mlx4_en_priv *priv) 683 { 684 struct mlx4_en_dev *mdev = priv->mdev; 685 struct mlx4_dev *dev = mdev->dev; 686 int qpn = priv->base_qpn; 687 688 if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { 689 u64 mac = ether_addr_to_u64(priv->dev->dev_addr); 690 en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n", 691 priv->dev->dev_addr); 692 mlx4_unregister_mac(dev, priv->port, mac); 693 } else { 694 en_dbg(DRV, priv, "Releasing qp: port %d, qpn %d\n", 695 priv->port, qpn); 696 mlx4_qp_release_range(dev, qpn, 1); 697 priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; 698 } 699 } 700 701 static int mlx4_en_replace_mac(struct mlx4_en_priv *priv, int qpn, 702 unsigned char *new_mac, unsigned char *prev_mac) 703 { 704 struct mlx4_en_dev *mdev = priv->mdev; 705 struct mlx4_dev *dev = mdev->dev; 706 int err = 0; 707 u64 new_mac_u64 = ether_addr_to_u64(new_mac); 708 709 if (dev->caps.steering_mode != MLX4_STEERING_MODE_A0) { 710 struct hlist_head *bucket; 711 unsigned int mac_hash; 712 struct mlx4_mac_entry *entry; 713 struct hlist_node *tmp; 714 u64 prev_mac_u64 = ether_addr_to_u64(prev_mac); 715 716 bucket = &priv->mac_hash[prev_mac[MLX4_EN_MAC_HASH_IDX]]; 717 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { 718 if (ether_addr_equal_64bits(entry->mac, prev_mac)) { 719 mlx4_en_uc_steer_release(priv, entry->mac, 720 qpn, entry->reg_id); 721 mlx4_unregister_mac(dev, priv->port, 722 prev_mac_u64); 723 hlist_del_rcu(&entry->hlist); 724 synchronize_rcu(); 725 memcpy(entry->mac, new_mac, ETH_ALEN); 726 entry->reg_id = 0; 727 mac_hash = new_mac[MLX4_EN_MAC_HASH_IDX]; 728 hlist_add_head_rcu(&entry->hlist, 729 &priv->mac_hash[mac_hash]); 730 mlx4_register_mac(dev, priv->port, new_mac_u64); 731 err = mlx4_en_uc_steer_add(priv, new_mac, 732 &qpn, 733 &entry->reg_id); 734 if (err) 735 return err; 736 if (priv->tunnel_reg_id) { 737 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id); 738 priv->tunnel_reg_id = 0; 739 } 740 err = mlx4_en_tunnel_steer_add(priv, new_mac, qpn, 741 &priv->tunnel_reg_id); 742 return err; 743 } 744 } 745 return -EINVAL; 746 } 747 748 return __mlx4_replace_mac(dev, priv->port, qpn, new_mac_u64); 749 } 750 751 static void mlx4_en_update_user_mac(struct mlx4_en_priv *priv, 752 unsigned char new_mac[ETH_ALEN + 2]) 753 { 754 struct mlx4_en_dev *mdev = priv->mdev; 755 int err; 756 757 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_USER_MAC_EN)) 758 return; 759 760 err = mlx4_SET_PORT_user_mac(mdev->dev, priv->port, new_mac); 761 if (err) 762 en_err(priv, "Failed to pass user MAC(%pM) to Firmware for port %d, with error %d\n", 763 new_mac, priv->port, err); 764 } 765 766 static int mlx4_en_do_set_mac(struct mlx4_en_priv *priv, 767 unsigned char new_mac[ETH_ALEN + 2]) 768 { 769 int err = 0; 770 771 if (priv->port_up) { 772 /* Remove old MAC and insert the new one */ 773 err = mlx4_en_replace_mac(priv, priv->base_qpn, 774 new_mac, priv->current_mac); 775 if (err) 776 en_err(priv, "Failed changing HW MAC address\n"); 777 } else 778 en_dbg(HW, priv, "Port is down while registering mac, exiting...\n"); 779 780 if (!err) 781 memcpy(priv->current_mac, new_mac, sizeof(priv->current_mac)); 782 783 return err; 784 } 785 786 static int mlx4_en_set_mac(struct net_device *dev, void *addr) 787 { 788 struct mlx4_en_priv *priv = netdev_priv(dev); 789 struct mlx4_en_dev *mdev = priv->mdev; 790 struct sockaddr *saddr = addr; 791 unsigned char new_mac[ETH_ALEN + 2]; 792 int err; 793 794 if (!is_valid_ether_addr(saddr->sa_data)) 795 return -EADDRNOTAVAIL; 796 797 mutex_lock(&mdev->state_lock); 798 memcpy(new_mac, saddr->sa_data, ETH_ALEN); 799 err = mlx4_en_do_set_mac(priv, new_mac); 800 if (err) 801 goto out; 802 803 eth_hw_addr_set(dev, saddr->sa_data); 804 mlx4_en_update_user_mac(priv, new_mac); 805 out: 806 mutex_unlock(&mdev->state_lock); 807 808 return err; 809 } 810 811 static void mlx4_en_clear_list(struct net_device *dev) 812 { 813 struct mlx4_en_priv *priv = netdev_priv(dev); 814 struct mlx4_en_mc_list *tmp, *mc_to_del; 815 816 list_for_each_entry_safe(mc_to_del, tmp, &priv->mc_list, list) { 817 list_del(&mc_to_del->list); 818 kfree(mc_to_del); 819 } 820 } 821 822 static void mlx4_en_cache_mclist(struct net_device *dev) 823 { 824 struct mlx4_en_priv *priv = netdev_priv(dev); 825 struct netdev_hw_addr *ha; 826 struct mlx4_en_mc_list *tmp; 827 828 mlx4_en_clear_list(dev); 829 netdev_for_each_mc_addr(ha, dev) { 830 tmp = kzalloc(sizeof(struct mlx4_en_mc_list), GFP_ATOMIC); 831 if (!tmp) { 832 mlx4_en_clear_list(dev); 833 return; 834 } 835 memcpy(tmp->addr, ha->addr, ETH_ALEN); 836 list_add_tail(&tmp->list, &priv->mc_list); 837 } 838 } 839 840 static void update_mclist_flags(struct mlx4_en_priv *priv, 841 struct list_head *dst, 842 struct list_head *src) 843 { 844 struct mlx4_en_mc_list *dst_tmp, *src_tmp, *new_mc; 845 bool found; 846 847 /* Find all the entries that should be removed from dst, 848 * These are the entries that are not found in src 849 */ 850 list_for_each_entry(dst_tmp, dst, list) { 851 found = false; 852 list_for_each_entry(src_tmp, src, list) { 853 if (ether_addr_equal(dst_tmp->addr, src_tmp->addr)) { 854 found = true; 855 break; 856 } 857 } 858 if (!found) 859 dst_tmp->action = MCLIST_REM; 860 } 861 862 /* Add entries that exist in src but not in dst 863 * mark them as need to add 864 */ 865 list_for_each_entry(src_tmp, src, list) { 866 found = false; 867 list_for_each_entry(dst_tmp, dst, list) { 868 if (ether_addr_equal(dst_tmp->addr, src_tmp->addr)) { 869 dst_tmp->action = MCLIST_NONE; 870 found = true; 871 break; 872 } 873 } 874 if (!found) { 875 new_mc = kmemdup(src_tmp, 876 sizeof(struct mlx4_en_mc_list), 877 GFP_KERNEL); 878 if (!new_mc) 879 return; 880 881 new_mc->action = MCLIST_ADD; 882 list_add_tail(&new_mc->list, dst); 883 } 884 } 885 } 886 887 static void mlx4_en_set_rx_mode(struct net_device *dev) 888 { 889 struct mlx4_en_priv *priv = netdev_priv(dev); 890 891 if (!priv->port_up) 892 return; 893 894 queue_work(priv->mdev->workqueue, &priv->rx_mode_task); 895 } 896 897 static void mlx4_en_set_promisc_mode(struct mlx4_en_priv *priv, 898 struct mlx4_en_dev *mdev) 899 { 900 int err = 0; 901 902 if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) { 903 if (netif_msg_rx_status(priv)) 904 en_warn(priv, "Entering promiscuous mode\n"); 905 priv->flags |= MLX4_EN_FLAG_PROMISC; 906 907 /* Enable promiscouos mode */ 908 switch (mdev->dev->caps.steering_mode) { 909 case MLX4_STEERING_MODE_DEVICE_MANAGED: 910 err = mlx4_flow_steer_promisc_add(mdev->dev, 911 priv->port, 912 priv->base_qpn, 913 MLX4_FS_ALL_DEFAULT); 914 if (err) 915 en_err(priv, "Failed enabling promiscuous mode\n"); 916 priv->flags |= MLX4_EN_FLAG_MC_PROMISC; 917 break; 918 919 case MLX4_STEERING_MODE_B0: 920 err = mlx4_unicast_promisc_add(mdev->dev, 921 priv->base_qpn, 922 priv->port); 923 if (err) 924 en_err(priv, "Failed enabling unicast promiscuous mode\n"); 925 926 /* Add the default qp number as multicast 927 * promisc 928 */ 929 if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { 930 err = mlx4_multicast_promisc_add(mdev->dev, 931 priv->base_qpn, 932 priv->port); 933 if (err) 934 en_err(priv, "Failed enabling multicast promiscuous mode\n"); 935 priv->flags |= MLX4_EN_FLAG_MC_PROMISC; 936 } 937 break; 938 939 case MLX4_STEERING_MODE_A0: 940 err = mlx4_SET_PORT_qpn_calc(mdev->dev, 941 priv->port, 942 priv->base_qpn, 943 1); 944 if (err) 945 en_err(priv, "Failed enabling promiscuous mode\n"); 946 break; 947 } 948 949 /* Disable port multicast filter (unconditionally) */ 950 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 951 0, MLX4_MCAST_DISABLE); 952 if (err) 953 en_err(priv, "Failed disabling multicast filter\n"); 954 } 955 } 956 957 static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv *priv, 958 struct mlx4_en_dev *mdev) 959 { 960 int err = 0; 961 962 if (netif_msg_rx_status(priv)) 963 en_warn(priv, "Leaving promiscuous mode\n"); 964 priv->flags &= ~MLX4_EN_FLAG_PROMISC; 965 966 /* Disable promiscouos mode */ 967 switch (mdev->dev->caps.steering_mode) { 968 case MLX4_STEERING_MODE_DEVICE_MANAGED: 969 err = mlx4_flow_steer_promisc_remove(mdev->dev, 970 priv->port, 971 MLX4_FS_ALL_DEFAULT); 972 if (err) 973 en_err(priv, "Failed disabling promiscuous mode\n"); 974 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 975 break; 976 977 case MLX4_STEERING_MODE_B0: 978 err = mlx4_unicast_promisc_remove(mdev->dev, 979 priv->base_qpn, 980 priv->port); 981 if (err) 982 en_err(priv, "Failed disabling unicast promiscuous mode\n"); 983 /* Disable Multicast promisc */ 984 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { 985 err = mlx4_multicast_promisc_remove(mdev->dev, 986 priv->base_qpn, 987 priv->port); 988 if (err) 989 en_err(priv, "Failed disabling multicast promiscuous mode\n"); 990 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 991 } 992 break; 993 994 case MLX4_STEERING_MODE_A0: 995 err = mlx4_SET_PORT_qpn_calc(mdev->dev, 996 priv->port, 997 priv->base_qpn, 0); 998 if (err) 999 en_err(priv, "Failed disabling promiscuous mode\n"); 1000 break; 1001 } 1002 } 1003 1004 static void mlx4_en_do_multicast(struct mlx4_en_priv *priv, 1005 struct net_device *dev, 1006 struct mlx4_en_dev *mdev) 1007 { 1008 struct mlx4_en_mc_list *mclist, *tmp; 1009 u64 mcast_addr = 0; 1010 u8 mc_list[16] = {0}; 1011 int err = 0; 1012 1013 /* Enable/disable the multicast filter according to IFF_ALLMULTI */ 1014 if (dev->flags & IFF_ALLMULTI) { 1015 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1016 0, MLX4_MCAST_DISABLE); 1017 if (err) 1018 en_err(priv, "Failed disabling multicast filter\n"); 1019 1020 /* Add the default qp number as multicast promisc */ 1021 if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { 1022 switch (mdev->dev->caps.steering_mode) { 1023 case MLX4_STEERING_MODE_DEVICE_MANAGED: 1024 err = mlx4_flow_steer_promisc_add(mdev->dev, 1025 priv->port, 1026 priv->base_qpn, 1027 MLX4_FS_MC_DEFAULT); 1028 break; 1029 1030 case MLX4_STEERING_MODE_B0: 1031 err = mlx4_multicast_promisc_add(mdev->dev, 1032 priv->base_qpn, 1033 priv->port); 1034 break; 1035 1036 case MLX4_STEERING_MODE_A0: 1037 break; 1038 } 1039 if (err) 1040 en_err(priv, "Failed entering multicast promisc mode\n"); 1041 priv->flags |= MLX4_EN_FLAG_MC_PROMISC; 1042 } 1043 } else { 1044 /* Disable Multicast promisc */ 1045 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { 1046 switch (mdev->dev->caps.steering_mode) { 1047 case MLX4_STEERING_MODE_DEVICE_MANAGED: 1048 err = mlx4_flow_steer_promisc_remove(mdev->dev, 1049 priv->port, 1050 MLX4_FS_MC_DEFAULT); 1051 break; 1052 1053 case MLX4_STEERING_MODE_B0: 1054 err = mlx4_multicast_promisc_remove(mdev->dev, 1055 priv->base_qpn, 1056 priv->port); 1057 break; 1058 1059 case MLX4_STEERING_MODE_A0: 1060 break; 1061 } 1062 if (err) 1063 en_err(priv, "Failed disabling multicast promiscuous mode\n"); 1064 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 1065 } 1066 1067 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1068 0, MLX4_MCAST_DISABLE); 1069 if (err) 1070 en_err(priv, "Failed disabling multicast filter\n"); 1071 1072 /* Flush mcast filter and init it with broadcast address */ 1073 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, ETH_BCAST, 1074 1, MLX4_MCAST_CONFIG); 1075 1076 /* Update multicast list - we cache all addresses so they won't 1077 * change while HW is updated holding the command semaphore 1078 */ 1079 netif_addr_lock_bh(dev); 1080 mlx4_en_cache_mclist(dev); 1081 netif_addr_unlock_bh(dev); 1082 list_for_each_entry(mclist, &priv->mc_list, list) { 1083 mcast_addr = ether_addr_to_u64(mclist->addr); 1084 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 1085 mcast_addr, 0, MLX4_MCAST_CONFIG); 1086 } 1087 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1088 0, MLX4_MCAST_ENABLE); 1089 if (err) 1090 en_err(priv, "Failed enabling multicast filter\n"); 1091 1092 update_mclist_flags(priv, &priv->curr_list, &priv->mc_list); 1093 list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { 1094 if (mclist->action == MCLIST_REM) { 1095 /* detach this address and delete from list */ 1096 memcpy(&mc_list[10], mclist->addr, ETH_ALEN); 1097 mc_list[5] = priv->port; 1098 err = mlx4_multicast_detach(mdev->dev, 1099 priv->rss_map.indir_qp, 1100 mc_list, 1101 MLX4_PROT_ETH, 1102 mclist->reg_id); 1103 if (err) 1104 en_err(priv, "Fail to detach multicast address\n"); 1105 1106 if (mclist->tunnel_reg_id) { 1107 err = mlx4_flow_detach(priv->mdev->dev, mclist->tunnel_reg_id); 1108 if (err) 1109 en_err(priv, "Failed to detach multicast address\n"); 1110 } 1111 1112 /* remove from list */ 1113 list_del(&mclist->list); 1114 kfree(mclist); 1115 } else if (mclist->action == MCLIST_ADD) { 1116 /* attach the address */ 1117 memcpy(&mc_list[10], mclist->addr, ETH_ALEN); 1118 /* needed for B0 steering support */ 1119 mc_list[5] = priv->port; 1120 err = mlx4_multicast_attach(mdev->dev, 1121 priv->rss_map.indir_qp, 1122 mc_list, 1123 priv->port, 0, 1124 MLX4_PROT_ETH, 1125 &mclist->reg_id); 1126 if (err) 1127 en_err(priv, "Fail to attach multicast address\n"); 1128 1129 err = mlx4_en_tunnel_steer_add(priv, &mc_list[10], priv->base_qpn, 1130 &mclist->tunnel_reg_id); 1131 if (err) 1132 en_err(priv, "Failed to attach multicast address\n"); 1133 } 1134 } 1135 } 1136 } 1137 1138 static void mlx4_en_do_uc_filter(struct mlx4_en_priv *priv, 1139 struct net_device *dev, 1140 struct mlx4_en_dev *mdev) 1141 { 1142 struct netdev_hw_addr *ha; 1143 struct mlx4_mac_entry *entry; 1144 struct hlist_node *tmp; 1145 bool found; 1146 u64 mac; 1147 int err = 0; 1148 struct hlist_head *bucket; 1149 unsigned int i; 1150 int removed = 0; 1151 u32 prev_flags; 1152 1153 /* Note that we do not need to protect our mac_hash traversal with rcu, 1154 * since all modification code is protected by mdev->state_lock 1155 */ 1156 1157 /* find what to remove */ 1158 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { 1159 bucket = &priv->mac_hash[i]; 1160 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { 1161 found = false; 1162 netdev_for_each_uc_addr(ha, dev) { 1163 if (ether_addr_equal_64bits(entry->mac, 1164 ha->addr)) { 1165 found = true; 1166 break; 1167 } 1168 } 1169 1170 /* MAC address of the port is not in uc list */ 1171 if (ether_addr_equal_64bits(entry->mac, 1172 priv->current_mac)) 1173 found = true; 1174 1175 if (!found) { 1176 mac = ether_addr_to_u64(entry->mac); 1177 mlx4_en_uc_steer_release(priv, entry->mac, 1178 priv->base_qpn, 1179 entry->reg_id); 1180 mlx4_unregister_mac(mdev->dev, priv->port, mac); 1181 1182 hlist_del_rcu(&entry->hlist); 1183 en_dbg(DRV, priv, "Removed MAC %pM on port:%d\n", 1184 entry->mac, priv->port); 1185 kfree_rcu(entry, rcu); 1186 ++removed; 1187 } 1188 } 1189 } 1190 1191 /* if we didn't remove anything, there is no use in trying to add 1192 * again once we are in a forced promisc mode state 1193 */ 1194 if ((priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) && 0 == removed) 1195 return; 1196 1197 prev_flags = priv->flags; 1198 priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; 1199 1200 /* find what to add */ 1201 netdev_for_each_uc_addr(ha, dev) { 1202 found = false; 1203 bucket = &priv->mac_hash[ha->addr[MLX4_EN_MAC_HASH_IDX]]; 1204 hlist_for_each_entry(entry, bucket, hlist) { 1205 if (ether_addr_equal_64bits(entry->mac, ha->addr)) { 1206 found = true; 1207 break; 1208 } 1209 } 1210 1211 if (!found) { 1212 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1213 if (!entry) { 1214 en_err(priv, "Failed adding MAC %pM on port:%d (out of memory)\n", 1215 ha->addr, priv->port); 1216 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; 1217 break; 1218 } 1219 mac = ether_addr_to_u64(ha->addr); 1220 memcpy(entry->mac, ha->addr, ETH_ALEN); 1221 err = mlx4_register_mac(mdev->dev, priv->port, mac); 1222 if (err < 0) { 1223 en_err(priv, "Failed registering MAC %pM on port %d: %d\n", 1224 ha->addr, priv->port, err); 1225 kfree(entry); 1226 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; 1227 break; 1228 } 1229 err = mlx4_en_uc_steer_add(priv, ha->addr, 1230 &priv->base_qpn, 1231 &entry->reg_id); 1232 if (err) { 1233 en_err(priv, "Failed adding MAC %pM on port %d: %d\n", 1234 ha->addr, priv->port, err); 1235 mlx4_unregister_mac(mdev->dev, priv->port, mac); 1236 kfree(entry); 1237 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; 1238 break; 1239 } else { 1240 unsigned int mac_hash; 1241 en_dbg(DRV, priv, "Added MAC %pM on port:%d\n", 1242 ha->addr, priv->port); 1243 mac_hash = ha->addr[MLX4_EN_MAC_HASH_IDX]; 1244 bucket = &priv->mac_hash[mac_hash]; 1245 hlist_add_head_rcu(&entry->hlist, bucket); 1246 } 1247 } 1248 } 1249 1250 if (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) { 1251 en_warn(priv, "Forcing promiscuous mode on port:%d\n", 1252 priv->port); 1253 } else if (prev_flags & MLX4_EN_FLAG_FORCE_PROMISC) { 1254 en_warn(priv, "Stop forcing promiscuous mode on port:%d\n", 1255 priv->port); 1256 } 1257 } 1258 1259 static void mlx4_en_do_set_rx_mode(struct work_struct *work) 1260 { 1261 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 1262 rx_mode_task); 1263 struct mlx4_en_dev *mdev = priv->mdev; 1264 struct net_device *dev = priv->dev; 1265 1266 mutex_lock(&mdev->state_lock); 1267 if (!mdev->device_up) { 1268 en_dbg(HW, priv, "Card is not up, ignoring rx mode change.\n"); 1269 goto out; 1270 } 1271 if (!priv->port_up) { 1272 en_dbg(HW, priv, "Port is down, ignoring rx mode change.\n"); 1273 goto out; 1274 } 1275 1276 if (!netif_carrier_ok(dev)) { 1277 if (!mlx4_en_QUERY_PORT(mdev, priv->port)) { 1278 if (priv->port_state.link_state) { 1279 netif_carrier_on(dev); 1280 en_dbg(LINK, priv, "Link Up\n"); 1281 } 1282 } 1283 } 1284 1285 if (dev->priv_flags & IFF_UNICAST_FLT) 1286 mlx4_en_do_uc_filter(priv, dev, mdev); 1287 1288 /* Promsicuous mode: disable all filters */ 1289 if ((dev->flags & IFF_PROMISC) || 1290 (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC)) { 1291 mlx4_en_set_promisc_mode(priv, mdev); 1292 goto out; 1293 } 1294 1295 /* Not in promiscuous mode */ 1296 if (priv->flags & MLX4_EN_FLAG_PROMISC) 1297 mlx4_en_clear_promisc_mode(priv, mdev); 1298 1299 mlx4_en_do_multicast(priv, dev, mdev); 1300 out: 1301 mutex_unlock(&mdev->state_lock); 1302 } 1303 1304 static int mlx4_en_set_rss_steer_rules(struct mlx4_en_priv *priv) 1305 { 1306 u64 reg_id; 1307 int err = 0; 1308 int *qpn = &priv->base_qpn; 1309 struct mlx4_mac_entry *entry; 1310 1311 err = mlx4_en_uc_steer_add(priv, priv->dev->dev_addr, qpn, ®_id); 1312 if (err) 1313 return err; 1314 1315 err = mlx4_en_tunnel_steer_add(priv, priv->dev->dev_addr, *qpn, 1316 &priv->tunnel_reg_id); 1317 if (err) 1318 goto tunnel_err; 1319 1320 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1321 if (!entry) { 1322 err = -ENOMEM; 1323 goto alloc_err; 1324 } 1325 1326 memcpy(entry->mac, priv->dev->dev_addr, sizeof(entry->mac)); 1327 memcpy(priv->current_mac, entry->mac, sizeof(priv->current_mac)); 1328 entry->reg_id = reg_id; 1329 hlist_add_head_rcu(&entry->hlist, 1330 &priv->mac_hash[entry->mac[MLX4_EN_MAC_HASH_IDX]]); 1331 1332 return 0; 1333 1334 alloc_err: 1335 if (priv->tunnel_reg_id) 1336 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id); 1337 1338 tunnel_err: 1339 mlx4_en_uc_steer_release(priv, priv->dev->dev_addr, *qpn, reg_id); 1340 return err; 1341 } 1342 1343 static void mlx4_en_delete_rss_steer_rules(struct mlx4_en_priv *priv) 1344 { 1345 u64 mac; 1346 unsigned int i; 1347 int qpn = priv->base_qpn; 1348 struct hlist_head *bucket; 1349 struct hlist_node *tmp; 1350 struct mlx4_mac_entry *entry; 1351 1352 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { 1353 bucket = &priv->mac_hash[i]; 1354 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { 1355 mac = ether_addr_to_u64(entry->mac); 1356 en_dbg(DRV, priv, "Registering MAC:%pM for deleting\n", 1357 entry->mac); 1358 mlx4_en_uc_steer_release(priv, entry->mac, 1359 qpn, entry->reg_id); 1360 1361 mlx4_unregister_mac(priv->mdev->dev, priv->port, mac); 1362 hlist_del_rcu(&entry->hlist); 1363 kfree_rcu(entry, rcu); 1364 } 1365 } 1366 1367 if (priv->tunnel_reg_id) { 1368 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id); 1369 priv->tunnel_reg_id = 0; 1370 } 1371 } 1372 1373 static void mlx4_en_tx_timeout(struct net_device *dev, unsigned int txqueue) 1374 { 1375 struct mlx4_en_priv *priv = netdev_priv(dev); 1376 struct mlx4_en_dev *mdev = priv->mdev; 1377 struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[TX][txqueue]; 1378 1379 if (netif_msg_timer(priv)) 1380 en_warn(priv, "Tx timeout called on port:%d\n", priv->port); 1381 1382 en_warn(priv, "TX timeout on queue: %d, QP: 0x%x, CQ: 0x%x, Cons: 0x%x, Prod: 0x%x\n", 1383 txqueue, tx_ring->qpn, tx_ring->sp_cqn, 1384 tx_ring->cons, tx_ring->prod); 1385 1386 priv->port_stats.tx_timeout++; 1387 if (!test_and_set_bit(MLX4_EN_STATE_FLAG_RESTARTING, &priv->state)) { 1388 en_dbg(DRV, priv, "Scheduling port restart\n"); 1389 queue_work(mdev->workqueue, &priv->restart_task); 1390 } 1391 } 1392 1393 1394 static void 1395 mlx4_en_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 1396 { 1397 struct mlx4_en_priv *priv = netdev_priv(dev); 1398 1399 spin_lock_bh(&priv->stats_lock); 1400 mlx4_en_fold_software_stats(dev); 1401 netdev_stats_to_stats64(stats, &dev->stats); 1402 spin_unlock_bh(&priv->stats_lock); 1403 } 1404 1405 static void mlx4_en_set_default_moderation(struct mlx4_en_priv *priv) 1406 { 1407 struct mlx4_en_cq *cq; 1408 int i, t; 1409 1410 /* If we haven't received a specific coalescing setting 1411 * (module param), we set the moderation parameters as follows: 1412 * - moder_cnt is set to the number of mtu sized packets to 1413 * satisfy our coalescing target. 1414 * - moder_time is set to a fixed value. 1415 */ 1416 priv->rx_frames = MLX4_EN_RX_COAL_TARGET; 1417 priv->rx_usecs = MLX4_EN_RX_COAL_TIME; 1418 priv->tx_frames = MLX4_EN_TX_COAL_PKTS; 1419 priv->tx_usecs = MLX4_EN_TX_COAL_TIME; 1420 en_dbg(INTR, priv, "Default coalescing params for mtu:%d - rx_frames:%d rx_usecs:%d\n", 1421 priv->dev->mtu, priv->rx_frames, priv->rx_usecs); 1422 1423 /* Setup cq moderation params */ 1424 for (i = 0; i < priv->rx_ring_num; i++) { 1425 cq = priv->rx_cq[i]; 1426 cq->moder_cnt = priv->rx_frames; 1427 cq->moder_time = priv->rx_usecs; 1428 priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; 1429 priv->last_moder_packets[i] = 0; 1430 priv->last_moder_bytes[i] = 0; 1431 } 1432 1433 for (t = 0 ; t < MLX4_EN_NUM_TX_TYPES; t++) { 1434 for (i = 0; i < priv->tx_ring_num[t]; i++) { 1435 cq = priv->tx_cq[t][i]; 1436 cq->moder_cnt = priv->tx_frames; 1437 cq->moder_time = priv->tx_usecs; 1438 } 1439 } 1440 1441 /* Reset auto-moderation params */ 1442 priv->pkt_rate_low = MLX4_EN_RX_RATE_LOW; 1443 priv->rx_usecs_low = MLX4_EN_RX_COAL_TIME_LOW; 1444 priv->pkt_rate_high = MLX4_EN_RX_RATE_HIGH; 1445 priv->rx_usecs_high = MLX4_EN_RX_COAL_TIME_HIGH; 1446 priv->sample_interval = MLX4_EN_SAMPLE_INTERVAL; 1447 priv->adaptive_rx_coal = 1; 1448 priv->last_moder_jiffies = 0; 1449 priv->last_moder_tx_packets = 0; 1450 } 1451 1452 static void mlx4_en_auto_moderation(struct mlx4_en_priv *priv) 1453 { 1454 unsigned long period = (unsigned long) (jiffies - priv->last_moder_jiffies); 1455 u32 pkt_rate_high, pkt_rate_low; 1456 struct mlx4_en_cq *cq; 1457 unsigned long packets; 1458 unsigned long rate; 1459 unsigned long avg_pkt_size; 1460 unsigned long rx_packets; 1461 unsigned long rx_bytes; 1462 unsigned long rx_pkt_diff; 1463 int moder_time; 1464 int ring, err; 1465 1466 if (!priv->adaptive_rx_coal || period < priv->sample_interval * HZ) 1467 return; 1468 1469 pkt_rate_low = READ_ONCE(priv->pkt_rate_low); 1470 pkt_rate_high = READ_ONCE(priv->pkt_rate_high); 1471 1472 for (ring = 0; ring < priv->rx_ring_num; ring++) { 1473 rx_packets = READ_ONCE(priv->rx_ring[ring]->packets); 1474 rx_bytes = READ_ONCE(priv->rx_ring[ring]->bytes); 1475 1476 rx_pkt_diff = rx_packets - priv->last_moder_packets[ring]; 1477 packets = rx_pkt_diff; 1478 rate = packets * HZ / period; 1479 avg_pkt_size = packets ? (rx_bytes - 1480 priv->last_moder_bytes[ring]) / packets : 0; 1481 1482 /* Apply auto-moderation only when packet rate 1483 * exceeds a rate that it matters */ 1484 if (rate > (MLX4_EN_RX_RATE_THRESH / priv->rx_ring_num) && 1485 avg_pkt_size > MLX4_EN_AVG_PKT_SMALL) { 1486 if (rate <= pkt_rate_low) 1487 moder_time = priv->rx_usecs_low; 1488 else if (rate >= pkt_rate_high) 1489 moder_time = priv->rx_usecs_high; 1490 else 1491 moder_time = (rate - pkt_rate_low) * 1492 (priv->rx_usecs_high - priv->rx_usecs_low) / 1493 (pkt_rate_high - pkt_rate_low) + 1494 priv->rx_usecs_low; 1495 } else { 1496 moder_time = priv->rx_usecs_low; 1497 } 1498 1499 cq = priv->rx_cq[ring]; 1500 if (moder_time != priv->last_moder_time[ring] || 1501 cq->moder_cnt != priv->rx_frames) { 1502 priv->last_moder_time[ring] = moder_time; 1503 cq->moder_time = moder_time; 1504 cq->moder_cnt = priv->rx_frames; 1505 err = mlx4_en_set_cq_moder(priv, cq); 1506 if (err) 1507 en_err(priv, "Failed modifying moderation for cq:%d\n", 1508 ring); 1509 } 1510 priv->last_moder_packets[ring] = rx_packets; 1511 priv->last_moder_bytes[ring] = rx_bytes; 1512 } 1513 1514 priv->last_moder_jiffies = jiffies; 1515 } 1516 1517 static void mlx4_en_do_get_stats(struct work_struct *work) 1518 { 1519 struct delayed_work *delay = to_delayed_work(work); 1520 struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, 1521 stats_task); 1522 struct mlx4_en_dev *mdev = priv->mdev; 1523 int err; 1524 1525 mutex_lock(&mdev->state_lock); 1526 if (mdev->device_up) { 1527 if (priv->port_up) { 1528 err = mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 0); 1529 if (err) 1530 en_dbg(HW, priv, "Could not update stats\n"); 1531 1532 mlx4_en_auto_moderation(priv); 1533 } 1534 1535 queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); 1536 } 1537 if (mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port]) { 1538 mlx4_en_do_set_mac(priv, priv->current_mac); 1539 mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port] = 0; 1540 } 1541 mutex_unlock(&mdev->state_lock); 1542 } 1543 1544 /* mlx4_en_service_task - Run service task for tasks that needed to be done 1545 * periodically 1546 */ 1547 static void mlx4_en_service_task(struct work_struct *work) 1548 { 1549 struct delayed_work *delay = to_delayed_work(work); 1550 struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, 1551 service_task); 1552 struct mlx4_en_dev *mdev = priv->mdev; 1553 1554 mutex_lock(&mdev->state_lock); 1555 if (mdev->device_up) { 1556 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) 1557 mlx4_en_ptp_overflow_check(mdev); 1558 1559 mlx4_en_recover_from_oom(priv); 1560 queue_delayed_work(mdev->workqueue, &priv->service_task, 1561 SERVICE_TASK_DELAY); 1562 } 1563 mutex_unlock(&mdev->state_lock); 1564 } 1565 1566 static void mlx4_en_linkstate(struct mlx4_en_priv *priv) 1567 { 1568 struct mlx4_en_port_state *port_state = &priv->port_state; 1569 struct mlx4_en_dev *mdev = priv->mdev; 1570 struct net_device *dev = priv->dev; 1571 bool up; 1572 1573 if (mlx4_en_QUERY_PORT(mdev, priv->port)) 1574 port_state->link_state = MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN; 1575 1576 up = port_state->link_state == MLX4_PORT_STATE_DEV_EVENT_PORT_UP; 1577 if (up == netif_carrier_ok(dev)) 1578 netif_carrier_event(dev); 1579 if (!up) { 1580 en_info(priv, "Link Down\n"); 1581 netif_carrier_off(dev); 1582 } else { 1583 en_info(priv, "Link Up\n"); 1584 netif_carrier_on(dev); 1585 } 1586 } 1587 1588 static void mlx4_en_linkstate_work(struct work_struct *work) 1589 { 1590 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 1591 linkstate_task); 1592 struct mlx4_en_dev *mdev = priv->mdev; 1593 1594 mutex_lock(&mdev->state_lock); 1595 mlx4_en_linkstate(priv); 1596 mutex_unlock(&mdev->state_lock); 1597 } 1598 1599 static int mlx4_en_init_affinity_hint(struct mlx4_en_priv *priv, int ring_idx) 1600 { 1601 struct mlx4_en_rx_ring *ring = priv->rx_ring[ring_idx]; 1602 int numa_node = priv->mdev->dev->numa_node; 1603 1604 if (!zalloc_cpumask_var(&ring->affinity_mask, GFP_KERNEL)) 1605 return -ENOMEM; 1606 1607 cpumask_set_cpu(cpumask_local_spread(ring_idx, numa_node), 1608 ring->affinity_mask); 1609 return 0; 1610 } 1611 1612 static void mlx4_en_free_affinity_hint(struct mlx4_en_priv *priv, int ring_idx) 1613 { 1614 free_cpumask_var(priv->rx_ring[ring_idx]->affinity_mask); 1615 } 1616 1617 static void mlx4_en_init_recycle_ring(struct mlx4_en_priv *priv, 1618 int tx_ring_idx) 1619 { 1620 struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[TX_XDP][tx_ring_idx]; 1621 int rr_index = tx_ring_idx; 1622 1623 tx_ring->free_tx_desc = mlx4_en_recycle_tx_desc; 1624 tx_ring->recycle_ring = priv->rx_ring[rr_index]; 1625 en_dbg(DRV, priv, "Set tx_ring[%d][%d]->recycle_ring = rx_ring[%d]\n", 1626 TX_XDP, tx_ring_idx, rr_index); 1627 } 1628 1629 int mlx4_en_start_port(struct net_device *dev) 1630 { 1631 struct mlx4_en_priv *priv = netdev_priv(dev); 1632 struct mlx4_en_dev *mdev = priv->mdev; 1633 struct mlx4_en_cq *cq; 1634 struct mlx4_en_tx_ring *tx_ring; 1635 int rx_index = 0; 1636 int err = 0; 1637 int i, t; 1638 int j; 1639 u8 mc_list[16] = {0}; 1640 1641 if (priv->port_up) { 1642 en_dbg(DRV, priv, "start port called while port already up\n"); 1643 return 0; 1644 } 1645 1646 INIT_LIST_HEAD(&priv->mc_list); 1647 INIT_LIST_HEAD(&priv->curr_list); 1648 INIT_LIST_HEAD(&priv->ethtool_list); 1649 memset(&priv->ethtool_rules[0], 0, 1650 sizeof(struct ethtool_flow_id) * MAX_NUM_OF_FS_RULES); 1651 1652 /* Calculate Rx buf size */ 1653 WRITE_ONCE(dev->mtu, min(dev->mtu, priv->max_mtu)); 1654 mlx4_en_calc_rx_buf(dev); 1655 en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_skb_size); 1656 1657 /* Configure rx cq's and rings */ 1658 err = mlx4_en_activate_rx_rings(priv); 1659 if (err) { 1660 en_err(priv, "Failed to activate RX rings\n"); 1661 return err; 1662 } 1663 for (i = 0; i < priv->rx_ring_num; i++) { 1664 cq = priv->rx_cq[i]; 1665 1666 err = mlx4_en_init_affinity_hint(priv, i); 1667 if (err) { 1668 en_err(priv, "Failed preparing IRQ affinity hint\n"); 1669 goto cq_err; 1670 } 1671 1672 err = mlx4_en_activate_cq(priv, cq, i); 1673 if (err) { 1674 en_err(priv, "Failed activating Rx CQ\n"); 1675 mlx4_en_free_affinity_hint(priv, i); 1676 goto cq_err; 1677 } 1678 1679 for (j = 0; j < cq->size; j++) { 1680 struct mlx4_cqe *cqe = NULL; 1681 1682 cqe = mlx4_en_get_cqe(cq->buf, j, priv->cqe_size) + 1683 priv->cqe_factor; 1684 cqe->owner_sr_opcode = MLX4_CQE_OWNER_MASK; 1685 } 1686 1687 err = mlx4_en_set_cq_moder(priv, cq); 1688 if (err) { 1689 en_err(priv, "Failed setting cq moderation parameters\n"); 1690 mlx4_en_deactivate_cq(priv, cq); 1691 mlx4_en_free_affinity_hint(priv, i); 1692 goto cq_err; 1693 } 1694 mlx4_en_arm_cq(priv, cq); 1695 priv->rx_ring[i]->cqn = cq->mcq.cqn; 1696 ++rx_index; 1697 } 1698 1699 /* Set qp number */ 1700 en_dbg(DRV, priv, "Getting qp number for port %d\n", priv->port); 1701 err = mlx4_en_get_qp(priv); 1702 if (err) { 1703 en_err(priv, "Failed getting eth qp\n"); 1704 goto cq_err; 1705 } 1706 mdev->mac_removed[priv->port] = 0; 1707 1708 priv->counter_index = 1709 mlx4_get_default_counter_index(mdev->dev, priv->port); 1710 1711 err = mlx4_en_config_rss_steer(priv); 1712 if (err) { 1713 en_err(priv, "Failed configuring rss steering\n"); 1714 goto mac_err; 1715 } 1716 1717 err = mlx4_en_create_drop_qp(priv); 1718 if (err) 1719 goto rss_err; 1720 1721 /* Configure tx cq's and rings */ 1722 for (t = 0 ; t < MLX4_EN_NUM_TX_TYPES; t++) { 1723 u8 num_tx_rings_p_up = t == TX ? 1724 priv->num_tx_rings_p_up : priv->tx_ring_num[t]; 1725 1726 for (i = 0; i < priv->tx_ring_num[t]; i++) { 1727 /* Configure cq */ 1728 cq = priv->tx_cq[t][i]; 1729 err = mlx4_en_activate_cq(priv, cq, i); 1730 if (err) { 1731 en_err(priv, "Failed allocating Tx CQ\n"); 1732 goto tx_err; 1733 } 1734 err = mlx4_en_set_cq_moder(priv, cq); 1735 if (err) { 1736 en_err(priv, "Failed setting cq moderation parameters\n"); 1737 mlx4_en_deactivate_cq(priv, cq); 1738 goto tx_err; 1739 } 1740 en_dbg(DRV, priv, 1741 "Resetting index of collapsed CQ:%d to -1\n", i); 1742 cq->buf->wqe_index = cpu_to_be16(0xffff); 1743 1744 /* Configure ring */ 1745 tx_ring = priv->tx_ring[t][i]; 1746 err = mlx4_en_activate_tx_ring(priv, tx_ring, 1747 cq->mcq.cqn, 1748 i / num_tx_rings_p_up); 1749 if (err) { 1750 en_err(priv, "Failed allocating Tx ring\n"); 1751 mlx4_en_deactivate_cq(priv, cq); 1752 goto tx_err; 1753 } 1754 clear_bit(MLX4_EN_TX_RING_STATE_RECOVERING, &tx_ring->state); 1755 if (t != TX_XDP) { 1756 tx_ring->tx_queue = netdev_get_tx_queue(dev, i); 1757 tx_ring->recycle_ring = NULL; 1758 1759 /* Arm CQ for TX completions */ 1760 mlx4_en_arm_cq(priv, cq); 1761 1762 } else { 1763 mlx4_en_init_tx_xdp_ring_descs(priv, tx_ring); 1764 mlx4_en_init_recycle_ring(priv, i); 1765 /* XDP TX CQ should never be armed */ 1766 } 1767 1768 /* Set initial ownership of all Tx TXBBs to SW (1) */ 1769 for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE) 1770 *((u32 *)(tx_ring->buf + j)) = 0xffffffff; 1771 } 1772 } 1773 1774 /* Configure port */ 1775 err = mlx4_SET_PORT_general(mdev->dev, priv->port, 1776 priv->rx_skb_size + ETH_FCS_LEN, 1777 priv->prof->tx_pause, 1778 priv->prof->tx_ppp, 1779 priv->prof->rx_pause, 1780 priv->prof->rx_ppp); 1781 if (err) { 1782 en_err(priv, "Failed setting port general configurations for port %d, with error %d\n", 1783 priv->port, err); 1784 goto tx_err; 1785 } 1786 1787 err = mlx4_SET_PORT_user_mtu(mdev->dev, priv->port, dev->mtu); 1788 if (err) { 1789 en_err(priv, "Failed to pass user MTU(%d) to Firmware for port %d, with error %d\n", 1790 dev->mtu, priv->port, err); 1791 goto tx_err; 1792 } 1793 1794 /* Set default qp number */ 1795 err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, priv->base_qpn, 0); 1796 if (err) { 1797 en_err(priv, "Failed setting default qp numbers\n"); 1798 goto tx_err; 1799 } 1800 1801 if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { 1802 err = mlx4_SET_PORT_VXLAN(mdev->dev, priv->port, VXLAN_STEER_BY_OUTER_MAC, 1); 1803 if (err) { 1804 en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n", 1805 err); 1806 goto tx_err; 1807 } 1808 } 1809 1810 /* Init port */ 1811 en_dbg(HW, priv, "Initializing port\n"); 1812 err = mlx4_INIT_PORT(mdev->dev, priv->port); 1813 if (err) { 1814 en_err(priv, "Failed Initializing port\n"); 1815 goto tx_err; 1816 } 1817 1818 /* Set Unicast and VXLAN steering rules */ 1819 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0 && 1820 mlx4_en_set_rss_steer_rules(priv)) 1821 mlx4_warn(mdev, "Failed setting steering rules\n"); 1822 1823 /* Attach rx QP to broadcast address */ 1824 eth_broadcast_addr(&mc_list[10]); 1825 mc_list[5] = priv->port; /* needed for B0 steering support */ 1826 if (mlx4_multicast_attach(mdev->dev, priv->rss_map.indir_qp, mc_list, 1827 priv->port, 0, MLX4_PROT_ETH, 1828 &priv->broadcast_id)) 1829 mlx4_warn(mdev, "Failed Attaching Broadcast\n"); 1830 1831 /* Must redo promiscuous mode setup. */ 1832 priv->flags &= ~(MLX4_EN_FLAG_PROMISC | MLX4_EN_FLAG_MC_PROMISC); 1833 1834 /* Schedule multicast task to populate multicast list */ 1835 queue_work(mdev->workqueue, &priv->rx_mode_task); 1836 1837 if (priv->mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) 1838 udp_tunnel_nic_reset_ntf(dev); 1839 1840 priv->port_up = true; 1841 1842 /* Process all completions if exist to prevent 1843 * the queues freezing if they are full 1844 */ 1845 for (i = 0; i < priv->rx_ring_num; i++) { 1846 local_bh_disable(); 1847 napi_schedule(&priv->rx_cq[i]->napi); 1848 local_bh_enable(); 1849 } 1850 1851 clear_bit(MLX4_EN_STATE_FLAG_RESTARTING, &priv->state); 1852 netif_tx_start_all_queues(dev); 1853 netif_device_attach(dev); 1854 1855 return 0; 1856 1857 tx_err: 1858 if (t == MLX4_EN_NUM_TX_TYPES) { 1859 t--; 1860 i = priv->tx_ring_num[t]; 1861 } 1862 while (t >= 0) { 1863 while (i--) { 1864 mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[t][i]); 1865 mlx4_en_deactivate_cq(priv, priv->tx_cq[t][i]); 1866 } 1867 if (!t--) 1868 break; 1869 i = priv->tx_ring_num[t]; 1870 } 1871 mlx4_en_destroy_drop_qp(priv); 1872 rss_err: 1873 mlx4_en_release_rss_steer(priv); 1874 mac_err: 1875 mlx4_en_put_qp(priv); 1876 cq_err: 1877 while (rx_index--) { 1878 mlx4_en_deactivate_cq(priv, priv->rx_cq[rx_index]); 1879 mlx4_en_free_affinity_hint(priv, rx_index); 1880 } 1881 for (i = 0; i < priv->rx_ring_num; i++) 1882 mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); 1883 1884 return err; /* need to close devices */ 1885 } 1886 1887 1888 void mlx4_en_stop_port(struct net_device *dev, int detach) 1889 { 1890 struct mlx4_en_priv *priv = netdev_priv(dev); 1891 struct mlx4_en_dev *mdev = priv->mdev; 1892 struct mlx4_en_mc_list *mclist, *tmp; 1893 struct ethtool_flow_id *flow, *tmp_flow; 1894 int i, t; 1895 u8 mc_list[16] = {0}; 1896 1897 if (!priv->port_up) { 1898 en_dbg(DRV, priv, "stop port called while port already down\n"); 1899 return; 1900 } 1901 1902 /* close port*/ 1903 mlx4_CLOSE_PORT(mdev->dev, priv->port); 1904 1905 /* Synchronize with tx routine */ 1906 netif_tx_lock_bh(dev); 1907 if (detach) 1908 netif_device_detach(dev); 1909 netif_tx_stop_all_queues(dev); 1910 netif_tx_unlock_bh(dev); 1911 1912 netif_tx_disable(dev); 1913 1914 spin_lock_bh(&priv->stats_lock); 1915 mlx4_en_fold_software_stats(dev); 1916 /* Set port as not active */ 1917 priv->port_up = false; 1918 spin_unlock_bh(&priv->stats_lock); 1919 1920 priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev); 1921 1922 /* Promsicuous mode */ 1923 if (mdev->dev->caps.steering_mode == 1924 MLX4_STEERING_MODE_DEVICE_MANAGED) { 1925 priv->flags &= ~(MLX4_EN_FLAG_PROMISC | 1926 MLX4_EN_FLAG_MC_PROMISC); 1927 mlx4_flow_steer_promisc_remove(mdev->dev, 1928 priv->port, 1929 MLX4_FS_ALL_DEFAULT); 1930 mlx4_flow_steer_promisc_remove(mdev->dev, 1931 priv->port, 1932 MLX4_FS_MC_DEFAULT); 1933 } else if (priv->flags & MLX4_EN_FLAG_PROMISC) { 1934 priv->flags &= ~MLX4_EN_FLAG_PROMISC; 1935 1936 /* Disable promiscouos mode */ 1937 mlx4_unicast_promisc_remove(mdev->dev, priv->base_qpn, 1938 priv->port); 1939 1940 /* Disable Multicast promisc */ 1941 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { 1942 mlx4_multicast_promisc_remove(mdev->dev, priv->base_qpn, 1943 priv->port); 1944 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 1945 } 1946 } 1947 1948 /* Detach All multicasts */ 1949 eth_broadcast_addr(&mc_list[10]); 1950 mc_list[5] = priv->port; /* needed for B0 steering support */ 1951 mlx4_multicast_detach(mdev->dev, priv->rss_map.indir_qp, mc_list, 1952 MLX4_PROT_ETH, priv->broadcast_id); 1953 list_for_each_entry(mclist, &priv->curr_list, list) { 1954 memcpy(&mc_list[10], mclist->addr, ETH_ALEN); 1955 mc_list[5] = priv->port; 1956 mlx4_multicast_detach(mdev->dev, priv->rss_map.indir_qp, 1957 mc_list, MLX4_PROT_ETH, mclist->reg_id); 1958 if (mclist->tunnel_reg_id) 1959 mlx4_flow_detach(mdev->dev, mclist->tunnel_reg_id); 1960 } 1961 mlx4_en_clear_list(dev); 1962 list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { 1963 list_del(&mclist->list); 1964 kfree(mclist); 1965 } 1966 1967 /* Flush multicast filter */ 1968 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1, MLX4_MCAST_CONFIG); 1969 1970 /* Remove flow steering rules for the port*/ 1971 if (mdev->dev->caps.steering_mode == 1972 MLX4_STEERING_MODE_DEVICE_MANAGED) { 1973 ASSERT_RTNL(); 1974 list_for_each_entry_safe(flow, tmp_flow, 1975 &priv->ethtool_list, list) { 1976 mlx4_flow_detach(mdev->dev, flow->id); 1977 list_del(&flow->list); 1978 } 1979 } 1980 1981 mlx4_en_destroy_drop_qp(priv); 1982 1983 /* Free TX Rings */ 1984 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 1985 for (i = 0; i < priv->tx_ring_num[t]; i++) { 1986 mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[t][i]); 1987 mlx4_en_deactivate_cq(priv, priv->tx_cq[t][i]); 1988 } 1989 } 1990 msleep(10); 1991 1992 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) 1993 for (i = 0; i < priv->tx_ring_num[t]; i++) 1994 mlx4_en_free_tx_buf(dev, priv->tx_ring[t][i]); 1995 1996 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0) 1997 mlx4_en_delete_rss_steer_rules(priv); 1998 1999 /* Free RSS qps */ 2000 mlx4_en_release_rss_steer(priv); 2001 2002 /* Unregister Mac address for the port */ 2003 mlx4_en_put_qp(priv); 2004 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_REASSIGN_MAC_EN)) 2005 mdev->mac_removed[priv->port] = 1; 2006 2007 /* Free RX Rings */ 2008 for (i = 0; i < priv->rx_ring_num; i++) { 2009 struct mlx4_en_cq *cq = priv->rx_cq[i]; 2010 2011 napi_synchronize(&cq->napi); 2012 mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); 2013 mlx4_en_deactivate_cq(priv, cq); 2014 2015 mlx4_en_free_affinity_hint(priv, i); 2016 } 2017 } 2018 2019 static void mlx4_en_restart(struct work_struct *work) 2020 { 2021 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 2022 restart_task); 2023 struct mlx4_en_dev *mdev = priv->mdev; 2024 struct net_device *dev = priv->dev; 2025 2026 en_dbg(DRV, priv, "Watchdog task called for port %d\n", priv->port); 2027 2028 rtnl_lock(); 2029 mutex_lock(&mdev->state_lock); 2030 if (priv->port_up) { 2031 mlx4_en_stop_port(dev, 1); 2032 if (mlx4_en_start_port(dev)) 2033 en_err(priv, "Failed restarting port %d\n", priv->port); 2034 } 2035 mutex_unlock(&mdev->state_lock); 2036 rtnl_unlock(); 2037 } 2038 2039 static void mlx4_en_clear_stats(struct net_device *dev) 2040 { 2041 struct mlx4_en_priv *priv = netdev_priv(dev); 2042 struct mlx4_en_dev *mdev = priv->mdev; 2043 struct mlx4_en_tx_ring **tx_ring; 2044 int i; 2045 2046 if (!mlx4_is_slave(mdev->dev)) 2047 if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1)) 2048 en_dbg(HW, priv, "Failed dumping statistics\n"); 2049 2050 memset(&priv->pkstats, 0, sizeof(priv->pkstats)); 2051 memset(&priv->port_stats, 0, sizeof(priv->port_stats)); 2052 memset(&priv->rx_flowstats, 0, sizeof(priv->rx_flowstats)); 2053 memset(&priv->tx_flowstats, 0, sizeof(priv->tx_flowstats)); 2054 memset(&priv->rx_priority_flowstats, 0, 2055 sizeof(priv->rx_priority_flowstats)); 2056 memset(&priv->tx_priority_flowstats, 0, 2057 sizeof(priv->tx_priority_flowstats)); 2058 memset(&priv->pf_stats, 0, sizeof(priv->pf_stats)); 2059 2060 tx_ring = priv->tx_ring[TX]; 2061 for (i = 0; i < priv->tx_ring_num[TX]; i++) { 2062 tx_ring[i]->bytes = 0; 2063 tx_ring[i]->packets = 0; 2064 tx_ring[i]->tx_csum = 0; 2065 tx_ring[i]->tx_dropped = 0; 2066 tx_ring[i]->queue_stopped = 0; 2067 tx_ring[i]->wake_queue = 0; 2068 tx_ring[i]->tso_packets = 0; 2069 tx_ring[i]->xmit_more = 0; 2070 } 2071 for (i = 0; i < priv->rx_ring_num; i++) { 2072 priv->rx_ring[i]->bytes = 0; 2073 priv->rx_ring[i]->packets = 0; 2074 priv->rx_ring[i]->csum_ok = 0; 2075 priv->rx_ring[i]->csum_none = 0; 2076 priv->rx_ring[i]->csum_complete = 0; 2077 priv->rx_ring[i]->alloc_fail = 0; 2078 } 2079 } 2080 2081 static int mlx4_en_open(struct net_device *dev) 2082 { 2083 struct mlx4_en_priv *priv = netdev_priv(dev); 2084 struct mlx4_en_dev *mdev = priv->mdev; 2085 int err = 0; 2086 2087 mutex_lock(&mdev->state_lock); 2088 2089 if (!mdev->device_up) { 2090 en_err(priv, "Cannot open - device down/disabled\n"); 2091 err = -EBUSY; 2092 goto out; 2093 } 2094 2095 /* Reset HW statistics and SW counters */ 2096 mlx4_en_clear_stats(dev); 2097 2098 err = mlx4_en_start_port(dev); 2099 if (err) { 2100 en_err(priv, "Failed starting port:%d\n", priv->port); 2101 goto out; 2102 } 2103 mlx4_en_linkstate(priv); 2104 out: 2105 mutex_unlock(&mdev->state_lock); 2106 return err; 2107 } 2108 2109 2110 static int mlx4_en_close(struct net_device *dev) 2111 { 2112 struct mlx4_en_priv *priv = netdev_priv(dev); 2113 struct mlx4_en_dev *mdev = priv->mdev; 2114 2115 en_dbg(IFDOWN, priv, "Close port called\n"); 2116 2117 mutex_lock(&mdev->state_lock); 2118 2119 mlx4_en_stop_port(dev, 0); 2120 netif_carrier_off(dev); 2121 2122 mutex_unlock(&mdev->state_lock); 2123 return 0; 2124 } 2125 2126 static void mlx4_en_free_resources(struct mlx4_en_priv *priv) 2127 { 2128 int i, t; 2129 2130 #ifdef CONFIG_RFS_ACCEL 2131 priv->dev->rx_cpu_rmap = NULL; 2132 #endif 2133 2134 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2135 for (i = 0; i < priv->tx_ring_num[t]; i++) { 2136 if (priv->tx_ring[t] && priv->tx_ring[t][i]) 2137 mlx4_en_destroy_tx_ring(priv, 2138 &priv->tx_ring[t][i]); 2139 if (priv->tx_cq[t] && priv->tx_cq[t][i]) 2140 mlx4_en_destroy_cq(priv, &priv->tx_cq[t][i]); 2141 } 2142 kfree(priv->tx_ring[t]); 2143 kfree(priv->tx_cq[t]); 2144 } 2145 2146 for (i = 0; i < priv->rx_ring_num; i++) { 2147 if (priv->rx_ring[i]) 2148 mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], 2149 priv->prof->rx_ring_size, priv->stride); 2150 if (priv->rx_cq[i]) 2151 mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); 2152 } 2153 2154 } 2155 2156 static int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) 2157 { 2158 struct mlx4_en_port_profile *prof = priv->prof; 2159 int i, t; 2160 int node; 2161 2162 /* Create tx Rings */ 2163 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2164 for (i = 0; i < priv->tx_ring_num[t]; i++) { 2165 node = cpu_to_node(i % num_online_cpus()); 2166 if (mlx4_en_create_cq(priv, &priv->tx_cq[t][i], 2167 prof->tx_ring_size, i, t, node)) 2168 goto err; 2169 2170 if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[t][i], 2171 prof->tx_ring_size, 2172 TXBB_SIZE, node, i)) 2173 goto err; 2174 } 2175 } 2176 2177 /* Create rx Rings */ 2178 for (i = 0; i < priv->rx_ring_num; i++) { 2179 node = cpu_to_node(i % num_online_cpus()); 2180 if (mlx4_en_create_cq(priv, &priv->rx_cq[i], 2181 prof->rx_ring_size, i, RX, node)) 2182 goto err; 2183 2184 if (mlx4_en_create_rx_ring(priv, &priv->rx_ring[i], 2185 prof->rx_ring_size, priv->stride, 2186 node, i)) 2187 goto err; 2188 2189 } 2190 2191 #ifdef CONFIG_RFS_ACCEL 2192 priv->dev->rx_cpu_rmap = mlx4_get_cpu_rmap(priv->mdev->dev, priv->port); 2193 #endif 2194 2195 return 0; 2196 2197 err: 2198 en_err(priv, "Failed to allocate NIC resources\n"); 2199 for (i = 0; i < priv->rx_ring_num; i++) { 2200 if (priv->rx_ring[i]) 2201 mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], 2202 prof->rx_ring_size, 2203 priv->stride); 2204 if (priv->rx_cq[i]) 2205 mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); 2206 } 2207 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2208 for (i = 0; i < priv->tx_ring_num[t]; i++) { 2209 if (priv->tx_ring[t][i]) 2210 mlx4_en_destroy_tx_ring(priv, 2211 &priv->tx_ring[t][i]); 2212 if (priv->tx_cq[t][i]) 2213 mlx4_en_destroy_cq(priv, &priv->tx_cq[t][i]); 2214 } 2215 } 2216 return -ENOMEM; 2217 } 2218 2219 2220 static int mlx4_en_copy_priv(struct mlx4_en_priv *dst, 2221 struct mlx4_en_priv *src, 2222 struct mlx4_en_port_profile *prof) 2223 { 2224 int t; 2225 2226 memcpy(&dst->hwtstamp_config, &prof->hwtstamp_config, 2227 sizeof(dst->hwtstamp_config)); 2228 dst->num_tx_rings_p_up = prof->num_tx_rings_p_up; 2229 dst->rx_ring_num = prof->rx_ring_num; 2230 dst->flags = prof->flags; 2231 dst->mdev = src->mdev; 2232 dst->port = src->port; 2233 dst->dev = src->dev; 2234 dst->prof = prof; 2235 dst->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + 2236 DS_SIZE * MLX4_EN_MAX_RX_FRAGS); 2237 2238 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2239 dst->tx_ring_num[t] = prof->tx_ring_num[t]; 2240 if (!dst->tx_ring_num[t]) 2241 continue; 2242 2243 dst->tx_ring[t] = kcalloc(MAX_TX_RINGS, 2244 sizeof(struct mlx4_en_tx_ring *), 2245 GFP_KERNEL); 2246 if (!dst->tx_ring[t]) 2247 goto err_free_tx; 2248 2249 dst->tx_cq[t] = kcalloc(MAX_TX_RINGS, 2250 sizeof(struct mlx4_en_cq *), 2251 GFP_KERNEL); 2252 if (!dst->tx_cq[t]) { 2253 kfree(dst->tx_ring[t]); 2254 goto err_free_tx; 2255 } 2256 } 2257 2258 return 0; 2259 2260 err_free_tx: 2261 while (t--) { 2262 kfree(dst->tx_ring[t]); 2263 kfree(dst->tx_cq[t]); 2264 } 2265 return -ENOMEM; 2266 } 2267 2268 static void mlx4_en_update_priv(struct mlx4_en_priv *dst, 2269 struct mlx4_en_priv *src) 2270 { 2271 int t; 2272 memcpy(dst->rx_ring, src->rx_ring, 2273 sizeof(struct mlx4_en_rx_ring *) * src->rx_ring_num); 2274 memcpy(dst->rx_cq, src->rx_cq, 2275 sizeof(struct mlx4_en_cq *) * src->rx_ring_num); 2276 memcpy(&dst->hwtstamp_config, &src->hwtstamp_config, 2277 sizeof(dst->hwtstamp_config)); 2278 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2279 dst->tx_ring_num[t] = src->tx_ring_num[t]; 2280 dst->tx_ring[t] = src->tx_ring[t]; 2281 dst->tx_cq[t] = src->tx_cq[t]; 2282 } 2283 dst->num_tx_rings_p_up = src->num_tx_rings_p_up; 2284 dst->rx_ring_num = src->rx_ring_num; 2285 memcpy(dst->prof, src->prof, sizeof(struct mlx4_en_port_profile)); 2286 } 2287 2288 int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv, 2289 struct mlx4_en_priv *tmp, 2290 struct mlx4_en_port_profile *prof, 2291 bool carry_xdp_prog) 2292 { 2293 struct bpf_prog *xdp_prog; 2294 int i, t, ret; 2295 2296 ret = mlx4_en_copy_priv(tmp, priv, prof); 2297 if (ret) { 2298 en_warn(priv, "%s: mlx4_en_copy_priv() failed, return\n", 2299 __func__); 2300 return ret; 2301 } 2302 2303 if (mlx4_en_alloc_resources(tmp)) { 2304 en_warn(priv, 2305 "%s: Resource allocation failed, using previous configuration\n", 2306 __func__); 2307 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2308 kfree(tmp->tx_ring[t]); 2309 kfree(tmp->tx_cq[t]); 2310 } 2311 return -ENOMEM; 2312 } 2313 2314 /* All rx_rings has the same xdp_prog. Pick the first one. */ 2315 xdp_prog = rcu_dereference_protected( 2316 priv->rx_ring[0]->xdp_prog, 2317 lockdep_is_held(&priv->mdev->state_lock)); 2318 2319 if (xdp_prog && carry_xdp_prog) { 2320 bpf_prog_add(xdp_prog, tmp->rx_ring_num); 2321 for (i = 0; i < tmp->rx_ring_num; i++) 2322 rcu_assign_pointer(tmp->rx_ring[i]->xdp_prog, 2323 xdp_prog); 2324 } 2325 2326 return 0; 2327 } 2328 2329 void mlx4_en_safe_replace_resources(struct mlx4_en_priv *priv, 2330 struct mlx4_en_priv *tmp) 2331 { 2332 mlx4_en_free_resources(priv); 2333 mlx4_en_update_priv(priv, tmp); 2334 } 2335 2336 void mlx4_en_destroy_netdev(struct net_device *dev) 2337 { 2338 struct mlx4_en_priv *priv = netdev_priv(dev); 2339 struct mlx4_en_dev *mdev = priv->mdev; 2340 2341 en_dbg(DRV, priv, "Destroying netdev on port:%d\n", priv->port); 2342 2343 /* Unregister device - this will close the port if it was up */ 2344 if (priv->registered) 2345 unregister_netdev(dev); 2346 2347 if (priv->allocated) 2348 mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE); 2349 2350 cancel_delayed_work(&priv->stats_task); 2351 cancel_delayed_work(&priv->service_task); 2352 /* flush any pending task for this netdev */ 2353 flush_workqueue(mdev->workqueue); 2354 2355 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) 2356 mlx4_en_remove_timestamp(mdev); 2357 2358 /* Detach the netdev so tasks would not attempt to access it */ 2359 mutex_lock(&mdev->state_lock); 2360 mdev->pndev[priv->port] = NULL; 2361 mdev->upper[priv->port] = NULL; 2362 2363 #ifdef CONFIG_RFS_ACCEL 2364 mlx4_en_cleanup_filters(priv); 2365 #endif 2366 2367 mlx4_en_free_resources(priv); 2368 mutex_unlock(&mdev->state_lock); 2369 2370 free_netdev(dev); 2371 } 2372 2373 static bool mlx4_en_check_xdp_mtu(struct net_device *dev, int mtu) 2374 { 2375 struct mlx4_en_priv *priv = netdev_priv(dev); 2376 2377 if (mtu > MLX4_EN_MAX_XDP_MTU) { 2378 en_err(priv, "mtu:%d > max:%d when XDP prog is attached\n", 2379 mtu, MLX4_EN_MAX_XDP_MTU); 2380 return false; 2381 } 2382 2383 return true; 2384 } 2385 2386 static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu) 2387 { 2388 struct mlx4_en_priv *priv = netdev_priv(dev); 2389 struct mlx4_en_dev *mdev = priv->mdev; 2390 int err = 0; 2391 2392 en_dbg(DRV, priv, "Change MTU called - current:%d new:%d\n", 2393 dev->mtu, new_mtu); 2394 2395 if (priv->tx_ring_num[TX_XDP] && 2396 !mlx4_en_check_xdp_mtu(dev, new_mtu)) 2397 return -EOPNOTSUPP; 2398 2399 WRITE_ONCE(dev->mtu, new_mtu); 2400 2401 if (netif_running(dev)) { 2402 mutex_lock(&mdev->state_lock); 2403 if (!mdev->device_up) { 2404 /* NIC is probably restarting - let restart task reset 2405 * the port */ 2406 en_dbg(DRV, priv, "Change MTU called with card down!?\n"); 2407 } else { 2408 mlx4_en_stop_port(dev, 1); 2409 err = mlx4_en_start_port(dev); 2410 if (err) { 2411 en_err(priv, "Failed restarting port:%d\n", 2412 priv->port); 2413 if (!test_and_set_bit(MLX4_EN_STATE_FLAG_RESTARTING, 2414 &priv->state)) 2415 queue_work(mdev->workqueue, &priv->restart_task); 2416 } 2417 } 2418 mutex_unlock(&mdev->state_lock); 2419 } 2420 return 0; 2421 } 2422 2423 static int mlx4_en_hwtstamp_set(struct net_device *dev, 2424 struct kernel_hwtstamp_config *config, 2425 struct netlink_ext_ack *extack) 2426 { 2427 struct mlx4_en_priv *priv = netdev_priv(dev); 2428 struct mlx4_en_dev *mdev = priv->mdev; 2429 2430 /* device doesn't support time stamping */ 2431 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS)) { 2432 NL_SET_ERR_MSG_MOD(extack, 2433 "device doesn't support time stamping"); 2434 return -EINVAL; 2435 } 2436 2437 /* TX HW timestamp */ 2438 switch (config->tx_type) { 2439 case HWTSTAMP_TX_OFF: 2440 case HWTSTAMP_TX_ON: 2441 break; 2442 default: 2443 return -ERANGE; 2444 } 2445 2446 /* RX HW timestamp */ 2447 switch (config->rx_filter) { 2448 case HWTSTAMP_FILTER_NONE: 2449 break; 2450 case HWTSTAMP_FILTER_ALL: 2451 case HWTSTAMP_FILTER_SOME: 2452 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 2453 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 2454 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 2455 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 2456 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 2457 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 2458 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 2459 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 2460 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 2461 case HWTSTAMP_FILTER_PTP_V2_EVENT: 2462 case HWTSTAMP_FILTER_PTP_V2_SYNC: 2463 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 2464 case HWTSTAMP_FILTER_NTP_ALL: 2465 config->rx_filter = HWTSTAMP_FILTER_ALL; 2466 break; 2467 default: 2468 return -ERANGE; 2469 } 2470 2471 if (mlx4_en_reset_config(dev, config, dev->features)) { 2472 config->tx_type = HWTSTAMP_TX_OFF; 2473 config->rx_filter = HWTSTAMP_FILTER_NONE; 2474 } 2475 2476 return 0; 2477 } 2478 2479 static int mlx4_en_hwtstamp_get(struct net_device *dev, 2480 struct kernel_hwtstamp_config *config) 2481 { 2482 struct mlx4_en_priv *priv = netdev_priv(dev); 2483 2484 *config = priv->hwtstamp_config; 2485 return 0; 2486 } 2487 2488 static netdev_features_t mlx4_en_fix_features(struct net_device *netdev, 2489 netdev_features_t features) 2490 { 2491 struct mlx4_en_priv *en_priv = netdev_priv(netdev); 2492 struct mlx4_en_dev *mdev = en_priv->mdev; 2493 2494 /* Since there is no support for separate RX C-TAG/S-TAG vlan accel 2495 * enable/disable make sure S-TAG flag is always in same state as 2496 * C-TAG. 2497 */ 2498 if (features & NETIF_F_HW_VLAN_CTAG_RX && 2499 !(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) 2500 features |= NETIF_F_HW_VLAN_STAG_RX; 2501 else 2502 features &= ~NETIF_F_HW_VLAN_STAG_RX; 2503 2504 return features; 2505 } 2506 2507 static int mlx4_en_set_features(struct net_device *netdev, 2508 netdev_features_t features) 2509 { 2510 struct mlx4_en_priv *priv = netdev_priv(netdev); 2511 bool reset = false; 2512 int ret = 0; 2513 2514 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXFCS)) { 2515 en_info(priv, "Turn %s RX-FCS\n", 2516 (features & NETIF_F_RXFCS) ? "ON" : "OFF"); 2517 reset = true; 2518 } 2519 2520 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXALL)) { 2521 u8 ignore_fcs_value = (features & NETIF_F_RXALL) ? 1 : 0; 2522 2523 en_info(priv, "Turn %s RX-ALL\n", 2524 ignore_fcs_value ? "ON" : "OFF"); 2525 ret = mlx4_SET_PORT_fcs_check(priv->mdev->dev, 2526 priv->port, ignore_fcs_value); 2527 if (ret) 2528 return ret; 2529 } 2530 2531 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_RX)) { 2532 en_info(priv, "Turn %s RX vlan strip offload\n", 2533 (features & NETIF_F_HW_VLAN_CTAG_RX) ? "ON" : "OFF"); 2534 reset = true; 2535 } 2536 2537 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_TX)) 2538 en_info(priv, "Turn %s TX vlan strip offload\n", 2539 (features & NETIF_F_HW_VLAN_CTAG_TX) ? "ON" : "OFF"); 2540 2541 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_STAG_TX)) 2542 en_info(priv, "Turn %s TX S-VLAN strip offload\n", 2543 (features & NETIF_F_HW_VLAN_STAG_TX) ? "ON" : "OFF"); 2544 2545 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_LOOPBACK)) { 2546 en_info(priv, "Turn %s loopback\n", 2547 (features & NETIF_F_LOOPBACK) ? "ON" : "OFF"); 2548 mlx4_en_update_loopback_state(netdev, features); 2549 } 2550 2551 if (reset) { 2552 ret = mlx4_en_reset_config(netdev, &priv->hwtstamp_config, 2553 features); 2554 if (ret) 2555 return ret; 2556 } 2557 2558 return 0; 2559 } 2560 2561 static int mlx4_en_set_vf_mac(struct net_device *dev, int queue, u8 *mac) 2562 { 2563 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2564 struct mlx4_en_dev *mdev = en_priv->mdev; 2565 2566 return mlx4_set_vf_mac(mdev->dev, en_priv->port, queue, mac); 2567 } 2568 2569 static int mlx4_en_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 2570 __be16 vlan_proto) 2571 { 2572 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2573 struct mlx4_en_dev *mdev = en_priv->mdev; 2574 2575 return mlx4_set_vf_vlan(mdev->dev, en_priv->port, vf, vlan, qos, 2576 vlan_proto); 2577 } 2578 2579 static int mlx4_en_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 2580 int max_tx_rate) 2581 { 2582 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2583 struct mlx4_en_dev *mdev = en_priv->mdev; 2584 2585 return mlx4_set_vf_rate(mdev->dev, en_priv->port, vf, min_tx_rate, 2586 max_tx_rate); 2587 } 2588 2589 static int mlx4_en_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 2590 { 2591 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2592 struct mlx4_en_dev *mdev = en_priv->mdev; 2593 2594 return mlx4_set_vf_spoofchk(mdev->dev, en_priv->port, vf, setting); 2595 } 2596 2597 static int mlx4_en_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivf) 2598 { 2599 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2600 struct mlx4_en_dev *mdev = en_priv->mdev; 2601 2602 return mlx4_get_vf_config(mdev->dev, en_priv->port, vf, ivf); 2603 } 2604 2605 static int mlx4_en_set_vf_link_state(struct net_device *dev, int vf, int link_state) 2606 { 2607 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2608 struct mlx4_en_dev *mdev = en_priv->mdev; 2609 2610 return mlx4_set_vf_link_state(mdev->dev, en_priv->port, vf, link_state); 2611 } 2612 2613 static int mlx4_en_get_vf_stats(struct net_device *dev, int vf, 2614 struct ifla_vf_stats *vf_stats) 2615 { 2616 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2617 struct mlx4_en_dev *mdev = en_priv->mdev; 2618 2619 return mlx4_get_vf_stats(mdev->dev, en_priv->port, vf, vf_stats); 2620 } 2621 2622 #define PORT_ID_BYTE_LEN 8 2623 static int mlx4_en_get_phys_port_id(struct net_device *dev, 2624 struct netdev_phys_item_id *ppid) 2625 { 2626 struct mlx4_en_priv *priv = netdev_priv(dev); 2627 struct mlx4_dev *mdev = priv->mdev->dev; 2628 int i; 2629 u64 phys_port_id = mdev->caps.phys_port_id[priv->port]; 2630 2631 if (!phys_port_id) 2632 return -EOPNOTSUPP; 2633 2634 ppid->id_len = sizeof(phys_port_id); 2635 for (i = PORT_ID_BYTE_LEN - 1; i >= 0; --i) { 2636 ppid->id[i] = phys_port_id & 0xff; 2637 phys_port_id >>= 8; 2638 } 2639 return 0; 2640 } 2641 2642 static int mlx4_udp_tunnel_sync(struct net_device *dev, unsigned int table) 2643 { 2644 struct mlx4_en_priv *priv = netdev_priv(dev); 2645 struct udp_tunnel_info ti; 2646 int ret; 2647 2648 udp_tunnel_nic_get_port(dev, table, 0, &ti); 2649 priv->vxlan_port = ti.port; 2650 2651 ret = mlx4_config_vxlan_port(priv->mdev->dev, priv->vxlan_port); 2652 if (ret) 2653 return ret; 2654 2655 return mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port, 2656 VXLAN_STEER_BY_OUTER_MAC, 2657 !!priv->vxlan_port); 2658 } 2659 2660 static const struct udp_tunnel_nic_info mlx4_udp_tunnels = { 2661 .sync_table = mlx4_udp_tunnel_sync, 2662 .flags = UDP_TUNNEL_NIC_INFO_IPV4_ONLY, 2663 .tables = { 2664 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 2665 }, 2666 }; 2667 2668 static netdev_features_t mlx4_en_features_check(struct sk_buff *skb, 2669 struct net_device *dev, 2670 netdev_features_t features) 2671 { 2672 features = vlan_features_check(skb, features); 2673 features = vxlan_features_check(skb, features); 2674 2675 /* The ConnectX-3 doesn't support outer IPv6 checksums but it does 2676 * support inner IPv6 checksums and segmentation so we need to 2677 * strip that feature if this is an IPv6 encapsulated frame. 2678 */ 2679 if (skb->encapsulation && 2680 (skb->ip_summed == CHECKSUM_PARTIAL)) { 2681 struct mlx4_en_priv *priv = netdev_priv(dev); 2682 2683 if (!priv->vxlan_port || 2684 (ip_hdr(skb)->version != 4) || 2685 (udp_hdr(skb)->dest != priv->vxlan_port)) 2686 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 2687 } 2688 2689 return features; 2690 } 2691 2692 static int mlx4_en_set_tx_maxrate(struct net_device *dev, int queue_index, u32 maxrate) 2693 { 2694 struct mlx4_en_priv *priv = netdev_priv(dev); 2695 struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[TX][queue_index]; 2696 struct mlx4_update_qp_params params; 2697 int err; 2698 2699 if (!(priv->mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QP_RATE_LIMIT)) 2700 return -EOPNOTSUPP; 2701 2702 /* rate provided to us in Mbs, check if it fits into 12 bits, if not use Gbs */ 2703 if (maxrate >> 12) { 2704 params.rate_unit = MLX4_QP_RATE_LIMIT_GBS; 2705 params.rate_val = maxrate / 1000; 2706 } else if (maxrate) { 2707 params.rate_unit = MLX4_QP_RATE_LIMIT_MBS; 2708 params.rate_val = maxrate; 2709 } else { /* zero serves to revoke the QP rate-limitation */ 2710 params.rate_unit = 0; 2711 params.rate_val = 0; 2712 } 2713 2714 err = mlx4_update_qp(priv->mdev->dev, tx_ring->qpn, MLX4_UPDATE_QP_RATE_LIMIT, 2715 ¶ms); 2716 return err; 2717 } 2718 2719 static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) 2720 { 2721 struct mlx4_en_priv *priv = netdev_priv(dev); 2722 struct mlx4_en_dev *mdev = priv->mdev; 2723 struct mlx4_en_port_profile new_prof; 2724 struct bpf_prog *old_prog; 2725 struct mlx4_en_priv *tmp; 2726 int tx_changed = 0; 2727 int xdp_ring_num; 2728 int port_up = 0; 2729 int err; 2730 int i; 2731 2732 xdp_ring_num = prog ? priv->rx_ring_num : 0; 2733 2734 /* No need to reconfigure buffers when simply swapping the 2735 * program for a new one. 2736 */ 2737 if (priv->tx_ring_num[TX_XDP] == xdp_ring_num) { 2738 if (prog) 2739 bpf_prog_add(prog, priv->rx_ring_num - 1); 2740 2741 mutex_lock(&mdev->state_lock); 2742 for (i = 0; i < priv->rx_ring_num; i++) { 2743 old_prog = rcu_dereference_protected( 2744 priv->rx_ring[i]->xdp_prog, 2745 lockdep_is_held(&mdev->state_lock)); 2746 rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog); 2747 if (old_prog) 2748 bpf_prog_put(old_prog); 2749 } 2750 mutex_unlock(&mdev->state_lock); 2751 return 0; 2752 } 2753 2754 if (!mlx4_en_check_xdp_mtu(dev, dev->mtu)) 2755 return -EOPNOTSUPP; 2756 2757 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); 2758 if (!tmp) 2759 return -ENOMEM; 2760 2761 if (prog) 2762 bpf_prog_add(prog, priv->rx_ring_num - 1); 2763 2764 mutex_lock(&mdev->state_lock); 2765 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); 2766 new_prof.tx_ring_num[TX_XDP] = xdp_ring_num; 2767 2768 if (priv->tx_ring_num[TX] + xdp_ring_num > MAX_TX_RINGS) { 2769 tx_changed = 1; 2770 new_prof.tx_ring_num[TX] = 2771 MAX_TX_RINGS - ALIGN(xdp_ring_num, priv->prof->num_up); 2772 en_warn(priv, "Reducing the number of TX rings, to not exceed the max total rings number.\n"); 2773 } 2774 2775 err = mlx4_en_try_alloc_resources(priv, tmp, &new_prof, false); 2776 if (err) { 2777 if (prog) 2778 bpf_prog_sub(prog, priv->rx_ring_num - 1); 2779 goto unlock_out; 2780 } 2781 2782 if (priv->port_up) { 2783 port_up = 1; 2784 mlx4_en_stop_port(dev, 1); 2785 } 2786 2787 mlx4_en_safe_replace_resources(priv, tmp); 2788 if (tx_changed) 2789 netif_set_real_num_tx_queues(dev, priv->tx_ring_num[TX]); 2790 2791 for (i = 0; i < priv->rx_ring_num; i++) { 2792 old_prog = rcu_dereference_protected( 2793 priv->rx_ring[i]->xdp_prog, 2794 lockdep_is_held(&mdev->state_lock)); 2795 rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog); 2796 if (old_prog) 2797 bpf_prog_put(old_prog); 2798 } 2799 2800 if (port_up) { 2801 err = mlx4_en_start_port(dev); 2802 if (err) { 2803 en_err(priv, "Failed starting port %d for XDP change\n", 2804 priv->port); 2805 if (!test_and_set_bit(MLX4_EN_STATE_FLAG_RESTARTING, &priv->state)) 2806 queue_work(mdev->workqueue, &priv->restart_task); 2807 } 2808 } 2809 2810 unlock_out: 2811 mutex_unlock(&mdev->state_lock); 2812 kfree(tmp); 2813 return err; 2814 } 2815 2816 static int mlx4_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2817 { 2818 switch (xdp->command) { 2819 case XDP_SETUP_PROG: 2820 return mlx4_xdp_set(dev, xdp->prog); 2821 default: 2822 return -EINVAL; 2823 } 2824 } 2825 2826 static const struct net_device_ops mlx4_netdev_ops = { 2827 .ndo_open = mlx4_en_open, 2828 .ndo_stop = mlx4_en_close, 2829 .ndo_start_xmit = mlx4_en_xmit, 2830 .ndo_select_queue = mlx4_en_select_queue, 2831 .ndo_get_stats64 = mlx4_en_get_stats64, 2832 .ndo_set_rx_mode = mlx4_en_set_rx_mode, 2833 .ndo_set_mac_address = mlx4_en_set_mac, 2834 .ndo_validate_addr = eth_validate_addr, 2835 .ndo_change_mtu = mlx4_en_change_mtu, 2836 .ndo_tx_timeout = mlx4_en_tx_timeout, 2837 .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid, 2838 .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid, 2839 .ndo_set_features = mlx4_en_set_features, 2840 .ndo_fix_features = mlx4_en_fix_features, 2841 .ndo_setup_tc = __mlx4_en_setup_tc, 2842 #ifdef CONFIG_RFS_ACCEL 2843 .ndo_rx_flow_steer = mlx4_en_filter_rfs, 2844 #endif 2845 .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, 2846 .ndo_features_check = mlx4_en_features_check, 2847 .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, 2848 .ndo_bpf = mlx4_xdp, 2849 .ndo_hwtstamp_get = mlx4_en_hwtstamp_get, 2850 .ndo_hwtstamp_set = mlx4_en_hwtstamp_set, 2851 }; 2852 2853 static const struct net_device_ops mlx4_netdev_ops_master = { 2854 .ndo_open = mlx4_en_open, 2855 .ndo_stop = mlx4_en_close, 2856 .ndo_start_xmit = mlx4_en_xmit, 2857 .ndo_select_queue = mlx4_en_select_queue, 2858 .ndo_get_stats64 = mlx4_en_get_stats64, 2859 .ndo_set_rx_mode = mlx4_en_set_rx_mode, 2860 .ndo_set_mac_address = mlx4_en_set_mac, 2861 .ndo_validate_addr = eth_validate_addr, 2862 .ndo_change_mtu = mlx4_en_change_mtu, 2863 .ndo_tx_timeout = mlx4_en_tx_timeout, 2864 .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid, 2865 .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid, 2866 .ndo_set_vf_mac = mlx4_en_set_vf_mac, 2867 .ndo_set_vf_vlan = mlx4_en_set_vf_vlan, 2868 .ndo_set_vf_rate = mlx4_en_set_vf_rate, 2869 .ndo_set_vf_spoofchk = mlx4_en_set_vf_spoofchk, 2870 .ndo_set_vf_link_state = mlx4_en_set_vf_link_state, 2871 .ndo_get_vf_stats = mlx4_en_get_vf_stats, 2872 .ndo_get_vf_config = mlx4_en_get_vf_config, 2873 .ndo_set_features = mlx4_en_set_features, 2874 .ndo_fix_features = mlx4_en_fix_features, 2875 .ndo_setup_tc = __mlx4_en_setup_tc, 2876 #ifdef CONFIG_RFS_ACCEL 2877 .ndo_rx_flow_steer = mlx4_en_filter_rfs, 2878 #endif 2879 .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, 2880 .ndo_features_check = mlx4_en_features_check, 2881 .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, 2882 .ndo_bpf = mlx4_xdp, 2883 }; 2884 2885 static const struct xdp_metadata_ops mlx4_xdp_metadata_ops = { 2886 .xmo_rx_timestamp = mlx4_en_xdp_rx_timestamp, 2887 .xmo_rx_hash = mlx4_en_xdp_rx_hash, 2888 }; 2889 2890 int mlx4_en_netdev_event(struct notifier_block *this, 2891 unsigned long event, void *ptr) 2892 { 2893 struct net_device *ndev = netdev_notifier_info_to_dev(ptr); 2894 u8 port = 0; 2895 struct mlx4_en_dev *mdev; 2896 struct mlx4_dev *dev; 2897 int i, num_eth_ports = 0; 2898 bool do_bond = true; 2899 u8 v2p_port1 = 0; 2900 u8 v2p_port2 = 0; 2901 2902 if (!net_eq(dev_net(ndev), &init_net)) 2903 return NOTIFY_DONE; 2904 2905 mdev = container_of(this, struct mlx4_en_dev, netdev_nb); 2906 dev = mdev->dev; 2907 2908 /* Go into this mode only when two network devices set on two ports 2909 * of the same mlx4 device are slaves of the same bonding master 2910 */ 2911 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { 2912 ++num_eth_ports; 2913 if (!port && (mdev->pndev[i] == ndev)) 2914 port = i; 2915 mdev->upper[i] = mdev->pndev[i] ? 2916 netdev_master_upper_dev_get(mdev->pndev[i]) : NULL; 2917 /* condition not met: network device is a slave */ 2918 if (!mdev->upper[i]) 2919 do_bond = false; 2920 if (num_eth_ports < 2) 2921 continue; 2922 /* condition not met: same master */ 2923 if (mdev->upper[i] != mdev->upper[i-1]) 2924 do_bond = false; 2925 } 2926 /* condition not met: 2 salves */ 2927 do_bond = (num_eth_ports == 2) ? do_bond : false; 2928 2929 /* handle only events that come with enough info */ 2930 if ((do_bond && (event != NETDEV_BONDING_INFO)) || !port) 2931 return NOTIFY_DONE; 2932 2933 if (do_bond) { 2934 struct netdev_notifier_bonding_info *notifier_info = ptr; 2935 struct netdev_bonding_info *bonding_info = 2936 ¬ifier_info->bonding_info; 2937 2938 /* required mode 1, 2 or 4 */ 2939 if ((bonding_info->master.bond_mode != BOND_MODE_ACTIVEBACKUP) && 2940 (bonding_info->master.bond_mode != BOND_MODE_XOR) && 2941 (bonding_info->master.bond_mode != BOND_MODE_8023AD)) 2942 do_bond = false; 2943 2944 /* require exactly 2 slaves */ 2945 if (bonding_info->master.num_slaves != 2) 2946 do_bond = false; 2947 2948 /* calc v2p */ 2949 if (do_bond) { 2950 if (bonding_info->master.bond_mode == 2951 BOND_MODE_ACTIVEBACKUP) { 2952 /* in active-backup mode virtual ports are 2953 * mapped to the physical port of the active 2954 * slave */ 2955 if (bonding_info->slave.state == 2956 BOND_STATE_BACKUP) { 2957 if (port == 1) { 2958 v2p_port1 = 2; 2959 v2p_port2 = 2; 2960 } else { 2961 v2p_port1 = 1; 2962 v2p_port2 = 1; 2963 } 2964 } else { /* BOND_STATE_ACTIVE */ 2965 if (port == 1) { 2966 v2p_port1 = 1; 2967 v2p_port2 = 1; 2968 } else { 2969 v2p_port1 = 2; 2970 v2p_port2 = 2; 2971 } 2972 } 2973 } else { /* Active-Active */ 2974 /* in active-active mode a virtual port is 2975 * mapped to the native physical port if and only 2976 * if the physical port is up */ 2977 __s8 link = bonding_info->slave.link; 2978 2979 if (port == 1) 2980 v2p_port2 = 2; 2981 else 2982 v2p_port1 = 1; 2983 if ((link == BOND_LINK_UP) || 2984 (link == BOND_LINK_FAIL)) { 2985 if (port == 1) 2986 v2p_port1 = 1; 2987 else 2988 v2p_port2 = 2; 2989 } else { /* BOND_LINK_DOWN || BOND_LINK_BACK */ 2990 if (port == 1) 2991 v2p_port1 = 2; 2992 else 2993 v2p_port2 = 1; 2994 } 2995 } 2996 } 2997 } 2998 2999 mlx4_queue_bond_work(dev, do_bond, v2p_port1, v2p_port2); 3000 3001 return NOTIFY_DONE; 3002 } 3003 3004 void mlx4_en_update_pfc_stats_bitmap(struct mlx4_dev *dev, 3005 struct mlx4_en_stats_bitmap *stats_bitmap, 3006 u8 rx_ppp, u8 rx_pause, 3007 u8 tx_ppp, u8 tx_pause) 3008 { 3009 int last_i = NUM_MAIN_STATS + NUM_PORT_STATS + NUM_PF_STATS; 3010 3011 if (!mlx4_is_slave(dev) && 3012 (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN)) { 3013 mutex_lock(&stats_bitmap->mutex); 3014 bitmap_clear(stats_bitmap->bitmap, last_i, NUM_FLOW_STATS); 3015 3016 if (rx_ppp) 3017 bitmap_set(stats_bitmap->bitmap, last_i, 3018 NUM_FLOW_PRIORITY_STATS_RX); 3019 last_i += NUM_FLOW_PRIORITY_STATS_RX; 3020 3021 if (rx_pause && !(rx_ppp)) 3022 bitmap_set(stats_bitmap->bitmap, last_i, 3023 NUM_FLOW_STATS_RX); 3024 last_i += NUM_FLOW_STATS_RX; 3025 3026 if (tx_ppp) 3027 bitmap_set(stats_bitmap->bitmap, last_i, 3028 NUM_FLOW_PRIORITY_STATS_TX); 3029 last_i += NUM_FLOW_PRIORITY_STATS_TX; 3030 3031 if (tx_pause && !(tx_ppp)) 3032 bitmap_set(stats_bitmap->bitmap, last_i, 3033 NUM_FLOW_STATS_TX); 3034 last_i += NUM_FLOW_STATS_TX; 3035 3036 mutex_unlock(&stats_bitmap->mutex); 3037 } 3038 } 3039 3040 void mlx4_en_set_stats_bitmap(struct mlx4_dev *dev, 3041 struct mlx4_en_stats_bitmap *stats_bitmap, 3042 u8 rx_ppp, u8 rx_pause, 3043 u8 tx_ppp, u8 tx_pause) 3044 { 3045 int last_i = 0; 3046 3047 mutex_init(&stats_bitmap->mutex); 3048 bitmap_zero(stats_bitmap->bitmap, NUM_ALL_STATS); 3049 3050 if (mlx4_is_slave(dev)) { 3051 bitmap_set(stats_bitmap->bitmap, last_i + 3052 MLX4_FIND_NETDEV_STAT(rx_packets), 1); 3053 bitmap_set(stats_bitmap->bitmap, last_i + 3054 MLX4_FIND_NETDEV_STAT(tx_packets), 1); 3055 bitmap_set(stats_bitmap->bitmap, last_i + 3056 MLX4_FIND_NETDEV_STAT(rx_bytes), 1); 3057 bitmap_set(stats_bitmap->bitmap, last_i + 3058 MLX4_FIND_NETDEV_STAT(tx_bytes), 1); 3059 bitmap_set(stats_bitmap->bitmap, last_i + 3060 MLX4_FIND_NETDEV_STAT(rx_dropped), 1); 3061 bitmap_set(stats_bitmap->bitmap, last_i + 3062 MLX4_FIND_NETDEV_STAT(tx_dropped), 1); 3063 } else { 3064 bitmap_set(stats_bitmap->bitmap, last_i, NUM_MAIN_STATS); 3065 } 3066 last_i += NUM_MAIN_STATS; 3067 3068 bitmap_set(stats_bitmap->bitmap, last_i, NUM_PORT_STATS); 3069 last_i += NUM_PORT_STATS; 3070 3071 if (mlx4_is_master(dev)) 3072 bitmap_set(stats_bitmap->bitmap, last_i, 3073 NUM_PF_STATS); 3074 last_i += NUM_PF_STATS; 3075 3076 mlx4_en_update_pfc_stats_bitmap(dev, stats_bitmap, 3077 rx_ppp, rx_pause, 3078 tx_ppp, tx_pause); 3079 last_i += NUM_FLOW_STATS; 3080 3081 if (!mlx4_is_slave(dev)) 3082 bitmap_set(stats_bitmap->bitmap, last_i, NUM_PKT_STATS); 3083 last_i += NUM_PKT_STATS; 3084 3085 bitmap_set(stats_bitmap->bitmap, last_i, NUM_XDP_STATS); 3086 last_i += NUM_XDP_STATS; 3087 3088 if (!mlx4_is_slave(dev)) 3089 bitmap_set(stats_bitmap->bitmap, last_i, NUM_PHY_STATS); 3090 last_i += NUM_PHY_STATS; 3091 } 3092 3093 static void mlx4_get_queue_stats_rx(struct net_device *dev, int i, 3094 struct netdev_queue_stats_rx *stats) 3095 { 3096 struct mlx4_en_priv *priv = netdev_priv(dev); 3097 const struct mlx4_en_rx_ring *ring; 3098 3099 spin_lock_bh(&priv->stats_lock); 3100 3101 if (!priv->port_up || mlx4_is_master(priv->mdev->dev)) 3102 goto out_unlock; 3103 3104 ring = priv->rx_ring[i]; 3105 stats->packets = READ_ONCE(ring->packets); 3106 stats->bytes = READ_ONCE(ring->bytes); 3107 stats->alloc_fail = READ_ONCE(ring->alloc_fail); 3108 3109 out_unlock: 3110 spin_unlock_bh(&priv->stats_lock); 3111 } 3112 3113 static void mlx4_get_queue_stats_tx(struct net_device *dev, int i, 3114 struct netdev_queue_stats_tx *stats) 3115 { 3116 struct mlx4_en_priv *priv = netdev_priv(dev); 3117 const struct mlx4_en_tx_ring *ring; 3118 3119 spin_lock_bh(&priv->stats_lock); 3120 3121 if (!priv->port_up || mlx4_is_master(priv->mdev->dev)) 3122 goto out_unlock; 3123 3124 ring = priv->tx_ring[TX][i]; 3125 stats->packets = READ_ONCE(ring->packets); 3126 stats->bytes = READ_ONCE(ring->bytes); 3127 3128 out_unlock: 3129 spin_unlock_bh(&priv->stats_lock); 3130 } 3131 3132 static void mlx4_get_base_stats(struct net_device *dev, 3133 struct netdev_queue_stats_rx *rx, 3134 struct netdev_queue_stats_tx *tx) 3135 { 3136 struct mlx4_en_priv *priv = netdev_priv(dev); 3137 3138 spin_lock_bh(&priv->stats_lock); 3139 3140 if (!priv->port_up || mlx4_is_master(priv->mdev->dev)) 3141 goto out_unlock; 3142 3143 if (priv->rx_ring_num) { 3144 rx->packets = 0; 3145 rx->bytes = 0; 3146 rx->alloc_fail = 0; 3147 } 3148 3149 if (priv->tx_ring_num[TX]) { 3150 tx->packets = 0; 3151 tx->bytes = 0; 3152 } 3153 3154 out_unlock: 3155 spin_unlock_bh(&priv->stats_lock); 3156 } 3157 3158 static const struct netdev_stat_ops mlx4_stat_ops = { 3159 .get_queue_stats_rx = mlx4_get_queue_stats_rx, 3160 .get_queue_stats_tx = mlx4_get_queue_stats_tx, 3161 .get_base_stats = mlx4_get_base_stats, 3162 }; 3163 3164 int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, 3165 struct mlx4_en_port_profile *prof) 3166 { 3167 struct net_device *dev; 3168 struct mlx4_en_priv *priv; 3169 int i, t; 3170 int err; 3171 3172 dev = alloc_etherdev_mqs(sizeof(struct mlx4_en_priv), 3173 MAX_TX_RINGS, MAX_RX_RINGS); 3174 if (dev == NULL) 3175 return -ENOMEM; 3176 3177 netif_set_real_num_tx_queues(dev, prof->tx_ring_num[TX]); 3178 netif_set_real_num_rx_queues(dev, prof->rx_ring_num); 3179 3180 SET_NETDEV_DEV(dev, &mdev->dev->persist->pdev->dev); 3181 dev->dev_port = port - 1; 3182 3183 /* 3184 * Initialize driver private data 3185 */ 3186 3187 priv = netdev_priv(dev); 3188 memset(priv, 0, sizeof(struct mlx4_en_priv)); 3189 priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev); 3190 spin_lock_init(&priv->stats_lock); 3191 INIT_WORK(&priv->rx_mode_task, mlx4_en_do_set_rx_mode); 3192 INIT_WORK(&priv->restart_task, mlx4_en_restart); 3193 INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate_work); 3194 INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats); 3195 INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task); 3196 #ifdef CONFIG_RFS_ACCEL 3197 INIT_LIST_HEAD(&priv->filters); 3198 spin_lock_init(&priv->filters_lock); 3199 #endif 3200 3201 priv->dev = dev; 3202 priv->mdev = mdev; 3203 priv->ddev = &mdev->pdev->dev; 3204 priv->prof = prof; 3205 priv->port = port; 3206 priv->port_up = false; 3207 priv->flags = prof->flags; 3208 priv->pflags = MLX4_EN_PRIV_FLAGS_BLUEFLAME; 3209 priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | 3210 MLX4_WQE_CTRL_SOLICITED); 3211 priv->num_tx_rings_p_up = mdev->profile.max_num_tx_rings_p_up; 3212 priv->tx_work_limit = MLX4_EN_DEFAULT_TX_WORK; 3213 netdev_rss_key_fill(priv->rss_key, sizeof(priv->rss_key)); 3214 3215 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 3216 priv->tx_ring_num[t] = prof->tx_ring_num[t]; 3217 if (!priv->tx_ring_num[t]) 3218 continue; 3219 3220 priv->tx_ring[t] = kcalloc(MAX_TX_RINGS, 3221 sizeof(struct mlx4_en_tx_ring *), 3222 GFP_KERNEL); 3223 if (!priv->tx_ring[t]) { 3224 err = -ENOMEM; 3225 goto out; 3226 } 3227 priv->tx_cq[t] = kcalloc(MAX_TX_RINGS, 3228 sizeof(struct mlx4_en_cq *), 3229 GFP_KERNEL); 3230 if (!priv->tx_cq[t]) { 3231 err = -ENOMEM; 3232 goto out; 3233 } 3234 } 3235 priv->rx_ring_num = prof->rx_ring_num; 3236 priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0; 3237 priv->cqe_size = mdev->dev->caps.cqe_size; 3238 priv->mac_index = -1; 3239 priv->msg_enable = MLX4_EN_MSG_LEVEL; 3240 #ifdef CONFIG_MLX4_EN_DCB 3241 if (!mlx4_is_slave(priv->mdev->dev)) { 3242 u8 prio; 3243 3244 for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; ++prio) { 3245 priv->ets.prio_tc[prio] = prio; 3246 priv->ets.tc_tsa[prio] = IEEE_8021QAZ_TSA_VENDOR; 3247 } 3248 3249 priv->dcbx_cap = DCB_CAP_DCBX_VER_CEE | DCB_CAP_DCBX_HOST | 3250 DCB_CAP_DCBX_VER_IEEE; 3251 priv->flags |= MLX4_EN_DCB_ENABLED; 3252 priv->cee_config.pfc_state = false; 3253 3254 for (i = 0; i < MLX4_EN_NUM_UP_HIGH; i++) 3255 priv->cee_config.dcb_pfc[i] = pfc_disabled; 3256 3257 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) { 3258 dev->dcbnl_ops = &mlx4_en_dcbnl_ops; 3259 } else { 3260 en_info(priv, "enabling only PFC DCB ops\n"); 3261 dev->dcbnl_ops = &mlx4_en_dcbnl_pfc_ops; 3262 } 3263 } 3264 #endif 3265 3266 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) 3267 INIT_HLIST_HEAD(&priv->mac_hash[i]); 3268 3269 /* Query for default mac and max mtu */ 3270 priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; 3271 3272 if (mdev->dev->caps.rx_checksum_flags_port[priv->port] & 3273 MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP) 3274 priv->flags |= MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP; 3275 3276 /* Set default MAC */ 3277 dev->addr_len = ETH_ALEN; 3278 mlx4_en_u64_to_mac(dev, mdev->dev->caps.def_mac[priv->port]); 3279 if (!is_valid_ether_addr(dev->dev_addr)) { 3280 en_err(priv, "Port: %d, invalid mac burned: %pM, quitting\n", 3281 priv->port, dev->dev_addr); 3282 err = -EINVAL; 3283 goto out; 3284 } else if (mlx4_is_slave(priv->mdev->dev) && 3285 (priv->mdev->dev->port_random_macs & 1 << priv->port)) { 3286 /* Random MAC was assigned in mlx4_slave_cap 3287 * in mlx4_core module 3288 */ 3289 dev->addr_assign_type |= NET_ADDR_RANDOM; 3290 en_warn(priv, "Assigned random MAC address %pM\n", dev->dev_addr); 3291 } 3292 3293 memcpy(priv->current_mac, dev->dev_addr, sizeof(priv->current_mac)); 3294 3295 priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + 3296 DS_SIZE * MLX4_EN_MAX_RX_FRAGS); 3297 err = mlx4_en_alloc_resources(priv); 3298 if (err) 3299 goto out; 3300 3301 /* Initialize time stamping config */ 3302 priv->hwtstamp_config.flags = 0; 3303 priv->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; 3304 priv->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; 3305 3306 /* Allocate page for receive rings */ 3307 err = mlx4_alloc_hwq_res(mdev->dev, &priv->res, 3308 MLX4_EN_PAGE_SIZE); 3309 if (err) { 3310 en_err(priv, "Failed to allocate page for rx qps\n"); 3311 goto out; 3312 } 3313 priv->allocated = 1; 3314 3315 /* 3316 * Initialize netdev entry points 3317 */ 3318 if (mlx4_is_master(priv->mdev->dev)) 3319 dev->netdev_ops = &mlx4_netdev_ops_master; 3320 else 3321 dev->netdev_ops = &mlx4_netdev_ops; 3322 dev->xdp_metadata_ops = &mlx4_xdp_metadata_ops; 3323 dev->watchdog_timeo = MLX4_EN_WATCHDOG_TIMEOUT; 3324 netif_set_real_num_tx_queues(dev, priv->tx_ring_num[TX]); 3325 netif_set_real_num_rx_queues(dev, priv->rx_ring_num); 3326 3327 dev->stat_ops = &mlx4_stat_ops; 3328 dev->ethtool_ops = &mlx4_en_ethtool_ops; 3329 3330 /* 3331 * Set driver features 3332 */ 3333 dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 3334 if (mdev->LSO_support) 3335 dev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 3336 3337 if (mdev->dev->caps.tunnel_offload_mode == 3338 MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { 3339 dev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3340 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3341 NETIF_F_GSO_PARTIAL; 3342 dev->features |= NETIF_F_GSO_UDP_TUNNEL | 3343 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3344 NETIF_F_GSO_PARTIAL; 3345 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 3346 dev->hw_enc_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 3347 NETIF_F_RXCSUM | 3348 NETIF_F_TSO | NETIF_F_TSO6 | 3349 NETIF_F_GSO_UDP_TUNNEL | 3350 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3351 NETIF_F_GSO_PARTIAL; 3352 3353 dev->udp_tunnel_nic_info = &mlx4_udp_tunnels; 3354 } 3355 3356 dev->vlan_features = dev->hw_features; 3357 3358 dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_RXHASH; 3359 dev->features = dev->hw_features | NETIF_F_HIGHDMA | 3360 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 3361 NETIF_F_HW_VLAN_CTAG_FILTER; 3362 dev->hw_features |= NETIF_F_LOOPBACK | 3363 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 3364 3365 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) { 3366 dev->features |= NETIF_F_HW_VLAN_STAG_RX | 3367 NETIF_F_HW_VLAN_STAG_FILTER; 3368 dev->hw_features |= NETIF_F_HW_VLAN_STAG_RX; 3369 } 3370 3371 if (mlx4_is_slave(mdev->dev)) { 3372 bool vlan_offload_disabled; 3373 int phv; 3374 3375 err = get_phv_bit(mdev->dev, port, &phv); 3376 if (!err && phv) { 3377 dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 3378 priv->pflags |= MLX4_EN_PRIV_FLAGS_PHV; 3379 } 3380 err = mlx4_get_is_vlan_offload_disabled(mdev->dev, port, 3381 &vlan_offload_disabled); 3382 if (!err && vlan_offload_disabled) { 3383 dev->hw_features &= ~(NETIF_F_HW_VLAN_CTAG_TX | 3384 NETIF_F_HW_VLAN_CTAG_RX | 3385 NETIF_F_HW_VLAN_STAG_TX | 3386 NETIF_F_HW_VLAN_STAG_RX); 3387 dev->features &= ~(NETIF_F_HW_VLAN_CTAG_TX | 3388 NETIF_F_HW_VLAN_CTAG_RX | 3389 NETIF_F_HW_VLAN_STAG_TX | 3390 NETIF_F_HW_VLAN_STAG_RX); 3391 } 3392 } else { 3393 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN && 3394 !(mdev->dev->caps.flags2 & 3395 MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) 3396 dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 3397 } 3398 3399 if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) 3400 dev->hw_features |= NETIF_F_RXFCS; 3401 3402 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS) 3403 dev->hw_features |= NETIF_F_RXALL; 3404 3405 if (mdev->dev->caps.steering_mode == 3406 MLX4_STEERING_MODE_DEVICE_MANAGED && 3407 mdev->dev->caps.dmfs_high_steer_mode != MLX4_STEERING_DMFS_A0_STATIC) 3408 dev->hw_features |= NETIF_F_NTUPLE; 3409 3410 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0) 3411 dev->priv_flags |= IFF_UNICAST_FLT; 3412 3413 /* Setting a default hash function value */ 3414 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP) { 3415 priv->rss_hash_fn = ETH_RSS_HASH_TOP; 3416 } else if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_XOR) { 3417 priv->rss_hash_fn = ETH_RSS_HASH_XOR; 3418 } else { 3419 en_warn(priv, 3420 "No RSS hash capabilities exposed, using Toeplitz\n"); 3421 priv->rss_hash_fn = ETH_RSS_HASH_TOP; 3422 } 3423 3424 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT; 3425 3426 /* MTU range: 68 - hw-specific max */ 3427 dev->min_mtu = ETH_MIN_MTU; 3428 dev->max_mtu = priv->max_mtu; 3429 3430 /* supports LSOv2 packets. */ 3431 netif_set_tso_max_size(dev, GSO_MAX_SIZE); 3432 3433 mdev->pndev[port] = dev; 3434 mdev->upper[port] = NULL; 3435 3436 netif_carrier_off(dev); 3437 mlx4_en_set_default_moderation(priv); 3438 3439 en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num[TX]); 3440 en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); 3441 3442 mlx4_en_update_loopback_state(priv->dev, priv->dev->features); 3443 3444 /* Configure port */ 3445 mlx4_en_calc_rx_buf(dev); 3446 err = mlx4_SET_PORT_general(mdev->dev, priv->port, 3447 priv->rx_skb_size + ETH_FCS_LEN, 3448 prof->tx_pause, prof->tx_ppp, 3449 prof->rx_pause, prof->rx_ppp); 3450 if (err) { 3451 en_err(priv, "Failed setting port general configurations for port %d, with error %d\n", 3452 priv->port, err); 3453 goto out; 3454 } 3455 3456 if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { 3457 err = mlx4_SET_PORT_VXLAN(mdev->dev, priv->port, VXLAN_STEER_BY_OUTER_MAC, 1); 3458 if (err) { 3459 en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n", 3460 err); 3461 goto out; 3462 } 3463 } 3464 3465 /* Init port */ 3466 en_warn(priv, "Initializing port\n"); 3467 err = mlx4_INIT_PORT(mdev->dev, priv->port); 3468 if (err) { 3469 en_err(priv, "Failed Initializing port\n"); 3470 goto out; 3471 } 3472 queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); 3473 3474 /* Initialize time stamp mechanism */ 3475 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) 3476 mlx4_en_init_timestamp(mdev); 3477 3478 queue_delayed_work(mdev->workqueue, &priv->service_task, 3479 SERVICE_TASK_DELAY); 3480 3481 mlx4_en_set_stats_bitmap(mdev->dev, &priv->stats_bitmap, 3482 mdev->profile.prof[priv->port].rx_ppp, 3483 mdev->profile.prof[priv->port].rx_pause, 3484 mdev->profile.prof[priv->port].tx_ppp, 3485 mdev->profile.prof[priv->port].tx_pause); 3486 3487 SET_NETDEV_DEVLINK_PORT(dev, 3488 mlx4_get_devlink_port(mdev->dev, priv->port)); 3489 err = register_netdev(dev); 3490 if (err) { 3491 en_err(priv, "Netdev registration failed for port %d\n", port); 3492 goto out; 3493 } 3494 3495 priv->registered = 1; 3496 3497 return 0; 3498 3499 out: 3500 mlx4_en_destroy_netdev(dev); 3501 return err; 3502 } 3503 3504 int mlx4_en_reset_config(struct net_device *dev, 3505 struct kernel_hwtstamp_config *ts_config, 3506 netdev_features_t features) 3507 { 3508 struct mlx4_en_priv *priv = netdev_priv(dev); 3509 struct mlx4_en_dev *mdev = priv->mdev; 3510 struct mlx4_en_port_profile new_prof; 3511 struct mlx4_en_priv *tmp; 3512 int port_up = 0; 3513 int err = 0; 3514 3515 if (priv->hwtstamp_config.tx_type == ts_config->tx_type && 3516 priv->hwtstamp_config.rx_filter == ts_config->rx_filter && 3517 !DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) && 3518 !DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS)) 3519 return 0; /* Nothing to change */ 3520 3521 if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) && 3522 (features & NETIF_F_HW_VLAN_CTAG_RX) && 3523 (priv->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE)) { 3524 en_warn(priv, "Can't turn ON rx vlan offload while time-stamping rx filter is ON\n"); 3525 return -EINVAL; 3526 } 3527 3528 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); 3529 if (!tmp) 3530 return -ENOMEM; 3531 3532 mutex_lock(&mdev->state_lock); 3533 3534 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); 3535 memcpy(&new_prof.hwtstamp_config, ts_config, sizeof(*ts_config)); 3536 3537 err = mlx4_en_try_alloc_resources(priv, tmp, &new_prof, true); 3538 if (err) 3539 goto out; 3540 3541 if (priv->port_up) { 3542 port_up = 1; 3543 mlx4_en_stop_port(dev, 1); 3544 } 3545 3546 mlx4_en_safe_replace_resources(priv, tmp); 3547 3548 if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX)) { 3549 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3550 dev->features |= NETIF_F_HW_VLAN_CTAG_RX; 3551 else 3552 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3553 } else if (ts_config->rx_filter == HWTSTAMP_FILTER_NONE) { 3554 /* RX time-stamping is OFF, update the RX vlan offload 3555 * to the latest wanted state 3556 */ 3557 if (dev->wanted_features & NETIF_F_HW_VLAN_CTAG_RX) 3558 dev->features |= NETIF_F_HW_VLAN_CTAG_RX; 3559 else 3560 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3561 } 3562 3563 if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS)) { 3564 if (features & NETIF_F_RXFCS) 3565 dev->features |= NETIF_F_RXFCS; 3566 else 3567 dev->features &= ~NETIF_F_RXFCS; 3568 } 3569 3570 /* RX vlan offload and RX time-stamping can't co-exist ! 3571 * Regardless of the caller's choice, 3572 * Turn Off RX vlan offload in case of time-stamping is ON 3573 */ 3574 if (ts_config->rx_filter != HWTSTAMP_FILTER_NONE) { 3575 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 3576 en_warn(priv, "Turning off RX vlan offload since RX time-stamping is ON\n"); 3577 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3578 } 3579 3580 if (port_up) { 3581 err = mlx4_en_start_port(dev); 3582 if (err) 3583 en_err(priv, "Failed starting port\n"); 3584 } 3585 3586 if (!err) 3587 err = mlx4_en_moderation_update(priv); 3588 out: 3589 mutex_unlock(&mdev->state_lock); 3590 kfree(tmp); 3591 if (!err) 3592 netdev_features_change(dev); 3593 return err; 3594 } 3595