1 /* 2 * Copyright 2008-2010 Cisco Systems, Inc. All rights reserved. 3 * Copyright 2007 Nuova Systems, Inc. All rights reserved. 4 * 5 * This program is free software; you may redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; version 2 of the License. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 10 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 11 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 12 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 13 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 14 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 15 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 16 * SOFTWARE. 17 * 18 */ 19 20 #include <linux/module.h> 21 #include <linux/kernel.h> 22 #include <linux/string.h> 23 #include <linux/errno.h> 24 #include <linux/types.h> 25 #include <linux/init.h> 26 #include <linux/interrupt.h> 27 #include <linux/workqueue.h> 28 #include <linux/pci.h> 29 #include <linux/netdevice.h> 30 #include <linux/etherdevice.h> 31 #include <linux/if.h> 32 #include <linux/if_ether.h> 33 #include <linux/if_vlan.h> 34 #include <linux/in.h> 35 #include <linux/ip.h> 36 #include <linux/ipv6.h> 37 #include <linux/tcp.h> 38 #include <linux/rtnetlink.h> 39 #include <linux/prefetch.h> 40 #include <net/ip6_checksum.h> 41 #include <linux/ktime.h> 42 #include <linux/numa.h> 43 #ifdef CONFIG_RFS_ACCEL 44 #include <linux/cpu_rmap.h> 45 #endif 46 #include <linux/crash_dump.h> 47 #include <net/busy_poll.h> 48 #include <net/vxlan.h> 49 #include <net/netdev_queues.h> 50 51 #include "cq_enet_desc.h" 52 #include "vnic_dev.h" 53 #include "vnic_intr.h" 54 #include "vnic_stats.h" 55 #include "vnic_vic.h" 56 #include "enic_res.h" 57 #include "enic.h" 58 #include "enic_dev.h" 59 #include "enic_pp.h" 60 #include "enic_clsf.h" 61 62 #define ENIC_NOTIFY_TIMER_PERIOD (2 * HZ) 63 #define WQ_ENET_MAX_DESC_LEN (1 << WQ_ENET_LEN_BITS) 64 #define MAX_TSO (1 << 16) 65 #define ENIC_DESC_MAX_SPLITS (MAX_TSO / WQ_ENET_MAX_DESC_LEN + 1) 66 67 #define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */ 68 #define PCI_DEVICE_ID_CISCO_VIC_ENET_DYN 0x0044 /* enet dynamic vnic */ 69 #define PCI_DEVICE_ID_CISCO_VIC_ENET_VF 0x0071 /* enet SRIOV VF */ 70 71 #define RX_COPYBREAK_DEFAULT 256 72 73 /* Supported devices */ 74 static const struct pci_device_id enic_id_table[] = { 75 { PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET) }, 76 { PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_DYN) }, 77 { PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_VF) }, 78 { 0, } /* end of table */ 79 }; 80 81 MODULE_DESCRIPTION(DRV_DESCRIPTION); 82 MODULE_AUTHOR("Scott Feldman <scofeldm@cisco.com>"); 83 MODULE_LICENSE("GPL"); 84 MODULE_DEVICE_TABLE(pci, enic_id_table); 85 86 #define ENIC_LARGE_PKT_THRESHOLD 1000 87 #define ENIC_MAX_COALESCE_TIMERS 10 88 /* Interrupt moderation table, which will be used to decide the 89 * coalescing timer values 90 * {rx_rate in Mbps, mapping percentage of the range} 91 */ 92 static struct enic_intr_mod_table mod_table[ENIC_MAX_COALESCE_TIMERS + 1] = { 93 {4000, 0}, 94 {4400, 10}, 95 {5060, 20}, 96 {5230, 30}, 97 {5540, 40}, 98 {5820, 50}, 99 {6120, 60}, 100 {6435, 70}, 101 {6745, 80}, 102 {7000, 90}, 103 {0xFFFFFFFF, 100} 104 }; 105 106 /* This table helps the driver to pick different ranges for rx coalescing 107 * timer depending on the link speed. 108 */ 109 static struct enic_intr_mod_range mod_range[ENIC_MAX_LINK_SPEEDS] = { 110 {0, 0}, /* 0 - 4 Gbps */ 111 {0, 3}, /* 4 - 10 Gbps */ 112 {3, 6}, /* 10+ Gbps */ 113 }; 114 115 static void enic_init_affinity_hint(struct enic *enic) 116 { 117 int numa_node = dev_to_node(&enic->pdev->dev); 118 int i; 119 120 for (i = 0; i < enic->intr_count; i++) { 121 if (enic_is_err_intr(enic, i) || enic_is_notify_intr(enic, i) || 122 (cpumask_available(enic->msix[i].affinity_mask) && 123 !cpumask_empty(enic->msix[i].affinity_mask))) 124 continue; 125 if (zalloc_cpumask_var(&enic->msix[i].affinity_mask, 126 GFP_KERNEL)) 127 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 128 enic->msix[i].affinity_mask); 129 } 130 } 131 132 static void enic_free_affinity_hint(struct enic *enic) 133 { 134 int i; 135 136 for (i = 0; i < enic->intr_count; i++) { 137 if (enic_is_err_intr(enic, i) || enic_is_notify_intr(enic, i)) 138 continue; 139 free_cpumask_var(enic->msix[i].affinity_mask); 140 } 141 } 142 143 static void enic_set_affinity_hint(struct enic *enic) 144 { 145 int i; 146 int err; 147 148 for (i = 0; i < enic->intr_count; i++) { 149 if (enic_is_err_intr(enic, i) || 150 enic_is_notify_intr(enic, i) || 151 !cpumask_available(enic->msix[i].affinity_mask) || 152 cpumask_empty(enic->msix[i].affinity_mask)) 153 continue; 154 err = irq_update_affinity_hint(enic->msix_entry[i].vector, 155 enic->msix[i].affinity_mask); 156 if (err) 157 netdev_warn(enic->netdev, "irq_update_affinity_hint failed, err %d\n", 158 err); 159 } 160 161 for (i = 0; i < enic->wq_count; i++) { 162 int wq_intr = enic_msix_wq_intr(enic, i); 163 164 if (cpumask_available(enic->msix[wq_intr].affinity_mask) && 165 !cpumask_empty(enic->msix[wq_intr].affinity_mask)) 166 netif_set_xps_queue(enic->netdev, 167 enic->msix[wq_intr].affinity_mask, 168 i); 169 } 170 } 171 172 static void enic_unset_affinity_hint(struct enic *enic) 173 { 174 int i; 175 176 for (i = 0; i < enic->intr_count; i++) 177 irq_update_affinity_hint(enic->msix_entry[i].vector, NULL); 178 } 179 180 static int enic_udp_tunnel_set_port(struct net_device *netdev, 181 unsigned int table, unsigned int entry, 182 struct udp_tunnel_info *ti) 183 { 184 struct enic *enic = netdev_priv(netdev); 185 int err; 186 187 spin_lock_bh(&enic->devcmd_lock); 188 189 err = vnic_dev_overlay_offload_cfg(enic->vdev, 190 OVERLAY_CFG_VXLAN_PORT_UPDATE, 191 ntohs(ti->port)); 192 if (err) 193 goto error; 194 195 err = vnic_dev_overlay_offload_ctrl(enic->vdev, OVERLAY_FEATURE_VXLAN, 196 enic->vxlan.patch_level); 197 if (err) 198 goto error; 199 200 enic->vxlan.vxlan_udp_port_number = ntohs(ti->port); 201 error: 202 spin_unlock_bh(&enic->devcmd_lock); 203 204 return err; 205 } 206 207 static int enic_udp_tunnel_unset_port(struct net_device *netdev, 208 unsigned int table, unsigned int entry, 209 struct udp_tunnel_info *ti) 210 { 211 struct enic *enic = netdev_priv(netdev); 212 int err; 213 214 spin_lock_bh(&enic->devcmd_lock); 215 216 err = vnic_dev_overlay_offload_ctrl(enic->vdev, OVERLAY_FEATURE_VXLAN, 217 OVERLAY_OFFLOAD_DISABLE); 218 if (err) 219 goto unlock; 220 221 enic->vxlan.vxlan_udp_port_number = 0; 222 223 unlock: 224 spin_unlock_bh(&enic->devcmd_lock); 225 226 return err; 227 } 228 229 static const struct udp_tunnel_nic_info enic_udp_tunnels = { 230 .set_port = enic_udp_tunnel_set_port, 231 .unset_port = enic_udp_tunnel_unset_port, 232 .tables = { 233 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 234 }, 235 }, enic_udp_tunnels_v4 = { 236 .set_port = enic_udp_tunnel_set_port, 237 .unset_port = enic_udp_tunnel_unset_port, 238 .flags = UDP_TUNNEL_NIC_INFO_IPV4_ONLY, 239 .tables = { 240 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 241 }, 242 }; 243 244 static netdev_features_t enic_features_check(struct sk_buff *skb, 245 struct net_device *dev, 246 netdev_features_t features) 247 { 248 const struct ethhdr *eth = (struct ethhdr *)skb_inner_mac_header(skb); 249 struct enic *enic = netdev_priv(dev); 250 struct udphdr *udph; 251 u16 port = 0; 252 u8 proto; 253 254 if (!skb->encapsulation) 255 return features; 256 257 features = vxlan_features_check(skb, features); 258 259 switch (vlan_get_protocol(skb)) { 260 case htons(ETH_P_IPV6): 261 if (!(enic->vxlan.flags & ENIC_VXLAN_OUTER_IPV6)) 262 goto out; 263 proto = ipv6_hdr(skb)->nexthdr; 264 break; 265 case htons(ETH_P_IP): 266 proto = ip_hdr(skb)->protocol; 267 break; 268 default: 269 goto out; 270 } 271 272 switch (eth->h_proto) { 273 case ntohs(ETH_P_IPV6): 274 if (!(enic->vxlan.flags & ENIC_VXLAN_INNER_IPV6)) 275 goto out; 276 fallthrough; 277 case ntohs(ETH_P_IP): 278 break; 279 default: 280 goto out; 281 } 282 283 284 if (proto == IPPROTO_UDP) { 285 udph = udp_hdr(skb); 286 port = be16_to_cpu(udph->dest); 287 } 288 289 /* HW supports offload of only one UDP port. Remove CSUM and GSO MASK 290 * for other UDP port tunnels 291 */ 292 if (port != enic->vxlan.vxlan_udp_port_number) 293 goto out; 294 295 return features; 296 297 out: 298 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 299 } 300 301 int enic_is_dynamic(struct enic *enic) 302 { 303 return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_DYN; 304 } 305 306 int enic_sriov_enabled(struct enic *enic) 307 { 308 return (enic->priv_flags & ENIC_SRIOV_ENABLED) ? 1 : 0; 309 } 310 311 static int enic_is_sriov_vf(struct enic *enic) 312 { 313 return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; 314 } 315 316 int enic_is_valid_vf(struct enic *enic, int vf) 317 { 318 #ifdef CONFIG_PCI_IOV 319 return vf >= 0 && vf < enic->num_vfs; 320 #else 321 return 0; 322 #endif 323 } 324 325 static void enic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf) 326 { 327 struct enic *enic = vnic_dev_priv(wq->vdev); 328 329 if (buf->sop) 330 dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len, 331 DMA_TO_DEVICE); 332 else 333 dma_unmap_page(&enic->pdev->dev, buf->dma_addr, buf->len, 334 DMA_TO_DEVICE); 335 336 if (buf->os_buf) 337 dev_kfree_skb_any(buf->os_buf); 338 } 339 340 static void enic_wq_free_buf(struct vnic_wq *wq, 341 struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque) 342 { 343 struct enic *enic = vnic_dev_priv(wq->vdev); 344 345 enic->wq[wq->index].stats.cq_work++; 346 enic->wq[wq->index].stats.cq_bytes += buf->len; 347 enic_free_wq_buf(wq, buf); 348 } 349 350 static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, 351 u8 type, u16 q_number, u16 completed_index, void *opaque) 352 { 353 struct enic *enic = vnic_dev_priv(vdev); 354 355 spin_lock(&enic->wq[q_number].lock); 356 357 vnic_wq_service(&enic->wq[q_number].vwq, cq_desc, 358 completed_index, enic_wq_free_buf, 359 opaque); 360 361 if (netif_tx_queue_stopped(netdev_get_tx_queue(enic->netdev, q_number)) && 362 vnic_wq_desc_avail(&enic->wq[q_number].vwq) >= 363 (MAX_SKB_FRAGS + ENIC_DESC_MAX_SPLITS)) { 364 netif_wake_subqueue(enic->netdev, q_number); 365 enic->wq[q_number].stats.wake++; 366 } 367 368 spin_unlock(&enic->wq[q_number].lock); 369 370 return 0; 371 } 372 373 static bool enic_log_q_error(struct enic *enic) 374 { 375 unsigned int i; 376 u32 error_status; 377 bool err = false; 378 379 for (i = 0; i < enic->wq_count; i++) { 380 error_status = vnic_wq_error_status(&enic->wq[i].vwq); 381 err |= error_status; 382 if (error_status) 383 netdev_err(enic->netdev, "WQ[%d] error_status %d\n", 384 i, error_status); 385 } 386 387 for (i = 0; i < enic->rq_count; i++) { 388 error_status = vnic_rq_error_status(&enic->rq[i].vrq); 389 err |= error_status; 390 if (error_status) 391 netdev_err(enic->netdev, "RQ[%d] error_status %d\n", 392 i, error_status); 393 } 394 395 return err; 396 } 397 398 static void enic_msglvl_check(struct enic *enic) 399 { 400 u32 msg_enable = vnic_dev_msg_lvl(enic->vdev); 401 402 if (msg_enable != enic->msg_enable) { 403 netdev_info(enic->netdev, "msg lvl changed from 0x%x to 0x%x\n", 404 enic->msg_enable, msg_enable); 405 enic->msg_enable = msg_enable; 406 } 407 } 408 409 static void enic_mtu_check(struct enic *enic) 410 { 411 u32 mtu = vnic_dev_mtu(enic->vdev); 412 struct net_device *netdev = enic->netdev; 413 414 if (mtu && mtu != enic->port_mtu) { 415 enic->port_mtu = mtu; 416 if (enic_is_dynamic(enic) || enic_is_sriov_vf(enic)) { 417 mtu = max_t(int, ENIC_MIN_MTU, 418 min_t(int, ENIC_MAX_MTU, mtu)); 419 if (mtu != netdev->mtu) 420 schedule_work(&enic->change_mtu_work); 421 } else { 422 if (mtu < netdev->mtu) 423 netdev_warn(netdev, 424 "interface MTU (%d) set higher " 425 "than switch port MTU (%d)\n", 426 netdev->mtu, mtu); 427 } 428 } 429 } 430 431 static void enic_set_rx_coal_setting(struct enic *enic) 432 { 433 unsigned int speed; 434 int index = -1; 435 struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting; 436 437 /* 1. Read the link speed from fw 438 * 2. Pick the default range for the speed 439 * 3. Update it in enic->rx_coalesce_setting 440 */ 441 speed = vnic_dev_port_speed(enic->vdev); 442 if (speed > ENIC_LINK_SPEED_10G) 443 index = ENIC_LINK_40G_INDEX; 444 else if (speed > ENIC_LINK_SPEED_4G) 445 index = ENIC_LINK_10G_INDEX; 446 else 447 index = ENIC_LINK_4G_INDEX; 448 449 rx_coal->small_pkt_range_start = mod_range[index].small_pkt_range_start; 450 rx_coal->large_pkt_range_start = mod_range[index].large_pkt_range_start; 451 rx_coal->range_end = ENIC_RX_COALESCE_RANGE_END; 452 453 /* Start with the value provided by UCSM */ 454 for (index = 0; index < enic->rq_count; index++) 455 enic->cq[index].cur_rx_coal_timeval = 456 enic->config.intr_timer_usec; 457 458 rx_coal->use_adaptive_rx_coalesce = 1; 459 } 460 461 static void enic_link_check(struct enic *enic) 462 { 463 int link_status = vnic_dev_link_status(enic->vdev); 464 int carrier_ok = netif_carrier_ok(enic->netdev); 465 466 if (link_status && !carrier_ok) { 467 netdev_info(enic->netdev, "Link UP\n"); 468 netif_carrier_on(enic->netdev); 469 enic_set_rx_coal_setting(enic); 470 } else if (!link_status && carrier_ok) { 471 netdev_info(enic->netdev, "Link DOWN\n"); 472 netif_carrier_off(enic->netdev); 473 } 474 } 475 476 static void enic_notify_check(struct enic *enic) 477 { 478 enic_msglvl_check(enic); 479 enic_mtu_check(enic); 480 enic_link_check(enic); 481 } 482 483 #define ENIC_TEST_INTR(pba, i) (pba & (1 << i)) 484 485 static irqreturn_t enic_isr_legacy(int irq, void *data) 486 { 487 struct net_device *netdev = data; 488 struct enic *enic = netdev_priv(netdev); 489 unsigned int io_intr = ENIC_LEGACY_IO_INTR; 490 unsigned int err_intr = ENIC_LEGACY_ERR_INTR; 491 unsigned int notify_intr = ENIC_LEGACY_NOTIFY_INTR; 492 u32 pba; 493 494 vnic_intr_mask(&enic->intr[io_intr]); 495 496 pba = vnic_intr_legacy_pba(enic->legacy_pba); 497 if (!pba) { 498 vnic_intr_unmask(&enic->intr[io_intr]); 499 return IRQ_NONE; /* not our interrupt */ 500 } 501 502 if (ENIC_TEST_INTR(pba, notify_intr)) { 503 enic_notify_check(enic); 504 vnic_intr_return_all_credits(&enic->intr[notify_intr]); 505 } 506 507 if (ENIC_TEST_INTR(pba, err_intr)) { 508 vnic_intr_return_all_credits(&enic->intr[err_intr]); 509 enic_log_q_error(enic); 510 /* schedule recovery from WQ/RQ error */ 511 schedule_work(&enic->reset); 512 return IRQ_HANDLED; 513 } 514 515 if (ENIC_TEST_INTR(pba, io_intr)) 516 napi_schedule_irqoff(&enic->napi[0]); 517 else 518 vnic_intr_unmask(&enic->intr[io_intr]); 519 520 return IRQ_HANDLED; 521 } 522 523 static irqreturn_t enic_isr_msi(int irq, void *data) 524 { 525 struct enic *enic = data; 526 527 /* With MSI, there is no sharing of interrupts, so this is 528 * our interrupt and there is no need to ack it. The device 529 * is not providing per-vector masking, so the OS will not 530 * write to PCI config space to mask/unmask the interrupt. 531 * We're using mask_on_assertion for MSI, so the device 532 * automatically masks the interrupt when the interrupt is 533 * generated. Later, when exiting polling, the interrupt 534 * will be unmasked (see enic_poll). 535 * 536 * Also, the device uses the same PCIe Traffic Class (TC) 537 * for Memory Write data and MSI, so there are no ordering 538 * issues; the MSI will always arrive at the Root Complex 539 * _after_ corresponding Memory Writes (i.e. descriptor 540 * writes). 541 */ 542 543 napi_schedule_irqoff(&enic->napi[0]); 544 545 return IRQ_HANDLED; 546 } 547 548 static irqreturn_t enic_isr_msix(int irq, void *data) 549 { 550 struct napi_struct *napi = data; 551 552 napi_schedule_irqoff(napi); 553 554 return IRQ_HANDLED; 555 } 556 557 static irqreturn_t enic_isr_msix_err(int irq, void *data) 558 { 559 struct enic *enic = data; 560 unsigned int intr = enic_msix_err_intr(enic); 561 562 vnic_intr_return_all_credits(&enic->intr[intr]); 563 564 if (enic_log_q_error(enic)) 565 /* schedule recovery from WQ/RQ error */ 566 schedule_work(&enic->reset); 567 568 return IRQ_HANDLED; 569 } 570 571 static irqreturn_t enic_isr_msix_notify(int irq, void *data) 572 { 573 struct enic *enic = data; 574 unsigned int intr = enic_msix_notify_intr(enic); 575 576 enic_notify_check(enic); 577 vnic_intr_return_all_credits(&enic->intr[intr]); 578 579 return IRQ_HANDLED; 580 } 581 582 static int enic_queue_wq_skb_cont(struct enic *enic, struct vnic_wq *wq, 583 struct sk_buff *skb, unsigned int len_left, 584 int loopback) 585 { 586 const skb_frag_t *frag; 587 dma_addr_t dma_addr; 588 589 /* Queue additional data fragments */ 590 for (frag = skb_shinfo(skb)->frags; len_left; frag++) { 591 len_left -= skb_frag_size(frag); 592 dma_addr = skb_frag_dma_map(&enic->pdev->dev, frag, 0, 593 skb_frag_size(frag), 594 DMA_TO_DEVICE); 595 if (unlikely(enic_dma_map_check(enic, dma_addr))) 596 return -ENOMEM; 597 enic_queue_wq_desc_cont(wq, skb, dma_addr, skb_frag_size(frag), 598 (len_left == 0), /* EOP? */ 599 loopback); 600 } 601 602 return 0; 603 } 604 605 static int enic_queue_wq_skb_vlan(struct enic *enic, struct vnic_wq *wq, 606 struct sk_buff *skb, int vlan_tag_insert, 607 unsigned int vlan_tag, int loopback) 608 { 609 unsigned int head_len = skb_headlen(skb); 610 unsigned int len_left = skb->len - head_len; 611 int eop = (len_left == 0); 612 dma_addr_t dma_addr; 613 int err = 0; 614 615 dma_addr = dma_map_single(&enic->pdev->dev, skb->data, head_len, 616 DMA_TO_DEVICE); 617 if (unlikely(enic_dma_map_check(enic, dma_addr))) 618 return -ENOMEM; 619 620 /* Queue the main skb fragment. The fragments are no larger 621 * than max MTU(9000)+ETH_HDR_LEN(14) bytes, which is less 622 * than WQ_ENET_MAX_DESC_LEN length. So only one descriptor 623 * per fragment is queued. 624 */ 625 enic_queue_wq_desc(wq, skb, dma_addr, head_len, vlan_tag_insert, 626 vlan_tag, eop, loopback); 627 628 if (!eop) 629 err = enic_queue_wq_skb_cont(enic, wq, skb, len_left, loopback); 630 631 /* The enic_queue_wq_desc() above does not do HW checksum */ 632 enic->wq[wq->index].stats.csum_none++; 633 enic->wq[wq->index].stats.packets++; 634 enic->wq[wq->index].stats.bytes += skb->len; 635 636 return err; 637 } 638 639 static int enic_queue_wq_skb_csum_l4(struct enic *enic, struct vnic_wq *wq, 640 struct sk_buff *skb, int vlan_tag_insert, 641 unsigned int vlan_tag, int loopback) 642 { 643 unsigned int head_len = skb_headlen(skb); 644 unsigned int len_left = skb->len - head_len; 645 unsigned int hdr_len = skb_checksum_start_offset(skb); 646 unsigned int csum_offset = hdr_len + skb->csum_offset; 647 int eop = (len_left == 0); 648 dma_addr_t dma_addr; 649 int err = 0; 650 651 dma_addr = dma_map_single(&enic->pdev->dev, skb->data, head_len, 652 DMA_TO_DEVICE); 653 if (unlikely(enic_dma_map_check(enic, dma_addr))) 654 return -ENOMEM; 655 656 /* Queue the main skb fragment. The fragments are no larger 657 * than max MTU(9000)+ETH_HDR_LEN(14) bytes, which is less 658 * than WQ_ENET_MAX_DESC_LEN length. So only one descriptor 659 * per fragment is queued. 660 */ 661 enic_queue_wq_desc_csum_l4(wq, skb, dma_addr, head_len, csum_offset, 662 hdr_len, vlan_tag_insert, vlan_tag, eop, 663 loopback); 664 665 if (!eop) 666 err = enic_queue_wq_skb_cont(enic, wq, skb, len_left, loopback); 667 668 enic->wq[wq->index].stats.csum_partial++; 669 enic->wq[wq->index].stats.packets++; 670 enic->wq[wq->index].stats.bytes += skb->len; 671 672 return err; 673 } 674 675 static void enic_preload_tcp_csum_encap(struct sk_buff *skb) 676 { 677 const struct ethhdr *eth = (struct ethhdr *)skb_inner_mac_header(skb); 678 679 switch (eth->h_proto) { 680 case ntohs(ETH_P_IP): 681 inner_ip_hdr(skb)->check = 0; 682 inner_tcp_hdr(skb)->check = 683 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr, 684 inner_ip_hdr(skb)->daddr, 0, 685 IPPROTO_TCP, 0); 686 break; 687 case ntohs(ETH_P_IPV6): 688 inner_tcp_hdr(skb)->check = 689 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr, 690 &inner_ipv6_hdr(skb)->daddr, 0, 691 IPPROTO_TCP, 0); 692 break; 693 default: 694 WARN_ONCE(1, "Non ipv4/ipv6 inner pkt for encap offload"); 695 break; 696 } 697 } 698 699 static void enic_preload_tcp_csum(struct sk_buff *skb) 700 { 701 /* Preload TCP csum field with IP pseudo hdr calculated 702 * with IP length set to zero. HW will later add in length 703 * to each TCP segment resulting from the TSO. 704 */ 705 706 if (skb->protocol == cpu_to_be16(ETH_P_IP)) { 707 ip_hdr(skb)->check = 0; 708 tcp_hdr(skb)->check = ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 709 ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); 710 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) { 711 tcp_v6_gso_csum_prep(skb); 712 } 713 } 714 715 static int enic_queue_wq_skb_tso(struct enic *enic, struct vnic_wq *wq, 716 struct sk_buff *skb, unsigned int mss, 717 int vlan_tag_insert, unsigned int vlan_tag, 718 int loopback) 719 { 720 unsigned int frag_len_left = skb_headlen(skb); 721 unsigned int len_left = skb->len - frag_len_left; 722 int eop = (len_left == 0); 723 unsigned int offset = 0; 724 unsigned int hdr_len; 725 dma_addr_t dma_addr; 726 unsigned int pkts; 727 unsigned int len; 728 skb_frag_t *frag; 729 730 if (skb->encapsulation) { 731 hdr_len = skb_inner_tcp_all_headers(skb); 732 enic_preload_tcp_csum_encap(skb); 733 enic->wq[wq->index].stats.encap_tso++; 734 } else { 735 hdr_len = skb_tcp_all_headers(skb); 736 enic_preload_tcp_csum(skb); 737 enic->wq[wq->index].stats.tso++; 738 } 739 740 /* Queue WQ_ENET_MAX_DESC_LEN length descriptors 741 * for the main skb fragment 742 */ 743 while (frag_len_left) { 744 len = min(frag_len_left, (unsigned int)WQ_ENET_MAX_DESC_LEN); 745 dma_addr = dma_map_single(&enic->pdev->dev, 746 skb->data + offset, len, 747 DMA_TO_DEVICE); 748 if (unlikely(enic_dma_map_check(enic, dma_addr))) 749 return -ENOMEM; 750 enic_queue_wq_desc_tso(wq, skb, dma_addr, len, mss, hdr_len, 751 vlan_tag_insert, vlan_tag, 752 eop && (len == frag_len_left), loopback); 753 frag_len_left -= len; 754 offset += len; 755 } 756 757 if (eop) 758 goto tso_out_stats; 759 760 /* Queue WQ_ENET_MAX_DESC_LEN length descriptors 761 * for additional data fragments 762 */ 763 for (frag = skb_shinfo(skb)->frags; len_left; frag++) { 764 len_left -= skb_frag_size(frag); 765 frag_len_left = skb_frag_size(frag); 766 offset = 0; 767 768 while (frag_len_left) { 769 len = min(frag_len_left, 770 (unsigned int)WQ_ENET_MAX_DESC_LEN); 771 dma_addr = skb_frag_dma_map(&enic->pdev->dev, frag, 772 offset, len, 773 DMA_TO_DEVICE); 774 if (unlikely(enic_dma_map_check(enic, dma_addr))) 775 return -ENOMEM; 776 enic_queue_wq_desc_cont(wq, skb, dma_addr, len, 777 (len_left == 0) && 778 (len == frag_len_left),/*EOP*/ 779 loopback); 780 frag_len_left -= len; 781 offset += len; 782 } 783 } 784 785 tso_out_stats: 786 /* calculate how many packets tso sent */ 787 len = skb->len - hdr_len; 788 pkts = len / mss; 789 if ((len % mss) > 0) 790 pkts++; 791 enic->wq[wq->index].stats.packets += pkts; 792 enic->wq[wq->index].stats.bytes += (len + (pkts * hdr_len)); 793 794 return 0; 795 } 796 797 static inline int enic_queue_wq_skb_encap(struct enic *enic, struct vnic_wq *wq, 798 struct sk_buff *skb, 799 int vlan_tag_insert, 800 unsigned int vlan_tag, int loopback) 801 { 802 unsigned int head_len = skb_headlen(skb); 803 unsigned int len_left = skb->len - head_len; 804 /* Hardware will overwrite the checksum fields, calculating from 805 * scratch and ignoring the value placed by software. 806 * Offload mode = 00 807 * mss[2], mss[1], mss[0] bits are set 808 */ 809 unsigned int mss_or_csum = 7; 810 int eop = (len_left == 0); 811 dma_addr_t dma_addr; 812 int err = 0; 813 814 dma_addr = dma_map_single(&enic->pdev->dev, skb->data, head_len, 815 DMA_TO_DEVICE); 816 if (unlikely(enic_dma_map_check(enic, dma_addr))) 817 return -ENOMEM; 818 819 enic_queue_wq_desc_ex(wq, skb, dma_addr, head_len, mss_or_csum, 0, 820 vlan_tag_insert, vlan_tag, 821 WQ_ENET_OFFLOAD_MODE_CSUM, eop, 1 /* SOP */, eop, 822 loopback); 823 if (!eop) 824 err = enic_queue_wq_skb_cont(enic, wq, skb, len_left, loopback); 825 826 enic->wq[wq->index].stats.encap_csum++; 827 enic->wq[wq->index].stats.packets++; 828 enic->wq[wq->index].stats.bytes += skb->len; 829 830 return err; 831 } 832 833 static inline int enic_queue_wq_skb(struct enic *enic, 834 struct vnic_wq *wq, struct sk_buff *skb) 835 { 836 unsigned int mss = skb_shinfo(skb)->gso_size; 837 unsigned int vlan_tag = 0; 838 int vlan_tag_insert = 0; 839 int loopback = 0; 840 int err; 841 842 if (skb_vlan_tag_present(skb)) { 843 /* VLAN tag from trunking driver */ 844 vlan_tag_insert = 1; 845 vlan_tag = skb_vlan_tag_get(skb); 846 enic->wq[wq->index].stats.add_vlan++; 847 } else if (enic->loop_enable) { 848 vlan_tag = enic->loop_tag; 849 loopback = 1; 850 } 851 852 if (mss) 853 err = enic_queue_wq_skb_tso(enic, wq, skb, mss, 854 vlan_tag_insert, vlan_tag, 855 loopback); 856 else if (skb->encapsulation) 857 err = enic_queue_wq_skb_encap(enic, wq, skb, vlan_tag_insert, 858 vlan_tag, loopback); 859 else if (skb->ip_summed == CHECKSUM_PARTIAL) 860 err = enic_queue_wq_skb_csum_l4(enic, wq, skb, vlan_tag_insert, 861 vlan_tag, loopback); 862 else 863 err = enic_queue_wq_skb_vlan(enic, wq, skb, vlan_tag_insert, 864 vlan_tag, loopback); 865 if (unlikely(err)) { 866 struct vnic_wq_buf *buf; 867 868 buf = wq->to_use->prev; 869 /* while not EOP of previous pkt && queue not empty. 870 * For all non EOP bufs, os_buf is NULL. 871 */ 872 while (!buf->os_buf && (buf->next != wq->to_clean)) { 873 enic_free_wq_buf(wq, buf); 874 wq->ring.desc_avail++; 875 buf = buf->prev; 876 } 877 wq->to_use = buf->next; 878 dev_kfree_skb(skb); 879 } 880 return err; 881 } 882 883 /* netif_tx_lock held, process context with BHs disabled, or BH */ 884 static netdev_tx_t enic_hard_start_xmit(struct sk_buff *skb, 885 struct net_device *netdev) 886 { 887 struct enic *enic = netdev_priv(netdev); 888 struct vnic_wq *wq; 889 unsigned int txq_map; 890 struct netdev_queue *txq; 891 892 txq_map = skb_get_queue_mapping(skb) % enic->wq_count; 893 wq = &enic->wq[txq_map].vwq; 894 895 if (skb->len <= 0) { 896 dev_kfree_skb_any(skb); 897 enic->wq[wq->index].stats.null_pkt++; 898 return NETDEV_TX_OK; 899 } 900 901 txq = netdev_get_tx_queue(netdev, txq_map); 902 903 /* Non-TSO sends must fit within ENIC_NON_TSO_MAX_DESC descs, 904 * which is very likely. In the off chance it's going to take 905 * more than * ENIC_NON_TSO_MAX_DESC, linearize the skb. 906 */ 907 908 if (skb_shinfo(skb)->gso_size == 0 && 909 skb_shinfo(skb)->nr_frags + 1 > ENIC_NON_TSO_MAX_DESC && 910 skb_linearize(skb)) { 911 dev_kfree_skb_any(skb); 912 enic->wq[wq->index].stats.skb_linear_fail++; 913 return NETDEV_TX_OK; 914 } 915 916 spin_lock(&enic->wq[txq_map].lock); 917 918 if (vnic_wq_desc_avail(wq) < 919 skb_shinfo(skb)->nr_frags + ENIC_DESC_MAX_SPLITS) { 920 netif_tx_stop_queue(txq); 921 /* This is a hard error, log it */ 922 netdev_err(netdev, "BUG! Tx ring full when queue awake!\n"); 923 spin_unlock(&enic->wq[txq_map].lock); 924 enic->wq[wq->index].stats.desc_full_awake++; 925 return NETDEV_TX_BUSY; 926 } 927 928 if (enic_queue_wq_skb(enic, wq, skb)) 929 goto error; 930 931 if (vnic_wq_desc_avail(wq) < MAX_SKB_FRAGS + ENIC_DESC_MAX_SPLITS) { 932 netif_tx_stop_queue(txq); 933 enic->wq[wq->index].stats.stopped++; 934 } 935 skb_tx_timestamp(skb); 936 if (!netdev_xmit_more() || netif_xmit_stopped(txq)) 937 vnic_wq_doorbell(wq); 938 939 error: 940 spin_unlock(&enic->wq[txq_map].lock); 941 942 return NETDEV_TX_OK; 943 } 944 945 /* rcu_read_lock potentially held, nominally process context */ 946 static void enic_get_stats(struct net_device *netdev, 947 struct rtnl_link_stats64 *net_stats) 948 { 949 struct enic *enic = netdev_priv(netdev); 950 struct vnic_stats *stats; 951 u64 pkt_truncated = 0; 952 u64 bad_fcs = 0; 953 int err; 954 int i; 955 956 err = enic_dev_stats_dump(enic, &stats); 957 /* return only when dma_alloc_coherent fails in vnic_dev_stats_dump 958 * For other failures, like devcmd failure, we return previously 959 * recorded stats. 960 */ 961 if (err == -ENOMEM) 962 return; 963 964 net_stats->tx_packets = stats->tx.tx_frames_ok; 965 net_stats->tx_bytes = stats->tx.tx_bytes_ok; 966 net_stats->tx_errors = stats->tx.tx_errors; 967 net_stats->tx_dropped = stats->tx.tx_drops; 968 969 net_stats->rx_packets = stats->rx.rx_frames_ok; 970 net_stats->rx_bytes = stats->rx.rx_bytes_ok; 971 net_stats->rx_errors = stats->rx.rx_errors; 972 net_stats->multicast = stats->rx.rx_multicast_frames_ok; 973 974 for (i = 0; i < enic->rq_count; i++) { 975 struct enic_rq_stats *rqs = &enic->rq[i].stats; 976 977 if (!enic->rq[i].vrq.ctrl) 978 break; 979 pkt_truncated += rqs->pkt_truncated; 980 bad_fcs += rqs->bad_fcs; 981 } 982 net_stats->rx_over_errors = pkt_truncated; 983 net_stats->rx_crc_errors = bad_fcs; 984 net_stats->rx_dropped = stats->rx.rx_no_bufs + stats->rx.rx_drop; 985 } 986 987 static int enic_mc_sync(struct net_device *netdev, const u8 *mc_addr) 988 { 989 struct enic *enic = netdev_priv(netdev); 990 991 if (enic->mc_count == ENIC_MULTICAST_PERFECT_FILTERS) { 992 unsigned int mc_count = netdev_mc_count(netdev); 993 994 netdev_warn(netdev, "Registering only %d out of %d multicast addresses\n", 995 ENIC_MULTICAST_PERFECT_FILTERS, mc_count); 996 997 return -ENOSPC; 998 } 999 1000 enic_dev_add_addr(enic, mc_addr); 1001 enic->mc_count++; 1002 1003 return 0; 1004 } 1005 1006 static int enic_mc_unsync(struct net_device *netdev, const u8 *mc_addr) 1007 { 1008 struct enic *enic = netdev_priv(netdev); 1009 1010 enic_dev_del_addr(enic, mc_addr); 1011 enic->mc_count--; 1012 1013 return 0; 1014 } 1015 1016 static int enic_uc_sync(struct net_device *netdev, const u8 *uc_addr) 1017 { 1018 struct enic *enic = netdev_priv(netdev); 1019 1020 if (enic->uc_count == ENIC_UNICAST_PERFECT_FILTERS) { 1021 unsigned int uc_count = netdev_uc_count(netdev); 1022 1023 netdev_warn(netdev, "Registering only %d out of %d unicast addresses\n", 1024 ENIC_UNICAST_PERFECT_FILTERS, uc_count); 1025 1026 return -ENOSPC; 1027 } 1028 1029 enic_dev_add_addr(enic, uc_addr); 1030 enic->uc_count++; 1031 1032 return 0; 1033 } 1034 1035 static int enic_uc_unsync(struct net_device *netdev, const u8 *uc_addr) 1036 { 1037 struct enic *enic = netdev_priv(netdev); 1038 1039 enic_dev_del_addr(enic, uc_addr); 1040 enic->uc_count--; 1041 1042 return 0; 1043 } 1044 1045 void enic_reset_addr_lists(struct enic *enic) 1046 { 1047 struct net_device *netdev = enic->netdev; 1048 1049 __dev_uc_unsync(netdev, NULL); 1050 __dev_mc_unsync(netdev, NULL); 1051 1052 enic->mc_count = 0; 1053 enic->uc_count = 0; 1054 enic->flags = 0; 1055 } 1056 1057 static int enic_set_mac_addr(struct net_device *netdev, char *addr) 1058 { 1059 struct enic *enic = netdev_priv(netdev); 1060 1061 if (enic_is_dynamic(enic) || enic_is_sriov_vf(enic)) { 1062 if (!is_valid_ether_addr(addr) && !is_zero_ether_addr(addr)) 1063 return -EADDRNOTAVAIL; 1064 } else { 1065 if (!is_valid_ether_addr(addr)) 1066 return -EADDRNOTAVAIL; 1067 } 1068 1069 eth_hw_addr_set(netdev, addr); 1070 1071 return 0; 1072 } 1073 1074 static int enic_set_mac_address_dynamic(struct net_device *netdev, void *p) 1075 { 1076 struct enic *enic = netdev_priv(netdev); 1077 struct sockaddr *saddr = p; 1078 char *addr = saddr->sa_data; 1079 int err; 1080 1081 if (netif_running(enic->netdev)) { 1082 err = enic_dev_del_station_addr(enic); 1083 if (err) 1084 return err; 1085 } 1086 1087 err = enic_set_mac_addr(netdev, addr); 1088 if (err) 1089 return err; 1090 1091 if (netif_running(enic->netdev)) { 1092 err = enic_dev_add_station_addr(enic); 1093 if (err) 1094 return err; 1095 } 1096 1097 return err; 1098 } 1099 1100 static int enic_set_mac_address(struct net_device *netdev, void *p) 1101 { 1102 struct sockaddr *saddr = p; 1103 char *addr = saddr->sa_data; 1104 struct enic *enic = netdev_priv(netdev); 1105 int err; 1106 1107 err = enic_dev_del_station_addr(enic); 1108 if (err) 1109 return err; 1110 1111 err = enic_set_mac_addr(netdev, addr); 1112 if (err) 1113 return err; 1114 1115 return enic_dev_add_station_addr(enic); 1116 } 1117 1118 /* netif_tx_lock held, BHs disabled */ 1119 static void enic_set_rx_mode(struct net_device *netdev) 1120 { 1121 struct enic *enic = netdev_priv(netdev); 1122 int directed = 1; 1123 int multicast = (netdev->flags & IFF_MULTICAST) ? 1 : 0; 1124 int broadcast = (netdev->flags & IFF_BROADCAST) ? 1 : 0; 1125 int promisc = (netdev->flags & IFF_PROMISC) || 1126 netdev_uc_count(netdev) > ENIC_UNICAST_PERFECT_FILTERS; 1127 int allmulti = (netdev->flags & IFF_ALLMULTI) || 1128 netdev_mc_count(netdev) > ENIC_MULTICAST_PERFECT_FILTERS; 1129 unsigned int flags = netdev->flags | 1130 (allmulti ? IFF_ALLMULTI : 0) | 1131 (promisc ? IFF_PROMISC : 0); 1132 1133 if (enic->flags != flags) { 1134 enic->flags = flags; 1135 enic_dev_packet_filter(enic, directed, 1136 multicast, broadcast, promisc, allmulti); 1137 } 1138 1139 if (!promisc) { 1140 __dev_uc_sync(netdev, enic_uc_sync, enic_uc_unsync); 1141 if (!allmulti) 1142 __dev_mc_sync(netdev, enic_mc_sync, enic_mc_unsync); 1143 } 1144 } 1145 1146 /* netif_tx_lock held, BHs disabled */ 1147 static void enic_tx_timeout(struct net_device *netdev, unsigned int txqueue) 1148 { 1149 struct enic *enic = netdev_priv(netdev); 1150 schedule_work(&enic->tx_hang_reset); 1151 } 1152 1153 static int enic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 1154 { 1155 struct enic *enic = netdev_priv(netdev); 1156 struct enic_port_profile *pp; 1157 int err; 1158 1159 ENIC_PP_BY_INDEX(enic, vf, pp, &err); 1160 if (err) 1161 return err; 1162 1163 if (is_valid_ether_addr(mac) || is_zero_ether_addr(mac)) { 1164 if (vf == PORT_SELF_VF) { 1165 memcpy(pp->vf_mac, mac, ETH_ALEN); 1166 return 0; 1167 } else { 1168 /* 1169 * For sriov vf's set the mac in hw 1170 */ 1171 ENIC_DEVCMD_PROXY_BY_INDEX(vf, err, enic, 1172 vnic_dev_set_mac_addr, mac); 1173 return enic_dev_status_to_errno(err); 1174 } 1175 } else 1176 return -EINVAL; 1177 } 1178 1179 static int enic_set_vf_port(struct net_device *netdev, int vf, 1180 struct nlattr *port[]) 1181 { 1182 static const u8 zero_addr[ETH_ALEN] = {}; 1183 struct enic *enic = netdev_priv(netdev); 1184 struct enic_port_profile prev_pp; 1185 struct enic_port_profile *pp; 1186 int err = 0, restore_pp = 1; 1187 1188 ENIC_PP_BY_INDEX(enic, vf, pp, &err); 1189 if (err) 1190 return err; 1191 1192 if (!port[IFLA_PORT_REQUEST]) 1193 return -EOPNOTSUPP; 1194 1195 memcpy(&prev_pp, pp, sizeof(*enic->pp)); 1196 memset(pp, 0, sizeof(*enic->pp)); 1197 1198 pp->set |= ENIC_SET_REQUEST; 1199 pp->request = nla_get_u8(port[IFLA_PORT_REQUEST]); 1200 1201 if (port[IFLA_PORT_PROFILE]) { 1202 if (nla_len(port[IFLA_PORT_PROFILE]) != PORT_PROFILE_MAX) { 1203 memcpy(pp, &prev_pp, sizeof(*pp)); 1204 return -EINVAL; 1205 } 1206 pp->set |= ENIC_SET_NAME; 1207 memcpy(pp->name, nla_data(port[IFLA_PORT_PROFILE]), 1208 PORT_PROFILE_MAX); 1209 } 1210 1211 if (port[IFLA_PORT_INSTANCE_UUID]) { 1212 if (nla_len(port[IFLA_PORT_INSTANCE_UUID]) != PORT_UUID_MAX) { 1213 memcpy(pp, &prev_pp, sizeof(*pp)); 1214 return -EINVAL; 1215 } 1216 pp->set |= ENIC_SET_INSTANCE; 1217 memcpy(pp->instance_uuid, 1218 nla_data(port[IFLA_PORT_INSTANCE_UUID]), PORT_UUID_MAX); 1219 } 1220 1221 if (port[IFLA_PORT_HOST_UUID]) { 1222 if (nla_len(port[IFLA_PORT_HOST_UUID]) != PORT_UUID_MAX) { 1223 memcpy(pp, &prev_pp, sizeof(*pp)); 1224 return -EINVAL; 1225 } 1226 pp->set |= ENIC_SET_HOST; 1227 memcpy(pp->host_uuid, 1228 nla_data(port[IFLA_PORT_HOST_UUID]), PORT_UUID_MAX); 1229 } 1230 1231 if (vf == PORT_SELF_VF) { 1232 /* Special case handling: mac came from IFLA_VF_MAC */ 1233 if (!is_zero_ether_addr(prev_pp.vf_mac)) 1234 memcpy(pp->mac_addr, prev_pp.vf_mac, ETH_ALEN); 1235 1236 if (is_zero_ether_addr(netdev->dev_addr)) 1237 eth_hw_addr_random(netdev); 1238 } else { 1239 /* SR-IOV VF: get mac from adapter */ 1240 ENIC_DEVCMD_PROXY_BY_INDEX(vf, err, enic, 1241 vnic_dev_get_mac_addr, pp->mac_addr); 1242 if (err) { 1243 netdev_err(netdev, "Error getting mac for vf %d\n", vf); 1244 memcpy(pp, &prev_pp, sizeof(*pp)); 1245 return enic_dev_status_to_errno(err); 1246 } 1247 } 1248 1249 err = enic_process_set_pp_request(enic, vf, &prev_pp, &restore_pp); 1250 if (err) { 1251 if (restore_pp) { 1252 /* Things are still the way they were: Implicit 1253 * DISASSOCIATE failed 1254 */ 1255 memcpy(pp, &prev_pp, sizeof(*pp)); 1256 } else { 1257 memset(pp, 0, sizeof(*pp)); 1258 if (vf == PORT_SELF_VF) 1259 eth_hw_addr_set(netdev, zero_addr); 1260 } 1261 } else { 1262 /* Set flag to indicate that the port assoc/disassoc 1263 * request has been sent out to fw 1264 */ 1265 pp->set |= ENIC_PORT_REQUEST_APPLIED; 1266 1267 /* If DISASSOCIATE, clean up all assigned/saved macaddresses */ 1268 if (pp->request == PORT_REQUEST_DISASSOCIATE) { 1269 eth_zero_addr(pp->mac_addr); 1270 if (vf == PORT_SELF_VF) 1271 eth_hw_addr_set(netdev, zero_addr); 1272 } 1273 } 1274 1275 if (vf == PORT_SELF_VF) 1276 eth_zero_addr(pp->vf_mac); 1277 1278 return err; 1279 } 1280 1281 static int enic_get_vf_port(struct net_device *netdev, int vf, 1282 struct sk_buff *skb) 1283 { 1284 struct enic *enic = netdev_priv(netdev); 1285 u16 response = PORT_PROFILE_RESPONSE_SUCCESS; 1286 struct enic_port_profile *pp; 1287 int err; 1288 1289 ENIC_PP_BY_INDEX(enic, vf, pp, &err); 1290 if (err) 1291 return err; 1292 1293 if (!(pp->set & ENIC_PORT_REQUEST_APPLIED)) 1294 return -ENODATA; 1295 1296 err = enic_process_get_pp_request(enic, vf, pp->request, &response); 1297 if (err) 1298 return err; 1299 1300 if (nla_put_u16(skb, IFLA_PORT_REQUEST, pp->request) || 1301 nla_put_u16(skb, IFLA_PORT_RESPONSE, response) || 1302 ((pp->set & ENIC_SET_NAME) && 1303 nla_put(skb, IFLA_PORT_PROFILE, PORT_PROFILE_MAX, pp->name)) || 1304 ((pp->set & ENIC_SET_INSTANCE) && 1305 nla_put(skb, IFLA_PORT_INSTANCE_UUID, PORT_UUID_MAX, 1306 pp->instance_uuid)) || 1307 ((pp->set & ENIC_SET_HOST) && 1308 nla_put(skb, IFLA_PORT_HOST_UUID, PORT_UUID_MAX, pp->host_uuid))) 1309 goto nla_put_failure; 1310 return 0; 1311 1312 nla_put_failure: 1313 return -EMSGSIZE; 1314 } 1315 1316 static void enic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf) 1317 { 1318 struct enic *enic = vnic_dev_priv(rq->vdev); 1319 1320 if (!buf->os_buf) 1321 return; 1322 1323 dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len, 1324 DMA_FROM_DEVICE); 1325 dev_kfree_skb_any(buf->os_buf); 1326 buf->os_buf = NULL; 1327 } 1328 1329 static int enic_rq_alloc_buf(struct vnic_rq *rq) 1330 { 1331 struct enic *enic = vnic_dev_priv(rq->vdev); 1332 struct net_device *netdev = enic->netdev; 1333 struct sk_buff *skb; 1334 unsigned int len = netdev->mtu + VLAN_ETH_HLEN; 1335 unsigned int os_buf_index = 0; 1336 dma_addr_t dma_addr; 1337 struct vnic_rq_buf *buf = rq->to_use; 1338 1339 if (buf->os_buf) { 1340 enic_queue_rq_desc(rq, buf->os_buf, os_buf_index, buf->dma_addr, 1341 buf->len); 1342 1343 return 0; 1344 } 1345 skb = netdev_alloc_skb_ip_align(netdev, len); 1346 if (!skb) { 1347 enic->rq[rq->index].stats.no_skb++; 1348 return -ENOMEM; 1349 } 1350 1351 dma_addr = dma_map_single(&enic->pdev->dev, skb->data, len, 1352 DMA_FROM_DEVICE); 1353 if (unlikely(enic_dma_map_check(enic, dma_addr))) { 1354 dev_kfree_skb(skb); 1355 return -ENOMEM; 1356 } 1357 1358 enic_queue_rq_desc(rq, skb, os_buf_index, 1359 dma_addr, len); 1360 1361 return 0; 1362 } 1363 1364 static void enic_intr_update_pkt_size(struct vnic_rx_bytes_counter *pkt_size, 1365 u32 pkt_len) 1366 { 1367 if (ENIC_LARGE_PKT_THRESHOLD <= pkt_len) 1368 pkt_size->large_pkt_bytes_cnt += pkt_len; 1369 else 1370 pkt_size->small_pkt_bytes_cnt += pkt_len; 1371 } 1372 1373 static bool enic_rxcopybreak(struct net_device *netdev, struct sk_buff **skb, 1374 struct vnic_rq_buf *buf, u16 len) 1375 { 1376 struct enic *enic = netdev_priv(netdev); 1377 struct sk_buff *new_skb; 1378 1379 if (len > enic->rx_copybreak) 1380 return false; 1381 new_skb = netdev_alloc_skb_ip_align(netdev, len); 1382 if (!new_skb) 1383 return false; 1384 dma_sync_single_for_cpu(&enic->pdev->dev, buf->dma_addr, len, 1385 DMA_FROM_DEVICE); 1386 memcpy(new_skb->data, (*skb)->data, len); 1387 *skb = new_skb; 1388 1389 return true; 1390 } 1391 1392 static void enic_rq_indicate_buf(struct vnic_rq *rq, 1393 struct cq_desc *cq_desc, struct vnic_rq_buf *buf, 1394 int skipped, void *opaque) 1395 { 1396 struct enic *enic = vnic_dev_priv(rq->vdev); 1397 struct net_device *netdev = enic->netdev; 1398 struct sk_buff *skb; 1399 struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)]; 1400 struct enic_rq_stats *rqstats = &enic->rq[rq->index].stats; 1401 1402 u8 type, color, eop, sop, ingress_port, vlan_stripped; 1403 u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; 1404 u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; 1405 u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; 1406 u8 packet_error; 1407 u16 q_number, completed_index, bytes_written, vlan_tci, checksum; 1408 u32 rss_hash; 1409 bool outer_csum_ok = true, encap = false; 1410 1411 rqstats->packets++; 1412 if (skipped) { 1413 rqstats->desc_skip++; 1414 return; 1415 } 1416 1417 skb = buf->os_buf; 1418 1419 cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, 1420 &type, &color, &q_number, &completed_index, 1421 &ingress_port, &fcoe, &eop, &sop, &rss_type, 1422 &csum_not_calc, &rss_hash, &bytes_written, 1423 &packet_error, &vlan_stripped, &vlan_tci, &checksum, 1424 &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error, 1425 &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp, 1426 &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment, 1427 &fcs_ok); 1428 1429 if (packet_error) { 1430 1431 if (!fcs_ok) { 1432 if (bytes_written > 0) 1433 rqstats->bad_fcs++; 1434 else if (bytes_written == 0) 1435 rqstats->pkt_truncated++; 1436 } 1437 1438 dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len, 1439 DMA_FROM_DEVICE); 1440 dev_kfree_skb_any(skb); 1441 buf->os_buf = NULL; 1442 1443 return; 1444 } 1445 1446 if (eop && bytes_written > 0) { 1447 1448 /* Good receive 1449 */ 1450 rqstats->bytes += bytes_written; 1451 if (!enic_rxcopybreak(netdev, &skb, buf, bytes_written)) { 1452 buf->os_buf = NULL; 1453 dma_unmap_single(&enic->pdev->dev, buf->dma_addr, 1454 buf->len, DMA_FROM_DEVICE); 1455 } 1456 prefetch(skb->data - NET_IP_ALIGN); 1457 1458 skb_put(skb, bytes_written); 1459 skb->protocol = eth_type_trans(skb, netdev); 1460 skb_record_rx_queue(skb, q_number); 1461 if ((netdev->features & NETIF_F_RXHASH) && rss_hash && 1462 (type == 3)) { 1463 switch (rss_type) { 1464 case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv4: 1465 case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6: 1466 case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6_EX: 1467 skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L4); 1468 rqstats->l4_rss_hash++; 1469 break; 1470 case CQ_ENET_RQ_DESC_RSS_TYPE_IPv4: 1471 case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6: 1472 case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6_EX: 1473 skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L3); 1474 rqstats->l3_rss_hash++; 1475 break; 1476 } 1477 } 1478 if (enic->vxlan.vxlan_udp_port_number) { 1479 switch (enic->vxlan.patch_level) { 1480 case 0: 1481 if (fcoe) { 1482 encap = true; 1483 outer_csum_ok = fcoe_fc_crc_ok; 1484 } 1485 break; 1486 case 2: 1487 if ((type == 7) && 1488 (rss_hash & BIT(0))) { 1489 encap = true; 1490 outer_csum_ok = (rss_hash & BIT(1)) && 1491 (rss_hash & BIT(2)); 1492 } 1493 break; 1494 } 1495 } 1496 1497 /* Hardware does not provide whole packet checksum. It only 1498 * provides pseudo checksum. Since hw validates the packet 1499 * checksum but not provide us the checksum value. use 1500 * CHECSUM_UNNECESSARY. 1501 * 1502 * In case of encap pkt tcp_udp_csum_ok/tcp_udp_csum_ok is 1503 * inner csum_ok. outer_csum_ok is set by hw when outer udp 1504 * csum is correct or is zero. 1505 */ 1506 if ((netdev->features & NETIF_F_RXCSUM) && !csum_not_calc && 1507 tcp_udp_csum_ok && outer_csum_ok && 1508 (ipv4_csum_ok || ipv6)) { 1509 skb->ip_summed = CHECKSUM_UNNECESSARY; 1510 skb->csum_level = encap; 1511 if (encap) 1512 rqstats->csum_unnecessary_encap++; 1513 else 1514 rqstats->csum_unnecessary++; 1515 } 1516 1517 if (vlan_stripped) { 1518 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); 1519 rqstats->vlan_stripped++; 1520 } 1521 skb_mark_napi_id(skb, &enic->napi[rq->index]); 1522 if (!(netdev->features & NETIF_F_GRO)) 1523 netif_receive_skb(skb); 1524 else 1525 napi_gro_receive(&enic->napi[q_number], skb); 1526 if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce) 1527 enic_intr_update_pkt_size(&cq->pkt_size_counter, 1528 bytes_written); 1529 } else { 1530 1531 /* Buffer overflow 1532 */ 1533 rqstats->pkt_truncated++; 1534 dma_unmap_single(&enic->pdev->dev, buf->dma_addr, buf->len, 1535 DMA_FROM_DEVICE); 1536 dev_kfree_skb_any(skb); 1537 buf->os_buf = NULL; 1538 } 1539 } 1540 1541 static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, 1542 u8 type, u16 q_number, u16 completed_index, void *opaque) 1543 { 1544 struct enic *enic = vnic_dev_priv(vdev); 1545 1546 vnic_rq_service(&enic->rq[q_number].vrq, cq_desc, 1547 completed_index, VNIC_RQ_RETURN_DESC, 1548 enic_rq_indicate_buf, opaque); 1549 1550 return 0; 1551 } 1552 1553 static void enic_set_int_moderation(struct enic *enic, struct vnic_rq *rq) 1554 { 1555 unsigned int intr = enic_msix_rq_intr(enic, rq->index); 1556 struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)]; 1557 u32 timer = cq->tobe_rx_coal_timeval; 1558 1559 if (cq->tobe_rx_coal_timeval != cq->cur_rx_coal_timeval) { 1560 vnic_intr_coalescing_timer_set(&enic->intr[intr], timer); 1561 cq->cur_rx_coal_timeval = cq->tobe_rx_coal_timeval; 1562 } 1563 } 1564 1565 static void enic_calc_int_moderation(struct enic *enic, struct vnic_rq *rq) 1566 { 1567 struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting; 1568 struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)]; 1569 struct vnic_rx_bytes_counter *pkt_size_counter = &cq->pkt_size_counter; 1570 int index; 1571 u32 timer; 1572 u32 range_start; 1573 u32 traffic; 1574 u64 delta; 1575 ktime_t now = ktime_get(); 1576 1577 delta = ktime_us_delta(now, cq->prev_ts); 1578 if (delta < ENIC_AIC_TS_BREAK) 1579 return; 1580 cq->prev_ts = now; 1581 1582 traffic = pkt_size_counter->large_pkt_bytes_cnt + 1583 pkt_size_counter->small_pkt_bytes_cnt; 1584 /* The table takes Mbps 1585 * traffic *= 8 => bits 1586 * traffic *= (10^6 / delta) => bps 1587 * traffic /= 10^6 => Mbps 1588 * 1589 * Combining, traffic *= (8 / delta) 1590 */ 1591 1592 traffic <<= 3; 1593 traffic = delta > UINT_MAX ? 0 : traffic / (u32)delta; 1594 1595 for (index = 0; index < ENIC_MAX_COALESCE_TIMERS; index++) 1596 if (traffic < mod_table[index].rx_rate) 1597 break; 1598 range_start = (pkt_size_counter->small_pkt_bytes_cnt > 1599 pkt_size_counter->large_pkt_bytes_cnt << 1) ? 1600 rx_coal->small_pkt_range_start : 1601 rx_coal->large_pkt_range_start; 1602 timer = range_start + ((rx_coal->range_end - range_start) * 1603 mod_table[index].range_percent / 100); 1604 /* Damping */ 1605 cq->tobe_rx_coal_timeval = (timer + cq->tobe_rx_coal_timeval) >> 1; 1606 1607 pkt_size_counter->large_pkt_bytes_cnt = 0; 1608 pkt_size_counter->small_pkt_bytes_cnt = 0; 1609 } 1610 1611 static int enic_poll(struct napi_struct *napi, int budget) 1612 { 1613 struct net_device *netdev = napi->dev; 1614 struct enic *enic = netdev_priv(netdev); 1615 unsigned int cq_rq = enic_cq_rq(enic, 0); 1616 unsigned int cq_wq = enic_cq_wq(enic, 0); 1617 unsigned int intr = ENIC_LEGACY_IO_INTR; 1618 unsigned int rq_work_to_do = budget; 1619 unsigned int wq_work_to_do = ENIC_WQ_NAPI_BUDGET; 1620 unsigned int work_done, rq_work_done = 0, wq_work_done; 1621 int err; 1622 1623 wq_work_done = vnic_cq_service(&enic->cq[cq_wq], wq_work_to_do, 1624 enic_wq_service, NULL); 1625 1626 if (budget > 0) 1627 rq_work_done = vnic_cq_service(&enic->cq[cq_rq], 1628 rq_work_to_do, enic_rq_service, NULL); 1629 1630 /* Accumulate intr event credits for this polling 1631 * cycle. An intr event is the completion of a 1632 * a WQ or RQ packet. 1633 */ 1634 1635 work_done = rq_work_done + wq_work_done; 1636 1637 if (work_done > 0) 1638 vnic_intr_return_credits(&enic->intr[intr], 1639 work_done, 1640 0 /* don't unmask intr */, 1641 0 /* don't reset intr timer */); 1642 1643 err = vnic_rq_fill(&enic->rq[0].vrq, enic_rq_alloc_buf); 1644 1645 /* Buffer allocation failed. Stay in polling 1646 * mode so we can try to fill the ring again. 1647 */ 1648 1649 if (err) 1650 rq_work_done = rq_work_to_do; 1651 if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce) 1652 /* Call the function which refreshes the intr coalescing timer 1653 * value based on the traffic. 1654 */ 1655 enic_calc_int_moderation(enic, &enic->rq[0].vrq); 1656 1657 if ((rq_work_done < budget) && napi_complete_done(napi, rq_work_done)) { 1658 1659 /* Some work done, but not enough to stay in polling, 1660 * exit polling 1661 */ 1662 1663 if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce) 1664 enic_set_int_moderation(enic, &enic->rq[0].vrq); 1665 vnic_intr_unmask(&enic->intr[intr]); 1666 enic->rq[0].stats.napi_complete++; 1667 } else { 1668 enic->rq[0].stats.napi_repoll++; 1669 } 1670 1671 return rq_work_done; 1672 } 1673 1674 #ifdef CONFIG_RFS_ACCEL 1675 static void enic_free_rx_cpu_rmap(struct enic *enic) 1676 { 1677 free_irq_cpu_rmap(enic->netdev->rx_cpu_rmap); 1678 enic->netdev->rx_cpu_rmap = NULL; 1679 } 1680 1681 static void enic_set_rx_cpu_rmap(struct enic *enic) 1682 { 1683 int i, res; 1684 1685 if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX) { 1686 enic->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(enic->rq_count); 1687 if (unlikely(!enic->netdev->rx_cpu_rmap)) 1688 return; 1689 for (i = 0; i < enic->rq_count; i++) { 1690 res = irq_cpu_rmap_add(enic->netdev->rx_cpu_rmap, 1691 enic->msix_entry[i].vector); 1692 if (unlikely(res)) { 1693 enic_free_rx_cpu_rmap(enic); 1694 return; 1695 } 1696 } 1697 } 1698 } 1699 1700 #else 1701 1702 static void enic_free_rx_cpu_rmap(struct enic *enic) 1703 { 1704 } 1705 1706 static void enic_set_rx_cpu_rmap(struct enic *enic) 1707 { 1708 } 1709 1710 #endif /* CONFIG_RFS_ACCEL */ 1711 1712 static int enic_poll_msix_wq(struct napi_struct *napi, int budget) 1713 { 1714 struct net_device *netdev = napi->dev; 1715 struct enic *enic = netdev_priv(netdev); 1716 unsigned int wq_index = (napi - &enic->napi[0]) - enic->rq_count; 1717 struct vnic_wq *wq = &enic->wq[wq_index].vwq; 1718 unsigned int cq; 1719 unsigned int intr; 1720 unsigned int wq_work_to_do = ENIC_WQ_NAPI_BUDGET; 1721 unsigned int wq_work_done; 1722 unsigned int wq_irq; 1723 1724 wq_irq = wq->index; 1725 cq = enic_cq_wq(enic, wq_irq); 1726 intr = enic_msix_wq_intr(enic, wq_irq); 1727 wq_work_done = vnic_cq_service(&enic->cq[cq], wq_work_to_do, 1728 enic_wq_service, NULL); 1729 1730 vnic_intr_return_credits(&enic->intr[intr], wq_work_done, 1731 0 /* don't unmask intr */, 1732 1 /* reset intr timer */); 1733 if (!wq_work_done) { 1734 napi_complete(napi); 1735 vnic_intr_unmask(&enic->intr[intr]); 1736 return 0; 1737 } 1738 1739 return budget; 1740 } 1741 1742 static int enic_poll_msix_rq(struct napi_struct *napi, int budget) 1743 { 1744 struct net_device *netdev = napi->dev; 1745 struct enic *enic = netdev_priv(netdev); 1746 unsigned int rq = (napi - &enic->napi[0]); 1747 unsigned int cq = enic_cq_rq(enic, rq); 1748 unsigned int intr = enic_msix_rq_intr(enic, rq); 1749 unsigned int work_to_do = budget; 1750 unsigned int work_done = 0; 1751 int err; 1752 1753 /* Service RQ 1754 */ 1755 1756 if (budget > 0) 1757 work_done = vnic_cq_service(&enic->cq[cq], 1758 work_to_do, enic_rq_service, NULL); 1759 1760 /* Return intr event credits for this polling 1761 * cycle. An intr event is the completion of a 1762 * RQ packet. 1763 */ 1764 1765 if (work_done > 0) 1766 vnic_intr_return_credits(&enic->intr[intr], 1767 work_done, 1768 0 /* don't unmask intr */, 1769 0 /* don't reset intr timer */); 1770 1771 err = vnic_rq_fill(&enic->rq[rq].vrq, enic_rq_alloc_buf); 1772 1773 /* Buffer allocation failed. Stay in polling mode 1774 * so we can try to fill the ring again. 1775 */ 1776 1777 if (err) 1778 work_done = work_to_do; 1779 if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce) 1780 /* Call the function which refreshes the intr coalescing timer 1781 * value based on the traffic. 1782 */ 1783 enic_calc_int_moderation(enic, &enic->rq[rq].vrq); 1784 1785 if ((work_done < budget) && napi_complete_done(napi, work_done)) { 1786 1787 /* Some work done, but not enough to stay in polling, 1788 * exit polling 1789 */ 1790 1791 if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce) 1792 enic_set_int_moderation(enic, &enic->rq[rq].vrq); 1793 vnic_intr_unmask(&enic->intr[intr]); 1794 enic->rq[rq].stats.napi_complete++; 1795 } else { 1796 enic->rq[rq].stats.napi_repoll++; 1797 } 1798 1799 return work_done; 1800 } 1801 1802 static void enic_notify_timer(struct timer_list *t) 1803 { 1804 struct enic *enic = from_timer(enic, t, notify_timer); 1805 1806 enic_notify_check(enic); 1807 1808 mod_timer(&enic->notify_timer, 1809 round_jiffies(jiffies + ENIC_NOTIFY_TIMER_PERIOD)); 1810 } 1811 1812 static void enic_free_intr(struct enic *enic) 1813 { 1814 struct net_device *netdev = enic->netdev; 1815 unsigned int i; 1816 1817 enic_free_rx_cpu_rmap(enic); 1818 switch (vnic_dev_get_intr_mode(enic->vdev)) { 1819 case VNIC_DEV_INTR_MODE_INTX: 1820 free_irq(enic->pdev->irq, netdev); 1821 break; 1822 case VNIC_DEV_INTR_MODE_MSI: 1823 free_irq(enic->pdev->irq, enic); 1824 break; 1825 case VNIC_DEV_INTR_MODE_MSIX: 1826 for (i = 0; i < enic->intr_count; i++) 1827 if (enic->msix[i].requested) 1828 free_irq(enic->msix_entry[i].vector, 1829 enic->msix[i].devid); 1830 break; 1831 default: 1832 break; 1833 } 1834 } 1835 1836 static int enic_request_intr(struct enic *enic) 1837 { 1838 struct net_device *netdev = enic->netdev; 1839 unsigned int i, intr; 1840 int err = 0; 1841 1842 enic_set_rx_cpu_rmap(enic); 1843 switch (vnic_dev_get_intr_mode(enic->vdev)) { 1844 1845 case VNIC_DEV_INTR_MODE_INTX: 1846 1847 err = request_irq(enic->pdev->irq, enic_isr_legacy, 1848 IRQF_SHARED, netdev->name, netdev); 1849 break; 1850 1851 case VNIC_DEV_INTR_MODE_MSI: 1852 1853 err = request_irq(enic->pdev->irq, enic_isr_msi, 1854 0, netdev->name, enic); 1855 break; 1856 1857 case VNIC_DEV_INTR_MODE_MSIX: 1858 1859 for (i = 0; i < enic->rq_count; i++) { 1860 intr = enic_msix_rq_intr(enic, i); 1861 snprintf(enic->msix[intr].devname, 1862 sizeof(enic->msix[intr].devname), 1863 "%s-rx-%u", netdev->name, i); 1864 enic->msix[intr].isr = enic_isr_msix; 1865 enic->msix[intr].devid = &enic->napi[i]; 1866 } 1867 1868 for (i = 0; i < enic->wq_count; i++) { 1869 int wq = enic_cq_wq(enic, i); 1870 1871 intr = enic_msix_wq_intr(enic, i); 1872 snprintf(enic->msix[intr].devname, 1873 sizeof(enic->msix[intr].devname), 1874 "%s-tx-%u", netdev->name, i); 1875 enic->msix[intr].isr = enic_isr_msix; 1876 enic->msix[intr].devid = &enic->napi[wq]; 1877 } 1878 1879 intr = enic_msix_err_intr(enic); 1880 snprintf(enic->msix[intr].devname, 1881 sizeof(enic->msix[intr].devname), 1882 "%s-err", netdev->name); 1883 enic->msix[intr].isr = enic_isr_msix_err; 1884 enic->msix[intr].devid = enic; 1885 1886 intr = enic_msix_notify_intr(enic); 1887 snprintf(enic->msix[intr].devname, 1888 sizeof(enic->msix[intr].devname), 1889 "%s-notify", netdev->name); 1890 enic->msix[intr].isr = enic_isr_msix_notify; 1891 enic->msix[intr].devid = enic; 1892 1893 for (i = 0; i < enic->intr_count; i++) 1894 enic->msix[i].requested = 0; 1895 1896 for (i = 0; i < enic->intr_count; i++) { 1897 err = request_irq(enic->msix_entry[i].vector, 1898 enic->msix[i].isr, 0, 1899 enic->msix[i].devname, 1900 enic->msix[i].devid); 1901 if (err) { 1902 enic_free_intr(enic); 1903 break; 1904 } 1905 enic->msix[i].requested = 1; 1906 } 1907 1908 break; 1909 1910 default: 1911 break; 1912 } 1913 1914 return err; 1915 } 1916 1917 static void enic_synchronize_irqs(struct enic *enic) 1918 { 1919 unsigned int i; 1920 1921 switch (vnic_dev_get_intr_mode(enic->vdev)) { 1922 case VNIC_DEV_INTR_MODE_INTX: 1923 case VNIC_DEV_INTR_MODE_MSI: 1924 synchronize_irq(enic->pdev->irq); 1925 break; 1926 case VNIC_DEV_INTR_MODE_MSIX: 1927 for (i = 0; i < enic->intr_count; i++) 1928 synchronize_irq(enic->msix_entry[i].vector); 1929 break; 1930 default: 1931 break; 1932 } 1933 } 1934 1935 static int enic_dev_notify_set(struct enic *enic) 1936 { 1937 int err; 1938 1939 spin_lock_bh(&enic->devcmd_lock); 1940 switch (vnic_dev_get_intr_mode(enic->vdev)) { 1941 case VNIC_DEV_INTR_MODE_INTX: 1942 err = vnic_dev_notify_set(enic->vdev, ENIC_LEGACY_NOTIFY_INTR); 1943 break; 1944 case VNIC_DEV_INTR_MODE_MSIX: 1945 err = vnic_dev_notify_set(enic->vdev, 1946 enic_msix_notify_intr(enic)); 1947 break; 1948 default: 1949 err = vnic_dev_notify_set(enic->vdev, -1 /* no intr */); 1950 break; 1951 } 1952 spin_unlock_bh(&enic->devcmd_lock); 1953 1954 return err; 1955 } 1956 1957 static void enic_notify_timer_start(struct enic *enic) 1958 { 1959 switch (vnic_dev_get_intr_mode(enic->vdev)) { 1960 case VNIC_DEV_INTR_MODE_MSI: 1961 mod_timer(&enic->notify_timer, jiffies); 1962 break; 1963 default: 1964 /* Using intr for notification for INTx/MSI-X */ 1965 break; 1966 } 1967 } 1968 1969 /* rtnl lock is held, process context */ 1970 static int enic_open(struct net_device *netdev) 1971 { 1972 struct enic *enic = netdev_priv(netdev); 1973 unsigned int i; 1974 int err, ret; 1975 1976 err = enic_request_intr(enic); 1977 if (err) { 1978 netdev_err(netdev, "Unable to request irq.\n"); 1979 return err; 1980 } 1981 enic_init_affinity_hint(enic); 1982 enic_set_affinity_hint(enic); 1983 1984 err = enic_dev_notify_set(enic); 1985 if (err) { 1986 netdev_err(netdev, 1987 "Failed to alloc notify buffer, aborting.\n"); 1988 goto err_out_free_intr; 1989 } 1990 1991 for (i = 0; i < enic->rq_count; i++) { 1992 /* enable rq before updating rq desc */ 1993 vnic_rq_enable(&enic->rq[i].vrq); 1994 vnic_rq_fill(&enic->rq[i].vrq, enic_rq_alloc_buf); 1995 /* Need at least one buffer on ring to get going */ 1996 if (vnic_rq_desc_used(&enic->rq[i].vrq) == 0) { 1997 netdev_err(netdev, "Unable to alloc receive buffers\n"); 1998 err = -ENOMEM; 1999 goto err_out_free_rq; 2000 } 2001 } 2002 2003 for (i = 0; i < enic->wq_count; i++) 2004 vnic_wq_enable(&enic->wq[i].vwq); 2005 2006 if (!enic_is_dynamic(enic) && !enic_is_sriov_vf(enic)) 2007 enic_dev_add_station_addr(enic); 2008 2009 enic_set_rx_mode(netdev); 2010 2011 netif_tx_wake_all_queues(netdev); 2012 2013 for (i = 0; i < enic->rq_count; i++) 2014 napi_enable(&enic->napi[i]); 2015 2016 if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX) 2017 for (i = 0; i < enic->wq_count; i++) 2018 napi_enable(&enic->napi[enic_cq_wq(enic, i)]); 2019 enic_dev_enable(enic); 2020 2021 for (i = 0; i < enic->intr_count; i++) 2022 vnic_intr_unmask(&enic->intr[i]); 2023 2024 enic_notify_timer_start(enic); 2025 enic_rfs_timer_start(enic); 2026 2027 return 0; 2028 2029 err_out_free_rq: 2030 for (i = 0; i < enic->rq_count; i++) { 2031 ret = vnic_rq_disable(&enic->rq[i].vrq); 2032 if (!ret) 2033 vnic_rq_clean(&enic->rq[i].vrq, enic_free_rq_buf); 2034 } 2035 enic_dev_notify_unset(enic); 2036 err_out_free_intr: 2037 enic_unset_affinity_hint(enic); 2038 enic_free_intr(enic); 2039 2040 return err; 2041 } 2042 2043 /* rtnl lock is held, process context */ 2044 static int enic_stop(struct net_device *netdev) 2045 { 2046 struct enic *enic = netdev_priv(netdev); 2047 unsigned int i; 2048 int err; 2049 2050 for (i = 0; i < enic->intr_count; i++) { 2051 vnic_intr_mask(&enic->intr[i]); 2052 (void)vnic_intr_masked(&enic->intr[i]); /* flush write */ 2053 } 2054 2055 enic_synchronize_irqs(enic); 2056 2057 del_timer_sync(&enic->notify_timer); 2058 enic_rfs_flw_tbl_free(enic); 2059 2060 enic_dev_disable(enic); 2061 2062 for (i = 0; i < enic->rq_count; i++) 2063 napi_disable(&enic->napi[i]); 2064 2065 netif_carrier_off(netdev); 2066 if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX) 2067 for (i = 0; i < enic->wq_count; i++) 2068 napi_disable(&enic->napi[enic_cq_wq(enic, i)]); 2069 netif_tx_disable(netdev); 2070 2071 if (!enic_is_dynamic(enic) && !enic_is_sriov_vf(enic)) 2072 enic_dev_del_station_addr(enic); 2073 2074 for (i = 0; i < enic->wq_count; i++) { 2075 err = vnic_wq_disable(&enic->wq[i].vwq); 2076 if (err) 2077 return err; 2078 } 2079 for (i = 0; i < enic->rq_count; i++) { 2080 err = vnic_rq_disable(&enic->rq[i].vrq); 2081 if (err) 2082 return err; 2083 } 2084 2085 enic_dev_notify_unset(enic); 2086 enic_unset_affinity_hint(enic); 2087 enic_free_intr(enic); 2088 2089 for (i = 0; i < enic->wq_count; i++) 2090 vnic_wq_clean(&enic->wq[i].vwq, enic_free_wq_buf); 2091 for (i = 0; i < enic->rq_count; i++) 2092 vnic_rq_clean(&enic->rq[i].vrq, enic_free_rq_buf); 2093 for (i = 0; i < enic->cq_count; i++) 2094 vnic_cq_clean(&enic->cq[i]); 2095 for (i = 0; i < enic->intr_count; i++) 2096 vnic_intr_clean(&enic->intr[i]); 2097 2098 return 0; 2099 } 2100 2101 static int _enic_change_mtu(struct net_device *netdev, int new_mtu) 2102 { 2103 bool running = netif_running(netdev); 2104 int err = 0; 2105 2106 ASSERT_RTNL(); 2107 if (running) { 2108 err = enic_stop(netdev); 2109 if (err) 2110 return err; 2111 } 2112 2113 WRITE_ONCE(netdev->mtu, new_mtu); 2114 2115 if (running) { 2116 err = enic_open(netdev); 2117 if (err) 2118 return err; 2119 } 2120 2121 return 0; 2122 } 2123 2124 static int enic_change_mtu(struct net_device *netdev, int new_mtu) 2125 { 2126 struct enic *enic = netdev_priv(netdev); 2127 2128 if (enic_is_dynamic(enic) || enic_is_sriov_vf(enic)) 2129 return -EOPNOTSUPP; 2130 2131 if (netdev->mtu > enic->port_mtu) 2132 netdev_warn(netdev, 2133 "interface MTU (%d) set higher than port MTU (%d)\n", 2134 netdev->mtu, enic->port_mtu); 2135 2136 return _enic_change_mtu(netdev, new_mtu); 2137 } 2138 2139 static void enic_change_mtu_work(struct work_struct *work) 2140 { 2141 struct enic *enic = container_of(work, struct enic, change_mtu_work); 2142 struct net_device *netdev = enic->netdev; 2143 int new_mtu = vnic_dev_mtu(enic->vdev); 2144 2145 rtnl_lock(); 2146 (void)_enic_change_mtu(netdev, new_mtu); 2147 rtnl_unlock(); 2148 2149 netdev_info(netdev, "interface MTU set as %d\n", netdev->mtu); 2150 } 2151 2152 #ifdef CONFIG_NET_POLL_CONTROLLER 2153 static void enic_poll_controller(struct net_device *netdev) 2154 { 2155 struct enic *enic = netdev_priv(netdev); 2156 struct vnic_dev *vdev = enic->vdev; 2157 unsigned int i, intr; 2158 2159 switch (vnic_dev_get_intr_mode(vdev)) { 2160 case VNIC_DEV_INTR_MODE_MSIX: 2161 for (i = 0; i < enic->rq_count; i++) { 2162 intr = enic_msix_rq_intr(enic, i); 2163 enic_isr_msix(enic->msix_entry[intr].vector, 2164 &enic->napi[i]); 2165 } 2166 2167 for (i = 0; i < enic->wq_count; i++) { 2168 intr = enic_msix_wq_intr(enic, i); 2169 enic_isr_msix(enic->msix_entry[intr].vector, 2170 &enic->napi[enic_cq_wq(enic, i)]); 2171 } 2172 2173 break; 2174 case VNIC_DEV_INTR_MODE_MSI: 2175 enic_isr_msi(enic->pdev->irq, enic); 2176 break; 2177 case VNIC_DEV_INTR_MODE_INTX: 2178 enic_isr_legacy(enic->pdev->irq, netdev); 2179 break; 2180 default: 2181 break; 2182 } 2183 } 2184 #endif 2185 2186 static int enic_dev_wait(struct vnic_dev *vdev, 2187 int (*start)(struct vnic_dev *, int), 2188 int (*finished)(struct vnic_dev *, int *), 2189 int arg) 2190 { 2191 unsigned long time; 2192 int done; 2193 int err; 2194 2195 err = start(vdev, arg); 2196 if (err) 2197 return err; 2198 2199 /* Wait for func to complete...2 seconds max 2200 */ 2201 2202 time = jiffies + (HZ * 2); 2203 do { 2204 2205 err = finished(vdev, &done); 2206 if (err) 2207 return err; 2208 2209 if (done) 2210 return 0; 2211 2212 schedule_timeout_uninterruptible(HZ / 10); 2213 2214 } while (time_after(time, jiffies)); 2215 2216 return -ETIMEDOUT; 2217 } 2218 2219 static int enic_dev_open(struct enic *enic) 2220 { 2221 int err; 2222 u32 flags = CMD_OPENF_IG_DESCCACHE; 2223 2224 err = enic_dev_wait(enic->vdev, vnic_dev_open, 2225 vnic_dev_open_done, flags); 2226 if (err) 2227 dev_err(enic_get_dev(enic), "vNIC device open failed, err %d\n", 2228 err); 2229 2230 return err; 2231 } 2232 2233 static int enic_dev_soft_reset(struct enic *enic) 2234 { 2235 int err; 2236 2237 err = enic_dev_wait(enic->vdev, vnic_dev_soft_reset, 2238 vnic_dev_soft_reset_done, 0); 2239 if (err) 2240 netdev_err(enic->netdev, "vNIC soft reset failed, err %d\n", 2241 err); 2242 2243 return err; 2244 } 2245 2246 static int enic_dev_hang_reset(struct enic *enic) 2247 { 2248 int err; 2249 2250 err = enic_dev_wait(enic->vdev, vnic_dev_hang_reset, 2251 vnic_dev_hang_reset_done, 0); 2252 if (err) 2253 netdev_err(enic->netdev, "vNIC hang reset failed, err %d\n", 2254 err); 2255 2256 return err; 2257 } 2258 2259 int __enic_set_rsskey(struct enic *enic) 2260 { 2261 union vnic_rss_key *rss_key_buf_va; 2262 dma_addr_t rss_key_buf_pa; 2263 int i, kidx, bidx, err; 2264 2265 rss_key_buf_va = dma_alloc_coherent(&enic->pdev->dev, 2266 sizeof(union vnic_rss_key), 2267 &rss_key_buf_pa, GFP_ATOMIC); 2268 if (!rss_key_buf_va) 2269 return -ENOMEM; 2270 2271 for (i = 0; i < ENIC_RSS_LEN; i++) { 2272 kidx = i / ENIC_RSS_BYTES_PER_KEY; 2273 bidx = i % ENIC_RSS_BYTES_PER_KEY; 2274 rss_key_buf_va->key[kidx].b[bidx] = enic->rss_key[i]; 2275 } 2276 spin_lock_bh(&enic->devcmd_lock); 2277 err = enic_set_rss_key(enic, 2278 rss_key_buf_pa, 2279 sizeof(union vnic_rss_key)); 2280 spin_unlock_bh(&enic->devcmd_lock); 2281 2282 dma_free_coherent(&enic->pdev->dev, sizeof(union vnic_rss_key), 2283 rss_key_buf_va, rss_key_buf_pa); 2284 2285 return err; 2286 } 2287 2288 static int enic_set_rsskey(struct enic *enic) 2289 { 2290 netdev_rss_key_fill(enic->rss_key, ENIC_RSS_LEN); 2291 2292 return __enic_set_rsskey(enic); 2293 } 2294 2295 static int enic_set_rsscpu(struct enic *enic, u8 rss_hash_bits) 2296 { 2297 dma_addr_t rss_cpu_buf_pa; 2298 union vnic_rss_cpu *rss_cpu_buf_va = NULL; 2299 unsigned int i; 2300 int err; 2301 2302 rss_cpu_buf_va = dma_alloc_coherent(&enic->pdev->dev, 2303 sizeof(union vnic_rss_cpu), 2304 &rss_cpu_buf_pa, GFP_ATOMIC); 2305 if (!rss_cpu_buf_va) 2306 return -ENOMEM; 2307 2308 for (i = 0; i < (1 << rss_hash_bits); i++) 2309 (*rss_cpu_buf_va).cpu[i/4].b[i%4] = i % enic->rq_count; 2310 2311 spin_lock_bh(&enic->devcmd_lock); 2312 err = enic_set_rss_cpu(enic, 2313 rss_cpu_buf_pa, 2314 sizeof(union vnic_rss_cpu)); 2315 spin_unlock_bh(&enic->devcmd_lock); 2316 2317 dma_free_coherent(&enic->pdev->dev, sizeof(union vnic_rss_cpu), 2318 rss_cpu_buf_va, rss_cpu_buf_pa); 2319 2320 return err; 2321 } 2322 2323 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu, 2324 u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable) 2325 { 2326 const u8 tso_ipid_split_en = 0; 2327 const u8 ig_vlan_strip_en = 1; 2328 int err; 2329 2330 /* Enable VLAN tag stripping. 2331 */ 2332 2333 spin_lock_bh(&enic->devcmd_lock); 2334 err = enic_set_nic_cfg(enic, 2335 rss_default_cpu, rss_hash_type, 2336 rss_hash_bits, rss_base_cpu, 2337 rss_enable, tso_ipid_split_en, 2338 ig_vlan_strip_en); 2339 spin_unlock_bh(&enic->devcmd_lock); 2340 2341 return err; 2342 } 2343 2344 static int enic_set_rss_nic_cfg(struct enic *enic) 2345 { 2346 struct device *dev = enic_get_dev(enic); 2347 const u8 rss_default_cpu = 0; 2348 const u8 rss_hash_bits = 7; 2349 const u8 rss_base_cpu = 0; 2350 u8 rss_hash_type; 2351 int res; 2352 u8 rss_enable = ENIC_SETTING(enic, RSS) && (enic->rq_count > 1); 2353 2354 spin_lock_bh(&enic->devcmd_lock); 2355 res = vnic_dev_capable_rss_hash_type(enic->vdev, &rss_hash_type); 2356 spin_unlock_bh(&enic->devcmd_lock); 2357 if (res) { 2358 /* defaults for old adapters 2359 */ 2360 rss_hash_type = NIC_CFG_RSS_HASH_TYPE_IPV4 | 2361 NIC_CFG_RSS_HASH_TYPE_TCP_IPV4 | 2362 NIC_CFG_RSS_HASH_TYPE_IPV6 | 2363 NIC_CFG_RSS_HASH_TYPE_TCP_IPV6; 2364 } 2365 2366 if (rss_enable) { 2367 if (!enic_set_rsskey(enic)) { 2368 if (enic_set_rsscpu(enic, rss_hash_bits)) { 2369 rss_enable = 0; 2370 dev_warn(dev, "RSS disabled, " 2371 "Failed to set RSS cpu indirection table."); 2372 } 2373 } else { 2374 rss_enable = 0; 2375 dev_warn(dev, "RSS disabled, Failed to set RSS key.\n"); 2376 } 2377 } 2378 2379 return enic_set_niccfg(enic, rss_default_cpu, rss_hash_type, 2380 rss_hash_bits, rss_base_cpu, rss_enable); 2381 } 2382 2383 static void enic_set_api_busy(struct enic *enic, bool busy) 2384 { 2385 spin_lock(&enic->enic_api_lock); 2386 enic->enic_api_busy = busy; 2387 spin_unlock(&enic->enic_api_lock); 2388 } 2389 2390 static void enic_reset(struct work_struct *work) 2391 { 2392 struct enic *enic = container_of(work, struct enic, reset); 2393 2394 if (!netif_running(enic->netdev)) 2395 return; 2396 2397 rtnl_lock(); 2398 2399 /* Stop any activity from infiniband */ 2400 enic_set_api_busy(enic, true); 2401 2402 enic_stop(enic->netdev); 2403 enic_dev_soft_reset(enic); 2404 enic_reset_addr_lists(enic); 2405 enic_init_vnic_resources(enic); 2406 enic_set_rss_nic_cfg(enic); 2407 enic_dev_set_ig_vlan_rewrite_mode(enic); 2408 enic_open(enic->netdev); 2409 2410 /* Allow infiniband to fiddle with the device again */ 2411 enic_set_api_busy(enic, false); 2412 2413 call_netdevice_notifiers(NETDEV_REBOOT, enic->netdev); 2414 2415 rtnl_unlock(); 2416 } 2417 2418 static void enic_tx_hang_reset(struct work_struct *work) 2419 { 2420 struct enic *enic = container_of(work, struct enic, tx_hang_reset); 2421 2422 rtnl_lock(); 2423 2424 /* Stop any activity from infiniband */ 2425 enic_set_api_busy(enic, true); 2426 2427 enic_dev_hang_notify(enic); 2428 enic_stop(enic->netdev); 2429 enic_dev_hang_reset(enic); 2430 enic_reset_addr_lists(enic); 2431 enic_init_vnic_resources(enic); 2432 enic_set_rss_nic_cfg(enic); 2433 enic_dev_set_ig_vlan_rewrite_mode(enic); 2434 enic_open(enic->netdev); 2435 2436 /* Allow infiniband to fiddle with the device again */ 2437 enic_set_api_busy(enic, false); 2438 2439 call_netdevice_notifiers(NETDEV_REBOOT, enic->netdev); 2440 2441 rtnl_unlock(); 2442 } 2443 2444 static int enic_set_intr_mode(struct enic *enic) 2445 { 2446 unsigned int i; 2447 int num_intr; 2448 2449 /* Set interrupt mode (INTx, MSI, MSI-X) depending 2450 * on system capabilities. 2451 * 2452 * Try MSI-X first 2453 */ 2454 2455 if (enic->config.intr_mode < 1 && 2456 enic->intr_avail >= ENIC_MSIX_MIN_INTR) { 2457 for (i = 0; i < enic->intr_avail; i++) 2458 enic->msix_entry[i].entry = i; 2459 2460 num_intr = pci_enable_msix_range(enic->pdev, enic->msix_entry, 2461 ENIC_MSIX_MIN_INTR, 2462 enic->intr_avail); 2463 if (num_intr > 0) { 2464 vnic_dev_set_intr_mode(enic->vdev, 2465 VNIC_DEV_INTR_MODE_MSIX); 2466 enic->intr_avail = num_intr; 2467 return 0; 2468 } 2469 } 2470 2471 /* Next try MSI 2472 * 2473 * We need 1 INTR 2474 */ 2475 2476 if (enic->config.intr_mode < 2 && 2477 enic->intr_avail >= 1 && 2478 !pci_enable_msi(enic->pdev)) { 2479 enic->intr_avail = 1; 2480 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSI); 2481 return 0; 2482 } 2483 2484 /* Next try INTx 2485 * 2486 * We need 3 INTRs 2487 * (the first INTR is used for WQ/RQ) 2488 * (the second INTR is used for WQ/RQ errors) 2489 * (the last INTR is used for notifications) 2490 */ 2491 2492 if (enic->config.intr_mode < 3 && 2493 enic->intr_avail >= 3) { 2494 enic->intr_avail = 3; 2495 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_INTX); 2496 return 0; 2497 } 2498 2499 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_UNKNOWN); 2500 2501 return -EINVAL; 2502 } 2503 2504 static void enic_clear_intr_mode(struct enic *enic) 2505 { 2506 switch (vnic_dev_get_intr_mode(enic->vdev)) { 2507 case VNIC_DEV_INTR_MODE_MSIX: 2508 pci_disable_msix(enic->pdev); 2509 break; 2510 case VNIC_DEV_INTR_MODE_MSI: 2511 pci_disable_msi(enic->pdev); 2512 break; 2513 default: 2514 break; 2515 } 2516 2517 vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_UNKNOWN); 2518 } 2519 2520 static int enic_adjust_resources(struct enic *enic) 2521 { 2522 unsigned int max_queues; 2523 unsigned int rq_default; 2524 unsigned int rq_avail; 2525 unsigned int wq_avail; 2526 2527 if (enic->rq_avail < 1 || enic->wq_avail < 1 || enic->cq_avail < 2) { 2528 dev_err(enic_get_dev(enic), 2529 "Not enough resources available rq: %d wq: %d cq: %d\n", 2530 enic->rq_avail, enic->wq_avail, 2531 enic->cq_avail); 2532 return -ENOSPC; 2533 } 2534 2535 if (is_kdump_kernel()) { 2536 dev_info(enic_get_dev(enic), "Running from within kdump kernel. Using minimal resources\n"); 2537 enic->rq_avail = 1; 2538 enic->wq_avail = 1; 2539 enic->config.rq_desc_count = ENIC_MIN_RQ_DESCS; 2540 enic->config.wq_desc_count = ENIC_MIN_WQ_DESCS; 2541 enic->config.mtu = min_t(u16, 1500, enic->config.mtu); 2542 } 2543 2544 /* if RSS isn't set, then we can only use one RQ */ 2545 if (!ENIC_SETTING(enic, RSS)) 2546 enic->rq_avail = 1; 2547 2548 switch (vnic_dev_get_intr_mode(enic->vdev)) { 2549 case VNIC_DEV_INTR_MODE_INTX: 2550 case VNIC_DEV_INTR_MODE_MSI: 2551 enic->rq_count = 1; 2552 enic->wq_count = 1; 2553 enic->cq_count = 2; 2554 enic->intr_count = enic->intr_avail; 2555 break; 2556 case VNIC_DEV_INTR_MODE_MSIX: 2557 /* Adjust the number of wqs/rqs/cqs/interrupts that will be 2558 * used based on which resource is the most constrained 2559 */ 2560 wq_avail = min(enic->wq_avail, ENIC_WQ_MAX); 2561 rq_default = netif_get_num_default_rss_queues(); 2562 rq_avail = min3(enic->rq_avail, ENIC_RQ_MAX, rq_default); 2563 max_queues = min(enic->cq_avail, 2564 enic->intr_avail - ENIC_MSIX_RESERVED_INTR); 2565 if (wq_avail + rq_avail <= max_queues) { 2566 enic->rq_count = rq_avail; 2567 enic->wq_count = wq_avail; 2568 } else { 2569 /* recalculate wq/rq count */ 2570 if (rq_avail < wq_avail) { 2571 enic->rq_count = min(rq_avail, max_queues / 2); 2572 enic->wq_count = max_queues - enic->rq_count; 2573 } else { 2574 enic->wq_count = min(wq_avail, max_queues / 2); 2575 enic->rq_count = max_queues - enic->wq_count; 2576 } 2577 } 2578 enic->cq_count = enic->rq_count + enic->wq_count; 2579 enic->intr_count = enic->cq_count + ENIC_MSIX_RESERVED_INTR; 2580 2581 break; 2582 default: 2583 dev_err(enic_get_dev(enic), "Unknown interrupt mode\n"); 2584 return -EINVAL; 2585 } 2586 2587 return 0; 2588 } 2589 2590 static void enic_get_queue_stats_rx(struct net_device *dev, int idx, 2591 struct netdev_queue_stats_rx *rxs) 2592 { 2593 struct enic *enic = netdev_priv(dev); 2594 struct enic_rq_stats *rqstats = &enic->rq[idx].stats; 2595 2596 rxs->bytes = rqstats->bytes; 2597 rxs->packets = rqstats->packets; 2598 rxs->hw_drops = rqstats->bad_fcs + rqstats->pkt_truncated; 2599 rxs->hw_drop_overruns = rqstats->pkt_truncated; 2600 rxs->csum_unnecessary = rqstats->csum_unnecessary + 2601 rqstats->csum_unnecessary_encap; 2602 } 2603 2604 static void enic_get_queue_stats_tx(struct net_device *dev, int idx, 2605 struct netdev_queue_stats_tx *txs) 2606 { 2607 struct enic *enic = netdev_priv(dev); 2608 struct enic_wq_stats *wqstats = &enic->wq[idx].stats; 2609 2610 txs->bytes = wqstats->bytes; 2611 txs->packets = wqstats->packets; 2612 txs->csum_none = wqstats->csum_none; 2613 txs->needs_csum = wqstats->csum_partial + wqstats->encap_csum + 2614 wqstats->tso; 2615 txs->hw_gso_packets = wqstats->tso; 2616 txs->stop = wqstats->stopped; 2617 txs->wake = wqstats->wake; 2618 } 2619 2620 static void enic_get_base_stats(struct net_device *dev, 2621 struct netdev_queue_stats_rx *rxs, 2622 struct netdev_queue_stats_tx *txs) 2623 { 2624 rxs->bytes = 0; 2625 rxs->packets = 0; 2626 rxs->hw_drops = 0; 2627 rxs->hw_drop_overruns = 0; 2628 rxs->csum_unnecessary = 0; 2629 txs->bytes = 0; 2630 txs->packets = 0; 2631 txs->csum_none = 0; 2632 txs->needs_csum = 0; 2633 txs->hw_gso_packets = 0; 2634 txs->stop = 0; 2635 txs->wake = 0; 2636 } 2637 2638 static const struct net_device_ops enic_netdev_dynamic_ops = { 2639 .ndo_open = enic_open, 2640 .ndo_stop = enic_stop, 2641 .ndo_start_xmit = enic_hard_start_xmit, 2642 .ndo_get_stats64 = enic_get_stats, 2643 .ndo_validate_addr = eth_validate_addr, 2644 .ndo_set_rx_mode = enic_set_rx_mode, 2645 .ndo_set_mac_address = enic_set_mac_address_dynamic, 2646 .ndo_change_mtu = enic_change_mtu, 2647 .ndo_vlan_rx_add_vid = enic_vlan_rx_add_vid, 2648 .ndo_vlan_rx_kill_vid = enic_vlan_rx_kill_vid, 2649 .ndo_tx_timeout = enic_tx_timeout, 2650 .ndo_set_vf_port = enic_set_vf_port, 2651 .ndo_get_vf_port = enic_get_vf_port, 2652 .ndo_set_vf_mac = enic_set_vf_mac, 2653 #ifdef CONFIG_NET_POLL_CONTROLLER 2654 .ndo_poll_controller = enic_poll_controller, 2655 #endif 2656 #ifdef CONFIG_RFS_ACCEL 2657 .ndo_rx_flow_steer = enic_rx_flow_steer, 2658 #endif 2659 .ndo_features_check = enic_features_check, 2660 }; 2661 2662 static const struct net_device_ops enic_netdev_ops = { 2663 .ndo_open = enic_open, 2664 .ndo_stop = enic_stop, 2665 .ndo_start_xmit = enic_hard_start_xmit, 2666 .ndo_get_stats64 = enic_get_stats, 2667 .ndo_validate_addr = eth_validate_addr, 2668 .ndo_set_mac_address = enic_set_mac_address, 2669 .ndo_set_rx_mode = enic_set_rx_mode, 2670 .ndo_change_mtu = enic_change_mtu, 2671 .ndo_vlan_rx_add_vid = enic_vlan_rx_add_vid, 2672 .ndo_vlan_rx_kill_vid = enic_vlan_rx_kill_vid, 2673 .ndo_tx_timeout = enic_tx_timeout, 2674 .ndo_set_vf_port = enic_set_vf_port, 2675 .ndo_get_vf_port = enic_get_vf_port, 2676 .ndo_set_vf_mac = enic_set_vf_mac, 2677 #ifdef CONFIG_NET_POLL_CONTROLLER 2678 .ndo_poll_controller = enic_poll_controller, 2679 #endif 2680 #ifdef CONFIG_RFS_ACCEL 2681 .ndo_rx_flow_steer = enic_rx_flow_steer, 2682 #endif 2683 .ndo_features_check = enic_features_check, 2684 }; 2685 2686 static const struct netdev_stat_ops enic_netdev_stat_ops = { 2687 .get_queue_stats_rx = enic_get_queue_stats_rx, 2688 .get_queue_stats_tx = enic_get_queue_stats_tx, 2689 .get_base_stats = enic_get_base_stats, 2690 }; 2691 2692 static void enic_free_enic_resources(struct enic *enic) 2693 { 2694 kfree(enic->wq); 2695 enic->wq = NULL; 2696 2697 kfree(enic->rq); 2698 enic->rq = NULL; 2699 2700 kfree(enic->cq); 2701 enic->cq = NULL; 2702 2703 kfree(enic->napi); 2704 enic->napi = NULL; 2705 2706 kfree(enic->msix_entry); 2707 enic->msix_entry = NULL; 2708 2709 kfree(enic->msix); 2710 enic->msix = NULL; 2711 2712 kfree(enic->intr); 2713 enic->intr = NULL; 2714 } 2715 2716 static int enic_alloc_enic_resources(struct enic *enic) 2717 { 2718 enic->wq = kcalloc(enic->wq_avail, sizeof(struct enic_wq), GFP_KERNEL); 2719 if (!enic->wq) 2720 goto free_queues; 2721 2722 enic->rq = kcalloc(enic->rq_avail, sizeof(struct enic_rq), GFP_KERNEL); 2723 if (!enic->rq) 2724 goto free_queues; 2725 2726 enic->cq = kcalloc(enic->cq_avail, sizeof(struct vnic_cq), GFP_KERNEL); 2727 if (!enic->cq) 2728 goto free_queues; 2729 2730 enic->napi = kcalloc(enic->wq_avail + enic->rq_avail, 2731 sizeof(struct napi_struct), GFP_KERNEL); 2732 if (!enic->napi) 2733 goto free_queues; 2734 2735 enic->msix_entry = kcalloc(enic->intr_avail, sizeof(struct msix_entry), 2736 GFP_KERNEL); 2737 if (!enic->msix_entry) 2738 goto free_queues; 2739 2740 enic->msix = kcalloc(enic->intr_avail, sizeof(struct enic_msix_entry), 2741 GFP_KERNEL); 2742 if (!enic->msix) 2743 goto free_queues; 2744 2745 enic->intr = kcalloc(enic->intr_avail, sizeof(struct vnic_intr), 2746 GFP_KERNEL); 2747 if (!enic->intr) 2748 goto free_queues; 2749 2750 return 0; 2751 2752 free_queues: 2753 enic_free_enic_resources(enic); 2754 return -ENOMEM; 2755 } 2756 2757 static void enic_dev_deinit(struct enic *enic) 2758 { 2759 unsigned int i; 2760 2761 for (i = 0; i < enic->rq_count; i++) 2762 __netif_napi_del(&enic->napi[i]); 2763 2764 if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX) 2765 for (i = 0; i < enic->wq_count; i++) 2766 __netif_napi_del(&enic->napi[enic_cq_wq(enic, i)]); 2767 2768 /* observe RCU grace period after __netif_napi_del() calls */ 2769 synchronize_net(); 2770 2771 enic_free_vnic_resources(enic); 2772 enic_clear_intr_mode(enic); 2773 enic_free_affinity_hint(enic); 2774 enic_free_enic_resources(enic); 2775 } 2776 2777 static int enic_dev_init(struct enic *enic) 2778 { 2779 struct device *dev = enic_get_dev(enic); 2780 struct net_device *netdev = enic->netdev; 2781 unsigned int i; 2782 int err; 2783 2784 /* Get interrupt coalesce timer info */ 2785 err = enic_dev_intr_coal_timer_info(enic); 2786 if (err) { 2787 dev_warn(dev, "Using default conversion factor for " 2788 "interrupt coalesce timer\n"); 2789 vnic_dev_intr_coal_timer_info_default(enic->vdev); 2790 } 2791 2792 /* Get vNIC configuration 2793 */ 2794 2795 err = enic_get_vnic_config(enic); 2796 if (err) { 2797 dev_err(dev, "Get vNIC configuration failed, aborting\n"); 2798 return err; 2799 } 2800 2801 /* Get available resource counts 2802 */ 2803 2804 enic_get_res_counts(enic); 2805 2806 err = enic_alloc_enic_resources(enic); 2807 if (err) { 2808 dev_err(dev, "Failed to allocate enic resources\n"); 2809 return err; 2810 } 2811 2812 /* Set interrupt mode based on system capabilities */ 2813 2814 err = enic_set_intr_mode(enic); 2815 if (err) { 2816 dev_err(dev, "Failed to set intr mode based on resource " 2817 "counts and system capabilities, aborting\n"); 2818 goto err_out_free_vnic_resources; 2819 } 2820 2821 /* Adjust resource counts based on most constrained resources */ 2822 err = enic_adjust_resources(enic); 2823 if (err) { 2824 dev_err(dev, "Failed to adjust resources\n"); 2825 goto err_out_free_vnic_resources; 2826 } 2827 2828 /* Allocate and configure vNIC resources 2829 */ 2830 2831 err = enic_alloc_vnic_resources(enic); 2832 if (err) { 2833 dev_err(dev, "Failed to alloc vNIC resources, aborting\n"); 2834 goto err_out_free_vnic_resources; 2835 } 2836 2837 enic_init_vnic_resources(enic); 2838 2839 err = enic_set_rss_nic_cfg(enic); 2840 if (err) { 2841 dev_err(dev, "Failed to config nic, aborting\n"); 2842 goto err_out_free_vnic_resources; 2843 } 2844 2845 switch (vnic_dev_get_intr_mode(enic->vdev)) { 2846 default: 2847 netif_napi_add(netdev, &enic->napi[0], enic_poll); 2848 break; 2849 case VNIC_DEV_INTR_MODE_MSIX: 2850 for (i = 0; i < enic->rq_count; i++) { 2851 netif_napi_add(netdev, &enic->napi[i], 2852 enic_poll_msix_rq); 2853 } 2854 for (i = 0; i < enic->wq_count; i++) 2855 netif_napi_add(netdev, 2856 &enic->napi[enic_cq_wq(enic, i)], 2857 enic_poll_msix_wq); 2858 break; 2859 } 2860 2861 return 0; 2862 2863 err_out_free_vnic_resources: 2864 enic_free_affinity_hint(enic); 2865 enic_clear_intr_mode(enic); 2866 enic_free_vnic_resources(enic); 2867 enic_free_enic_resources(enic); 2868 2869 return err; 2870 } 2871 2872 static void enic_iounmap(struct enic *enic) 2873 { 2874 unsigned int i; 2875 2876 for (i = 0; i < ARRAY_SIZE(enic->bar); i++) 2877 if (enic->bar[i].vaddr) 2878 iounmap(enic->bar[i].vaddr); 2879 } 2880 2881 static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 2882 { 2883 struct device *dev = &pdev->dev; 2884 struct net_device *netdev; 2885 struct enic *enic; 2886 int using_dac = 0; 2887 unsigned int i; 2888 int err; 2889 #ifdef CONFIG_PCI_IOV 2890 int pos = 0; 2891 #endif 2892 int num_pps = 1; 2893 2894 /* Allocate net device structure and initialize. Private 2895 * instance data is initialized to zero. 2896 */ 2897 2898 netdev = alloc_etherdev_mqs(sizeof(struct enic), 2899 ENIC_RQ_MAX, ENIC_WQ_MAX); 2900 if (!netdev) 2901 return -ENOMEM; 2902 2903 pci_set_drvdata(pdev, netdev); 2904 2905 SET_NETDEV_DEV(netdev, &pdev->dev); 2906 2907 enic = netdev_priv(netdev); 2908 enic->netdev = netdev; 2909 enic->pdev = pdev; 2910 2911 /* Setup PCI resources 2912 */ 2913 2914 err = pci_enable_device_mem(pdev); 2915 if (err) { 2916 dev_err(dev, "Cannot enable PCI device, aborting\n"); 2917 goto err_out_free_netdev; 2918 } 2919 2920 err = pci_request_regions(pdev, DRV_NAME); 2921 if (err) { 2922 dev_err(dev, "Cannot request PCI regions, aborting\n"); 2923 goto err_out_disable_device; 2924 } 2925 2926 pci_set_master(pdev); 2927 2928 /* Query PCI controller on system for DMA addressing 2929 * limitation for the device. Try 47-bit first, and 2930 * fail to 32-bit. 2931 */ 2932 2933 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(47)); 2934 if (err) { 2935 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 2936 if (err) { 2937 dev_err(dev, "No usable DMA configuration, aborting\n"); 2938 goto err_out_release_regions; 2939 } 2940 } else { 2941 using_dac = 1; 2942 } 2943 2944 /* Map vNIC resources from BAR0-5 2945 */ 2946 2947 for (i = 0; i < ARRAY_SIZE(enic->bar); i++) { 2948 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 2949 continue; 2950 enic->bar[i].len = pci_resource_len(pdev, i); 2951 enic->bar[i].vaddr = pci_iomap(pdev, i, enic->bar[i].len); 2952 if (!enic->bar[i].vaddr) { 2953 dev_err(dev, "Cannot memory-map BAR %d, aborting\n", i); 2954 err = -ENODEV; 2955 goto err_out_iounmap; 2956 } 2957 enic->bar[i].bus_addr = pci_resource_start(pdev, i); 2958 } 2959 2960 /* Register vNIC device 2961 */ 2962 2963 enic->vdev = vnic_dev_register(NULL, enic, pdev, enic->bar, 2964 ARRAY_SIZE(enic->bar)); 2965 if (!enic->vdev) { 2966 dev_err(dev, "vNIC registration failed, aborting\n"); 2967 err = -ENODEV; 2968 goto err_out_iounmap; 2969 } 2970 2971 err = vnic_devcmd_init(enic->vdev); 2972 2973 if (err) 2974 goto err_out_vnic_unregister; 2975 2976 #ifdef CONFIG_PCI_IOV 2977 /* Get number of subvnics */ 2978 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); 2979 if (pos) { 2980 pci_read_config_word(pdev, pos + PCI_SRIOV_TOTAL_VF, 2981 &enic->num_vfs); 2982 if (enic->num_vfs) { 2983 err = pci_enable_sriov(pdev, enic->num_vfs); 2984 if (err) { 2985 dev_err(dev, "SRIOV enable failed, aborting." 2986 " pci_enable_sriov() returned %d\n", 2987 err); 2988 goto err_out_vnic_unregister; 2989 } 2990 enic->priv_flags |= ENIC_SRIOV_ENABLED; 2991 num_pps = enic->num_vfs; 2992 } 2993 } 2994 #endif 2995 2996 /* Allocate structure for port profiles */ 2997 enic->pp = kcalloc(num_pps, sizeof(*enic->pp), GFP_KERNEL); 2998 if (!enic->pp) { 2999 err = -ENOMEM; 3000 goto err_out_disable_sriov_pp; 3001 } 3002 3003 /* Issue device open to get device in known state 3004 */ 3005 3006 err = enic_dev_open(enic); 3007 if (err) { 3008 dev_err(dev, "vNIC dev open failed, aborting\n"); 3009 goto err_out_disable_sriov; 3010 } 3011 3012 /* Setup devcmd lock 3013 */ 3014 3015 spin_lock_init(&enic->devcmd_lock); 3016 spin_lock_init(&enic->enic_api_lock); 3017 3018 /* 3019 * Set ingress vlan rewrite mode before vnic initialization 3020 */ 3021 3022 err = enic_dev_set_ig_vlan_rewrite_mode(enic); 3023 if (err) { 3024 dev_err(dev, 3025 "Failed to set ingress vlan rewrite mode, aborting.\n"); 3026 goto err_out_dev_close; 3027 } 3028 3029 /* Issue device init to initialize the vnic-to-switch link. 3030 * We'll start with carrier off and wait for link UP 3031 * notification later to turn on carrier. We don't need 3032 * to wait here for the vnic-to-switch link initialization 3033 * to complete; link UP notification is the indication that 3034 * the process is complete. 3035 */ 3036 3037 netif_carrier_off(netdev); 3038 3039 /* Do not call dev_init for a dynamic vnic. 3040 * For a dynamic vnic, init_prov_info will be 3041 * called later by an upper layer. 3042 */ 3043 3044 if (!enic_is_dynamic(enic)) { 3045 err = vnic_dev_init(enic->vdev, 0); 3046 if (err) { 3047 dev_err(dev, "vNIC dev init failed, aborting\n"); 3048 goto err_out_dev_close; 3049 } 3050 } 3051 3052 err = enic_dev_init(enic); 3053 if (err) { 3054 dev_err(dev, "Device initialization failed, aborting\n"); 3055 goto err_out_dev_close; 3056 } 3057 3058 netif_set_real_num_tx_queues(netdev, enic->wq_count); 3059 netif_set_real_num_rx_queues(netdev, enic->rq_count); 3060 3061 /* Setup notification timer, HW reset task, and wq locks 3062 */ 3063 3064 timer_setup(&enic->notify_timer, enic_notify_timer, 0); 3065 3066 enic_rfs_flw_tbl_init(enic); 3067 INIT_WORK(&enic->reset, enic_reset); 3068 INIT_WORK(&enic->tx_hang_reset, enic_tx_hang_reset); 3069 INIT_WORK(&enic->change_mtu_work, enic_change_mtu_work); 3070 3071 for (i = 0; i < enic->wq_count; i++) 3072 spin_lock_init(&enic->wq[i].lock); 3073 3074 /* Register net device 3075 */ 3076 3077 enic->port_mtu = enic->config.mtu; 3078 3079 err = enic_set_mac_addr(netdev, enic->mac_addr); 3080 if (err) { 3081 dev_err(dev, "Invalid MAC address, aborting\n"); 3082 goto err_out_dev_deinit; 3083 } 3084 3085 enic->tx_coalesce_usecs = enic->config.intr_timer_usec; 3086 /* rx coalesce time already got initialized. This gets used 3087 * if adaptive coal is turned off 3088 */ 3089 enic->rx_coalesce_usecs = enic->tx_coalesce_usecs; 3090 3091 if (enic_is_dynamic(enic) || enic_is_sriov_vf(enic)) 3092 netdev->netdev_ops = &enic_netdev_dynamic_ops; 3093 else 3094 netdev->netdev_ops = &enic_netdev_ops; 3095 netdev->stat_ops = &enic_netdev_stat_ops; 3096 3097 netdev->watchdog_timeo = 2 * HZ; 3098 enic_set_ethtool_ops(netdev); 3099 3100 netdev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 3101 if (ENIC_SETTING(enic, LOOP)) { 3102 netdev->features &= ~NETIF_F_HW_VLAN_CTAG_TX; 3103 enic->loop_enable = 1; 3104 enic->loop_tag = enic->config.loop_tag; 3105 dev_info(dev, "loopback tag=0x%04x\n", enic->loop_tag); 3106 } 3107 if (ENIC_SETTING(enic, TXCSUM)) 3108 netdev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM; 3109 if (ENIC_SETTING(enic, TSO)) 3110 netdev->hw_features |= NETIF_F_TSO | 3111 NETIF_F_TSO6 | NETIF_F_TSO_ECN; 3112 if (ENIC_SETTING(enic, RSS)) 3113 netdev->hw_features |= NETIF_F_RXHASH; 3114 if (ENIC_SETTING(enic, RXCSUM)) 3115 netdev->hw_features |= NETIF_F_RXCSUM; 3116 if (ENIC_SETTING(enic, VXLAN)) { 3117 u64 patch_level; 3118 u64 a1 = 0; 3119 3120 netdev->hw_enc_features |= NETIF_F_RXCSUM | 3121 NETIF_F_TSO | 3122 NETIF_F_TSO6 | 3123 NETIF_F_TSO_ECN | 3124 NETIF_F_GSO_UDP_TUNNEL | 3125 NETIF_F_HW_CSUM | 3126 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3127 netdev->hw_features |= netdev->hw_enc_features; 3128 /* get bit mask from hw about supported offload bit level 3129 * BIT(0) = fw supports patch_level 0 3130 * fcoe bit = encap 3131 * fcoe_fc_crc_ok = outer csum ok 3132 * BIT(1) = always set by fw 3133 * BIT(2) = fw supports patch_level 2 3134 * BIT(0) in rss_hash = encap 3135 * BIT(1,2) in rss_hash = outer_ip_csum_ok/ 3136 * outer_tcp_csum_ok 3137 * used in enic_rq_indicate_buf 3138 */ 3139 err = vnic_dev_get_supported_feature_ver(enic->vdev, 3140 VIC_FEATURE_VXLAN, 3141 &patch_level, &a1); 3142 if (err) 3143 patch_level = 0; 3144 enic->vxlan.flags = (u8)a1; 3145 /* mask bits that are supported by driver 3146 */ 3147 patch_level &= BIT_ULL(0) | BIT_ULL(2); 3148 patch_level = fls(patch_level); 3149 patch_level = patch_level ? patch_level - 1 : 0; 3150 enic->vxlan.patch_level = patch_level; 3151 3152 if (vnic_dev_get_res_count(enic->vdev, RES_TYPE_WQ) == 1 || 3153 enic->vxlan.flags & ENIC_VXLAN_MULTI_WQ) { 3154 netdev->udp_tunnel_nic_info = &enic_udp_tunnels_v4; 3155 if (enic->vxlan.flags & ENIC_VXLAN_OUTER_IPV6) 3156 netdev->udp_tunnel_nic_info = &enic_udp_tunnels; 3157 } 3158 } 3159 3160 netdev->features |= netdev->hw_features; 3161 netdev->vlan_features |= netdev->features; 3162 3163 #ifdef CONFIG_RFS_ACCEL 3164 netdev->hw_features |= NETIF_F_NTUPLE; 3165 #endif 3166 3167 if (using_dac) 3168 netdev->features |= NETIF_F_HIGHDMA; 3169 3170 netdev->priv_flags |= IFF_UNICAST_FLT; 3171 3172 /* MTU range: 68 - 9000 */ 3173 netdev->min_mtu = ENIC_MIN_MTU; 3174 netdev->max_mtu = ENIC_MAX_MTU; 3175 netdev->mtu = enic->port_mtu; 3176 3177 err = register_netdev(netdev); 3178 if (err) { 3179 dev_err(dev, "Cannot register net device, aborting\n"); 3180 goto err_out_dev_deinit; 3181 } 3182 enic->rx_copybreak = RX_COPYBREAK_DEFAULT; 3183 3184 return 0; 3185 3186 err_out_dev_deinit: 3187 enic_dev_deinit(enic); 3188 err_out_dev_close: 3189 vnic_dev_close(enic->vdev); 3190 err_out_disable_sriov: 3191 kfree(enic->pp); 3192 err_out_disable_sriov_pp: 3193 #ifdef CONFIG_PCI_IOV 3194 if (enic_sriov_enabled(enic)) { 3195 pci_disable_sriov(pdev); 3196 enic->priv_flags &= ~ENIC_SRIOV_ENABLED; 3197 } 3198 #endif 3199 err_out_vnic_unregister: 3200 vnic_dev_unregister(enic->vdev); 3201 err_out_iounmap: 3202 enic_iounmap(enic); 3203 err_out_release_regions: 3204 pci_release_regions(pdev); 3205 err_out_disable_device: 3206 pci_disable_device(pdev); 3207 err_out_free_netdev: 3208 free_netdev(netdev); 3209 3210 return err; 3211 } 3212 3213 static void enic_remove(struct pci_dev *pdev) 3214 { 3215 struct net_device *netdev = pci_get_drvdata(pdev); 3216 3217 if (netdev) { 3218 struct enic *enic = netdev_priv(netdev); 3219 3220 cancel_work_sync(&enic->reset); 3221 cancel_work_sync(&enic->change_mtu_work); 3222 unregister_netdev(netdev); 3223 enic_dev_deinit(enic); 3224 vnic_dev_close(enic->vdev); 3225 #ifdef CONFIG_PCI_IOV 3226 if (enic_sriov_enabled(enic)) { 3227 pci_disable_sriov(pdev); 3228 enic->priv_flags &= ~ENIC_SRIOV_ENABLED; 3229 } 3230 #endif 3231 kfree(enic->pp); 3232 vnic_dev_unregister(enic->vdev); 3233 enic_iounmap(enic); 3234 pci_release_regions(pdev); 3235 pci_disable_device(pdev); 3236 free_netdev(netdev); 3237 } 3238 } 3239 3240 static struct pci_driver enic_driver = { 3241 .name = DRV_NAME, 3242 .id_table = enic_id_table, 3243 .probe = enic_probe, 3244 .remove = enic_remove, 3245 }; 3246 3247 module_pci_driver(enic_driver); 3248