1 /* 2 * Copyright (C) 2005 - 2016 Broadcom 3 * All rights reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License version 2 7 * as published by the Free Software Foundation. The full GNU General 8 * Public License is included in this distribution in the file called COPYING. 9 * 10 * Contact Information: 11 * linux-drivers@emulex.com 12 * 13 * Emulex 14 * 3333 Susan Street 15 * Costa Mesa, CA 92626 16 */ 17 18 #include <linux/prefetch.h> 19 #include <linux/module.h> 20 #include "be.h" 21 #include "be_cmds.h" 22 #include <asm/div64.h> 23 #include <linux/aer.h> 24 #include <linux/if_bridge.h> 25 #include <net/busy_poll.h> 26 #include <net/vxlan.h> 27 28 MODULE_VERSION(DRV_VER); 29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER); 30 MODULE_AUTHOR("Emulex Corporation"); 31 MODULE_LICENSE("GPL"); 32 33 /* num_vfs module param is obsolete. 34 * Use sysfs method to enable/disable VFs. 35 */ 36 static unsigned int num_vfs; 37 module_param(num_vfs, uint, 0444); 38 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize"); 39 40 static ushort rx_frag_size = 2048; 41 module_param(rx_frag_size, ushort, 0444); 42 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data."); 43 44 /* Per-module error detection/recovery workq shared across all functions. 45 * Each function schedules its own work request on this shared workq. 46 */ 47 static struct workqueue_struct *be_err_recovery_workq; 48 49 static const struct pci_device_id be_dev_ids[] = { 50 #ifdef CONFIG_BE2NET_BE2 51 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 52 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 53 #endif /* CONFIG_BE2NET_BE2 */ 54 #ifdef CONFIG_BE2NET_BE3 55 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 56 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 57 #endif /* CONFIG_BE2NET_BE3 */ 58 #ifdef CONFIG_BE2NET_LANCER 59 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)}, 60 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)}, 61 #endif /* CONFIG_BE2NET_LANCER */ 62 #ifdef CONFIG_BE2NET_SKYHAWK 63 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)}, 64 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)}, 65 #endif /* CONFIG_BE2NET_SKYHAWK */ 66 { 0 } 67 }; 68 MODULE_DEVICE_TABLE(pci, be_dev_ids); 69 70 /* Workqueue used by all functions for defering cmd calls to the adapter */ 71 static struct workqueue_struct *be_wq; 72 73 /* UE Status Low CSR */ 74 static const char * const ue_status_low_desc[] = { 75 "CEV", 76 "CTX", 77 "DBUF", 78 "ERX", 79 "Host", 80 "MPU", 81 "NDMA", 82 "PTC ", 83 "RDMA ", 84 "RXF ", 85 "RXIPS ", 86 "RXULP0 ", 87 "RXULP1 ", 88 "RXULP2 ", 89 "TIM ", 90 "TPOST ", 91 "TPRE ", 92 "TXIPS ", 93 "TXULP0 ", 94 "TXULP1 ", 95 "UC ", 96 "WDMA ", 97 "TXULP2 ", 98 "HOST1 ", 99 "P0_OB_LINK ", 100 "P1_OB_LINK ", 101 "HOST_GPIO ", 102 "MBOX ", 103 "ERX2 ", 104 "SPARE ", 105 "JTAG ", 106 "MPU_INTPEND " 107 }; 108 109 /* UE Status High CSR */ 110 static const char * const ue_status_hi_desc[] = { 111 "LPCMEMHOST", 112 "MGMT_MAC", 113 "PCS0ONLINE", 114 "MPU_IRAM", 115 "PCS1ONLINE", 116 "PCTL0", 117 "PCTL1", 118 "PMEM", 119 "RR", 120 "TXPB", 121 "RXPP", 122 "XAUI", 123 "TXP", 124 "ARM", 125 "IPC", 126 "HOST2", 127 "HOST3", 128 "HOST4", 129 "HOST5", 130 "HOST6", 131 "HOST7", 132 "ECRC", 133 "Poison TLP", 134 "NETC", 135 "PERIPH", 136 "LLTXULP", 137 "D2P", 138 "RCON", 139 "LDMA", 140 "LLTXP", 141 "LLTXPB", 142 "Unknown" 143 }; 144 145 #define BE_VF_IF_EN_FLAGS (BE_IF_FLAGS_UNTAGGED | \ 146 BE_IF_FLAGS_BROADCAST | \ 147 BE_IF_FLAGS_MULTICAST | \ 148 BE_IF_FLAGS_PASS_L3L4_ERRORS) 149 150 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q) 151 { 152 struct be_dma_mem *mem = &q->dma_mem; 153 154 if (mem->va) { 155 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 156 mem->dma); 157 mem->va = NULL; 158 } 159 } 160 161 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q, 162 u16 len, u16 entry_size) 163 { 164 struct be_dma_mem *mem = &q->dma_mem; 165 166 memset(q, 0, sizeof(*q)); 167 q->len = len; 168 q->entry_size = entry_size; 169 mem->size = len * entry_size; 170 mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma, 171 GFP_KERNEL); 172 if (!mem->va) 173 return -ENOMEM; 174 return 0; 175 } 176 177 static void be_reg_intr_set(struct be_adapter *adapter, bool enable) 178 { 179 u32 reg, enabled; 180 181 pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, 182 ®); 183 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 184 185 if (!enabled && enable) 186 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 187 else if (enabled && !enable) 188 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 189 else 190 return; 191 192 pci_write_config_dword(adapter->pdev, 193 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg); 194 } 195 196 static void be_intr_set(struct be_adapter *adapter, bool enable) 197 { 198 int status = 0; 199 200 /* On lancer interrupts can't be controlled via this register */ 201 if (lancer_chip(adapter)) 202 return; 203 204 if (be_check_error(adapter, BE_ERROR_EEH)) 205 return; 206 207 status = be_cmd_intr_set(adapter, enable); 208 if (status) 209 be_reg_intr_set(adapter, enable); 210 } 211 212 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted) 213 { 214 u32 val = 0; 215 216 if (be_check_error(adapter, BE_ERROR_HW)) 217 return; 218 219 val |= qid & DB_RQ_RING_ID_MASK; 220 val |= posted << DB_RQ_NUM_POSTED_SHIFT; 221 222 wmb(); 223 iowrite32(val, adapter->db + DB_RQ_OFFSET); 224 } 225 226 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo, 227 u16 posted) 228 { 229 u32 val = 0; 230 231 if (be_check_error(adapter, BE_ERROR_HW)) 232 return; 233 234 val |= txo->q.id & DB_TXULP_RING_ID_MASK; 235 val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT; 236 237 wmb(); 238 iowrite32(val, adapter->db + txo->db_offset); 239 } 240 241 static void be_eq_notify(struct be_adapter *adapter, u16 qid, 242 bool arm, bool clear_int, u16 num_popped, 243 u32 eq_delay_mult_enc) 244 { 245 u32 val = 0; 246 247 val |= qid & DB_EQ_RING_ID_MASK; 248 val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT); 249 250 if (be_check_error(adapter, BE_ERROR_HW)) 251 return; 252 253 if (arm) 254 val |= 1 << DB_EQ_REARM_SHIFT; 255 if (clear_int) 256 val |= 1 << DB_EQ_CLR_SHIFT; 257 val |= 1 << DB_EQ_EVNT_SHIFT; 258 val |= num_popped << DB_EQ_NUM_POPPED_SHIFT; 259 val |= eq_delay_mult_enc << DB_EQ_R2I_DLY_SHIFT; 260 iowrite32(val, adapter->db + DB_EQ_OFFSET); 261 } 262 263 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped) 264 { 265 u32 val = 0; 266 267 val |= qid & DB_CQ_RING_ID_MASK; 268 val |= ((qid & DB_CQ_RING_ID_EXT_MASK) << 269 DB_CQ_RING_ID_EXT_MASK_SHIFT); 270 271 if (be_check_error(adapter, BE_ERROR_HW)) 272 return; 273 274 if (arm) 275 val |= 1 << DB_CQ_REARM_SHIFT; 276 val |= num_popped << DB_CQ_NUM_POPPED_SHIFT; 277 iowrite32(val, adapter->db + DB_CQ_OFFSET); 278 } 279 280 static int be_dev_mac_add(struct be_adapter *adapter, u8 *mac) 281 { 282 int i; 283 284 /* Check if mac has already been added as part of uc-list */ 285 for (i = 0; i < adapter->uc_macs; i++) { 286 if (ether_addr_equal(adapter->uc_list[i].mac, mac)) { 287 /* mac already added, skip addition */ 288 adapter->pmac_id[0] = adapter->pmac_id[i + 1]; 289 return 0; 290 } 291 } 292 293 return be_cmd_pmac_add(adapter, mac, adapter->if_handle, 294 &adapter->pmac_id[0], 0); 295 } 296 297 static void be_dev_mac_del(struct be_adapter *adapter, int pmac_id) 298 { 299 int i; 300 301 /* Skip deletion if the programmed mac is 302 * being used in uc-list 303 */ 304 for (i = 0; i < adapter->uc_macs; i++) { 305 if (adapter->pmac_id[i + 1] == pmac_id) 306 return; 307 } 308 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0); 309 } 310 311 static int be_mac_addr_set(struct net_device *netdev, void *p) 312 { 313 struct be_adapter *adapter = netdev_priv(netdev); 314 struct device *dev = &adapter->pdev->dev; 315 struct sockaddr *addr = p; 316 int status; 317 u8 mac[ETH_ALEN]; 318 u32 old_pmac_id = adapter->pmac_id[0]; 319 320 if (!is_valid_ether_addr(addr->sa_data)) 321 return -EADDRNOTAVAIL; 322 323 /* Proceed further only if, User provided MAC is different 324 * from active MAC 325 */ 326 if (ether_addr_equal(addr->sa_data, adapter->dev_mac)) 327 return 0; 328 329 /* BE3 VFs without FILTMGMT privilege are not allowed to set its MAC 330 * address 331 */ 332 if (BEx_chip(adapter) && be_virtfn(adapter) && 333 !check_privilege(adapter, BE_PRIV_FILTMGMT)) 334 return -EPERM; 335 336 /* if device is not running, copy MAC to netdev->dev_addr */ 337 if (!netif_running(netdev)) 338 goto done; 339 340 /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT 341 * privilege or if PF did not provision the new MAC address. 342 * On BE3, this cmd will always fail if the VF doesn't have the 343 * FILTMGMT privilege. This failure is OK, only if the PF programmed 344 * the MAC for the VF. 345 */ 346 mutex_lock(&adapter->rx_filter_lock); 347 status = be_dev_mac_add(adapter, (u8 *)addr->sa_data); 348 if (!status) { 349 350 /* Delete the old programmed MAC. This call may fail if the 351 * old MAC was already deleted by the PF driver. 352 */ 353 if (adapter->pmac_id[0] != old_pmac_id) 354 be_dev_mac_del(adapter, old_pmac_id); 355 } 356 357 mutex_unlock(&adapter->rx_filter_lock); 358 /* Decide if the new MAC is successfully activated only after 359 * querying the FW 360 */ 361 status = be_cmd_get_active_mac(adapter, adapter->pmac_id[0], mac, 362 adapter->if_handle, true, 0); 363 if (status) 364 goto err; 365 366 /* The MAC change did not happen, either due to lack of privilege 367 * or PF didn't pre-provision. 368 */ 369 if (!ether_addr_equal(addr->sa_data, mac)) { 370 status = -EPERM; 371 goto err; 372 } 373 374 /* Remember currently programmed MAC */ 375 ether_addr_copy(adapter->dev_mac, addr->sa_data); 376 done: 377 ether_addr_copy(netdev->dev_addr, addr->sa_data); 378 dev_info(dev, "MAC address changed to %pM\n", addr->sa_data); 379 return 0; 380 err: 381 dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data); 382 return status; 383 } 384 385 /* BE2 supports only v0 cmd */ 386 static void *hw_stats_from_cmd(struct be_adapter *adapter) 387 { 388 if (BE2_chip(adapter)) { 389 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va; 390 391 return &cmd->hw_stats; 392 } else if (BE3_chip(adapter)) { 393 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va; 394 395 return &cmd->hw_stats; 396 } else { 397 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va; 398 399 return &cmd->hw_stats; 400 } 401 } 402 403 /* BE2 supports only v0 cmd */ 404 static void *be_erx_stats_from_cmd(struct be_adapter *adapter) 405 { 406 if (BE2_chip(adapter)) { 407 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 408 409 return &hw_stats->erx; 410 } else if (BE3_chip(adapter)) { 411 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 412 413 return &hw_stats->erx; 414 } else { 415 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 416 417 return &hw_stats->erx; 418 } 419 } 420 421 static void populate_be_v0_stats(struct be_adapter *adapter) 422 { 423 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 424 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 425 struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf; 426 struct be_port_rxf_stats_v0 *port_stats = 427 &rxf_stats->port[adapter->port_num]; 428 struct be_drv_stats *drvs = &adapter->drv_stats; 429 430 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 431 drvs->rx_pause_frames = port_stats->rx_pause_frames; 432 drvs->rx_crc_errors = port_stats->rx_crc_errors; 433 drvs->rx_control_frames = port_stats->rx_control_frames; 434 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 435 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 436 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 437 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 438 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 439 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 440 drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow; 441 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 442 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 443 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 444 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 445 drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow; 446 drvs->rx_dropped_header_too_small = 447 port_stats->rx_dropped_header_too_small; 448 drvs->rx_address_filtered = 449 port_stats->rx_address_filtered + 450 port_stats->rx_vlan_filtered; 451 drvs->rx_alignment_symbol_errors = 452 port_stats->rx_alignment_symbol_errors; 453 454 drvs->tx_pauseframes = port_stats->tx_pauseframes; 455 drvs->tx_controlframes = port_stats->tx_controlframes; 456 457 if (adapter->port_num) 458 drvs->jabber_events = rxf_stats->port1_jabber_events; 459 else 460 drvs->jabber_events = rxf_stats->port0_jabber_events; 461 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 462 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 463 drvs->forwarded_packets = rxf_stats->forwarded_packets; 464 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 465 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 466 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 467 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 468 } 469 470 static void populate_be_v1_stats(struct be_adapter *adapter) 471 { 472 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 473 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 474 struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf; 475 struct be_port_rxf_stats_v1 *port_stats = 476 &rxf_stats->port[adapter->port_num]; 477 struct be_drv_stats *drvs = &adapter->drv_stats; 478 479 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 480 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 481 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 482 drvs->rx_pause_frames = port_stats->rx_pause_frames; 483 drvs->rx_crc_errors = port_stats->rx_crc_errors; 484 drvs->rx_control_frames = port_stats->rx_control_frames; 485 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 486 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 487 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 488 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 489 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 490 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 491 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 492 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 493 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 494 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 495 drvs->rx_dropped_header_too_small = 496 port_stats->rx_dropped_header_too_small; 497 drvs->rx_input_fifo_overflow_drop = 498 port_stats->rx_input_fifo_overflow_drop; 499 drvs->rx_address_filtered = port_stats->rx_address_filtered; 500 drvs->rx_alignment_symbol_errors = 501 port_stats->rx_alignment_symbol_errors; 502 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 503 drvs->tx_pauseframes = port_stats->tx_pauseframes; 504 drvs->tx_controlframes = port_stats->tx_controlframes; 505 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 506 drvs->jabber_events = port_stats->jabber_events; 507 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 508 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 509 drvs->forwarded_packets = rxf_stats->forwarded_packets; 510 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 511 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 512 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 513 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 514 } 515 516 static void populate_be_v2_stats(struct be_adapter *adapter) 517 { 518 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 519 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 520 struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf; 521 struct be_port_rxf_stats_v2 *port_stats = 522 &rxf_stats->port[adapter->port_num]; 523 struct be_drv_stats *drvs = &adapter->drv_stats; 524 525 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 526 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 527 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 528 drvs->rx_pause_frames = port_stats->rx_pause_frames; 529 drvs->rx_crc_errors = port_stats->rx_crc_errors; 530 drvs->rx_control_frames = port_stats->rx_control_frames; 531 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 532 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 533 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 534 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 535 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 536 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 537 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 538 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 539 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 540 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 541 drvs->rx_dropped_header_too_small = 542 port_stats->rx_dropped_header_too_small; 543 drvs->rx_input_fifo_overflow_drop = 544 port_stats->rx_input_fifo_overflow_drop; 545 drvs->rx_address_filtered = port_stats->rx_address_filtered; 546 drvs->rx_alignment_symbol_errors = 547 port_stats->rx_alignment_symbol_errors; 548 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 549 drvs->tx_pauseframes = port_stats->tx_pauseframes; 550 drvs->tx_controlframes = port_stats->tx_controlframes; 551 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 552 drvs->jabber_events = port_stats->jabber_events; 553 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 554 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 555 drvs->forwarded_packets = rxf_stats->forwarded_packets; 556 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 557 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 558 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 559 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 560 if (be_roce_supported(adapter)) { 561 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd; 562 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd; 563 drvs->rx_roce_frames = port_stats->roce_frames_received; 564 drvs->roce_drops_crc = port_stats->roce_drops_crc; 565 drvs->roce_drops_payload_len = 566 port_stats->roce_drops_payload_len; 567 } 568 } 569 570 static void populate_lancer_stats(struct be_adapter *adapter) 571 { 572 struct be_drv_stats *drvs = &adapter->drv_stats; 573 struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter); 574 575 be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats)); 576 drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo; 577 drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo; 578 drvs->rx_control_frames = pport_stats->rx_control_frames_lo; 579 drvs->rx_in_range_errors = pport_stats->rx_in_range_errors; 580 drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo; 581 drvs->rx_dropped_runt = pport_stats->rx_dropped_runt; 582 drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors; 583 drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors; 584 drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors; 585 drvs->rx_dropped_tcp_length = 586 pport_stats->rx_dropped_invalid_tcp_length; 587 drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small; 588 drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short; 589 drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors; 590 drvs->rx_dropped_header_too_small = 591 pport_stats->rx_dropped_header_too_small; 592 drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 593 drvs->rx_address_filtered = 594 pport_stats->rx_address_filtered + 595 pport_stats->rx_vlan_filtered; 596 drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo; 597 drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 598 drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo; 599 drvs->tx_controlframes = pport_stats->tx_control_frames_lo; 600 drvs->jabber_events = pport_stats->rx_jabbers; 601 drvs->forwarded_packets = pport_stats->num_forwards_lo; 602 drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo; 603 drvs->rx_drops_too_many_frags = 604 pport_stats->rx_drops_too_many_frags_lo; 605 } 606 607 static void accumulate_16bit_val(u32 *acc, u16 val) 608 { 609 #define lo(x) (x & 0xFFFF) 610 #define hi(x) (x & 0xFFFF0000) 611 bool wrapped = val < lo(*acc); 612 u32 newacc = hi(*acc) + val; 613 614 if (wrapped) 615 newacc += 65536; 616 WRITE_ONCE(*acc, newacc); 617 } 618 619 static void populate_erx_stats(struct be_adapter *adapter, 620 struct be_rx_obj *rxo, u32 erx_stat) 621 { 622 if (!BEx_chip(adapter)) 623 rx_stats(rxo)->rx_drops_no_frags = erx_stat; 624 else 625 /* below erx HW counter can actually wrap around after 626 * 65535. Driver accumulates a 32-bit value 627 */ 628 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags, 629 (u16)erx_stat); 630 } 631 632 void be_parse_stats(struct be_adapter *adapter) 633 { 634 struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter); 635 struct be_rx_obj *rxo; 636 int i; 637 u32 erx_stat; 638 639 if (lancer_chip(adapter)) { 640 populate_lancer_stats(adapter); 641 } else { 642 if (BE2_chip(adapter)) 643 populate_be_v0_stats(adapter); 644 else if (BE3_chip(adapter)) 645 /* for BE3 */ 646 populate_be_v1_stats(adapter); 647 else 648 populate_be_v2_stats(adapter); 649 650 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */ 651 for_all_rx_queues(adapter, rxo, i) { 652 erx_stat = erx->rx_drops_no_fragments[rxo->q.id]; 653 populate_erx_stats(adapter, rxo, erx_stat); 654 } 655 } 656 } 657 658 static void be_get_stats64(struct net_device *netdev, 659 struct rtnl_link_stats64 *stats) 660 { 661 struct be_adapter *adapter = netdev_priv(netdev); 662 struct be_drv_stats *drvs = &adapter->drv_stats; 663 struct be_rx_obj *rxo; 664 struct be_tx_obj *txo; 665 u64 pkts, bytes; 666 unsigned int start; 667 int i; 668 669 for_all_rx_queues(adapter, rxo, i) { 670 const struct be_rx_stats *rx_stats = rx_stats(rxo); 671 672 do { 673 start = u64_stats_fetch_begin_irq(&rx_stats->sync); 674 pkts = rx_stats(rxo)->rx_pkts; 675 bytes = rx_stats(rxo)->rx_bytes; 676 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start)); 677 stats->rx_packets += pkts; 678 stats->rx_bytes += bytes; 679 stats->multicast += rx_stats(rxo)->rx_mcast_pkts; 680 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs + 681 rx_stats(rxo)->rx_drops_no_frags; 682 } 683 684 for_all_tx_queues(adapter, txo, i) { 685 const struct be_tx_stats *tx_stats = tx_stats(txo); 686 687 do { 688 start = u64_stats_fetch_begin_irq(&tx_stats->sync); 689 pkts = tx_stats(txo)->tx_pkts; 690 bytes = tx_stats(txo)->tx_bytes; 691 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start)); 692 stats->tx_packets += pkts; 693 stats->tx_bytes += bytes; 694 } 695 696 /* bad pkts received */ 697 stats->rx_errors = drvs->rx_crc_errors + 698 drvs->rx_alignment_symbol_errors + 699 drvs->rx_in_range_errors + 700 drvs->rx_out_range_errors + 701 drvs->rx_frame_too_long + 702 drvs->rx_dropped_too_small + 703 drvs->rx_dropped_too_short + 704 drvs->rx_dropped_header_too_small + 705 drvs->rx_dropped_tcp_length + 706 drvs->rx_dropped_runt; 707 708 /* detailed rx errors */ 709 stats->rx_length_errors = drvs->rx_in_range_errors + 710 drvs->rx_out_range_errors + 711 drvs->rx_frame_too_long; 712 713 stats->rx_crc_errors = drvs->rx_crc_errors; 714 715 /* frame alignment errors */ 716 stats->rx_frame_errors = drvs->rx_alignment_symbol_errors; 717 718 /* receiver fifo overrun */ 719 /* drops_no_pbuf is no per i/f, it's per BE card */ 720 stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop + 721 drvs->rx_input_fifo_overflow_drop + 722 drvs->rx_drops_no_pbuf; 723 } 724 725 void be_link_status_update(struct be_adapter *adapter, u8 link_status) 726 { 727 struct net_device *netdev = adapter->netdev; 728 729 if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) { 730 netif_carrier_off(netdev); 731 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT; 732 } 733 734 if (link_status) 735 netif_carrier_on(netdev); 736 else 737 netif_carrier_off(netdev); 738 739 netdev_info(netdev, "Link is %s\n", link_status ? "Up" : "Down"); 740 } 741 742 static int be_gso_hdr_len(struct sk_buff *skb) 743 { 744 if (skb->encapsulation) 745 return skb_inner_transport_offset(skb) + 746 inner_tcp_hdrlen(skb); 747 return skb_transport_offset(skb) + tcp_hdrlen(skb); 748 } 749 750 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb) 751 { 752 struct be_tx_stats *stats = tx_stats(txo); 753 u32 tx_pkts = skb_shinfo(skb)->gso_segs ? : 1; 754 /* Account for headers which get duplicated in TSO pkt */ 755 u32 dup_hdr_len = tx_pkts > 1 ? be_gso_hdr_len(skb) * (tx_pkts - 1) : 0; 756 757 u64_stats_update_begin(&stats->sync); 758 stats->tx_reqs++; 759 stats->tx_bytes += skb->len + dup_hdr_len; 760 stats->tx_pkts += tx_pkts; 761 if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL) 762 stats->tx_vxlan_offload_pkts += tx_pkts; 763 u64_stats_update_end(&stats->sync); 764 } 765 766 /* Returns number of WRBs needed for the skb */ 767 static u32 skb_wrb_cnt(struct sk_buff *skb) 768 { 769 /* +1 for the header wrb */ 770 return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags; 771 } 772 773 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len) 774 { 775 wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr)); 776 wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr)); 777 wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK); 778 wrb->rsvd0 = 0; 779 } 780 781 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb 782 * to avoid the swap and shift/mask operations in wrb_fill(). 783 */ 784 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb) 785 { 786 wrb->frag_pa_hi = 0; 787 wrb->frag_pa_lo = 0; 788 wrb->frag_len = 0; 789 wrb->rsvd0 = 0; 790 } 791 792 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter, 793 struct sk_buff *skb) 794 { 795 u8 vlan_prio; 796 u16 vlan_tag; 797 798 vlan_tag = skb_vlan_tag_get(skb); 799 vlan_prio = skb_vlan_tag_get_prio(skb); 800 /* If vlan priority provided by OS is NOT in available bmap */ 801 if (!(adapter->vlan_prio_bmap & (1 << vlan_prio))) 802 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) | 803 adapter->recommended_prio_bits; 804 805 return vlan_tag; 806 } 807 808 /* Used only for IP tunnel packets */ 809 static u16 skb_inner_ip_proto(struct sk_buff *skb) 810 { 811 return (inner_ip_hdr(skb)->version == 4) ? 812 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr; 813 } 814 815 static u16 skb_ip_proto(struct sk_buff *skb) 816 { 817 return (ip_hdr(skb)->version == 4) ? 818 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr; 819 } 820 821 static inline bool be_is_txq_full(struct be_tx_obj *txo) 822 { 823 return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len; 824 } 825 826 static inline bool be_can_txq_wake(struct be_tx_obj *txo) 827 { 828 return atomic_read(&txo->q.used) < txo->q.len / 2; 829 } 830 831 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo) 832 { 833 return atomic_read(&txo->q.used) > txo->pend_wrb_cnt; 834 } 835 836 static void be_get_wrb_params_from_skb(struct be_adapter *adapter, 837 struct sk_buff *skb, 838 struct be_wrb_params *wrb_params) 839 { 840 u16 proto; 841 842 if (skb_is_gso(skb)) { 843 BE_WRB_F_SET(wrb_params->features, LSO, 1); 844 wrb_params->lso_mss = skb_shinfo(skb)->gso_size; 845 if (skb_is_gso_v6(skb) && !lancer_chip(adapter)) 846 BE_WRB_F_SET(wrb_params->features, LSO6, 1); 847 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 848 if (skb->encapsulation) { 849 BE_WRB_F_SET(wrb_params->features, IPCS, 1); 850 proto = skb_inner_ip_proto(skb); 851 } else { 852 proto = skb_ip_proto(skb); 853 } 854 if (proto == IPPROTO_TCP) 855 BE_WRB_F_SET(wrb_params->features, TCPCS, 1); 856 else if (proto == IPPROTO_UDP) 857 BE_WRB_F_SET(wrb_params->features, UDPCS, 1); 858 } 859 860 if (skb_vlan_tag_present(skb)) { 861 BE_WRB_F_SET(wrb_params->features, VLAN, 1); 862 wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb); 863 } 864 865 BE_WRB_F_SET(wrb_params->features, CRC, 1); 866 } 867 868 static void wrb_fill_hdr(struct be_adapter *adapter, 869 struct be_eth_hdr_wrb *hdr, 870 struct be_wrb_params *wrb_params, 871 struct sk_buff *skb) 872 { 873 memset(hdr, 0, sizeof(*hdr)); 874 875 SET_TX_WRB_HDR_BITS(crc, hdr, 876 BE_WRB_F_GET(wrb_params->features, CRC)); 877 SET_TX_WRB_HDR_BITS(ipcs, hdr, 878 BE_WRB_F_GET(wrb_params->features, IPCS)); 879 SET_TX_WRB_HDR_BITS(tcpcs, hdr, 880 BE_WRB_F_GET(wrb_params->features, TCPCS)); 881 SET_TX_WRB_HDR_BITS(udpcs, hdr, 882 BE_WRB_F_GET(wrb_params->features, UDPCS)); 883 884 SET_TX_WRB_HDR_BITS(lso, hdr, 885 BE_WRB_F_GET(wrb_params->features, LSO)); 886 SET_TX_WRB_HDR_BITS(lso6, hdr, 887 BE_WRB_F_GET(wrb_params->features, LSO6)); 888 SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss); 889 890 /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this 891 * hack is not needed, the evt bit is set while ringing DB. 892 */ 893 SET_TX_WRB_HDR_BITS(event, hdr, 894 BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW)); 895 SET_TX_WRB_HDR_BITS(vlan, hdr, 896 BE_WRB_F_GET(wrb_params->features, VLAN)); 897 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag); 898 899 SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb)); 900 SET_TX_WRB_HDR_BITS(len, hdr, skb->len); 901 SET_TX_WRB_HDR_BITS(mgmt, hdr, 902 BE_WRB_F_GET(wrb_params->features, OS2BMC)); 903 } 904 905 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb, 906 bool unmap_single) 907 { 908 dma_addr_t dma; 909 u32 frag_len = le32_to_cpu(wrb->frag_len); 910 911 912 dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 | 913 (u64)le32_to_cpu(wrb->frag_pa_lo); 914 if (frag_len) { 915 if (unmap_single) 916 dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE); 917 else 918 dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE); 919 } 920 } 921 922 /* Grab a WRB header for xmit */ 923 static u32 be_tx_get_wrb_hdr(struct be_tx_obj *txo) 924 { 925 u32 head = txo->q.head; 926 927 queue_head_inc(&txo->q); 928 return head; 929 } 930 931 /* Set up the WRB header for xmit */ 932 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter, 933 struct be_tx_obj *txo, 934 struct be_wrb_params *wrb_params, 935 struct sk_buff *skb, u16 head) 936 { 937 u32 num_frags = skb_wrb_cnt(skb); 938 struct be_queue_info *txq = &txo->q; 939 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head); 940 941 wrb_fill_hdr(adapter, hdr, wrb_params, skb); 942 be_dws_cpu_to_le(hdr, sizeof(*hdr)); 943 944 BUG_ON(txo->sent_skb_list[head]); 945 txo->sent_skb_list[head] = skb; 946 txo->last_req_hdr = head; 947 atomic_add(num_frags, &txq->used); 948 txo->last_req_wrb_cnt = num_frags; 949 txo->pend_wrb_cnt += num_frags; 950 } 951 952 /* Setup a WRB fragment (buffer descriptor) for xmit */ 953 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr, 954 int len) 955 { 956 struct be_eth_wrb *wrb; 957 struct be_queue_info *txq = &txo->q; 958 959 wrb = queue_head_node(txq); 960 wrb_fill(wrb, busaddr, len); 961 queue_head_inc(txq); 962 } 963 964 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine 965 * was invoked. The producer index is restored to the previous packet and the 966 * WRBs of the current packet are unmapped. Invoked to handle tx setup errors. 967 */ 968 static void be_xmit_restore(struct be_adapter *adapter, 969 struct be_tx_obj *txo, u32 head, bool map_single, 970 u32 copied) 971 { 972 struct device *dev; 973 struct be_eth_wrb *wrb; 974 struct be_queue_info *txq = &txo->q; 975 976 dev = &adapter->pdev->dev; 977 txq->head = head; 978 979 /* skip the first wrb (hdr); it's not mapped */ 980 queue_head_inc(txq); 981 while (copied) { 982 wrb = queue_head_node(txq); 983 unmap_tx_frag(dev, wrb, map_single); 984 map_single = false; 985 copied -= le32_to_cpu(wrb->frag_len); 986 queue_head_inc(txq); 987 } 988 989 txq->head = head; 990 } 991 992 /* Enqueue the given packet for transmit. This routine allocates WRBs for the 993 * packet, dma maps the packet buffers and sets up the WRBs. Returns the number 994 * of WRBs used up by the packet. 995 */ 996 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo, 997 struct sk_buff *skb, 998 struct be_wrb_params *wrb_params) 999 { 1000 u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb); 1001 struct device *dev = &adapter->pdev->dev; 1002 bool map_single = false; 1003 u32 head; 1004 dma_addr_t busaddr; 1005 int len; 1006 1007 head = be_tx_get_wrb_hdr(txo); 1008 1009 if (skb->len > skb->data_len) { 1010 len = skb_headlen(skb); 1011 1012 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE); 1013 if (dma_mapping_error(dev, busaddr)) 1014 goto dma_err; 1015 map_single = true; 1016 be_tx_setup_wrb_frag(txo, busaddr, len); 1017 copied += len; 1018 } 1019 1020 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1021 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; 1022 len = skb_frag_size(frag); 1023 1024 busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE); 1025 if (dma_mapping_error(dev, busaddr)) 1026 goto dma_err; 1027 be_tx_setup_wrb_frag(txo, busaddr, len); 1028 copied += len; 1029 } 1030 1031 be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head); 1032 1033 be_tx_stats_update(txo, skb); 1034 return wrb_cnt; 1035 1036 dma_err: 1037 adapter->drv_stats.dma_map_errors++; 1038 be_xmit_restore(adapter, txo, head, map_single, copied); 1039 return 0; 1040 } 1041 1042 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter) 1043 { 1044 return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD; 1045 } 1046 1047 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter, 1048 struct sk_buff *skb, 1049 struct be_wrb_params 1050 *wrb_params) 1051 { 1052 bool insert_vlan = false; 1053 u16 vlan_tag = 0; 1054 1055 skb = skb_share_check(skb, GFP_ATOMIC); 1056 if (unlikely(!skb)) 1057 return skb; 1058 1059 if (skb_vlan_tag_present(skb)) { 1060 vlan_tag = be_get_tx_vlan_tag(adapter, skb); 1061 insert_vlan = true; 1062 } 1063 1064 if (qnq_async_evt_rcvd(adapter) && adapter->pvid) { 1065 if (!insert_vlan) { 1066 vlan_tag = adapter->pvid; 1067 insert_vlan = true; 1068 } 1069 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to 1070 * skip VLAN insertion 1071 */ 1072 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1073 } 1074 1075 if (insert_vlan) { 1076 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 1077 vlan_tag); 1078 if (unlikely(!skb)) 1079 return skb; 1080 __vlan_hwaccel_clear_tag(skb); 1081 } 1082 1083 /* Insert the outer VLAN, if any */ 1084 if (adapter->qnq_vid) { 1085 vlan_tag = adapter->qnq_vid; 1086 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 1087 vlan_tag); 1088 if (unlikely(!skb)) 1089 return skb; 1090 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1091 } 1092 1093 return skb; 1094 } 1095 1096 static bool be_ipv6_exthdr_check(struct sk_buff *skb) 1097 { 1098 struct ethhdr *eh = (struct ethhdr *)skb->data; 1099 u16 offset = ETH_HLEN; 1100 1101 if (eh->h_proto == htons(ETH_P_IPV6)) { 1102 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset); 1103 1104 offset += sizeof(struct ipv6hdr); 1105 if (ip6h->nexthdr != NEXTHDR_TCP && 1106 ip6h->nexthdr != NEXTHDR_UDP) { 1107 struct ipv6_opt_hdr *ehdr = 1108 (struct ipv6_opt_hdr *)(skb->data + offset); 1109 1110 /* offending pkt: 2nd byte following IPv6 hdr is 0xff */ 1111 if (ehdr->hdrlen == 0xff) 1112 return true; 1113 } 1114 } 1115 return false; 1116 } 1117 1118 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb) 1119 { 1120 return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid; 1121 } 1122 1123 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb) 1124 { 1125 return BE3_chip(adapter) && be_ipv6_exthdr_check(skb); 1126 } 1127 1128 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter, 1129 struct sk_buff *skb, 1130 struct be_wrb_params 1131 *wrb_params) 1132 { 1133 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 1134 unsigned int eth_hdr_len; 1135 struct iphdr *ip; 1136 1137 /* For padded packets, BE HW modifies tot_len field in IP header 1138 * incorrecly when VLAN tag is inserted by HW. 1139 * For padded packets, Lancer computes incorrect checksum. 1140 */ 1141 eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ? 1142 VLAN_ETH_HLEN : ETH_HLEN; 1143 if (skb->len <= 60 && 1144 (lancer_chip(adapter) || skb_vlan_tag_present(skb)) && 1145 is_ipv4_pkt(skb)) { 1146 ip = (struct iphdr *)ip_hdr(skb); 1147 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len)); 1148 } 1149 1150 /* If vlan tag is already inlined in the packet, skip HW VLAN 1151 * tagging in pvid-tagging mode 1152 */ 1153 if (be_pvid_tagging_enabled(adapter) && 1154 veh->h_vlan_proto == htons(ETH_P_8021Q)) 1155 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1156 1157 /* HW has a bug wherein it will calculate CSUM for VLAN 1158 * pkts even though it is disabled. 1159 * Manually insert VLAN in pkt. 1160 */ 1161 if (skb->ip_summed != CHECKSUM_PARTIAL && 1162 skb_vlan_tag_present(skb)) { 1163 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1164 if (unlikely(!skb)) 1165 goto err; 1166 } 1167 1168 /* HW may lockup when VLAN HW tagging is requested on 1169 * certain ipv6 packets. Drop such pkts if the HW workaround to 1170 * skip HW tagging is not enabled by FW. 1171 */ 1172 if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) && 1173 (adapter->pvid || adapter->qnq_vid) && 1174 !qnq_async_evt_rcvd(adapter))) 1175 goto tx_drop; 1176 1177 /* Manual VLAN tag insertion to prevent: 1178 * ASIC lockup when the ASIC inserts VLAN tag into 1179 * certain ipv6 packets. Insert VLAN tags in driver, 1180 * and set event, completion, vlan bits accordingly 1181 * in the Tx WRB. 1182 */ 1183 if (be_ipv6_tx_stall_chk(adapter, skb) && 1184 be_vlan_tag_tx_chk(adapter, skb)) { 1185 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1186 if (unlikely(!skb)) 1187 goto err; 1188 } 1189 1190 return skb; 1191 tx_drop: 1192 dev_kfree_skb_any(skb); 1193 err: 1194 return NULL; 1195 } 1196 1197 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter, 1198 struct sk_buff *skb, 1199 struct be_wrb_params *wrb_params) 1200 { 1201 int err; 1202 1203 /* Lancer, SH and BE3 in SRIOV mode have a bug wherein 1204 * packets that are 32b or less may cause a transmit stall 1205 * on that port. The workaround is to pad such packets 1206 * (len <= 32 bytes) to a minimum length of 36b. 1207 */ 1208 if (skb->len <= 32) { 1209 if (skb_put_padto(skb, 36)) 1210 return NULL; 1211 } 1212 1213 if (BEx_chip(adapter) || lancer_chip(adapter)) { 1214 skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params); 1215 if (!skb) 1216 return NULL; 1217 } 1218 1219 /* The stack can send us skbs with length greater than 1220 * what the HW can handle. Trim the extra bytes. 1221 */ 1222 WARN_ON_ONCE(skb->len > BE_MAX_GSO_SIZE); 1223 err = pskb_trim(skb, BE_MAX_GSO_SIZE); 1224 WARN_ON(err); 1225 1226 return skb; 1227 } 1228 1229 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo) 1230 { 1231 struct be_queue_info *txq = &txo->q; 1232 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr); 1233 1234 /* Mark the last request eventable if it hasn't been marked already */ 1235 if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT))) 1236 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL); 1237 1238 /* compose a dummy wrb if there are odd set of wrbs to notify */ 1239 if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) { 1240 wrb_fill_dummy(queue_head_node(txq)); 1241 queue_head_inc(txq); 1242 atomic_inc(&txq->used); 1243 txo->pend_wrb_cnt++; 1244 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK << 1245 TX_HDR_WRB_NUM_SHIFT); 1246 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) << 1247 TX_HDR_WRB_NUM_SHIFT); 1248 } 1249 be_txq_notify(adapter, txo, txo->pend_wrb_cnt); 1250 txo->pend_wrb_cnt = 0; 1251 } 1252 1253 /* OS2BMC related */ 1254 1255 #define DHCP_CLIENT_PORT 68 1256 #define DHCP_SERVER_PORT 67 1257 #define NET_BIOS_PORT1 137 1258 #define NET_BIOS_PORT2 138 1259 #define DHCPV6_RAS_PORT 547 1260 1261 #define is_mc_allowed_on_bmc(adapter, eh) \ 1262 (!is_multicast_filt_enabled(adapter) && \ 1263 is_multicast_ether_addr(eh->h_dest) && \ 1264 !is_broadcast_ether_addr(eh->h_dest)) 1265 1266 #define is_bc_allowed_on_bmc(adapter, eh) \ 1267 (!is_broadcast_filt_enabled(adapter) && \ 1268 is_broadcast_ether_addr(eh->h_dest)) 1269 1270 #define is_arp_allowed_on_bmc(adapter, skb) \ 1271 (is_arp(skb) && is_arp_filt_enabled(adapter)) 1272 1273 #define is_broadcast_packet(eh, adapter) \ 1274 (is_multicast_ether_addr(eh->h_dest) && \ 1275 !compare_ether_addr(eh->h_dest, adapter->netdev->broadcast)) 1276 1277 #define is_arp(skb) (skb->protocol == htons(ETH_P_ARP)) 1278 1279 #define is_arp_filt_enabled(adapter) \ 1280 (adapter->bmc_filt_mask & (BMC_FILT_BROADCAST_ARP)) 1281 1282 #define is_dhcp_client_filt_enabled(adapter) \ 1283 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_CLIENT) 1284 1285 #define is_dhcp_srvr_filt_enabled(adapter) \ 1286 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_SERVER) 1287 1288 #define is_nbios_filt_enabled(adapter) \ 1289 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_NET_BIOS) 1290 1291 #define is_ipv6_na_filt_enabled(adapter) \ 1292 (adapter->bmc_filt_mask & \ 1293 BMC_FILT_MULTICAST_IPV6_NEIGH_ADVER) 1294 1295 #define is_ipv6_ra_filt_enabled(adapter) \ 1296 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RA) 1297 1298 #define is_ipv6_ras_filt_enabled(adapter) \ 1299 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RAS) 1300 1301 #define is_broadcast_filt_enabled(adapter) \ 1302 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST) 1303 1304 #define is_multicast_filt_enabled(adapter) \ 1305 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST) 1306 1307 static bool be_send_pkt_to_bmc(struct be_adapter *adapter, 1308 struct sk_buff **skb) 1309 { 1310 struct ethhdr *eh = (struct ethhdr *)(*skb)->data; 1311 bool os2bmc = false; 1312 1313 if (!be_is_os2bmc_enabled(adapter)) 1314 goto done; 1315 1316 if (!is_multicast_ether_addr(eh->h_dest)) 1317 goto done; 1318 1319 if (is_mc_allowed_on_bmc(adapter, eh) || 1320 is_bc_allowed_on_bmc(adapter, eh) || 1321 is_arp_allowed_on_bmc(adapter, (*skb))) { 1322 os2bmc = true; 1323 goto done; 1324 } 1325 1326 if ((*skb)->protocol == htons(ETH_P_IPV6)) { 1327 struct ipv6hdr *hdr = ipv6_hdr((*skb)); 1328 u8 nexthdr = hdr->nexthdr; 1329 1330 if (nexthdr == IPPROTO_ICMPV6) { 1331 struct icmp6hdr *icmp6 = icmp6_hdr((*skb)); 1332 1333 switch (icmp6->icmp6_type) { 1334 case NDISC_ROUTER_ADVERTISEMENT: 1335 os2bmc = is_ipv6_ra_filt_enabled(adapter); 1336 goto done; 1337 case NDISC_NEIGHBOUR_ADVERTISEMENT: 1338 os2bmc = is_ipv6_na_filt_enabled(adapter); 1339 goto done; 1340 default: 1341 break; 1342 } 1343 } 1344 } 1345 1346 if (is_udp_pkt((*skb))) { 1347 struct udphdr *udp = udp_hdr((*skb)); 1348 1349 switch (ntohs(udp->dest)) { 1350 case DHCP_CLIENT_PORT: 1351 os2bmc = is_dhcp_client_filt_enabled(adapter); 1352 goto done; 1353 case DHCP_SERVER_PORT: 1354 os2bmc = is_dhcp_srvr_filt_enabled(adapter); 1355 goto done; 1356 case NET_BIOS_PORT1: 1357 case NET_BIOS_PORT2: 1358 os2bmc = is_nbios_filt_enabled(adapter); 1359 goto done; 1360 case DHCPV6_RAS_PORT: 1361 os2bmc = is_ipv6_ras_filt_enabled(adapter); 1362 goto done; 1363 default: 1364 break; 1365 } 1366 } 1367 done: 1368 /* For packets over a vlan, which are destined 1369 * to BMC, asic expects the vlan to be inline in the packet. 1370 */ 1371 if (os2bmc) 1372 *skb = be_insert_vlan_in_pkt(adapter, *skb, NULL); 1373 1374 return os2bmc; 1375 } 1376 1377 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev) 1378 { 1379 struct be_adapter *adapter = netdev_priv(netdev); 1380 u16 q_idx = skb_get_queue_mapping(skb); 1381 struct be_tx_obj *txo = &adapter->tx_obj[q_idx]; 1382 struct be_wrb_params wrb_params = { 0 }; 1383 bool flush = !skb->xmit_more; 1384 u16 wrb_cnt; 1385 1386 skb = be_xmit_workarounds(adapter, skb, &wrb_params); 1387 if (unlikely(!skb)) 1388 goto drop; 1389 1390 be_get_wrb_params_from_skb(adapter, skb, &wrb_params); 1391 1392 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); 1393 if (unlikely(!wrb_cnt)) { 1394 dev_kfree_skb_any(skb); 1395 goto drop; 1396 } 1397 1398 /* if os2bmc is enabled and if the pkt is destined to bmc, 1399 * enqueue the pkt a 2nd time with mgmt bit set. 1400 */ 1401 if (be_send_pkt_to_bmc(adapter, &skb)) { 1402 BE_WRB_F_SET(wrb_params.features, OS2BMC, 1); 1403 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); 1404 if (unlikely(!wrb_cnt)) 1405 goto drop; 1406 else 1407 skb_get(skb); 1408 } 1409 1410 if (be_is_txq_full(txo)) { 1411 netif_stop_subqueue(netdev, q_idx); 1412 tx_stats(txo)->tx_stops++; 1413 } 1414 1415 if (flush || __netif_subqueue_stopped(netdev, q_idx)) 1416 be_xmit_flush(adapter, txo); 1417 1418 return NETDEV_TX_OK; 1419 drop: 1420 tx_stats(txo)->tx_drv_drops++; 1421 /* Flush the already enqueued tx requests */ 1422 if (flush && txo->pend_wrb_cnt) 1423 be_xmit_flush(adapter, txo); 1424 1425 return NETDEV_TX_OK; 1426 } 1427 1428 static void be_tx_timeout(struct net_device *netdev) 1429 { 1430 struct be_adapter *adapter = netdev_priv(netdev); 1431 struct device *dev = &adapter->pdev->dev; 1432 struct be_tx_obj *txo; 1433 struct sk_buff *skb; 1434 struct tcphdr *tcphdr; 1435 struct udphdr *udphdr; 1436 u32 *entry; 1437 int status; 1438 int i, j; 1439 1440 for_all_tx_queues(adapter, txo, i) { 1441 dev_info(dev, "TXQ Dump: %d H: %d T: %d used: %d, qid: 0x%x\n", 1442 i, txo->q.head, txo->q.tail, 1443 atomic_read(&txo->q.used), txo->q.id); 1444 1445 entry = txo->q.dma_mem.va; 1446 for (j = 0; j < TX_Q_LEN * 4; j += 4) { 1447 if (entry[j] != 0 || entry[j + 1] != 0 || 1448 entry[j + 2] != 0 || entry[j + 3] != 0) { 1449 dev_info(dev, "Entry %d 0x%x 0x%x 0x%x 0x%x\n", 1450 j, entry[j], entry[j + 1], 1451 entry[j + 2], entry[j + 3]); 1452 } 1453 } 1454 1455 entry = txo->cq.dma_mem.va; 1456 dev_info(dev, "TXCQ Dump: %d H: %d T: %d used: %d\n", 1457 i, txo->cq.head, txo->cq.tail, 1458 atomic_read(&txo->cq.used)); 1459 for (j = 0; j < TX_CQ_LEN * 4; j += 4) { 1460 if (entry[j] != 0 || entry[j + 1] != 0 || 1461 entry[j + 2] != 0 || entry[j + 3] != 0) { 1462 dev_info(dev, "Entry %d 0x%x 0x%x 0x%x 0x%x\n", 1463 j, entry[j], entry[j + 1], 1464 entry[j + 2], entry[j + 3]); 1465 } 1466 } 1467 1468 for (j = 0; j < TX_Q_LEN; j++) { 1469 if (txo->sent_skb_list[j]) { 1470 skb = txo->sent_skb_list[j]; 1471 if (ip_hdr(skb)->protocol == IPPROTO_TCP) { 1472 tcphdr = tcp_hdr(skb); 1473 dev_info(dev, "TCP source port %d\n", 1474 ntohs(tcphdr->source)); 1475 dev_info(dev, "TCP dest port %d\n", 1476 ntohs(tcphdr->dest)); 1477 dev_info(dev, "TCP sequence num %d\n", 1478 ntohs(tcphdr->seq)); 1479 dev_info(dev, "TCP ack_seq %d\n", 1480 ntohs(tcphdr->ack_seq)); 1481 } else if (ip_hdr(skb)->protocol == 1482 IPPROTO_UDP) { 1483 udphdr = udp_hdr(skb); 1484 dev_info(dev, "UDP source port %d\n", 1485 ntohs(udphdr->source)); 1486 dev_info(dev, "UDP dest port %d\n", 1487 ntohs(udphdr->dest)); 1488 } 1489 dev_info(dev, "skb[%d] %p len %d proto 0x%x\n", 1490 j, skb, skb->len, skb->protocol); 1491 } 1492 } 1493 } 1494 1495 if (lancer_chip(adapter)) { 1496 dev_info(dev, "Initiating reset due to tx timeout\n"); 1497 dev_info(dev, "Resetting adapter\n"); 1498 status = lancer_physdev_ctrl(adapter, 1499 PHYSDEV_CONTROL_FW_RESET_MASK); 1500 if (status) 1501 dev_err(dev, "Reset failed .. Reboot server\n"); 1502 } 1503 } 1504 1505 static inline bool be_in_all_promisc(struct be_adapter *adapter) 1506 { 1507 return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) == 1508 BE_IF_FLAGS_ALL_PROMISCUOUS; 1509 } 1510 1511 static int be_set_vlan_promisc(struct be_adapter *adapter) 1512 { 1513 struct device *dev = &adapter->pdev->dev; 1514 int status; 1515 1516 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) 1517 return 0; 1518 1519 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON); 1520 if (!status) { 1521 dev_info(dev, "Enabled VLAN promiscuous mode\n"); 1522 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS; 1523 } else { 1524 dev_err(dev, "Failed to enable VLAN promiscuous mode\n"); 1525 } 1526 return status; 1527 } 1528 1529 static int be_clear_vlan_promisc(struct be_adapter *adapter) 1530 { 1531 struct device *dev = &adapter->pdev->dev; 1532 int status; 1533 1534 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF); 1535 if (!status) { 1536 dev_info(dev, "Disabling VLAN promiscuous mode\n"); 1537 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 1538 } 1539 return status; 1540 } 1541 1542 /* 1543 * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE. 1544 * If the user configures more, place BE in vlan promiscuous mode. 1545 */ 1546 static int be_vid_config(struct be_adapter *adapter) 1547 { 1548 struct device *dev = &adapter->pdev->dev; 1549 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1550 u16 num = 0, i = 0; 1551 int status = 0; 1552 1553 /* No need to change the VLAN state if the I/F is in promiscuous */ 1554 if (adapter->netdev->flags & IFF_PROMISC) 1555 return 0; 1556 1557 if (adapter->vlans_added > be_max_vlans(adapter)) 1558 return be_set_vlan_promisc(adapter); 1559 1560 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 1561 status = be_clear_vlan_promisc(adapter); 1562 if (status) 1563 return status; 1564 } 1565 /* Construct VLAN Table to give to HW */ 1566 for_each_set_bit(i, adapter->vids, VLAN_N_VID) 1567 vids[num++] = cpu_to_le16(i); 1568 1569 status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0); 1570 if (status) { 1571 dev_err(dev, "Setting HW VLAN filtering failed\n"); 1572 /* Set to VLAN promisc mode as setting VLAN filter failed */ 1573 if (addl_status(status) == MCC_ADDL_STATUS_INSUFFICIENT_VLANS || 1574 addl_status(status) == 1575 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES) 1576 return be_set_vlan_promisc(adapter); 1577 } 1578 return status; 1579 } 1580 1581 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 1582 { 1583 struct be_adapter *adapter = netdev_priv(netdev); 1584 int status = 0; 1585 1586 mutex_lock(&adapter->rx_filter_lock); 1587 1588 /* Packets with VID 0 are always received by Lancer by default */ 1589 if (lancer_chip(adapter) && vid == 0) 1590 goto done; 1591 1592 if (test_bit(vid, adapter->vids)) 1593 goto done; 1594 1595 set_bit(vid, adapter->vids); 1596 adapter->vlans_added++; 1597 1598 status = be_vid_config(adapter); 1599 done: 1600 mutex_unlock(&adapter->rx_filter_lock); 1601 return status; 1602 } 1603 1604 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid) 1605 { 1606 struct be_adapter *adapter = netdev_priv(netdev); 1607 int status = 0; 1608 1609 mutex_lock(&adapter->rx_filter_lock); 1610 1611 /* Packets with VID 0 are always received by Lancer by default */ 1612 if (lancer_chip(adapter) && vid == 0) 1613 goto done; 1614 1615 if (!test_bit(vid, adapter->vids)) 1616 goto done; 1617 1618 clear_bit(vid, adapter->vids); 1619 adapter->vlans_added--; 1620 1621 status = be_vid_config(adapter); 1622 done: 1623 mutex_unlock(&adapter->rx_filter_lock); 1624 return status; 1625 } 1626 1627 static void be_set_all_promisc(struct be_adapter *adapter) 1628 { 1629 be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON); 1630 adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS; 1631 } 1632 1633 static void be_set_mc_promisc(struct be_adapter *adapter) 1634 { 1635 int status; 1636 1637 if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) 1638 return; 1639 1640 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON); 1641 if (!status) 1642 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS; 1643 } 1644 1645 static void be_set_uc_promisc(struct be_adapter *adapter) 1646 { 1647 int status; 1648 1649 if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) 1650 return; 1651 1652 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, ON); 1653 if (!status) 1654 adapter->if_flags |= BE_IF_FLAGS_PROMISCUOUS; 1655 } 1656 1657 static void be_clear_uc_promisc(struct be_adapter *adapter) 1658 { 1659 int status; 1660 1661 if (!(adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS)) 1662 return; 1663 1664 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, OFF); 1665 if (!status) 1666 adapter->if_flags &= ~BE_IF_FLAGS_PROMISCUOUS; 1667 } 1668 1669 /* The below 2 functions are the callback args for __dev_mc_sync/dev_uc_sync(). 1670 * We use a single callback function for both sync and unsync. We really don't 1671 * add/remove addresses through this callback. But, we use it to detect changes 1672 * to the uc/mc lists. The entire uc/mc list is programmed in be_set_rx_mode(). 1673 */ 1674 static int be_uc_list_update(struct net_device *netdev, 1675 const unsigned char *addr) 1676 { 1677 struct be_adapter *adapter = netdev_priv(netdev); 1678 1679 adapter->update_uc_list = true; 1680 return 0; 1681 } 1682 1683 static int be_mc_list_update(struct net_device *netdev, 1684 const unsigned char *addr) 1685 { 1686 struct be_adapter *adapter = netdev_priv(netdev); 1687 1688 adapter->update_mc_list = true; 1689 return 0; 1690 } 1691 1692 static void be_set_mc_list(struct be_adapter *adapter) 1693 { 1694 struct net_device *netdev = adapter->netdev; 1695 struct netdev_hw_addr *ha; 1696 bool mc_promisc = false; 1697 int status; 1698 1699 netif_addr_lock_bh(netdev); 1700 __dev_mc_sync(netdev, be_mc_list_update, be_mc_list_update); 1701 1702 if (netdev->flags & IFF_PROMISC) { 1703 adapter->update_mc_list = false; 1704 } else if (netdev->flags & IFF_ALLMULTI || 1705 netdev_mc_count(netdev) > be_max_mc(adapter)) { 1706 /* Enable multicast promisc if num configured exceeds 1707 * what we support 1708 */ 1709 mc_promisc = true; 1710 adapter->update_mc_list = false; 1711 } else if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) { 1712 /* Update mc-list unconditionally if the iface was previously 1713 * in mc-promisc mode and now is out of that mode. 1714 */ 1715 adapter->update_mc_list = true; 1716 } 1717 1718 if (adapter->update_mc_list) { 1719 int i = 0; 1720 1721 /* cache the mc-list in adapter */ 1722 netdev_for_each_mc_addr(ha, netdev) { 1723 ether_addr_copy(adapter->mc_list[i].mac, ha->addr); 1724 i++; 1725 } 1726 adapter->mc_count = netdev_mc_count(netdev); 1727 } 1728 netif_addr_unlock_bh(netdev); 1729 1730 if (mc_promisc) { 1731 be_set_mc_promisc(adapter); 1732 } else if (adapter->update_mc_list) { 1733 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON); 1734 if (!status) 1735 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS; 1736 else 1737 be_set_mc_promisc(adapter); 1738 1739 adapter->update_mc_list = false; 1740 } 1741 } 1742 1743 static void be_clear_mc_list(struct be_adapter *adapter) 1744 { 1745 struct net_device *netdev = adapter->netdev; 1746 1747 __dev_mc_unsync(netdev, NULL); 1748 be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, OFF); 1749 adapter->mc_count = 0; 1750 } 1751 1752 static int be_uc_mac_add(struct be_adapter *adapter, int uc_idx) 1753 { 1754 if (ether_addr_equal(adapter->uc_list[uc_idx].mac, adapter->dev_mac)) { 1755 adapter->pmac_id[uc_idx + 1] = adapter->pmac_id[0]; 1756 return 0; 1757 } 1758 1759 return be_cmd_pmac_add(adapter, adapter->uc_list[uc_idx].mac, 1760 adapter->if_handle, 1761 &adapter->pmac_id[uc_idx + 1], 0); 1762 } 1763 1764 static void be_uc_mac_del(struct be_adapter *adapter, int pmac_id) 1765 { 1766 if (pmac_id == adapter->pmac_id[0]) 1767 return; 1768 1769 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0); 1770 } 1771 1772 static void be_set_uc_list(struct be_adapter *adapter) 1773 { 1774 struct net_device *netdev = adapter->netdev; 1775 struct netdev_hw_addr *ha; 1776 bool uc_promisc = false; 1777 int curr_uc_macs = 0, i; 1778 1779 netif_addr_lock_bh(netdev); 1780 __dev_uc_sync(netdev, be_uc_list_update, be_uc_list_update); 1781 1782 if (netdev->flags & IFF_PROMISC) { 1783 adapter->update_uc_list = false; 1784 } else if (netdev_uc_count(netdev) > (be_max_uc(adapter) - 1)) { 1785 uc_promisc = true; 1786 adapter->update_uc_list = false; 1787 } else if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) { 1788 /* Update uc-list unconditionally if the iface was previously 1789 * in uc-promisc mode and now is out of that mode. 1790 */ 1791 adapter->update_uc_list = true; 1792 } 1793 1794 if (adapter->update_uc_list) { 1795 /* cache the uc-list in adapter array */ 1796 i = 0; 1797 netdev_for_each_uc_addr(ha, netdev) { 1798 ether_addr_copy(adapter->uc_list[i].mac, ha->addr); 1799 i++; 1800 } 1801 curr_uc_macs = netdev_uc_count(netdev); 1802 } 1803 netif_addr_unlock_bh(netdev); 1804 1805 if (uc_promisc) { 1806 be_set_uc_promisc(adapter); 1807 } else if (adapter->update_uc_list) { 1808 be_clear_uc_promisc(adapter); 1809 1810 for (i = 0; i < adapter->uc_macs; i++) 1811 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]); 1812 1813 for (i = 0; i < curr_uc_macs; i++) 1814 be_uc_mac_add(adapter, i); 1815 adapter->uc_macs = curr_uc_macs; 1816 adapter->update_uc_list = false; 1817 } 1818 } 1819 1820 static void be_clear_uc_list(struct be_adapter *adapter) 1821 { 1822 struct net_device *netdev = adapter->netdev; 1823 int i; 1824 1825 __dev_uc_unsync(netdev, NULL); 1826 for (i = 0; i < adapter->uc_macs; i++) 1827 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]); 1828 1829 adapter->uc_macs = 0; 1830 } 1831 1832 static void __be_set_rx_mode(struct be_adapter *adapter) 1833 { 1834 struct net_device *netdev = adapter->netdev; 1835 1836 mutex_lock(&adapter->rx_filter_lock); 1837 1838 if (netdev->flags & IFF_PROMISC) { 1839 if (!be_in_all_promisc(adapter)) 1840 be_set_all_promisc(adapter); 1841 } else if (be_in_all_promisc(adapter)) { 1842 /* We need to re-program the vlan-list or clear 1843 * vlan-promisc mode (if needed) when the interface 1844 * comes out of promisc mode. 1845 */ 1846 be_vid_config(adapter); 1847 } 1848 1849 be_set_uc_list(adapter); 1850 be_set_mc_list(adapter); 1851 1852 mutex_unlock(&adapter->rx_filter_lock); 1853 } 1854 1855 static void be_work_set_rx_mode(struct work_struct *work) 1856 { 1857 struct be_cmd_work *cmd_work = 1858 container_of(work, struct be_cmd_work, work); 1859 1860 __be_set_rx_mode(cmd_work->adapter); 1861 kfree(cmd_work); 1862 } 1863 1864 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 1865 { 1866 struct be_adapter *adapter = netdev_priv(netdev); 1867 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1868 int status; 1869 1870 if (!sriov_enabled(adapter)) 1871 return -EPERM; 1872 1873 if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs) 1874 return -EINVAL; 1875 1876 /* Proceed further only if user provided MAC is different 1877 * from active MAC 1878 */ 1879 if (ether_addr_equal(mac, vf_cfg->mac_addr)) 1880 return 0; 1881 1882 if (BEx_chip(adapter)) { 1883 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id, 1884 vf + 1); 1885 1886 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle, 1887 &vf_cfg->pmac_id, vf + 1); 1888 } else { 1889 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 1890 vf + 1); 1891 } 1892 1893 if (status) { 1894 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x", 1895 mac, vf, status); 1896 return be_cmd_status(status); 1897 } 1898 1899 ether_addr_copy(vf_cfg->mac_addr, mac); 1900 1901 return 0; 1902 } 1903 1904 static int be_get_vf_config(struct net_device *netdev, int vf, 1905 struct ifla_vf_info *vi) 1906 { 1907 struct be_adapter *adapter = netdev_priv(netdev); 1908 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1909 1910 if (!sriov_enabled(adapter)) 1911 return -EPERM; 1912 1913 if (vf >= adapter->num_vfs) 1914 return -EINVAL; 1915 1916 vi->vf = vf; 1917 vi->max_tx_rate = vf_cfg->tx_rate; 1918 vi->min_tx_rate = 0; 1919 vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK; 1920 vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT; 1921 memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN); 1922 vi->linkstate = adapter->vf_cfg[vf].plink_tracking; 1923 vi->spoofchk = adapter->vf_cfg[vf].spoofchk; 1924 1925 return 0; 1926 } 1927 1928 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan) 1929 { 1930 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1931 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1932 int vf_if_id = vf_cfg->if_handle; 1933 int status; 1934 1935 /* Enable Transparent VLAN Tagging */ 1936 status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0, 0); 1937 if (status) 1938 return status; 1939 1940 /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */ 1941 vids[0] = 0; 1942 status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1); 1943 if (!status) 1944 dev_info(&adapter->pdev->dev, 1945 "Cleared guest VLANs on VF%d", vf); 1946 1947 /* After TVT is enabled, disallow VFs to program VLAN filters */ 1948 if (vf_cfg->privileges & BE_PRIV_FILTMGMT) { 1949 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges & 1950 ~BE_PRIV_FILTMGMT, vf + 1); 1951 if (!status) 1952 vf_cfg->privileges &= ~BE_PRIV_FILTMGMT; 1953 } 1954 return 0; 1955 } 1956 1957 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf) 1958 { 1959 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1960 struct device *dev = &adapter->pdev->dev; 1961 int status; 1962 1963 /* Reset Transparent VLAN Tagging. */ 1964 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1, 1965 vf_cfg->if_handle, 0, 0); 1966 if (status) 1967 return status; 1968 1969 /* Allow VFs to program VLAN filtering */ 1970 if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 1971 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges | 1972 BE_PRIV_FILTMGMT, vf + 1); 1973 if (!status) { 1974 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 1975 dev_info(dev, "VF%d: FILTMGMT priv enabled", vf); 1976 } 1977 } 1978 1979 dev_info(dev, 1980 "Disable/re-enable i/f in VM to clear Transparent VLAN tag"); 1981 return 0; 1982 } 1983 1984 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos, 1985 __be16 vlan_proto) 1986 { 1987 struct be_adapter *adapter = netdev_priv(netdev); 1988 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1989 int status; 1990 1991 if (!sriov_enabled(adapter)) 1992 return -EPERM; 1993 1994 if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7) 1995 return -EINVAL; 1996 1997 if (vlan_proto != htons(ETH_P_8021Q)) 1998 return -EPROTONOSUPPORT; 1999 2000 if (vlan || qos) { 2001 vlan |= qos << VLAN_PRIO_SHIFT; 2002 status = be_set_vf_tvt(adapter, vf, vlan); 2003 } else { 2004 status = be_clear_vf_tvt(adapter, vf); 2005 } 2006 2007 if (status) { 2008 dev_err(&adapter->pdev->dev, 2009 "VLAN %d config on VF %d failed : %#x\n", vlan, vf, 2010 status); 2011 return be_cmd_status(status); 2012 } 2013 2014 vf_cfg->vlan_tag = vlan; 2015 return 0; 2016 } 2017 2018 static int be_set_vf_tx_rate(struct net_device *netdev, int vf, 2019 int min_tx_rate, int max_tx_rate) 2020 { 2021 struct be_adapter *adapter = netdev_priv(netdev); 2022 struct device *dev = &adapter->pdev->dev; 2023 int percent_rate, status = 0; 2024 u16 link_speed = 0; 2025 u8 link_status; 2026 2027 if (!sriov_enabled(adapter)) 2028 return -EPERM; 2029 2030 if (vf >= adapter->num_vfs) 2031 return -EINVAL; 2032 2033 if (min_tx_rate) 2034 return -EINVAL; 2035 2036 if (!max_tx_rate) 2037 goto config_qos; 2038 2039 status = be_cmd_link_status_query(adapter, &link_speed, 2040 &link_status, 0); 2041 if (status) 2042 goto err; 2043 2044 if (!link_status) { 2045 dev_err(dev, "TX-rate setting not allowed when link is down\n"); 2046 status = -ENETDOWN; 2047 goto err; 2048 } 2049 2050 if (max_tx_rate < 100 || max_tx_rate > link_speed) { 2051 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n", 2052 link_speed); 2053 status = -EINVAL; 2054 goto err; 2055 } 2056 2057 /* On Skyhawk the QOS setting must be done only as a % value */ 2058 percent_rate = link_speed / 100; 2059 if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) { 2060 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n", 2061 percent_rate); 2062 status = -EINVAL; 2063 goto err; 2064 } 2065 2066 config_qos: 2067 status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1); 2068 if (status) 2069 goto err; 2070 2071 adapter->vf_cfg[vf].tx_rate = max_tx_rate; 2072 return 0; 2073 2074 err: 2075 dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n", 2076 max_tx_rate, vf); 2077 return be_cmd_status(status); 2078 } 2079 2080 static int be_set_vf_link_state(struct net_device *netdev, int vf, 2081 int link_state) 2082 { 2083 struct be_adapter *adapter = netdev_priv(netdev); 2084 int status; 2085 2086 if (!sriov_enabled(adapter)) 2087 return -EPERM; 2088 2089 if (vf >= adapter->num_vfs) 2090 return -EINVAL; 2091 2092 status = be_cmd_set_logical_link_config(adapter, link_state, vf+1); 2093 if (status) { 2094 dev_err(&adapter->pdev->dev, 2095 "Link state change on VF %d failed: %#x\n", vf, status); 2096 return be_cmd_status(status); 2097 } 2098 2099 adapter->vf_cfg[vf].plink_tracking = link_state; 2100 2101 return 0; 2102 } 2103 2104 static int be_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable) 2105 { 2106 struct be_adapter *adapter = netdev_priv(netdev); 2107 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 2108 u8 spoofchk; 2109 int status; 2110 2111 if (!sriov_enabled(adapter)) 2112 return -EPERM; 2113 2114 if (vf >= adapter->num_vfs) 2115 return -EINVAL; 2116 2117 if (BEx_chip(adapter)) 2118 return -EOPNOTSUPP; 2119 2120 if (enable == vf_cfg->spoofchk) 2121 return 0; 2122 2123 spoofchk = enable ? ENABLE_MAC_SPOOFCHK : DISABLE_MAC_SPOOFCHK; 2124 2125 status = be_cmd_set_hsw_config(adapter, 0, vf + 1, vf_cfg->if_handle, 2126 0, spoofchk); 2127 if (status) { 2128 dev_err(&adapter->pdev->dev, 2129 "Spoofchk change on VF %d failed: %#x\n", vf, status); 2130 return be_cmd_status(status); 2131 } 2132 2133 vf_cfg->spoofchk = enable; 2134 return 0; 2135 } 2136 2137 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts, 2138 ulong now) 2139 { 2140 aic->rx_pkts_prev = rx_pkts; 2141 aic->tx_reqs_prev = tx_pkts; 2142 aic->jiffies = now; 2143 } 2144 2145 static int be_get_new_eqd(struct be_eq_obj *eqo) 2146 { 2147 struct be_adapter *adapter = eqo->adapter; 2148 int eqd, start; 2149 struct be_aic_obj *aic; 2150 struct be_rx_obj *rxo; 2151 struct be_tx_obj *txo; 2152 u64 rx_pkts = 0, tx_pkts = 0; 2153 ulong now; 2154 u32 pps, delta; 2155 int i; 2156 2157 aic = &adapter->aic_obj[eqo->idx]; 2158 if (!aic->enable) { 2159 if (aic->jiffies) 2160 aic->jiffies = 0; 2161 eqd = aic->et_eqd; 2162 return eqd; 2163 } 2164 2165 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 2166 do { 2167 start = u64_stats_fetch_begin_irq(&rxo->stats.sync); 2168 rx_pkts += rxo->stats.rx_pkts; 2169 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start)); 2170 } 2171 2172 for_all_tx_queues_on_eq(adapter, eqo, txo, i) { 2173 do { 2174 start = u64_stats_fetch_begin_irq(&txo->stats.sync); 2175 tx_pkts += txo->stats.tx_reqs; 2176 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start)); 2177 } 2178 2179 /* Skip, if wrapped around or first calculation */ 2180 now = jiffies; 2181 if (!aic->jiffies || time_before(now, aic->jiffies) || 2182 rx_pkts < aic->rx_pkts_prev || 2183 tx_pkts < aic->tx_reqs_prev) { 2184 be_aic_update(aic, rx_pkts, tx_pkts, now); 2185 return aic->prev_eqd; 2186 } 2187 2188 delta = jiffies_to_msecs(now - aic->jiffies); 2189 if (delta == 0) 2190 return aic->prev_eqd; 2191 2192 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) + 2193 (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta); 2194 eqd = (pps / 15000) << 2; 2195 2196 if (eqd < 8) 2197 eqd = 0; 2198 eqd = min_t(u32, eqd, aic->max_eqd); 2199 eqd = max_t(u32, eqd, aic->min_eqd); 2200 2201 be_aic_update(aic, rx_pkts, tx_pkts, now); 2202 2203 return eqd; 2204 } 2205 2206 /* For Skyhawk-R only */ 2207 static u32 be_get_eq_delay_mult_enc(struct be_eq_obj *eqo) 2208 { 2209 struct be_adapter *adapter = eqo->adapter; 2210 struct be_aic_obj *aic = &adapter->aic_obj[eqo->idx]; 2211 ulong now = jiffies; 2212 int eqd; 2213 u32 mult_enc; 2214 2215 if (!aic->enable) 2216 return 0; 2217 2218 if (jiffies_to_msecs(now - aic->jiffies) < 1) 2219 eqd = aic->prev_eqd; 2220 else 2221 eqd = be_get_new_eqd(eqo); 2222 2223 if (eqd > 100) 2224 mult_enc = R2I_DLY_ENC_1; 2225 else if (eqd > 60) 2226 mult_enc = R2I_DLY_ENC_2; 2227 else if (eqd > 20) 2228 mult_enc = R2I_DLY_ENC_3; 2229 else 2230 mult_enc = R2I_DLY_ENC_0; 2231 2232 aic->prev_eqd = eqd; 2233 2234 return mult_enc; 2235 } 2236 2237 void be_eqd_update(struct be_adapter *adapter, bool force_update) 2238 { 2239 struct be_set_eqd set_eqd[MAX_EVT_QS]; 2240 struct be_aic_obj *aic; 2241 struct be_eq_obj *eqo; 2242 int i, num = 0, eqd; 2243 2244 for_all_evt_queues(adapter, eqo, i) { 2245 aic = &adapter->aic_obj[eqo->idx]; 2246 eqd = be_get_new_eqd(eqo); 2247 if (force_update || eqd != aic->prev_eqd) { 2248 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2249 set_eqd[num].eq_id = eqo->q.id; 2250 aic->prev_eqd = eqd; 2251 num++; 2252 } 2253 } 2254 2255 if (num) 2256 be_cmd_modify_eqd(adapter, set_eqd, num); 2257 } 2258 2259 static void be_rx_stats_update(struct be_rx_obj *rxo, 2260 struct be_rx_compl_info *rxcp) 2261 { 2262 struct be_rx_stats *stats = rx_stats(rxo); 2263 2264 u64_stats_update_begin(&stats->sync); 2265 stats->rx_compl++; 2266 stats->rx_bytes += rxcp->pkt_size; 2267 stats->rx_pkts++; 2268 if (rxcp->tunneled) 2269 stats->rx_vxlan_offload_pkts++; 2270 if (rxcp->pkt_type == BE_MULTICAST_PACKET) 2271 stats->rx_mcast_pkts++; 2272 if (rxcp->err) 2273 stats->rx_compl_err++; 2274 u64_stats_update_end(&stats->sync); 2275 } 2276 2277 static inline bool csum_passed(struct be_rx_compl_info *rxcp) 2278 { 2279 /* L4 checksum is not reliable for non TCP/UDP packets. 2280 * Also ignore ipcksm for ipv6 pkts 2281 */ 2282 return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum && 2283 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err; 2284 } 2285 2286 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo) 2287 { 2288 struct be_adapter *adapter = rxo->adapter; 2289 struct be_rx_page_info *rx_page_info; 2290 struct be_queue_info *rxq = &rxo->q; 2291 u32 frag_idx = rxq->tail; 2292 2293 rx_page_info = &rxo->page_info_tbl[frag_idx]; 2294 BUG_ON(!rx_page_info->page); 2295 2296 if (rx_page_info->last_frag) { 2297 dma_unmap_page(&adapter->pdev->dev, 2298 dma_unmap_addr(rx_page_info, bus), 2299 adapter->big_page_size, DMA_FROM_DEVICE); 2300 rx_page_info->last_frag = false; 2301 } else { 2302 dma_sync_single_for_cpu(&adapter->pdev->dev, 2303 dma_unmap_addr(rx_page_info, bus), 2304 rx_frag_size, DMA_FROM_DEVICE); 2305 } 2306 2307 queue_tail_inc(rxq); 2308 atomic_dec(&rxq->used); 2309 return rx_page_info; 2310 } 2311 2312 /* Throwaway the data in the Rx completion */ 2313 static void be_rx_compl_discard(struct be_rx_obj *rxo, 2314 struct be_rx_compl_info *rxcp) 2315 { 2316 struct be_rx_page_info *page_info; 2317 u16 i, num_rcvd = rxcp->num_rcvd; 2318 2319 for (i = 0; i < num_rcvd; i++) { 2320 page_info = get_rx_page_info(rxo); 2321 put_page(page_info->page); 2322 memset(page_info, 0, sizeof(*page_info)); 2323 } 2324 } 2325 2326 /* 2327 * skb_fill_rx_data forms a complete skb for an ether frame 2328 * indicated by rxcp. 2329 */ 2330 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb, 2331 struct be_rx_compl_info *rxcp) 2332 { 2333 struct be_rx_page_info *page_info; 2334 u16 i, j; 2335 u16 hdr_len, curr_frag_len, remaining; 2336 u8 *start; 2337 2338 page_info = get_rx_page_info(rxo); 2339 start = page_address(page_info->page) + page_info->page_offset; 2340 prefetch(start); 2341 2342 /* Copy data in the first descriptor of this completion */ 2343 curr_frag_len = min(rxcp->pkt_size, rx_frag_size); 2344 2345 skb->len = curr_frag_len; 2346 if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */ 2347 memcpy(skb->data, start, curr_frag_len); 2348 /* Complete packet has now been moved to data */ 2349 put_page(page_info->page); 2350 skb->data_len = 0; 2351 skb->tail += curr_frag_len; 2352 } else { 2353 hdr_len = ETH_HLEN; 2354 memcpy(skb->data, start, hdr_len); 2355 skb_shinfo(skb)->nr_frags = 1; 2356 skb_frag_set_page(skb, 0, page_info->page); 2357 skb_shinfo(skb)->frags[0].page_offset = 2358 page_info->page_offset + hdr_len; 2359 skb_frag_size_set(&skb_shinfo(skb)->frags[0], 2360 curr_frag_len - hdr_len); 2361 skb->data_len = curr_frag_len - hdr_len; 2362 skb->truesize += rx_frag_size; 2363 skb->tail += hdr_len; 2364 } 2365 page_info->page = NULL; 2366 2367 if (rxcp->pkt_size <= rx_frag_size) { 2368 BUG_ON(rxcp->num_rcvd != 1); 2369 return; 2370 } 2371 2372 /* More frags present for this completion */ 2373 remaining = rxcp->pkt_size - curr_frag_len; 2374 for (i = 1, j = 0; i < rxcp->num_rcvd; i++) { 2375 page_info = get_rx_page_info(rxo); 2376 curr_frag_len = min(remaining, rx_frag_size); 2377 2378 /* Coalesce all frags from the same physical page in one slot */ 2379 if (page_info->page_offset == 0) { 2380 /* Fresh page */ 2381 j++; 2382 skb_frag_set_page(skb, j, page_info->page); 2383 skb_shinfo(skb)->frags[j].page_offset = 2384 page_info->page_offset; 2385 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 2386 skb_shinfo(skb)->nr_frags++; 2387 } else { 2388 put_page(page_info->page); 2389 } 2390 2391 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 2392 skb->len += curr_frag_len; 2393 skb->data_len += curr_frag_len; 2394 skb->truesize += rx_frag_size; 2395 remaining -= curr_frag_len; 2396 page_info->page = NULL; 2397 } 2398 BUG_ON(j > MAX_SKB_FRAGS); 2399 } 2400 2401 /* Process the RX completion indicated by rxcp when GRO is disabled */ 2402 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi, 2403 struct be_rx_compl_info *rxcp) 2404 { 2405 struct be_adapter *adapter = rxo->adapter; 2406 struct net_device *netdev = adapter->netdev; 2407 struct sk_buff *skb; 2408 2409 skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE); 2410 if (unlikely(!skb)) { 2411 rx_stats(rxo)->rx_drops_no_skbs++; 2412 be_rx_compl_discard(rxo, rxcp); 2413 return; 2414 } 2415 2416 skb_fill_rx_data(rxo, skb, rxcp); 2417 2418 if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp))) 2419 skb->ip_summed = CHECKSUM_UNNECESSARY; 2420 else 2421 skb_checksum_none_assert(skb); 2422 2423 skb->protocol = eth_type_trans(skb, netdev); 2424 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 2425 if (netdev->features & NETIF_F_RXHASH) 2426 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 2427 2428 skb->csum_level = rxcp->tunneled; 2429 skb_mark_napi_id(skb, napi); 2430 2431 if (rxcp->vlanf) 2432 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 2433 2434 netif_receive_skb(skb); 2435 } 2436 2437 /* Process the RX completion indicated by rxcp when GRO is enabled */ 2438 static void be_rx_compl_process_gro(struct be_rx_obj *rxo, 2439 struct napi_struct *napi, 2440 struct be_rx_compl_info *rxcp) 2441 { 2442 struct be_adapter *adapter = rxo->adapter; 2443 struct be_rx_page_info *page_info; 2444 struct sk_buff *skb = NULL; 2445 u16 remaining, curr_frag_len; 2446 u16 i, j; 2447 2448 skb = napi_get_frags(napi); 2449 if (!skb) { 2450 be_rx_compl_discard(rxo, rxcp); 2451 return; 2452 } 2453 2454 remaining = rxcp->pkt_size; 2455 for (i = 0, j = -1; i < rxcp->num_rcvd; i++) { 2456 page_info = get_rx_page_info(rxo); 2457 2458 curr_frag_len = min(remaining, rx_frag_size); 2459 2460 /* Coalesce all frags from the same physical page in one slot */ 2461 if (i == 0 || page_info->page_offset == 0) { 2462 /* First frag or Fresh page */ 2463 j++; 2464 skb_frag_set_page(skb, j, page_info->page); 2465 skb_shinfo(skb)->frags[j].page_offset = 2466 page_info->page_offset; 2467 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 2468 } else { 2469 put_page(page_info->page); 2470 } 2471 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 2472 skb->truesize += rx_frag_size; 2473 remaining -= curr_frag_len; 2474 memset(page_info, 0, sizeof(*page_info)); 2475 } 2476 BUG_ON(j > MAX_SKB_FRAGS); 2477 2478 skb_shinfo(skb)->nr_frags = j + 1; 2479 skb->len = rxcp->pkt_size; 2480 skb->data_len = rxcp->pkt_size; 2481 skb->ip_summed = CHECKSUM_UNNECESSARY; 2482 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 2483 if (adapter->netdev->features & NETIF_F_RXHASH) 2484 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 2485 2486 skb->csum_level = rxcp->tunneled; 2487 2488 if (rxcp->vlanf) 2489 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 2490 2491 napi_gro_frags(napi); 2492 } 2493 2494 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl, 2495 struct be_rx_compl_info *rxcp) 2496 { 2497 rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl); 2498 rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl); 2499 rxcp->err = GET_RX_COMPL_V1_BITS(err, compl); 2500 rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl); 2501 rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl); 2502 rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl); 2503 rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl); 2504 rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl); 2505 rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl); 2506 rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl); 2507 rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl); 2508 if (rxcp->vlanf) { 2509 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl); 2510 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl); 2511 } 2512 rxcp->port = GET_RX_COMPL_V1_BITS(port, compl); 2513 rxcp->tunneled = 2514 GET_RX_COMPL_V1_BITS(tunneled, compl); 2515 } 2516 2517 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl, 2518 struct be_rx_compl_info *rxcp) 2519 { 2520 rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl); 2521 rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl); 2522 rxcp->err = GET_RX_COMPL_V0_BITS(err, compl); 2523 rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl); 2524 rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl); 2525 rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl); 2526 rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl); 2527 rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl); 2528 rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl); 2529 rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl); 2530 rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl); 2531 if (rxcp->vlanf) { 2532 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl); 2533 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl); 2534 } 2535 rxcp->port = GET_RX_COMPL_V0_BITS(port, compl); 2536 rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl); 2537 } 2538 2539 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo) 2540 { 2541 struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq); 2542 struct be_rx_compl_info *rxcp = &rxo->rxcp; 2543 struct be_adapter *adapter = rxo->adapter; 2544 2545 /* For checking the valid bit it is Ok to use either definition as the 2546 * valid bit is at the same position in both v0 and v1 Rx compl */ 2547 if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0) 2548 return NULL; 2549 2550 rmb(); 2551 be_dws_le_to_cpu(compl, sizeof(*compl)); 2552 2553 if (adapter->be3_native) 2554 be_parse_rx_compl_v1(compl, rxcp); 2555 else 2556 be_parse_rx_compl_v0(compl, rxcp); 2557 2558 if (rxcp->ip_frag) 2559 rxcp->l4_csum = 0; 2560 2561 if (rxcp->vlanf) { 2562 /* In QNQ modes, if qnq bit is not set, then the packet was 2563 * tagged only with the transparent outer vlan-tag and must 2564 * not be treated as a vlan packet by host 2565 */ 2566 if (be_is_qnq_mode(adapter) && !rxcp->qnq) 2567 rxcp->vlanf = 0; 2568 2569 if (!lancer_chip(adapter)) 2570 rxcp->vlan_tag = swab16(rxcp->vlan_tag); 2571 2572 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) && 2573 !test_bit(rxcp->vlan_tag, adapter->vids)) 2574 rxcp->vlanf = 0; 2575 } 2576 2577 /* As the compl has been parsed, reset it; we wont touch it again */ 2578 compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0; 2579 2580 queue_tail_inc(&rxo->cq); 2581 return rxcp; 2582 } 2583 2584 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp) 2585 { 2586 u32 order = get_order(size); 2587 2588 if (order > 0) 2589 gfp |= __GFP_COMP; 2590 return alloc_pages(gfp, order); 2591 } 2592 2593 /* 2594 * Allocate a page, split it to fragments of size rx_frag_size and post as 2595 * receive buffers to BE 2596 */ 2597 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed) 2598 { 2599 struct be_adapter *adapter = rxo->adapter; 2600 struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL; 2601 struct be_queue_info *rxq = &rxo->q; 2602 struct page *pagep = NULL; 2603 struct device *dev = &adapter->pdev->dev; 2604 struct be_eth_rx_d *rxd; 2605 u64 page_dmaaddr = 0, frag_dmaaddr; 2606 u32 posted, page_offset = 0, notify = 0; 2607 2608 page_info = &rxo->page_info_tbl[rxq->head]; 2609 for (posted = 0; posted < frags_needed && !page_info->page; posted++) { 2610 if (!pagep) { 2611 pagep = be_alloc_pages(adapter->big_page_size, gfp); 2612 if (unlikely(!pagep)) { 2613 rx_stats(rxo)->rx_post_fail++; 2614 break; 2615 } 2616 page_dmaaddr = dma_map_page(dev, pagep, 0, 2617 adapter->big_page_size, 2618 DMA_FROM_DEVICE); 2619 if (dma_mapping_error(dev, page_dmaaddr)) { 2620 put_page(pagep); 2621 pagep = NULL; 2622 adapter->drv_stats.dma_map_errors++; 2623 break; 2624 } 2625 page_offset = 0; 2626 } else { 2627 get_page(pagep); 2628 page_offset += rx_frag_size; 2629 } 2630 page_info->page_offset = page_offset; 2631 page_info->page = pagep; 2632 2633 rxd = queue_head_node(rxq); 2634 frag_dmaaddr = page_dmaaddr + page_info->page_offset; 2635 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF); 2636 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr)); 2637 2638 /* Any space left in the current big page for another frag? */ 2639 if ((page_offset + rx_frag_size + rx_frag_size) > 2640 adapter->big_page_size) { 2641 pagep = NULL; 2642 page_info->last_frag = true; 2643 dma_unmap_addr_set(page_info, bus, page_dmaaddr); 2644 } else { 2645 dma_unmap_addr_set(page_info, bus, frag_dmaaddr); 2646 } 2647 2648 prev_page_info = page_info; 2649 queue_head_inc(rxq); 2650 page_info = &rxo->page_info_tbl[rxq->head]; 2651 } 2652 2653 /* Mark the last frag of a page when we break out of the above loop 2654 * with no more slots available in the RXQ 2655 */ 2656 if (pagep) { 2657 prev_page_info->last_frag = true; 2658 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr); 2659 } 2660 2661 if (posted) { 2662 atomic_add(posted, &rxq->used); 2663 if (rxo->rx_post_starved) 2664 rxo->rx_post_starved = false; 2665 do { 2666 notify = min(MAX_NUM_POST_ERX_DB, posted); 2667 be_rxq_notify(adapter, rxq->id, notify); 2668 posted -= notify; 2669 } while (posted); 2670 } else if (atomic_read(&rxq->used) == 0) { 2671 /* Let be_worker replenish when memory is available */ 2672 rxo->rx_post_starved = true; 2673 } 2674 } 2675 2676 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status) 2677 { 2678 switch (status) { 2679 case BE_TX_COMP_HDR_PARSE_ERR: 2680 tx_stats(txo)->tx_hdr_parse_err++; 2681 break; 2682 case BE_TX_COMP_NDMA_ERR: 2683 tx_stats(txo)->tx_dma_err++; 2684 break; 2685 case BE_TX_COMP_ACL_ERR: 2686 tx_stats(txo)->tx_spoof_check_err++; 2687 break; 2688 } 2689 } 2690 2691 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status) 2692 { 2693 switch (status) { 2694 case LANCER_TX_COMP_LSO_ERR: 2695 tx_stats(txo)->tx_tso_err++; 2696 break; 2697 case LANCER_TX_COMP_HSW_DROP_MAC_ERR: 2698 case LANCER_TX_COMP_HSW_DROP_VLAN_ERR: 2699 tx_stats(txo)->tx_spoof_check_err++; 2700 break; 2701 case LANCER_TX_COMP_QINQ_ERR: 2702 tx_stats(txo)->tx_qinq_err++; 2703 break; 2704 case LANCER_TX_COMP_PARITY_ERR: 2705 tx_stats(txo)->tx_internal_parity_err++; 2706 break; 2707 case LANCER_TX_COMP_DMA_ERR: 2708 tx_stats(txo)->tx_dma_err++; 2709 break; 2710 case LANCER_TX_COMP_SGE_ERR: 2711 tx_stats(txo)->tx_sge_err++; 2712 break; 2713 } 2714 } 2715 2716 static struct be_tx_compl_info *be_tx_compl_get(struct be_adapter *adapter, 2717 struct be_tx_obj *txo) 2718 { 2719 struct be_queue_info *tx_cq = &txo->cq; 2720 struct be_tx_compl_info *txcp = &txo->txcp; 2721 struct be_eth_tx_compl *compl = queue_tail_node(tx_cq); 2722 2723 if (compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0) 2724 return NULL; 2725 2726 /* Ensure load ordering of valid bit dword and other dwords below */ 2727 rmb(); 2728 be_dws_le_to_cpu(compl, sizeof(*compl)); 2729 2730 txcp->status = GET_TX_COMPL_BITS(status, compl); 2731 txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl); 2732 2733 if (txcp->status) { 2734 if (lancer_chip(adapter)) { 2735 lancer_update_tx_err(txo, txcp->status); 2736 /* Reset the adapter incase of TSO, 2737 * SGE or Parity error 2738 */ 2739 if (txcp->status == LANCER_TX_COMP_LSO_ERR || 2740 txcp->status == LANCER_TX_COMP_PARITY_ERR || 2741 txcp->status == LANCER_TX_COMP_SGE_ERR) 2742 be_set_error(adapter, BE_ERROR_TX); 2743 } else { 2744 be_update_tx_err(txo, txcp->status); 2745 } 2746 } 2747 2748 if (be_check_error(adapter, BE_ERROR_TX)) 2749 return NULL; 2750 2751 compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0; 2752 queue_tail_inc(tx_cq); 2753 return txcp; 2754 } 2755 2756 static u16 be_tx_compl_process(struct be_adapter *adapter, 2757 struct be_tx_obj *txo, u16 last_index) 2758 { 2759 struct sk_buff **sent_skbs = txo->sent_skb_list; 2760 struct be_queue_info *txq = &txo->q; 2761 struct sk_buff *skb = NULL; 2762 bool unmap_skb_hdr = false; 2763 struct be_eth_wrb *wrb; 2764 u16 num_wrbs = 0; 2765 u32 frag_index; 2766 2767 do { 2768 if (sent_skbs[txq->tail]) { 2769 /* Free skb from prev req */ 2770 if (skb) 2771 dev_consume_skb_any(skb); 2772 skb = sent_skbs[txq->tail]; 2773 sent_skbs[txq->tail] = NULL; 2774 queue_tail_inc(txq); /* skip hdr wrb */ 2775 num_wrbs++; 2776 unmap_skb_hdr = true; 2777 } 2778 wrb = queue_tail_node(txq); 2779 frag_index = txq->tail; 2780 unmap_tx_frag(&adapter->pdev->dev, wrb, 2781 (unmap_skb_hdr && skb_headlen(skb))); 2782 unmap_skb_hdr = false; 2783 queue_tail_inc(txq); 2784 num_wrbs++; 2785 } while (frag_index != last_index); 2786 dev_consume_skb_any(skb); 2787 2788 return num_wrbs; 2789 } 2790 2791 /* Return the number of events in the event queue */ 2792 static inline int events_get(struct be_eq_obj *eqo) 2793 { 2794 struct be_eq_entry *eqe; 2795 int num = 0; 2796 2797 do { 2798 eqe = queue_tail_node(&eqo->q); 2799 if (eqe->evt == 0) 2800 break; 2801 2802 rmb(); 2803 eqe->evt = 0; 2804 num++; 2805 queue_tail_inc(&eqo->q); 2806 } while (true); 2807 2808 return num; 2809 } 2810 2811 /* Leaves the EQ is disarmed state */ 2812 static void be_eq_clean(struct be_eq_obj *eqo) 2813 { 2814 int num = events_get(eqo); 2815 2816 be_eq_notify(eqo->adapter, eqo->q.id, false, true, num, 0); 2817 } 2818 2819 /* Free posted rx buffers that were not used */ 2820 static void be_rxq_clean(struct be_rx_obj *rxo) 2821 { 2822 struct be_queue_info *rxq = &rxo->q; 2823 struct be_rx_page_info *page_info; 2824 2825 while (atomic_read(&rxq->used) > 0) { 2826 page_info = get_rx_page_info(rxo); 2827 put_page(page_info->page); 2828 memset(page_info, 0, sizeof(*page_info)); 2829 } 2830 BUG_ON(atomic_read(&rxq->used)); 2831 rxq->tail = 0; 2832 rxq->head = 0; 2833 } 2834 2835 static void be_rx_cq_clean(struct be_rx_obj *rxo) 2836 { 2837 struct be_queue_info *rx_cq = &rxo->cq; 2838 struct be_rx_compl_info *rxcp; 2839 struct be_adapter *adapter = rxo->adapter; 2840 int flush_wait = 0; 2841 2842 /* Consume pending rx completions. 2843 * Wait for the flush completion (identified by zero num_rcvd) 2844 * to arrive. Notify CQ even when there are no more CQ entries 2845 * for HW to flush partially coalesced CQ entries. 2846 * In Lancer, there is no need to wait for flush compl. 2847 */ 2848 for (;;) { 2849 rxcp = be_rx_compl_get(rxo); 2850 if (!rxcp) { 2851 if (lancer_chip(adapter)) 2852 break; 2853 2854 if (flush_wait++ > 50 || 2855 be_check_error(adapter, 2856 BE_ERROR_HW)) { 2857 dev_warn(&adapter->pdev->dev, 2858 "did not receive flush compl\n"); 2859 break; 2860 } 2861 be_cq_notify(adapter, rx_cq->id, true, 0); 2862 mdelay(1); 2863 } else { 2864 be_rx_compl_discard(rxo, rxcp); 2865 be_cq_notify(adapter, rx_cq->id, false, 1); 2866 if (rxcp->num_rcvd == 0) 2867 break; 2868 } 2869 } 2870 2871 /* After cleanup, leave the CQ in unarmed state */ 2872 be_cq_notify(adapter, rx_cq->id, false, 0); 2873 } 2874 2875 static void be_tx_compl_clean(struct be_adapter *adapter) 2876 { 2877 struct device *dev = &adapter->pdev->dev; 2878 u16 cmpl = 0, timeo = 0, num_wrbs = 0; 2879 struct be_tx_compl_info *txcp; 2880 struct be_queue_info *txq; 2881 u32 end_idx, notified_idx; 2882 struct be_tx_obj *txo; 2883 int i, pending_txqs; 2884 2885 /* Stop polling for compls when HW has been silent for 10ms */ 2886 do { 2887 pending_txqs = adapter->num_tx_qs; 2888 2889 for_all_tx_queues(adapter, txo, i) { 2890 cmpl = 0; 2891 num_wrbs = 0; 2892 txq = &txo->q; 2893 while ((txcp = be_tx_compl_get(adapter, txo))) { 2894 num_wrbs += 2895 be_tx_compl_process(adapter, txo, 2896 txcp->end_index); 2897 cmpl++; 2898 } 2899 if (cmpl) { 2900 be_cq_notify(adapter, txo->cq.id, false, cmpl); 2901 atomic_sub(num_wrbs, &txq->used); 2902 timeo = 0; 2903 } 2904 if (!be_is_tx_compl_pending(txo)) 2905 pending_txqs--; 2906 } 2907 2908 if (pending_txqs == 0 || ++timeo > 10 || 2909 be_check_error(adapter, BE_ERROR_HW)) 2910 break; 2911 2912 mdelay(1); 2913 } while (true); 2914 2915 /* Free enqueued TX that was never notified to HW */ 2916 for_all_tx_queues(adapter, txo, i) { 2917 txq = &txo->q; 2918 2919 if (atomic_read(&txq->used)) { 2920 dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n", 2921 i, atomic_read(&txq->used)); 2922 notified_idx = txq->tail; 2923 end_idx = txq->tail; 2924 index_adv(&end_idx, atomic_read(&txq->used) - 1, 2925 txq->len); 2926 /* Use the tx-compl process logic to handle requests 2927 * that were not sent to the HW. 2928 */ 2929 num_wrbs = be_tx_compl_process(adapter, txo, end_idx); 2930 atomic_sub(num_wrbs, &txq->used); 2931 BUG_ON(atomic_read(&txq->used)); 2932 txo->pend_wrb_cnt = 0; 2933 /* Since hw was never notified of these requests, 2934 * reset TXQ indices 2935 */ 2936 txq->head = notified_idx; 2937 txq->tail = notified_idx; 2938 } 2939 } 2940 } 2941 2942 static void be_evt_queues_destroy(struct be_adapter *adapter) 2943 { 2944 struct be_eq_obj *eqo; 2945 int i; 2946 2947 for_all_evt_queues(adapter, eqo, i) { 2948 if (eqo->q.created) { 2949 be_eq_clean(eqo); 2950 be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ); 2951 netif_napi_del(&eqo->napi); 2952 free_cpumask_var(eqo->affinity_mask); 2953 } 2954 be_queue_free(adapter, &eqo->q); 2955 } 2956 } 2957 2958 static int be_evt_queues_create(struct be_adapter *adapter) 2959 { 2960 struct be_queue_info *eq; 2961 struct be_eq_obj *eqo; 2962 struct be_aic_obj *aic; 2963 int i, rc; 2964 2965 /* need enough EQs to service both RX and TX queues */ 2966 adapter->num_evt_qs = min_t(u16, num_irqs(adapter), 2967 max(adapter->cfg_num_rx_irqs, 2968 adapter->cfg_num_tx_irqs)); 2969 2970 for_all_evt_queues(adapter, eqo, i) { 2971 int numa_node = dev_to_node(&adapter->pdev->dev); 2972 2973 aic = &adapter->aic_obj[i]; 2974 eqo->adapter = adapter; 2975 eqo->idx = i; 2976 aic->max_eqd = BE_MAX_EQD; 2977 aic->enable = true; 2978 2979 eq = &eqo->q; 2980 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN, 2981 sizeof(struct be_eq_entry)); 2982 if (rc) 2983 return rc; 2984 2985 rc = be_cmd_eq_create(adapter, eqo); 2986 if (rc) 2987 return rc; 2988 2989 if (!zalloc_cpumask_var(&eqo->affinity_mask, GFP_KERNEL)) 2990 return -ENOMEM; 2991 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 2992 eqo->affinity_mask); 2993 netif_napi_add(adapter->netdev, &eqo->napi, be_poll, 2994 BE_NAPI_WEIGHT); 2995 } 2996 return 0; 2997 } 2998 2999 static void be_mcc_queues_destroy(struct be_adapter *adapter) 3000 { 3001 struct be_queue_info *q; 3002 3003 q = &adapter->mcc_obj.q; 3004 if (q->created) 3005 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ); 3006 be_queue_free(adapter, q); 3007 3008 q = &adapter->mcc_obj.cq; 3009 if (q->created) 3010 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 3011 be_queue_free(adapter, q); 3012 } 3013 3014 /* Must be called only after TX qs are created as MCC shares TX EQ */ 3015 static int be_mcc_queues_create(struct be_adapter *adapter) 3016 { 3017 struct be_queue_info *q, *cq; 3018 3019 cq = &adapter->mcc_obj.cq; 3020 if (be_queue_alloc(adapter, cq, MCC_CQ_LEN, 3021 sizeof(struct be_mcc_compl))) 3022 goto err; 3023 3024 /* Use the default EQ for MCC completions */ 3025 if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0)) 3026 goto mcc_cq_free; 3027 3028 q = &adapter->mcc_obj.q; 3029 if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 3030 goto mcc_cq_destroy; 3031 3032 if (be_cmd_mccq_create(adapter, q, cq)) 3033 goto mcc_q_free; 3034 3035 return 0; 3036 3037 mcc_q_free: 3038 be_queue_free(adapter, q); 3039 mcc_cq_destroy: 3040 be_cmd_q_destroy(adapter, cq, QTYPE_CQ); 3041 mcc_cq_free: 3042 be_queue_free(adapter, cq); 3043 err: 3044 return -1; 3045 } 3046 3047 static void be_tx_queues_destroy(struct be_adapter *adapter) 3048 { 3049 struct be_queue_info *q; 3050 struct be_tx_obj *txo; 3051 u8 i; 3052 3053 for_all_tx_queues(adapter, txo, i) { 3054 q = &txo->q; 3055 if (q->created) 3056 be_cmd_q_destroy(adapter, q, QTYPE_TXQ); 3057 be_queue_free(adapter, q); 3058 3059 q = &txo->cq; 3060 if (q->created) 3061 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 3062 be_queue_free(adapter, q); 3063 } 3064 } 3065 3066 static int be_tx_qs_create(struct be_adapter *adapter) 3067 { 3068 struct be_queue_info *cq; 3069 struct be_tx_obj *txo; 3070 struct be_eq_obj *eqo; 3071 int status, i; 3072 3073 adapter->num_tx_qs = min(adapter->num_evt_qs, adapter->cfg_num_tx_irqs); 3074 3075 for_all_tx_queues(adapter, txo, i) { 3076 cq = &txo->cq; 3077 status = be_queue_alloc(adapter, cq, TX_CQ_LEN, 3078 sizeof(struct be_eth_tx_compl)); 3079 if (status) 3080 return status; 3081 3082 u64_stats_init(&txo->stats.sync); 3083 u64_stats_init(&txo->stats.sync_compl); 3084 3085 /* If num_evt_qs is less than num_tx_qs, then more than 3086 * one txq share an eq 3087 */ 3088 eqo = &adapter->eq_obj[i % adapter->num_evt_qs]; 3089 status = be_cmd_cq_create(adapter, cq, &eqo->q, false, 3); 3090 if (status) 3091 return status; 3092 3093 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN, 3094 sizeof(struct be_eth_wrb)); 3095 if (status) 3096 return status; 3097 3098 status = be_cmd_txq_create(adapter, txo); 3099 if (status) 3100 return status; 3101 3102 netif_set_xps_queue(adapter->netdev, eqo->affinity_mask, 3103 eqo->idx); 3104 } 3105 3106 dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n", 3107 adapter->num_tx_qs); 3108 return 0; 3109 } 3110 3111 static void be_rx_cqs_destroy(struct be_adapter *adapter) 3112 { 3113 struct be_queue_info *q; 3114 struct be_rx_obj *rxo; 3115 int i; 3116 3117 for_all_rx_queues(adapter, rxo, i) { 3118 q = &rxo->cq; 3119 if (q->created) 3120 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 3121 be_queue_free(adapter, q); 3122 } 3123 } 3124 3125 static int be_rx_cqs_create(struct be_adapter *adapter) 3126 { 3127 struct be_queue_info *eq, *cq; 3128 struct be_rx_obj *rxo; 3129 int rc, i; 3130 3131 adapter->num_rss_qs = 3132 min(adapter->num_evt_qs, adapter->cfg_num_rx_irqs); 3133 3134 /* We'll use RSS only if atleast 2 RSS rings are supported. */ 3135 if (adapter->num_rss_qs < 2) 3136 adapter->num_rss_qs = 0; 3137 3138 adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq; 3139 3140 /* When the interface is not capable of RSS rings (and there is no 3141 * need to create a default RXQ) we'll still need one RXQ 3142 */ 3143 if (adapter->num_rx_qs == 0) 3144 adapter->num_rx_qs = 1; 3145 3146 adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE; 3147 for_all_rx_queues(adapter, rxo, i) { 3148 rxo->adapter = adapter; 3149 cq = &rxo->cq; 3150 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN, 3151 sizeof(struct be_eth_rx_compl)); 3152 if (rc) 3153 return rc; 3154 3155 u64_stats_init(&rxo->stats.sync); 3156 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; 3157 rc = be_cmd_cq_create(adapter, cq, eq, false, 3); 3158 if (rc) 3159 return rc; 3160 } 3161 3162 dev_info(&adapter->pdev->dev, 3163 "created %d RX queue(s)\n", adapter->num_rx_qs); 3164 return 0; 3165 } 3166 3167 static irqreturn_t be_intx(int irq, void *dev) 3168 { 3169 struct be_eq_obj *eqo = dev; 3170 struct be_adapter *adapter = eqo->adapter; 3171 int num_evts = 0; 3172 3173 /* IRQ is not expected when NAPI is scheduled as the EQ 3174 * will not be armed. 3175 * But, this can happen on Lancer INTx where it takes 3176 * a while to de-assert INTx or in BE2 where occasionaly 3177 * an interrupt may be raised even when EQ is unarmed. 3178 * If NAPI is already scheduled, then counting & notifying 3179 * events will orphan them. 3180 */ 3181 if (napi_schedule_prep(&eqo->napi)) { 3182 num_evts = events_get(eqo); 3183 __napi_schedule(&eqo->napi); 3184 if (num_evts) 3185 eqo->spurious_intr = 0; 3186 } 3187 be_eq_notify(adapter, eqo->q.id, false, true, num_evts, 0); 3188 3189 /* Return IRQ_HANDLED only for the the first spurious intr 3190 * after a valid intr to stop the kernel from branding 3191 * this irq as a bad one! 3192 */ 3193 if (num_evts || eqo->spurious_intr++ == 0) 3194 return IRQ_HANDLED; 3195 else 3196 return IRQ_NONE; 3197 } 3198 3199 static irqreturn_t be_msix(int irq, void *dev) 3200 { 3201 struct be_eq_obj *eqo = dev; 3202 3203 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0); 3204 napi_schedule(&eqo->napi); 3205 return IRQ_HANDLED; 3206 } 3207 3208 static inline bool do_gro(struct be_rx_compl_info *rxcp) 3209 { 3210 return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false; 3211 } 3212 3213 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi, 3214 int budget) 3215 { 3216 struct be_adapter *adapter = rxo->adapter; 3217 struct be_queue_info *rx_cq = &rxo->cq; 3218 struct be_rx_compl_info *rxcp; 3219 u32 work_done; 3220 u32 frags_consumed = 0; 3221 3222 for (work_done = 0; work_done < budget; work_done++) { 3223 rxcp = be_rx_compl_get(rxo); 3224 if (!rxcp) 3225 break; 3226 3227 /* Is it a flush compl that has no data */ 3228 if (unlikely(rxcp->num_rcvd == 0)) 3229 goto loop_continue; 3230 3231 /* Discard compl with partial DMA Lancer B0 */ 3232 if (unlikely(!rxcp->pkt_size)) { 3233 be_rx_compl_discard(rxo, rxcp); 3234 goto loop_continue; 3235 } 3236 3237 /* On BE drop pkts that arrive due to imperfect filtering in 3238 * promiscuous mode on some skews 3239 */ 3240 if (unlikely(rxcp->port != adapter->port_num && 3241 !lancer_chip(adapter))) { 3242 be_rx_compl_discard(rxo, rxcp); 3243 goto loop_continue; 3244 } 3245 3246 if (do_gro(rxcp)) 3247 be_rx_compl_process_gro(rxo, napi, rxcp); 3248 else 3249 be_rx_compl_process(rxo, napi, rxcp); 3250 3251 loop_continue: 3252 frags_consumed += rxcp->num_rcvd; 3253 be_rx_stats_update(rxo, rxcp); 3254 } 3255 3256 if (work_done) { 3257 be_cq_notify(adapter, rx_cq->id, true, work_done); 3258 3259 /* When an rx-obj gets into post_starved state, just 3260 * let be_worker do the posting. 3261 */ 3262 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM && 3263 !rxo->rx_post_starved) 3264 be_post_rx_frags(rxo, GFP_ATOMIC, 3265 max_t(u32, MAX_RX_POST, 3266 frags_consumed)); 3267 } 3268 3269 return work_done; 3270 } 3271 3272 3273 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo, 3274 int idx) 3275 { 3276 int num_wrbs = 0, work_done = 0; 3277 struct be_tx_compl_info *txcp; 3278 3279 while ((txcp = be_tx_compl_get(adapter, txo))) { 3280 num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index); 3281 work_done++; 3282 } 3283 3284 if (work_done) { 3285 be_cq_notify(adapter, txo->cq.id, true, work_done); 3286 atomic_sub(num_wrbs, &txo->q.used); 3287 3288 /* As Tx wrbs have been freed up, wake up netdev queue 3289 * if it was stopped due to lack of tx wrbs. */ 3290 if (__netif_subqueue_stopped(adapter->netdev, idx) && 3291 be_can_txq_wake(txo)) { 3292 netif_wake_subqueue(adapter->netdev, idx); 3293 } 3294 3295 u64_stats_update_begin(&tx_stats(txo)->sync_compl); 3296 tx_stats(txo)->tx_compl += work_done; 3297 u64_stats_update_end(&tx_stats(txo)->sync_compl); 3298 } 3299 } 3300 3301 int be_poll(struct napi_struct *napi, int budget) 3302 { 3303 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 3304 struct be_adapter *adapter = eqo->adapter; 3305 int max_work = 0, work, i, num_evts; 3306 struct be_rx_obj *rxo; 3307 struct be_tx_obj *txo; 3308 u32 mult_enc = 0; 3309 3310 num_evts = events_get(eqo); 3311 3312 for_all_tx_queues_on_eq(adapter, eqo, txo, i) 3313 be_process_tx(adapter, txo, i); 3314 3315 /* This loop will iterate twice for EQ0 in which 3316 * completions of the last RXQ (default one) are also processed 3317 * For other EQs the loop iterates only once 3318 */ 3319 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 3320 work = be_process_rx(rxo, napi, budget); 3321 max_work = max(work, max_work); 3322 } 3323 3324 if (is_mcc_eqo(eqo)) 3325 be_process_mcc(adapter); 3326 3327 if (max_work < budget) { 3328 napi_complete_done(napi, max_work); 3329 3330 /* Skyhawk EQ_DB has a provision to set the rearm to interrupt 3331 * delay via a delay multiplier encoding value 3332 */ 3333 if (skyhawk_chip(adapter)) 3334 mult_enc = be_get_eq_delay_mult_enc(eqo); 3335 3336 be_eq_notify(adapter, eqo->q.id, true, false, num_evts, 3337 mult_enc); 3338 } else { 3339 /* As we'll continue in polling mode, count and clear events */ 3340 be_eq_notify(adapter, eqo->q.id, false, false, num_evts, 0); 3341 } 3342 return max_work; 3343 } 3344 3345 void be_detect_error(struct be_adapter *adapter) 3346 { 3347 u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0; 3348 u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 3349 struct device *dev = &adapter->pdev->dev; 3350 u16 val; 3351 u32 i; 3352 3353 if (be_check_error(adapter, BE_ERROR_HW)) 3354 return; 3355 3356 if (lancer_chip(adapter)) { 3357 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET); 3358 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 3359 be_set_error(adapter, BE_ERROR_UE); 3360 sliport_err1 = ioread32(adapter->db + 3361 SLIPORT_ERROR1_OFFSET); 3362 sliport_err2 = ioread32(adapter->db + 3363 SLIPORT_ERROR2_OFFSET); 3364 /* Do not log error messages if its a FW reset */ 3365 if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 && 3366 sliport_err2 == SLIPORT_ERROR_FW_RESET2) { 3367 dev_info(dev, "Reset is in progress\n"); 3368 } else { 3369 dev_err(dev, "Error detected in the card\n"); 3370 dev_err(dev, "ERR: sliport status 0x%x\n", 3371 sliport_status); 3372 dev_err(dev, "ERR: sliport error1 0x%x\n", 3373 sliport_err1); 3374 dev_err(dev, "ERR: sliport error2 0x%x\n", 3375 sliport_err2); 3376 } 3377 } 3378 } else { 3379 ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW); 3380 ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH); 3381 ue_lo_mask = ioread32(adapter->pcicfg + 3382 PCICFG_UE_STATUS_LOW_MASK); 3383 ue_hi_mask = ioread32(adapter->pcicfg + 3384 PCICFG_UE_STATUS_HI_MASK); 3385 3386 ue_lo = (ue_lo & ~ue_lo_mask); 3387 ue_hi = (ue_hi & ~ue_hi_mask); 3388 3389 if (ue_lo || ue_hi) { 3390 /* On certain platforms BE3 hardware can indicate 3391 * spurious UEs. In case of a UE in the chip, 3392 * the POST register correctly reports either a 3393 * FAT_LOG_START state (FW is currently dumping 3394 * FAT log data) or a ARMFW_UE state. Check for the 3395 * above states to ascertain if the UE is valid or not. 3396 */ 3397 if (BE3_chip(adapter)) { 3398 val = be_POST_stage_get(adapter); 3399 if ((val & POST_STAGE_FAT_LOG_START) 3400 != POST_STAGE_FAT_LOG_START && 3401 (val & POST_STAGE_ARMFW_UE) 3402 != POST_STAGE_ARMFW_UE && 3403 (val & POST_STAGE_RECOVERABLE_ERR) 3404 != POST_STAGE_RECOVERABLE_ERR) 3405 return; 3406 } 3407 3408 dev_err(dev, "Error detected in the adapter"); 3409 be_set_error(adapter, BE_ERROR_UE); 3410 3411 for (i = 0; ue_lo; ue_lo >>= 1, i++) { 3412 if (ue_lo & 1) 3413 dev_err(dev, "UE: %s bit set\n", 3414 ue_status_low_desc[i]); 3415 } 3416 for (i = 0; ue_hi; ue_hi >>= 1, i++) { 3417 if (ue_hi & 1) 3418 dev_err(dev, "UE: %s bit set\n", 3419 ue_status_hi_desc[i]); 3420 } 3421 } 3422 } 3423 } 3424 3425 static void be_msix_disable(struct be_adapter *adapter) 3426 { 3427 if (msix_enabled(adapter)) { 3428 pci_disable_msix(adapter->pdev); 3429 adapter->num_msix_vec = 0; 3430 adapter->num_msix_roce_vec = 0; 3431 } 3432 } 3433 3434 static int be_msix_enable(struct be_adapter *adapter) 3435 { 3436 unsigned int i, max_roce_eqs; 3437 struct device *dev = &adapter->pdev->dev; 3438 int num_vec; 3439 3440 /* If RoCE is supported, program the max number of vectors that 3441 * could be used for NIC and RoCE, else, just program the number 3442 * we'll use initially. 3443 */ 3444 if (be_roce_supported(adapter)) { 3445 max_roce_eqs = 3446 be_max_func_eqs(adapter) - be_max_nic_eqs(adapter); 3447 max_roce_eqs = min(max_roce_eqs, num_online_cpus()); 3448 num_vec = be_max_any_irqs(adapter) + max_roce_eqs; 3449 } else { 3450 num_vec = max(adapter->cfg_num_rx_irqs, 3451 adapter->cfg_num_tx_irqs); 3452 } 3453 3454 for (i = 0; i < num_vec; i++) 3455 adapter->msix_entries[i].entry = i; 3456 3457 num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 3458 MIN_MSIX_VECTORS, num_vec); 3459 if (num_vec < 0) 3460 goto fail; 3461 3462 if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) { 3463 adapter->num_msix_roce_vec = num_vec / 2; 3464 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n", 3465 adapter->num_msix_roce_vec); 3466 } 3467 3468 adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec; 3469 3470 dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n", 3471 adapter->num_msix_vec); 3472 return 0; 3473 3474 fail: 3475 dev_warn(dev, "MSIx enable failed\n"); 3476 3477 /* INTx is not supported in VFs, so fail probe if enable_msix fails */ 3478 if (be_virtfn(adapter)) 3479 return num_vec; 3480 return 0; 3481 } 3482 3483 static inline int be_msix_vec_get(struct be_adapter *adapter, 3484 struct be_eq_obj *eqo) 3485 { 3486 return adapter->msix_entries[eqo->msix_idx].vector; 3487 } 3488 3489 static int be_msix_register(struct be_adapter *adapter) 3490 { 3491 struct net_device *netdev = adapter->netdev; 3492 struct be_eq_obj *eqo; 3493 int status, i, vec; 3494 3495 for_all_evt_queues(adapter, eqo, i) { 3496 sprintf(eqo->desc, "%s-q%d", netdev->name, i); 3497 vec = be_msix_vec_get(adapter, eqo); 3498 status = request_irq(vec, be_msix, 0, eqo->desc, eqo); 3499 if (status) 3500 goto err_msix; 3501 3502 irq_set_affinity_hint(vec, eqo->affinity_mask); 3503 } 3504 3505 return 0; 3506 err_msix: 3507 for (i--; i >= 0; i--) { 3508 eqo = &adapter->eq_obj[i]; 3509 free_irq(be_msix_vec_get(adapter, eqo), eqo); 3510 } 3511 dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n", 3512 status); 3513 be_msix_disable(adapter); 3514 return status; 3515 } 3516 3517 static int be_irq_register(struct be_adapter *adapter) 3518 { 3519 struct net_device *netdev = adapter->netdev; 3520 int status; 3521 3522 if (msix_enabled(adapter)) { 3523 status = be_msix_register(adapter); 3524 if (status == 0) 3525 goto done; 3526 /* INTx is not supported for VF */ 3527 if (be_virtfn(adapter)) 3528 return status; 3529 } 3530 3531 /* INTx: only the first EQ is used */ 3532 netdev->irq = adapter->pdev->irq; 3533 status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name, 3534 &adapter->eq_obj[0]); 3535 if (status) { 3536 dev_err(&adapter->pdev->dev, 3537 "INTx request IRQ failed - err %d\n", status); 3538 return status; 3539 } 3540 done: 3541 adapter->isr_registered = true; 3542 return 0; 3543 } 3544 3545 static void be_irq_unregister(struct be_adapter *adapter) 3546 { 3547 struct net_device *netdev = adapter->netdev; 3548 struct be_eq_obj *eqo; 3549 int i, vec; 3550 3551 if (!adapter->isr_registered) 3552 return; 3553 3554 /* INTx */ 3555 if (!msix_enabled(adapter)) { 3556 free_irq(netdev->irq, &adapter->eq_obj[0]); 3557 goto done; 3558 } 3559 3560 /* MSIx */ 3561 for_all_evt_queues(adapter, eqo, i) { 3562 vec = be_msix_vec_get(adapter, eqo); 3563 irq_set_affinity_hint(vec, NULL); 3564 free_irq(vec, eqo); 3565 } 3566 3567 done: 3568 adapter->isr_registered = false; 3569 } 3570 3571 static void be_rx_qs_destroy(struct be_adapter *adapter) 3572 { 3573 struct rss_info *rss = &adapter->rss_info; 3574 struct be_queue_info *q; 3575 struct be_rx_obj *rxo; 3576 int i; 3577 3578 for_all_rx_queues(adapter, rxo, i) { 3579 q = &rxo->q; 3580 if (q->created) { 3581 /* If RXQs are destroyed while in an "out of buffer" 3582 * state, there is a possibility of an HW stall on 3583 * Lancer. So, post 64 buffers to each queue to relieve 3584 * the "out of buffer" condition. 3585 * Make sure there's space in the RXQ before posting. 3586 */ 3587 if (lancer_chip(adapter)) { 3588 be_rx_cq_clean(rxo); 3589 if (atomic_read(&q->used) == 0) 3590 be_post_rx_frags(rxo, GFP_KERNEL, 3591 MAX_RX_POST); 3592 } 3593 3594 be_cmd_rxq_destroy(adapter, q); 3595 be_rx_cq_clean(rxo); 3596 be_rxq_clean(rxo); 3597 } 3598 be_queue_free(adapter, q); 3599 } 3600 3601 if (rss->rss_flags) { 3602 rss->rss_flags = RSS_ENABLE_NONE; 3603 be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3604 128, rss->rss_hkey); 3605 } 3606 } 3607 3608 static void be_disable_if_filters(struct be_adapter *adapter) 3609 { 3610 /* Don't delete MAC on BE3 VFs without FILTMGMT privilege */ 3611 if (!BEx_chip(adapter) || !be_virtfn(adapter) || 3612 check_privilege(adapter, BE_PRIV_FILTMGMT)) { 3613 be_dev_mac_del(adapter, adapter->pmac_id[0]); 3614 eth_zero_addr(adapter->dev_mac); 3615 } 3616 3617 be_clear_uc_list(adapter); 3618 be_clear_mc_list(adapter); 3619 3620 /* The IFACE flags are enabled in the open path and cleared 3621 * in the close path. When a VF gets detached from the host and 3622 * assigned to a VM the following happens: 3623 * - VF's IFACE flags get cleared in the detach path 3624 * - IFACE create is issued by the VF in the attach path 3625 * Due to a bug in the BE3/Skyhawk-R FW 3626 * (Lancer FW doesn't have the bug), the IFACE capability flags 3627 * specified along with the IFACE create cmd issued by a VF are not 3628 * honoured by FW. As a consequence, if a *new* driver 3629 * (that enables/disables IFACE flags in open/close) 3630 * is loaded in the host and an *old* driver is * used by a VM/VF, 3631 * the IFACE gets created *without* the needed flags. 3632 * To avoid this, disable RX-filter flags only for Lancer. 3633 */ 3634 if (lancer_chip(adapter)) { 3635 be_cmd_rx_filter(adapter, BE_IF_ALL_FILT_FLAGS, OFF); 3636 adapter->if_flags &= ~BE_IF_ALL_FILT_FLAGS; 3637 } 3638 } 3639 3640 static int be_close(struct net_device *netdev) 3641 { 3642 struct be_adapter *adapter = netdev_priv(netdev); 3643 struct be_eq_obj *eqo; 3644 int i; 3645 3646 /* This protection is needed as be_close() may be called even when the 3647 * adapter is in cleared state (after eeh perm failure) 3648 */ 3649 if (!(adapter->flags & BE_FLAGS_SETUP_DONE)) 3650 return 0; 3651 3652 /* Before attempting cleanup ensure all the pending cmds in the 3653 * config_wq have finished execution 3654 */ 3655 flush_workqueue(be_wq); 3656 3657 be_disable_if_filters(adapter); 3658 3659 if (adapter->flags & BE_FLAGS_NAPI_ENABLED) { 3660 for_all_evt_queues(adapter, eqo, i) { 3661 napi_disable(&eqo->napi); 3662 } 3663 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED; 3664 } 3665 3666 be_async_mcc_disable(adapter); 3667 3668 /* Wait for all pending tx completions to arrive so that 3669 * all tx skbs are freed. 3670 */ 3671 netif_tx_disable(netdev); 3672 be_tx_compl_clean(adapter); 3673 3674 be_rx_qs_destroy(adapter); 3675 3676 for_all_evt_queues(adapter, eqo, i) { 3677 if (msix_enabled(adapter)) 3678 synchronize_irq(be_msix_vec_get(adapter, eqo)); 3679 else 3680 synchronize_irq(netdev->irq); 3681 be_eq_clean(eqo); 3682 } 3683 3684 be_irq_unregister(adapter); 3685 3686 return 0; 3687 } 3688 3689 static int be_rx_qs_create(struct be_adapter *adapter) 3690 { 3691 struct rss_info *rss = &adapter->rss_info; 3692 u8 rss_key[RSS_HASH_KEY_LEN]; 3693 struct be_rx_obj *rxo; 3694 int rc, i, j; 3695 3696 for_all_rx_queues(adapter, rxo, i) { 3697 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN, 3698 sizeof(struct be_eth_rx_d)); 3699 if (rc) 3700 return rc; 3701 } 3702 3703 if (adapter->need_def_rxq || !adapter->num_rss_qs) { 3704 rxo = default_rxo(adapter); 3705 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3706 rx_frag_size, adapter->if_handle, 3707 false, &rxo->rss_id); 3708 if (rc) 3709 return rc; 3710 } 3711 3712 for_all_rss_queues(adapter, rxo, i) { 3713 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3714 rx_frag_size, adapter->if_handle, 3715 true, &rxo->rss_id); 3716 if (rc) 3717 return rc; 3718 } 3719 3720 if (be_multi_rxq(adapter)) { 3721 for (j = 0; j < RSS_INDIR_TABLE_LEN; j += adapter->num_rss_qs) { 3722 for_all_rss_queues(adapter, rxo, i) { 3723 if ((j + i) >= RSS_INDIR_TABLE_LEN) 3724 break; 3725 rss->rsstable[j + i] = rxo->rss_id; 3726 rss->rss_queue[j + i] = i; 3727 } 3728 } 3729 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 | 3730 RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6; 3731 3732 if (!BEx_chip(adapter)) 3733 rss->rss_flags |= RSS_ENABLE_UDP_IPV4 | 3734 RSS_ENABLE_UDP_IPV6; 3735 3736 netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN); 3737 rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3738 RSS_INDIR_TABLE_LEN, rss_key); 3739 if (rc) { 3740 rss->rss_flags = RSS_ENABLE_NONE; 3741 return rc; 3742 } 3743 3744 memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN); 3745 } else { 3746 /* Disable RSS, if only default RX Q is created */ 3747 rss->rss_flags = RSS_ENABLE_NONE; 3748 } 3749 3750 3751 /* Post 1 less than RXQ-len to avoid head being equal to tail, 3752 * which is a queue empty condition 3753 */ 3754 for_all_rx_queues(adapter, rxo, i) 3755 be_post_rx_frags(rxo, GFP_KERNEL, RX_Q_LEN - 1); 3756 3757 return 0; 3758 } 3759 3760 static int be_enable_if_filters(struct be_adapter *adapter) 3761 { 3762 int status; 3763 3764 status = be_cmd_rx_filter(adapter, BE_IF_FILT_FLAGS_BASIC, ON); 3765 if (status) 3766 return status; 3767 3768 /* Normally this condition usually true as the ->dev_mac is zeroed. 3769 * But on BE3 VFs the initial MAC is pre-programmed by PF and 3770 * subsequent be_dev_mac_add() can fail (after fresh boot) 3771 */ 3772 if (!ether_addr_equal(adapter->dev_mac, adapter->netdev->dev_addr)) { 3773 int old_pmac_id = -1; 3774 3775 /* Remember old programmed MAC if any - can happen on BE3 VF */ 3776 if (!is_zero_ether_addr(adapter->dev_mac)) 3777 old_pmac_id = adapter->pmac_id[0]; 3778 3779 status = be_dev_mac_add(adapter, adapter->netdev->dev_addr); 3780 if (status) 3781 return status; 3782 3783 /* Delete the old programmed MAC as we successfully programmed 3784 * a new MAC 3785 */ 3786 if (old_pmac_id >= 0 && old_pmac_id != adapter->pmac_id[0]) 3787 be_dev_mac_del(adapter, old_pmac_id); 3788 3789 ether_addr_copy(adapter->dev_mac, adapter->netdev->dev_addr); 3790 } 3791 3792 if (adapter->vlans_added) 3793 be_vid_config(adapter); 3794 3795 __be_set_rx_mode(adapter); 3796 3797 return 0; 3798 } 3799 3800 static int be_open(struct net_device *netdev) 3801 { 3802 struct be_adapter *adapter = netdev_priv(netdev); 3803 struct be_eq_obj *eqo; 3804 struct be_rx_obj *rxo; 3805 struct be_tx_obj *txo; 3806 u8 link_status; 3807 int status, i; 3808 3809 status = be_rx_qs_create(adapter); 3810 if (status) 3811 goto err; 3812 3813 status = be_enable_if_filters(adapter); 3814 if (status) 3815 goto err; 3816 3817 status = be_irq_register(adapter); 3818 if (status) 3819 goto err; 3820 3821 for_all_rx_queues(adapter, rxo, i) 3822 be_cq_notify(adapter, rxo->cq.id, true, 0); 3823 3824 for_all_tx_queues(adapter, txo, i) 3825 be_cq_notify(adapter, txo->cq.id, true, 0); 3826 3827 be_async_mcc_enable(adapter); 3828 3829 for_all_evt_queues(adapter, eqo, i) { 3830 napi_enable(&eqo->napi); 3831 be_eq_notify(adapter, eqo->q.id, true, true, 0, 0); 3832 } 3833 adapter->flags |= BE_FLAGS_NAPI_ENABLED; 3834 3835 status = be_cmd_link_status_query(adapter, NULL, &link_status, 0); 3836 if (!status) 3837 be_link_status_update(adapter, link_status); 3838 3839 netif_tx_start_all_queues(netdev); 3840 if (skyhawk_chip(adapter)) 3841 udp_tunnel_get_rx_info(netdev); 3842 3843 return 0; 3844 err: 3845 be_close(adapter->netdev); 3846 return -EIO; 3847 } 3848 3849 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac) 3850 { 3851 u32 addr; 3852 3853 addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0); 3854 3855 mac[5] = (u8)(addr & 0xFF); 3856 mac[4] = (u8)((addr >> 8) & 0xFF); 3857 mac[3] = (u8)((addr >> 16) & 0xFF); 3858 /* Use the OUI from the current MAC address */ 3859 memcpy(mac, adapter->netdev->dev_addr, 3); 3860 } 3861 3862 /* 3863 * Generate a seed MAC address from the PF MAC Address using jhash. 3864 * MAC Address for VFs are assigned incrementally starting from the seed. 3865 * These addresses are programmed in the ASIC by the PF and the VF driver 3866 * queries for the MAC address during its probe. 3867 */ 3868 static int be_vf_eth_addr_config(struct be_adapter *adapter) 3869 { 3870 u32 vf; 3871 int status = 0; 3872 u8 mac[ETH_ALEN]; 3873 struct be_vf_cfg *vf_cfg; 3874 3875 be_vf_eth_addr_generate(adapter, mac); 3876 3877 for_all_vfs(adapter, vf_cfg, vf) { 3878 if (BEx_chip(adapter)) 3879 status = be_cmd_pmac_add(adapter, mac, 3880 vf_cfg->if_handle, 3881 &vf_cfg->pmac_id, vf + 1); 3882 else 3883 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 3884 vf + 1); 3885 3886 if (status) 3887 dev_err(&adapter->pdev->dev, 3888 "Mac address assignment failed for VF %d\n", 3889 vf); 3890 else 3891 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3892 3893 mac[5] += 1; 3894 } 3895 return status; 3896 } 3897 3898 static int be_vfs_mac_query(struct be_adapter *adapter) 3899 { 3900 int status, vf; 3901 u8 mac[ETH_ALEN]; 3902 struct be_vf_cfg *vf_cfg; 3903 3904 for_all_vfs(adapter, vf_cfg, vf) { 3905 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id, 3906 mac, vf_cfg->if_handle, 3907 false, vf+1); 3908 if (status) 3909 return status; 3910 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3911 } 3912 return 0; 3913 } 3914 3915 static void be_vf_clear(struct be_adapter *adapter) 3916 { 3917 struct be_vf_cfg *vf_cfg; 3918 u32 vf; 3919 3920 if (pci_vfs_assigned(adapter->pdev)) { 3921 dev_warn(&adapter->pdev->dev, 3922 "VFs are assigned to VMs: not disabling VFs\n"); 3923 goto done; 3924 } 3925 3926 pci_disable_sriov(adapter->pdev); 3927 3928 for_all_vfs(adapter, vf_cfg, vf) { 3929 if (BEx_chip(adapter)) 3930 be_cmd_pmac_del(adapter, vf_cfg->if_handle, 3931 vf_cfg->pmac_id, vf + 1); 3932 else 3933 be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle, 3934 vf + 1); 3935 3936 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1); 3937 } 3938 3939 if (BE3_chip(adapter)) 3940 be_cmd_set_hsw_config(adapter, 0, 0, 3941 adapter->if_handle, 3942 PORT_FWD_TYPE_PASSTHRU, 0); 3943 done: 3944 kfree(adapter->vf_cfg); 3945 adapter->num_vfs = 0; 3946 adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED; 3947 } 3948 3949 static void be_clear_queues(struct be_adapter *adapter) 3950 { 3951 be_mcc_queues_destroy(adapter); 3952 be_rx_cqs_destroy(adapter); 3953 be_tx_queues_destroy(adapter); 3954 be_evt_queues_destroy(adapter); 3955 } 3956 3957 static void be_cancel_worker(struct be_adapter *adapter) 3958 { 3959 if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) { 3960 cancel_delayed_work_sync(&adapter->work); 3961 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED; 3962 } 3963 } 3964 3965 static void be_cancel_err_detection(struct be_adapter *adapter) 3966 { 3967 struct be_error_recovery *err_rec = &adapter->error_recovery; 3968 3969 if (!be_err_recovery_workq) 3970 return; 3971 3972 if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) { 3973 cancel_delayed_work_sync(&err_rec->err_detection_work); 3974 adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED; 3975 } 3976 } 3977 3978 static int be_enable_vxlan_offloads(struct be_adapter *adapter) 3979 { 3980 struct net_device *netdev = adapter->netdev; 3981 struct device *dev = &adapter->pdev->dev; 3982 struct be_vxlan_port *vxlan_port; 3983 __be16 port; 3984 int status; 3985 3986 vxlan_port = list_first_entry(&adapter->vxlan_port_list, 3987 struct be_vxlan_port, list); 3988 port = vxlan_port->port; 3989 3990 status = be_cmd_manage_iface(adapter, adapter->if_handle, 3991 OP_CONVERT_NORMAL_TO_TUNNEL); 3992 if (status) { 3993 dev_warn(dev, "Failed to convert normal interface to tunnel\n"); 3994 return status; 3995 } 3996 adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS; 3997 3998 status = be_cmd_set_vxlan_port(adapter, port); 3999 if (status) { 4000 dev_warn(dev, "Failed to add VxLAN port\n"); 4001 return status; 4002 } 4003 adapter->vxlan_port = port; 4004 4005 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4006 NETIF_F_TSO | NETIF_F_TSO6 | 4007 NETIF_F_GSO_UDP_TUNNEL; 4008 4009 dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n", 4010 be16_to_cpu(port)); 4011 return 0; 4012 } 4013 4014 static void be_disable_vxlan_offloads(struct be_adapter *adapter) 4015 { 4016 struct net_device *netdev = adapter->netdev; 4017 4018 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) 4019 be_cmd_manage_iface(adapter, adapter->if_handle, 4020 OP_CONVERT_TUNNEL_TO_NORMAL); 4021 4022 if (adapter->vxlan_port) 4023 be_cmd_set_vxlan_port(adapter, 0); 4024 4025 adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS; 4026 adapter->vxlan_port = 0; 4027 4028 netdev->hw_enc_features = 0; 4029 } 4030 4031 static void be_calculate_vf_res(struct be_adapter *adapter, u16 num_vfs, 4032 struct be_resources *vft_res) 4033 { 4034 struct be_resources res = adapter->pool_res; 4035 u32 vf_if_cap_flags = res.vf_if_cap_flags; 4036 struct be_resources res_mod = {0}; 4037 u16 num_vf_qs = 1; 4038 4039 /* Distribute the queue resources among the PF and it's VFs */ 4040 if (num_vfs) { 4041 /* Divide the rx queues evenly among the VFs and the PF, capped 4042 * at VF-EQ-count. Any remainder queues belong to the PF. 4043 */ 4044 num_vf_qs = min(SH_VF_MAX_NIC_EQS, 4045 res.max_rss_qs / (num_vfs + 1)); 4046 4047 /* Skyhawk-R chip supports only MAX_PORT_RSS_TABLES 4048 * RSS Tables per port. Provide RSS on VFs, only if number of 4049 * VFs requested is less than it's PF Pool's RSS Tables limit. 4050 */ 4051 if (num_vfs >= be_max_pf_pool_rss_tables(adapter)) 4052 num_vf_qs = 1; 4053 } 4054 4055 /* Resource with fields set to all '1's by GET_PROFILE_CONFIG cmd, 4056 * which are modifiable using SET_PROFILE_CONFIG cmd. 4057 */ 4058 be_cmd_get_profile_config(adapter, &res_mod, NULL, ACTIVE_PROFILE_TYPE, 4059 RESOURCE_MODIFIABLE, 0); 4060 4061 /* If RSS IFACE capability flags are modifiable for a VF, set the 4062 * capability flag as valid and set RSS and DEFQ_RSS IFACE flags if 4063 * more than 1 RSSQ is available for a VF. 4064 * Otherwise, provision only 1 queue pair for VF. 4065 */ 4066 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_RSS) { 4067 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT); 4068 if (num_vf_qs > 1) { 4069 vf_if_cap_flags |= BE_IF_FLAGS_RSS; 4070 if (res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS) 4071 vf_if_cap_flags |= BE_IF_FLAGS_DEFQ_RSS; 4072 } else { 4073 vf_if_cap_flags &= ~(BE_IF_FLAGS_RSS | 4074 BE_IF_FLAGS_DEFQ_RSS); 4075 } 4076 } else { 4077 num_vf_qs = 1; 4078 } 4079 4080 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 4081 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT); 4082 vf_if_cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 4083 } 4084 4085 vft_res->vf_if_cap_flags = vf_if_cap_flags; 4086 vft_res->max_rx_qs = num_vf_qs; 4087 vft_res->max_rss_qs = num_vf_qs; 4088 vft_res->max_tx_qs = res.max_tx_qs / (num_vfs + 1); 4089 vft_res->max_cq_count = res.max_cq_count / (num_vfs + 1); 4090 4091 /* Distribute unicast MACs, VLANs, IFACE count and MCCQ count equally 4092 * among the PF and it's VFs, if the fields are changeable 4093 */ 4094 if (res_mod.max_uc_mac == FIELD_MODIFIABLE) 4095 vft_res->max_uc_mac = res.max_uc_mac / (num_vfs + 1); 4096 4097 if (res_mod.max_vlans == FIELD_MODIFIABLE) 4098 vft_res->max_vlans = res.max_vlans / (num_vfs + 1); 4099 4100 if (res_mod.max_iface_count == FIELD_MODIFIABLE) 4101 vft_res->max_iface_count = res.max_iface_count / (num_vfs + 1); 4102 4103 if (res_mod.max_mcc_count == FIELD_MODIFIABLE) 4104 vft_res->max_mcc_count = res.max_mcc_count / (num_vfs + 1); 4105 } 4106 4107 static void be_if_destroy(struct be_adapter *adapter) 4108 { 4109 be_cmd_if_destroy(adapter, adapter->if_handle, 0); 4110 4111 kfree(adapter->pmac_id); 4112 adapter->pmac_id = NULL; 4113 4114 kfree(adapter->mc_list); 4115 adapter->mc_list = NULL; 4116 4117 kfree(adapter->uc_list); 4118 adapter->uc_list = NULL; 4119 } 4120 4121 static int be_clear(struct be_adapter *adapter) 4122 { 4123 struct pci_dev *pdev = adapter->pdev; 4124 struct be_resources vft_res = {0}; 4125 4126 be_cancel_worker(adapter); 4127 4128 flush_workqueue(be_wq); 4129 4130 if (sriov_enabled(adapter)) 4131 be_vf_clear(adapter); 4132 4133 /* Re-configure FW to distribute resources evenly across max-supported 4134 * number of VFs, only when VFs are not already enabled. 4135 */ 4136 if (skyhawk_chip(adapter) && be_physfn(adapter) && 4137 !pci_vfs_assigned(pdev)) { 4138 be_calculate_vf_res(adapter, 4139 pci_sriov_get_totalvfs(pdev), 4140 &vft_res); 4141 be_cmd_set_sriov_config(adapter, adapter->pool_res, 4142 pci_sriov_get_totalvfs(pdev), 4143 &vft_res); 4144 } 4145 4146 be_disable_vxlan_offloads(adapter); 4147 4148 be_if_destroy(adapter); 4149 4150 be_clear_queues(adapter); 4151 4152 be_msix_disable(adapter); 4153 adapter->flags &= ~BE_FLAGS_SETUP_DONE; 4154 return 0; 4155 } 4156 4157 static int be_vfs_if_create(struct be_adapter *adapter) 4158 { 4159 struct be_resources res = {0}; 4160 u32 cap_flags, en_flags, vf; 4161 struct be_vf_cfg *vf_cfg; 4162 int status; 4163 4164 /* If a FW profile exists, then cap_flags are updated */ 4165 cap_flags = BE_VF_IF_EN_FLAGS; 4166 4167 for_all_vfs(adapter, vf_cfg, vf) { 4168 if (!BE3_chip(adapter)) { 4169 status = be_cmd_get_profile_config(adapter, &res, NULL, 4170 ACTIVE_PROFILE_TYPE, 4171 RESOURCE_LIMITS, 4172 vf + 1); 4173 if (!status) { 4174 cap_flags = res.if_cap_flags; 4175 /* Prevent VFs from enabling VLAN promiscuous 4176 * mode 4177 */ 4178 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 4179 } 4180 } 4181 4182 /* PF should enable IF flags during proxy if_create call */ 4183 en_flags = cap_flags & BE_VF_IF_EN_FLAGS; 4184 status = be_cmd_if_create(adapter, cap_flags, en_flags, 4185 &vf_cfg->if_handle, vf + 1); 4186 if (status) 4187 return status; 4188 } 4189 4190 return 0; 4191 } 4192 4193 static int be_vf_setup_init(struct be_adapter *adapter) 4194 { 4195 struct be_vf_cfg *vf_cfg; 4196 int vf; 4197 4198 adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg), 4199 GFP_KERNEL); 4200 if (!adapter->vf_cfg) 4201 return -ENOMEM; 4202 4203 for_all_vfs(adapter, vf_cfg, vf) { 4204 vf_cfg->if_handle = -1; 4205 vf_cfg->pmac_id = -1; 4206 } 4207 return 0; 4208 } 4209 4210 static int be_vf_setup(struct be_adapter *adapter) 4211 { 4212 struct device *dev = &adapter->pdev->dev; 4213 struct be_vf_cfg *vf_cfg; 4214 int status, old_vfs, vf; 4215 bool spoofchk; 4216 4217 old_vfs = pci_num_vf(adapter->pdev); 4218 4219 status = be_vf_setup_init(adapter); 4220 if (status) 4221 goto err; 4222 4223 if (old_vfs) { 4224 for_all_vfs(adapter, vf_cfg, vf) { 4225 status = be_cmd_get_if_id(adapter, vf_cfg, vf); 4226 if (status) 4227 goto err; 4228 } 4229 4230 status = be_vfs_mac_query(adapter); 4231 if (status) 4232 goto err; 4233 } else { 4234 status = be_vfs_if_create(adapter); 4235 if (status) 4236 goto err; 4237 4238 status = be_vf_eth_addr_config(adapter); 4239 if (status) 4240 goto err; 4241 } 4242 4243 for_all_vfs(adapter, vf_cfg, vf) { 4244 /* Allow VFs to programs MAC/VLAN filters */ 4245 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges, 4246 vf + 1); 4247 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 4248 status = be_cmd_set_fn_privileges(adapter, 4249 vf_cfg->privileges | 4250 BE_PRIV_FILTMGMT, 4251 vf + 1); 4252 if (!status) { 4253 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 4254 dev_info(dev, "VF%d has FILTMGMT privilege\n", 4255 vf); 4256 } 4257 } 4258 4259 /* Allow full available bandwidth */ 4260 if (!old_vfs) 4261 be_cmd_config_qos(adapter, 0, 0, vf + 1); 4262 4263 status = be_cmd_get_hsw_config(adapter, NULL, vf + 1, 4264 vf_cfg->if_handle, NULL, 4265 &spoofchk); 4266 if (!status) 4267 vf_cfg->spoofchk = spoofchk; 4268 4269 if (!old_vfs) { 4270 be_cmd_enable_vf(adapter, vf + 1); 4271 be_cmd_set_logical_link_config(adapter, 4272 IFLA_VF_LINK_STATE_AUTO, 4273 vf+1); 4274 } 4275 } 4276 4277 if (!old_vfs) { 4278 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs); 4279 if (status) { 4280 dev_err(dev, "SRIOV enable failed\n"); 4281 adapter->num_vfs = 0; 4282 goto err; 4283 } 4284 } 4285 4286 if (BE3_chip(adapter)) { 4287 /* On BE3, enable VEB only when SRIOV is enabled */ 4288 status = be_cmd_set_hsw_config(adapter, 0, 0, 4289 adapter->if_handle, 4290 PORT_FWD_TYPE_VEB, 0); 4291 if (status) 4292 goto err; 4293 } 4294 4295 adapter->flags |= BE_FLAGS_SRIOV_ENABLED; 4296 return 0; 4297 err: 4298 dev_err(dev, "VF setup failed\n"); 4299 be_vf_clear(adapter); 4300 return status; 4301 } 4302 4303 /* Converting function_mode bits on BE3 to SH mc_type enums */ 4304 4305 static u8 be_convert_mc_type(u32 function_mode) 4306 { 4307 if (function_mode & VNIC_MODE && function_mode & QNQ_MODE) 4308 return vNIC1; 4309 else if (function_mode & QNQ_MODE) 4310 return FLEX10; 4311 else if (function_mode & VNIC_MODE) 4312 return vNIC2; 4313 else if (function_mode & UMC_ENABLED) 4314 return UMC; 4315 else 4316 return MC_NONE; 4317 } 4318 4319 /* On BE2/BE3 FW does not suggest the supported limits */ 4320 static void BEx_get_resources(struct be_adapter *adapter, 4321 struct be_resources *res) 4322 { 4323 bool use_sriov = adapter->num_vfs ? 1 : 0; 4324 4325 if (be_physfn(adapter)) 4326 res->max_uc_mac = BE_UC_PMAC_COUNT; 4327 else 4328 res->max_uc_mac = BE_VF_UC_PMAC_COUNT; 4329 4330 adapter->mc_type = be_convert_mc_type(adapter->function_mode); 4331 4332 if (be_is_mc(adapter)) { 4333 /* Assuming that there are 4 channels per port, 4334 * when multi-channel is enabled 4335 */ 4336 if (be_is_qnq_mode(adapter)) 4337 res->max_vlans = BE_NUM_VLANS_SUPPORTED/8; 4338 else 4339 /* In a non-qnq multichannel mode, the pvid 4340 * takes up one vlan entry 4341 */ 4342 res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1; 4343 } else { 4344 res->max_vlans = BE_NUM_VLANS_SUPPORTED; 4345 } 4346 4347 res->max_mcast_mac = BE_MAX_MC; 4348 4349 /* 1) For BE3 1Gb ports, FW does not support multiple TXQs 4350 * 2) Create multiple TX rings on a BE3-R multi-channel interface 4351 * *only* if it is RSS-capable. 4352 */ 4353 if (BE2_chip(adapter) || use_sriov || (adapter->port_num > 1) || 4354 be_virtfn(adapter) || 4355 (be_is_mc(adapter) && 4356 !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) { 4357 res->max_tx_qs = 1; 4358 } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) { 4359 struct be_resources super_nic_res = {0}; 4360 4361 /* On a SuperNIC profile, the driver needs to use the 4362 * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits 4363 */ 4364 be_cmd_get_profile_config(adapter, &super_nic_res, NULL, 4365 ACTIVE_PROFILE_TYPE, RESOURCE_LIMITS, 4366 0); 4367 /* Some old versions of BE3 FW don't report max_tx_qs value */ 4368 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS; 4369 } else { 4370 res->max_tx_qs = BE3_MAX_TX_QS; 4371 } 4372 4373 if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) && 4374 !use_sriov && be_physfn(adapter)) 4375 res->max_rss_qs = (adapter->be3_native) ? 4376 BE3_MAX_RSS_QS : BE2_MAX_RSS_QS; 4377 res->max_rx_qs = res->max_rss_qs + 1; 4378 4379 if (be_physfn(adapter)) 4380 res->max_evt_qs = (be_max_vfs(adapter) > 0) ? 4381 BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS; 4382 else 4383 res->max_evt_qs = 1; 4384 4385 res->if_cap_flags = BE_IF_CAP_FLAGS_WANT; 4386 res->if_cap_flags &= ~BE_IF_FLAGS_DEFQ_RSS; 4387 if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS)) 4388 res->if_cap_flags &= ~BE_IF_FLAGS_RSS; 4389 } 4390 4391 static void be_setup_init(struct be_adapter *adapter) 4392 { 4393 adapter->vlan_prio_bmap = 0xff; 4394 adapter->phy.link_speed = -1; 4395 adapter->if_handle = -1; 4396 adapter->be3_native = false; 4397 adapter->if_flags = 0; 4398 adapter->phy_state = BE_UNKNOWN_PHY_STATE; 4399 if (be_physfn(adapter)) 4400 adapter->cmd_privileges = MAX_PRIVILEGES; 4401 else 4402 adapter->cmd_privileges = MIN_PRIVILEGES; 4403 } 4404 4405 /* HW supports only MAX_PORT_RSS_TABLES RSS Policy Tables per port. 4406 * However, this HW limitation is not exposed to the host via any SLI cmd. 4407 * As a result, in the case of SRIOV and in particular multi-partition configs 4408 * the driver needs to calcuate a proportional share of RSS Tables per PF-pool 4409 * for distribution between the VFs. This self-imposed limit will determine the 4410 * no: of VFs for which RSS can be enabled. 4411 */ 4412 static void be_calculate_pf_pool_rss_tables(struct be_adapter *adapter) 4413 { 4414 struct be_port_resources port_res = {0}; 4415 u8 rss_tables_on_port; 4416 u16 max_vfs = be_max_vfs(adapter); 4417 4418 be_cmd_get_profile_config(adapter, NULL, &port_res, SAVED_PROFILE_TYPE, 4419 RESOURCE_LIMITS, 0); 4420 4421 rss_tables_on_port = MAX_PORT_RSS_TABLES - port_res.nic_pfs; 4422 4423 /* Each PF Pool's RSS Tables limit = 4424 * PF's Max VFs / Total_Max_VFs on Port * RSS Tables on Port 4425 */ 4426 adapter->pool_res.max_rss_tables = 4427 max_vfs * rss_tables_on_port / port_res.max_vfs; 4428 } 4429 4430 static int be_get_sriov_config(struct be_adapter *adapter) 4431 { 4432 struct be_resources res = {0}; 4433 int max_vfs, old_vfs; 4434 4435 be_cmd_get_profile_config(adapter, &res, NULL, ACTIVE_PROFILE_TYPE, 4436 RESOURCE_LIMITS, 0); 4437 4438 /* Some old versions of BE3 FW don't report max_vfs value */ 4439 if (BE3_chip(adapter) && !res.max_vfs) { 4440 max_vfs = pci_sriov_get_totalvfs(adapter->pdev); 4441 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0; 4442 } 4443 4444 adapter->pool_res = res; 4445 4446 /* If during previous unload of the driver, the VFs were not disabled, 4447 * then we cannot rely on the PF POOL limits for the TotalVFs value. 4448 * Instead use the TotalVFs value stored in the pci-dev struct. 4449 */ 4450 old_vfs = pci_num_vf(adapter->pdev); 4451 if (old_vfs) { 4452 dev_info(&adapter->pdev->dev, "%d VFs are already enabled\n", 4453 old_vfs); 4454 4455 adapter->pool_res.max_vfs = 4456 pci_sriov_get_totalvfs(adapter->pdev); 4457 adapter->num_vfs = old_vfs; 4458 } 4459 4460 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) { 4461 be_calculate_pf_pool_rss_tables(adapter); 4462 dev_info(&adapter->pdev->dev, 4463 "RSS can be enabled for all VFs if num_vfs <= %d\n", 4464 be_max_pf_pool_rss_tables(adapter)); 4465 } 4466 return 0; 4467 } 4468 4469 static void be_alloc_sriov_res(struct be_adapter *adapter) 4470 { 4471 int old_vfs = pci_num_vf(adapter->pdev); 4472 struct be_resources vft_res = {0}; 4473 int status; 4474 4475 be_get_sriov_config(adapter); 4476 4477 if (!old_vfs) 4478 pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter)); 4479 4480 /* When the HW is in SRIOV capable configuration, the PF-pool 4481 * resources are given to PF during driver load, if there are no 4482 * old VFs. This facility is not available in BE3 FW. 4483 * Also, this is done by FW in Lancer chip. 4484 */ 4485 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) { 4486 be_calculate_vf_res(adapter, 0, &vft_res); 4487 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0, 4488 &vft_res); 4489 if (status) 4490 dev_err(&adapter->pdev->dev, 4491 "Failed to optimize SRIOV resources\n"); 4492 } 4493 } 4494 4495 static int be_get_resources(struct be_adapter *adapter) 4496 { 4497 struct device *dev = &adapter->pdev->dev; 4498 struct be_resources res = {0}; 4499 int status; 4500 4501 /* For Lancer, SH etc read per-function resource limits from FW. 4502 * GET_FUNC_CONFIG returns per function guaranteed limits. 4503 * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits 4504 */ 4505 if (BEx_chip(adapter)) { 4506 BEx_get_resources(adapter, &res); 4507 } else { 4508 status = be_cmd_get_func_config(adapter, &res); 4509 if (status) 4510 return status; 4511 4512 /* If a deafault RXQ must be created, we'll use up one RSSQ*/ 4513 if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs && 4514 !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS)) 4515 res.max_rss_qs -= 1; 4516 } 4517 4518 /* If RoCE is supported stash away half the EQs for RoCE */ 4519 res.max_nic_evt_qs = be_roce_supported(adapter) ? 4520 res.max_evt_qs / 2 : res.max_evt_qs; 4521 adapter->res = res; 4522 4523 /* If FW supports RSS default queue, then skip creating non-RSS 4524 * queue for non-IP traffic. 4525 */ 4526 adapter->need_def_rxq = (be_if_cap_flags(adapter) & 4527 BE_IF_FLAGS_DEFQ_RSS) ? 0 : 1; 4528 4529 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n", 4530 be_max_txqs(adapter), be_max_rxqs(adapter), 4531 be_max_rss(adapter), be_max_nic_eqs(adapter), 4532 be_max_vfs(adapter)); 4533 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n", 4534 be_max_uc(adapter), be_max_mc(adapter), 4535 be_max_vlans(adapter)); 4536 4537 /* Ensure RX and TX queues are created in pairs at init time */ 4538 adapter->cfg_num_rx_irqs = 4539 min_t(u16, netif_get_num_default_rss_queues(), 4540 be_max_qp_irqs(adapter)); 4541 adapter->cfg_num_tx_irqs = adapter->cfg_num_rx_irqs; 4542 return 0; 4543 } 4544 4545 static int be_get_config(struct be_adapter *adapter) 4546 { 4547 int status, level; 4548 u16 profile_id; 4549 4550 status = be_cmd_get_cntl_attributes(adapter); 4551 if (status) 4552 return status; 4553 4554 status = be_cmd_query_fw_cfg(adapter); 4555 if (status) 4556 return status; 4557 4558 if (!lancer_chip(adapter) && be_physfn(adapter)) 4559 be_cmd_get_fat_dump_len(adapter, &adapter->fat_dump_len); 4560 4561 if (BEx_chip(adapter)) { 4562 level = be_cmd_get_fw_log_level(adapter); 4563 adapter->msg_enable = 4564 level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0; 4565 } 4566 4567 be_cmd_get_acpi_wol_cap(adapter); 4568 pci_enable_wake(adapter->pdev, PCI_D3hot, adapter->wol_en); 4569 pci_enable_wake(adapter->pdev, PCI_D3cold, adapter->wol_en); 4570 4571 be_cmd_query_port_name(adapter); 4572 4573 if (be_physfn(adapter)) { 4574 status = be_cmd_get_active_profile(adapter, &profile_id); 4575 if (!status) 4576 dev_info(&adapter->pdev->dev, 4577 "Using profile 0x%x\n", profile_id); 4578 } 4579 4580 return 0; 4581 } 4582 4583 static int be_mac_setup(struct be_adapter *adapter) 4584 { 4585 u8 mac[ETH_ALEN]; 4586 int status; 4587 4588 if (is_zero_ether_addr(adapter->netdev->dev_addr)) { 4589 status = be_cmd_get_perm_mac(adapter, mac); 4590 if (status) 4591 return status; 4592 4593 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN); 4594 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN); 4595 4596 /* Initial MAC for BE3 VFs is already programmed by PF */ 4597 if (BEx_chip(adapter) && be_virtfn(adapter)) 4598 memcpy(adapter->dev_mac, mac, ETH_ALEN); 4599 } 4600 4601 return 0; 4602 } 4603 4604 static void be_schedule_worker(struct be_adapter *adapter) 4605 { 4606 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000)); 4607 adapter->flags |= BE_FLAGS_WORKER_SCHEDULED; 4608 } 4609 4610 static void be_destroy_err_recovery_workq(void) 4611 { 4612 if (!be_err_recovery_workq) 4613 return; 4614 4615 flush_workqueue(be_err_recovery_workq); 4616 destroy_workqueue(be_err_recovery_workq); 4617 be_err_recovery_workq = NULL; 4618 } 4619 4620 static void be_schedule_err_detection(struct be_adapter *adapter, u32 delay) 4621 { 4622 struct be_error_recovery *err_rec = &adapter->error_recovery; 4623 4624 if (!be_err_recovery_workq) 4625 return; 4626 4627 queue_delayed_work(be_err_recovery_workq, &err_rec->err_detection_work, 4628 msecs_to_jiffies(delay)); 4629 adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED; 4630 } 4631 4632 static int be_setup_queues(struct be_adapter *adapter) 4633 { 4634 struct net_device *netdev = adapter->netdev; 4635 int status; 4636 4637 status = be_evt_queues_create(adapter); 4638 if (status) 4639 goto err; 4640 4641 status = be_tx_qs_create(adapter); 4642 if (status) 4643 goto err; 4644 4645 status = be_rx_cqs_create(adapter); 4646 if (status) 4647 goto err; 4648 4649 status = be_mcc_queues_create(adapter); 4650 if (status) 4651 goto err; 4652 4653 status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs); 4654 if (status) 4655 goto err; 4656 4657 status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs); 4658 if (status) 4659 goto err; 4660 4661 return 0; 4662 err: 4663 dev_err(&adapter->pdev->dev, "queue_setup failed\n"); 4664 return status; 4665 } 4666 4667 static int be_if_create(struct be_adapter *adapter) 4668 { 4669 u32 en_flags = BE_IF_FLAGS_RSS | BE_IF_FLAGS_DEFQ_RSS; 4670 u32 cap_flags = be_if_cap_flags(adapter); 4671 int status; 4672 4673 /* alloc required memory for other filtering fields */ 4674 adapter->pmac_id = kcalloc(be_max_uc(adapter), 4675 sizeof(*adapter->pmac_id), GFP_KERNEL); 4676 if (!adapter->pmac_id) 4677 return -ENOMEM; 4678 4679 adapter->mc_list = kcalloc(be_max_mc(adapter), 4680 sizeof(*adapter->mc_list), GFP_KERNEL); 4681 if (!adapter->mc_list) 4682 return -ENOMEM; 4683 4684 adapter->uc_list = kcalloc(be_max_uc(adapter), 4685 sizeof(*adapter->uc_list), GFP_KERNEL); 4686 if (!adapter->uc_list) 4687 return -ENOMEM; 4688 4689 if (adapter->cfg_num_rx_irqs == 1) 4690 cap_flags &= ~(BE_IF_FLAGS_DEFQ_RSS | BE_IF_FLAGS_RSS); 4691 4692 en_flags &= cap_flags; 4693 /* will enable all the needed filter flags in be_open() */ 4694 status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags, 4695 &adapter->if_handle, 0); 4696 4697 if (status) 4698 return status; 4699 4700 return 0; 4701 } 4702 4703 int be_update_queues(struct be_adapter *adapter) 4704 { 4705 struct net_device *netdev = adapter->netdev; 4706 int status; 4707 4708 if (netif_running(netdev)) 4709 be_close(netdev); 4710 4711 be_cancel_worker(adapter); 4712 4713 /* If any vectors have been shared with RoCE we cannot re-program 4714 * the MSIx table. 4715 */ 4716 if (!adapter->num_msix_roce_vec) 4717 be_msix_disable(adapter); 4718 4719 be_clear_queues(adapter); 4720 status = be_cmd_if_destroy(adapter, adapter->if_handle, 0); 4721 if (status) 4722 return status; 4723 4724 if (!msix_enabled(adapter)) { 4725 status = be_msix_enable(adapter); 4726 if (status) 4727 return status; 4728 } 4729 4730 status = be_if_create(adapter); 4731 if (status) 4732 return status; 4733 4734 status = be_setup_queues(adapter); 4735 if (status) 4736 return status; 4737 4738 be_schedule_worker(adapter); 4739 4740 /* The IF was destroyed and re-created. We need to clear 4741 * all promiscuous flags valid for the destroyed IF. 4742 * Without this promisc mode is not restored during 4743 * be_open() because the driver thinks that it is 4744 * already enabled in HW. 4745 */ 4746 adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS; 4747 4748 if (netif_running(netdev)) 4749 status = be_open(netdev); 4750 4751 return status; 4752 } 4753 4754 static inline int fw_major_num(const char *fw_ver) 4755 { 4756 int fw_major = 0, i; 4757 4758 i = sscanf(fw_ver, "%d.", &fw_major); 4759 if (i != 1) 4760 return 0; 4761 4762 return fw_major; 4763 } 4764 4765 /* If it is error recovery, FLR the PF 4766 * Else if any VFs are already enabled don't FLR the PF 4767 */ 4768 static bool be_reset_required(struct be_adapter *adapter) 4769 { 4770 if (be_error_recovering(adapter)) 4771 return true; 4772 else 4773 return pci_num_vf(adapter->pdev) == 0; 4774 } 4775 4776 /* Wait for the FW to be ready and perform the required initialization */ 4777 static int be_func_init(struct be_adapter *adapter) 4778 { 4779 int status; 4780 4781 status = be_fw_wait_ready(adapter); 4782 if (status) 4783 return status; 4784 4785 /* FW is now ready; clear errors to allow cmds/doorbell */ 4786 be_clear_error(adapter, BE_CLEAR_ALL); 4787 4788 if (be_reset_required(adapter)) { 4789 status = be_cmd_reset_function(adapter); 4790 if (status) 4791 return status; 4792 4793 /* Wait for interrupts to quiesce after an FLR */ 4794 msleep(100); 4795 } 4796 4797 /* Tell FW we're ready to fire cmds */ 4798 status = be_cmd_fw_init(adapter); 4799 if (status) 4800 return status; 4801 4802 /* Allow interrupts for other ULPs running on NIC function */ 4803 be_intr_set(adapter, true); 4804 4805 return 0; 4806 } 4807 4808 static int be_setup(struct be_adapter *adapter) 4809 { 4810 struct device *dev = &adapter->pdev->dev; 4811 int status; 4812 4813 status = be_func_init(adapter); 4814 if (status) 4815 return status; 4816 4817 be_setup_init(adapter); 4818 4819 if (!lancer_chip(adapter)) 4820 be_cmd_req_native_mode(adapter); 4821 4822 /* invoke this cmd first to get pf_num and vf_num which are needed 4823 * for issuing profile related cmds 4824 */ 4825 if (!BEx_chip(adapter)) { 4826 status = be_cmd_get_func_config(adapter, NULL); 4827 if (status) 4828 return status; 4829 } 4830 4831 status = be_get_config(adapter); 4832 if (status) 4833 goto err; 4834 4835 if (!BE2_chip(adapter) && be_physfn(adapter)) 4836 be_alloc_sriov_res(adapter); 4837 4838 status = be_get_resources(adapter); 4839 if (status) 4840 goto err; 4841 4842 status = be_msix_enable(adapter); 4843 if (status) 4844 goto err; 4845 4846 /* will enable all the needed filter flags in be_open() */ 4847 status = be_if_create(adapter); 4848 if (status) 4849 goto err; 4850 4851 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 4852 rtnl_lock(); 4853 status = be_setup_queues(adapter); 4854 rtnl_unlock(); 4855 if (status) 4856 goto err; 4857 4858 be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0); 4859 4860 status = be_mac_setup(adapter); 4861 if (status) 4862 goto err; 4863 4864 be_cmd_get_fw_ver(adapter); 4865 dev_info(dev, "FW version is %s\n", adapter->fw_ver); 4866 4867 if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) { 4868 dev_err(dev, "Firmware on card is old(%s), IRQs may not work", 4869 adapter->fw_ver); 4870 dev_err(dev, "Please upgrade firmware to version >= 4.0\n"); 4871 } 4872 4873 status = be_cmd_set_flow_control(adapter, adapter->tx_fc, 4874 adapter->rx_fc); 4875 if (status) 4876 be_cmd_get_flow_control(adapter, &adapter->tx_fc, 4877 &adapter->rx_fc); 4878 4879 dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n", 4880 adapter->tx_fc, adapter->rx_fc); 4881 4882 if (be_physfn(adapter)) 4883 be_cmd_set_logical_link_config(adapter, 4884 IFLA_VF_LINK_STATE_AUTO, 0); 4885 4886 /* BE3 EVB echoes broadcast/multicast packets back to PF's vport 4887 * confusing a linux bridge or OVS that it might be connected to. 4888 * Set the EVB to PASSTHRU mode which effectively disables the EVB 4889 * when SRIOV is not enabled. 4890 */ 4891 if (BE3_chip(adapter)) 4892 be_cmd_set_hsw_config(adapter, 0, 0, adapter->if_handle, 4893 PORT_FWD_TYPE_PASSTHRU, 0); 4894 4895 if (adapter->num_vfs) 4896 be_vf_setup(adapter); 4897 4898 status = be_cmd_get_phy_info(adapter); 4899 if (!status && be_pause_supported(adapter)) 4900 adapter->phy.fc_autoneg = 1; 4901 4902 if (be_physfn(adapter) && !lancer_chip(adapter)) 4903 be_cmd_set_features(adapter); 4904 4905 be_schedule_worker(adapter); 4906 adapter->flags |= BE_FLAGS_SETUP_DONE; 4907 return 0; 4908 err: 4909 be_clear(adapter); 4910 return status; 4911 } 4912 4913 #ifdef CONFIG_NET_POLL_CONTROLLER 4914 static void be_netpoll(struct net_device *netdev) 4915 { 4916 struct be_adapter *adapter = netdev_priv(netdev); 4917 struct be_eq_obj *eqo; 4918 int i; 4919 4920 for_all_evt_queues(adapter, eqo, i) { 4921 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0); 4922 napi_schedule(&eqo->napi); 4923 } 4924 } 4925 #endif 4926 4927 int be_load_fw(struct be_adapter *adapter, u8 *fw_file) 4928 { 4929 const struct firmware *fw; 4930 int status; 4931 4932 if (!netif_running(adapter->netdev)) { 4933 dev_err(&adapter->pdev->dev, 4934 "Firmware load not allowed (interface is down)\n"); 4935 return -ENETDOWN; 4936 } 4937 4938 status = request_firmware(&fw, fw_file, &adapter->pdev->dev); 4939 if (status) 4940 goto fw_exit; 4941 4942 dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file); 4943 4944 if (lancer_chip(adapter)) 4945 status = lancer_fw_download(adapter, fw); 4946 else 4947 status = be_fw_download(adapter, fw); 4948 4949 if (!status) 4950 be_cmd_get_fw_ver(adapter); 4951 4952 fw_exit: 4953 release_firmware(fw); 4954 return status; 4955 } 4956 4957 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4958 u16 flags, struct netlink_ext_ack *extack) 4959 { 4960 struct be_adapter *adapter = netdev_priv(dev); 4961 struct nlattr *attr, *br_spec; 4962 int rem; 4963 int status = 0; 4964 u16 mode = 0; 4965 4966 if (!sriov_enabled(adapter)) 4967 return -EOPNOTSUPP; 4968 4969 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4970 if (!br_spec) 4971 return -EINVAL; 4972 4973 nla_for_each_nested(attr, br_spec, rem) { 4974 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4975 continue; 4976 4977 if (nla_len(attr) < sizeof(mode)) 4978 return -EINVAL; 4979 4980 mode = nla_get_u16(attr); 4981 if (BE3_chip(adapter) && mode == BRIDGE_MODE_VEPA) 4982 return -EOPNOTSUPP; 4983 4984 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 4985 return -EINVAL; 4986 4987 status = be_cmd_set_hsw_config(adapter, 0, 0, 4988 adapter->if_handle, 4989 mode == BRIDGE_MODE_VEPA ? 4990 PORT_FWD_TYPE_VEPA : 4991 PORT_FWD_TYPE_VEB, 0); 4992 if (status) 4993 goto err; 4994 4995 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n", 4996 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 4997 4998 return status; 4999 } 5000 err: 5001 dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n", 5002 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 5003 5004 return status; 5005 } 5006 5007 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 5008 struct net_device *dev, u32 filter_mask, 5009 int nlflags) 5010 { 5011 struct be_adapter *adapter = netdev_priv(dev); 5012 int status = 0; 5013 u8 hsw_mode; 5014 5015 /* BE and Lancer chips support VEB mode only */ 5016 if (BEx_chip(adapter) || lancer_chip(adapter)) { 5017 /* VEB is disabled in non-SR-IOV profiles on BE3/Lancer */ 5018 if (!pci_sriov_get_totalvfs(adapter->pdev)) 5019 return 0; 5020 hsw_mode = PORT_FWD_TYPE_VEB; 5021 } else { 5022 status = be_cmd_get_hsw_config(adapter, NULL, 0, 5023 adapter->if_handle, &hsw_mode, 5024 NULL); 5025 if (status) 5026 return 0; 5027 5028 if (hsw_mode == PORT_FWD_TYPE_PASSTHRU) 5029 return 0; 5030 } 5031 5032 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 5033 hsw_mode == PORT_FWD_TYPE_VEPA ? 5034 BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB, 5035 0, 0, nlflags, filter_mask, NULL); 5036 } 5037 5038 static struct be_cmd_work *be_alloc_work(struct be_adapter *adapter, 5039 void (*func)(struct work_struct *)) 5040 { 5041 struct be_cmd_work *work; 5042 5043 work = kzalloc(sizeof(*work), GFP_ATOMIC); 5044 if (!work) { 5045 dev_err(&adapter->pdev->dev, 5046 "be_work memory allocation failed\n"); 5047 return NULL; 5048 } 5049 5050 INIT_WORK(&work->work, func); 5051 work->adapter = adapter; 5052 return work; 5053 } 5054 5055 /* VxLAN offload Notes: 5056 * 5057 * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't 5058 * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload 5059 * is expected to work across all types of IP tunnels once exported. Skyhawk 5060 * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN 5061 * offloads in hw_enc_features only when a VxLAN port is added. If other (non 5062 * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for 5063 * those other tunnels are unexported on the fly through ndo_features_check(). 5064 * 5065 * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack 5066 * adds more than one port, disable offloads and re-enable them again when 5067 * there's only one port left. We maintain a list of ports for this purpose. 5068 */ 5069 static void be_work_add_vxlan_port(struct work_struct *work) 5070 { 5071 struct be_cmd_work *cmd_work = 5072 container_of(work, struct be_cmd_work, work); 5073 struct be_adapter *adapter = cmd_work->adapter; 5074 struct device *dev = &adapter->pdev->dev; 5075 __be16 port = cmd_work->info.vxlan_port; 5076 struct be_vxlan_port *vxlan_port; 5077 int status; 5078 5079 /* Bump up the alias count if it is an existing port */ 5080 list_for_each_entry(vxlan_port, &adapter->vxlan_port_list, list) { 5081 if (vxlan_port->port == port) { 5082 vxlan_port->port_aliases++; 5083 goto done; 5084 } 5085 } 5086 5087 /* Add a new port to our list. We don't need a lock here since port 5088 * add/delete are done only in the context of a single-threaded work 5089 * queue (be_wq). 5090 */ 5091 vxlan_port = kzalloc(sizeof(*vxlan_port), GFP_KERNEL); 5092 if (!vxlan_port) 5093 goto done; 5094 5095 vxlan_port->port = port; 5096 INIT_LIST_HEAD(&vxlan_port->list); 5097 list_add_tail(&vxlan_port->list, &adapter->vxlan_port_list); 5098 adapter->vxlan_port_count++; 5099 5100 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) { 5101 dev_info(dev, 5102 "Only one UDP port supported for VxLAN offloads\n"); 5103 dev_info(dev, "Disabling VxLAN offloads\n"); 5104 goto err; 5105 } 5106 5107 if (adapter->vxlan_port_count > 1) 5108 goto done; 5109 5110 status = be_enable_vxlan_offloads(adapter); 5111 if (!status) 5112 goto done; 5113 5114 err: 5115 be_disable_vxlan_offloads(adapter); 5116 done: 5117 kfree(cmd_work); 5118 return; 5119 } 5120 5121 static void be_work_del_vxlan_port(struct work_struct *work) 5122 { 5123 struct be_cmd_work *cmd_work = 5124 container_of(work, struct be_cmd_work, work); 5125 struct be_adapter *adapter = cmd_work->adapter; 5126 __be16 port = cmd_work->info.vxlan_port; 5127 struct be_vxlan_port *vxlan_port; 5128 5129 /* Nothing to be done if a port alias is being deleted */ 5130 list_for_each_entry(vxlan_port, &adapter->vxlan_port_list, list) { 5131 if (vxlan_port->port == port) { 5132 if (vxlan_port->port_aliases) { 5133 vxlan_port->port_aliases--; 5134 goto done; 5135 } 5136 break; 5137 } 5138 } 5139 5140 /* No port aliases left; delete the port from the list */ 5141 list_del(&vxlan_port->list); 5142 adapter->vxlan_port_count--; 5143 5144 /* Disable VxLAN offload if this is the offloaded port */ 5145 if (adapter->vxlan_port == vxlan_port->port) { 5146 WARN_ON(adapter->vxlan_port_count); 5147 be_disable_vxlan_offloads(adapter); 5148 dev_info(&adapter->pdev->dev, 5149 "Disabled VxLAN offloads for UDP port %d\n", 5150 be16_to_cpu(port)); 5151 goto out; 5152 } 5153 5154 /* If only 1 port is left, re-enable VxLAN offload */ 5155 if (adapter->vxlan_port_count == 1) 5156 be_enable_vxlan_offloads(adapter); 5157 5158 out: 5159 kfree(vxlan_port); 5160 done: 5161 kfree(cmd_work); 5162 } 5163 5164 static void be_cfg_vxlan_port(struct net_device *netdev, 5165 struct udp_tunnel_info *ti, 5166 void (*func)(struct work_struct *)) 5167 { 5168 struct be_adapter *adapter = netdev_priv(netdev); 5169 struct be_cmd_work *cmd_work; 5170 5171 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 5172 return; 5173 5174 if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter)) 5175 return; 5176 5177 cmd_work = be_alloc_work(adapter, func); 5178 if (cmd_work) { 5179 cmd_work->info.vxlan_port = ti->port; 5180 queue_work(be_wq, &cmd_work->work); 5181 } 5182 } 5183 5184 static void be_del_vxlan_port(struct net_device *netdev, 5185 struct udp_tunnel_info *ti) 5186 { 5187 be_cfg_vxlan_port(netdev, ti, be_work_del_vxlan_port); 5188 } 5189 5190 static void be_add_vxlan_port(struct net_device *netdev, 5191 struct udp_tunnel_info *ti) 5192 { 5193 be_cfg_vxlan_port(netdev, ti, be_work_add_vxlan_port); 5194 } 5195 5196 static netdev_features_t be_features_check(struct sk_buff *skb, 5197 struct net_device *dev, 5198 netdev_features_t features) 5199 { 5200 struct be_adapter *adapter = netdev_priv(dev); 5201 u8 l4_hdr = 0; 5202 5203 if (skb_is_gso(skb)) { 5204 /* IPv6 TSO requests with extension hdrs are a problem 5205 * to Lancer and BE3 HW. Disable TSO6 feature. 5206 */ 5207 if (!skyhawk_chip(adapter) && is_ipv6_ext_hdr(skb)) 5208 features &= ~NETIF_F_TSO6; 5209 5210 /* Lancer cannot handle the packet with MSS less than 256. 5211 * Also it can't handle a TSO packet with a single segment 5212 * Disable the GSO support in such cases 5213 */ 5214 if (lancer_chip(adapter) && 5215 (skb_shinfo(skb)->gso_size < 256 || 5216 skb_shinfo(skb)->gso_segs == 1)) 5217 features &= ~NETIF_F_GSO_MASK; 5218 } 5219 5220 /* The code below restricts offload features for some tunneled and 5221 * Q-in-Q packets. 5222 * Offload features for normal (non tunnel) packets are unchanged. 5223 */ 5224 features = vlan_features_check(skb, features); 5225 if (!skb->encapsulation || 5226 !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)) 5227 return features; 5228 5229 /* It's an encapsulated packet and VxLAN offloads are enabled. We 5230 * should disable tunnel offload features if it's not a VxLAN packet, 5231 * as tunnel offloads have been enabled only for VxLAN. This is done to 5232 * allow other tunneled traffic like GRE work fine while VxLAN 5233 * offloads are configured in Skyhawk-R. 5234 */ 5235 switch (vlan_get_protocol(skb)) { 5236 case htons(ETH_P_IP): 5237 l4_hdr = ip_hdr(skb)->protocol; 5238 break; 5239 case htons(ETH_P_IPV6): 5240 l4_hdr = ipv6_hdr(skb)->nexthdr; 5241 break; 5242 default: 5243 return features; 5244 } 5245 5246 if (l4_hdr != IPPROTO_UDP || 5247 skb->inner_protocol_type != ENCAP_TYPE_ETHER || 5248 skb->inner_protocol != htons(ETH_P_TEB) || 5249 skb_inner_mac_header(skb) - skb_transport_header(skb) != 5250 sizeof(struct udphdr) + sizeof(struct vxlanhdr) || 5251 !adapter->vxlan_port || 5252 udp_hdr(skb)->dest != adapter->vxlan_port) 5253 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 5254 5255 return features; 5256 } 5257 5258 static int be_get_phys_port_id(struct net_device *dev, 5259 struct netdev_phys_item_id *ppid) 5260 { 5261 int i, id_len = CNTL_SERIAL_NUM_WORDS * CNTL_SERIAL_NUM_WORD_SZ + 1; 5262 struct be_adapter *adapter = netdev_priv(dev); 5263 u8 *id; 5264 5265 if (MAX_PHYS_ITEM_ID_LEN < id_len) 5266 return -ENOSPC; 5267 5268 ppid->id[0] = adapter->hba_port_num + 1; 5269 id = &ppid->id[1]; 5270 for (i = CNTL_SERIAL_NUM_WORDS - 1; i >= 0; 5271 i--, id += CNTL_SERIAL_NUM_WORD_SZ) 5272 memcpy(id, &adapter->serial_num[i], CNTL_SERIAL_NUM_WORD_SZ); 5273 5274 ppid->id_len = id_len; 5275 5276 return 0; 5277 } 5278 5279 static void be_set_rx_mode(struct net_device *dev) 5280 { 5281 struct be_adapter *adapter = netdev_priv(dev); 5282 struct be_cmd_work *work; 5283 5284 work = be_alloc_work(adapter, be_work_set_rx_mode); 5285 if (work) 5286 queue_work(be_wq, &work->work); 5287 } 5288 5289 static const struct net_device_ops be_netdev_ops = { 5290 .ndo_open = be_open, 5291 .ndo_stop = be_close, 5292 .ndo_start_xmit = be_xmit, 5293 .ndo_set_rx_mode = be_set_rx_mode, 5294 .ndo_set_mac_address = be_mac_addr_set, 5295 .ndo_get_stats64 = be_get_stats64, 5296 .ndo_validate_addr = eth_validate_addr, 5297 .ndo_vlan_rx_add_vid = be_vlan_add_vid, 5298 .ndo_vlan_rx_kill_vid = be_vlan_rem_vid, 5299 .ndo_set_vf_mac = be_set_vf_mac, 5300 .ndo_set_vf_vlan = be_set_vf_vlan, 5301 .ndo_set_vf_rate = be_set_vf_tx_rate, 5302 .ndo_get_vf_config = be_get_vf_config, 5303 .ndo_set_vf_link_state = be_set_vf_link_state, 5304 .ndo_set_vf_spoofchk = be_set_vf_spoofchk, 5305 .ndo_tx_timeout = be_tx_timeout, 5306 #ifdef CONFIG_NET_POLL_CONTROLLER 5307 .ndo_poll_controller = be_netpoll, 5308 #endif 5309 .ndo_bridge_setlink = be_ndo_bridge_setlink, 5310 .ndo_bridge_getlink = be_ndo_bridge_getlink, 5311 .ndo_udp_tunnel_add = be_add_vxlan_port, 5312 .ndo_udp_tunnel_del = be_del_vxlan_port, 5313 .ndo_features_check = be_features_check, 5314 .ndo_get_phys_port_id = be_get_phys_port_id, 5315 }; 5316 5317 static void be_netdev_init(struct net_device *netdev) 5318 { 5319 struct be_adapter *adapter = netdev_priv(netdev); 5320 5321 netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5322 NETIF_F_GSO_UDP_TUNNEL | 5323 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | 5324 NETIF_F_HW_VLAN_CTAG_TX; 5325 if ((be_if_cap_flags(adapter) & BE_IF_FLAGS_RSS)) 5326 netdev->hw_features |= NETIF_F_RXHASH; 5327 5328 netdev->features |= netdev->hw_features | 5329 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER; 5330 5331 netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5332 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 5333 5334 netdev->priv_flags |= IFF_UNICAST_FLT; 5335 5336 netdev->flags |= IFF_MULTICAST; 5337 5338 netif_set_gso_max_size(netdev, BE_MAX_GSO_SIZE - ETH_HLEN); 5339 5340 netdev->netdev_ops = &be_netdev_ops; 5341 5342 netdev->ethtool_ops = &be_ethtool_ops; 5343 5344 /* MTU range: 256 - 9000 */ 5345 netdev->min_mtu = BE_MIN_MTU; 5346 netdev->max_mtu = BE_MAX_MTU; 5347 } 5348 5349 static void be_cleanup(struct be_adapter *adapter) 5350 { 5351 struct net_device *netdev = adapter->netdev; 5352 5353 rtnl_lock(); 5354 netif_device_detach(netdev); 5355 if (netif_running(netdev)) 5356 be_close(netdev); 5357 rtnl_unlock(); 5358 5359 be_clear(adapter); 5360 } 5361 5362 static int be_resume(struct be_adapter *adapter) 5363 { 5364 struct net_device *netdev = adapter->netdev; 5365 int status; 5366 5367 status = be_setup(adapter); 5368 if (status) 5369 return status; 5370 5371 rtnl_lock(); 5372 if (netif_running(netdev)) 5373 status = be_open(netdev); 5374 rtnl_unlock(); 5375 5376 if (status) 5377 return status; 5378 5379 netif_device_attach(netdev); 5380 5381 return 0; 5382 } 5383 5384 static void be_soft_reset(struct be_adapter *adapter) 5385 { 5386 u32 val; 5387 5388 dev_info(&adapter->pdev->dev, "Initiating chip soft reset\n"); 5389 val = ioread32(adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET); 5390 val |= SLIPORT_SOFTRESET_SR_MASK; 5391 iowrite32(val, adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET); 5392 } 5393 5394 static bool be_err_is_recoverable(struct be_adapter *adapter) 5395 { 5396 struct be_error_recovery *err_rec = &adapter->error_recovery; 5397 unsigned long initial_idle_time = 5398 msecs_to_jiffies(ERR_RECOVERY_IDLE_TIME); 5399 unsigned long recovery_interval = 5400 msecs_to_jiffies(ERR_RECOVERY_INTERVAL); 5401 u16 ue_err_code; 5402 u32 val; 5403 5404 val = be_POST_stage_get(adapter); 5405 if ((val & POST_STAGE_RECOVERABLE_ERR) != POST_STAGE_RECOVERABLE_ERR) 5406 return false; 5407 ue_err_code = val & POST_ERR_RECOVERY_CODE_MASK; 5408 if (ue_err_code == 0) 5409 return false; 5410 5411 dev_err(&adapter->pdev->dev, "Recoverable HW error code: 0x%x\n", 5412 ue_err_code); 5413 5414 if (time_before_eq(jiffies - err_rec->probe_time, initial_idle_time)) { 5415 dev_err(&adapter->pdev->dev, 5416 "Cannot recover within %lu sec from driver load\n", 5417 jiffies_to_msecs(initial_idle_time) / MSEC_PER_SEC); 5418 return false; 5419 } 5420 5421 if (err_rec->last_recovery_time && time_before_eq( 5422 jiffies - err_rec->last_recovery_time, recovery_interval)) { 5423 dev_err(&adapter->pdev->dev, 5424 "Cannot recover within %lu sec from last recovery\n", 5425 jiffies_to_msecs(recovery_interval) / MSEC_PER_SEC); 5426 return false; 5427 } 5428 5429 if (ue_err_code == err_rec->last_err_code) { 5430 dev_err(&adapter->pdev->dev, 5431 "Cannot recover from a consecutive TPE error\n"); 5432 return false; 5433 } 5434 5435 err_rec->last_recovery_time = jiffies; 5436 err_rec->last_err_code = ue_err_code; 5437 return true; 5438 } 5439 5440 static int be_tpe_recover(struct be_adapter *adapter) 5441 { 5442 struct be_error_recovery *err_rec = &adapter->error_recovery; 5443 int status = -EAGAIN; 5444 u32 val; 5445 5446 switch (err_rec->recovery_state) { 5447 case ERR_RECOVERY_ST_NONE: 5448 err_rec->recovery_state = ERR_RECOVERY_ST_DETECT; 5449 err_rec->resched_delay = ERR_RECOVERY_UE_DETECT_DURATION; 5450 break; 5451 5452 case ERR_RECOVERY_ST_DETECT: 5453 val = be_POST_stage_get(adapter); 5454 if ((val & POST_STAGE_RECOVERABLE_ERR) != 5455 POST_STAGE_RECOVERABLE_ERR) { 5456 dev_err(&adapter->pdev->dev, 5457 "Unrecoverable HW error detected: 0x%x\n", val); 5458 status = -EINVAL; 5459 err_rec->resched_delay = 0; 5460 break; 5461 } 5462 5463 dev_err(&adapter->pdev->dev, "Recoverable HW error detected\n"); 5464 5465 /* Only PF0 initiates Chip Soft Reset. But PF0 must wait UE2SR 5466 * milliseconds before it checks for final error status in 5467 * SLIPORT_SEMAPHORE to determine if recovery criteria is met. 5468 * If it does, then PF0 initiates a Soft Reset. 5469 */ 5470 if (adapter->pf_num == 0) { 5471 err_rec->recovery_state = ERR_RECOVERY_ST_RESET; 5472 err_rec->resched_delay = err_rec->ue_to_reset_time - 5473 ERR_RECOVERY_UE_DETECT_DURATION; 5474 break; 5475 } 5476 5477 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL; 5478 err_rec->resched_delay = err_rec->ue_to_poll_time - 5479 ERR_RECOVERY_UE_DETECT_DURATION; 5480 break; 5481 5482 case ERR_RECOVERY_ST_RESET: 5483 if (!be_err_is_recoverable(adapter)) { 5484 dev_err(&adapter->pdev->dev, 5485 "Failed to meet recovery criteria\n"); 5486 status = -EIO; 5487 err_rec->resched_delay = 0; 5488 break; 5489 } 5490 be_soft_reset(adapter); 5491 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL; 5492 err_rec->resched_delay = err_rec->ue_to_poll_time - 5493 err_rec->ue_to_reset_time; 5494 break; 5495 5496 case ERR_RECOVERY_ST_PRE_POLL: 5497 err_rec->recovery_state = ERR_RECOVERY_ST_REINIT; 5498 err_rec->resched_delay = 0; 5499 status = 0; /* done */ 5500 break; 5501 5502 default: 5503 status = -EINVAL; 5504 err_rec->resched_delay = 0; 5505 break; 5506 } 5507 5508 return status; 5509 } 5510 5511 static int be_err_recover(struct be_adapter *adapter) 5512 { 5513 int status; 5514 5515 if (!lancer_chip(adapter)) { 5516 if (!adapter->error_recovery.recovery_supported || 5517 adapter->priv_flags & BE_DISABLE_TPE_RECOVERY) 5518 return -EIO; 5519 status = be_tpe_recover(adapter); 5520 if (status) 5521 goto err; 5522 } 5523 5524 /* Wait for adapter to reach quiescent state before 5525 * destroying queues 5526 */ 5527 status = be_fw_wait_ready(adapter); 5528 if (status) 5529 goto err; 5530 5531 adapter->flags |= BE_FLAGS_TRY_RECOVERY; 5532 5533 be_cleanup(adapter); 5534 5535 status = be_resume(adapter); 5536 if (status) 5537 goto err; 5538 5539 adapter->flags &= ~BE_FLAGS_TRY_RECOVERY; 5540 5541 err: 5542 return status; 5543 } 5544 5545 static void be_err_detection_task(struct work_struct *work) 5546 { 5547 struct be_error_recovery *err_rec = 5548 container_of(work, struct be_error_recovery, 5549 err_detection_work.work); 5550 struct be_adapter *adapter = 5551 container_of(err_rec, struct be_adapter, 5552 error_recovery); 5553 u32 resched_delay = ERR_RECOVERY_DETECTION_DELAY; 5554 struct device *dev = &adapter->pdev->dev; 5555 int recovery_status; 5556 5557 be_detect_error(adapter); 5558 if (!be_check_error(adapter, BE_ERROR_HW)) 5559 goto reschedule_task; 5560 5561 recovery_status = be_err_recover(adapter); 5562 if (!recovery_status) { 5563 err_rec->recovery_retries = 0; 5564 err_rec->recovery_state = ERR_RECOVERY_ST_NONE; 5565 dev_info(dev, "Adapter recovery successful\n"); 5566 goto reschedule_task; 5567 } else if (!lancer_chip(adapter) && err_rec->resched_delay) { 5568 /* BEx/SH recovery state machine */ 5569 if (adapter->pf_num == 0 && 5570 err_rec->recovery_state > ERR_RECOVERY_ST_DETECT) 5571 dev_err(&adapter->pdev->dev, 5572 "Adapter recovery in progress\n"); 5573 resched_delay = err_rec->resched_delay; 5574 goto reschedule_task; 5575 } else if (lancer_chip(adapter) && be_virtfn(adapter)) { 5576 /* For VFs, check if PF have allocated resources 5577 * every second. 5578 */ 5579 dev_err(dev, "Re-trying adapter recovery\n"); 5580 goto reschedule_task; 5581 } else if (lancer_chip(adapter) && err_rec->recovery_retries++ < 5582 ERR_RECOVERY_MAX_RETRY_COUNT) { 5583 /* In case of another error during recovery, it takes 30 sec 5584 * for adapter to come out of error. Retry error recovery after 5585 * this time interval. 5586 */ 5587 dev_err(&adapter->pdev->dev, "Re-trying adapter recovery\n"); 5588 resched_delay = ERR_RECOVERY_RETRY_DELAY; 5589 goto reschedule_task; 5590 } else { 5591 dev_err(dev, "Adapter recovery failed\n"); 5592 dev_err(dev, "Please reboot server to recover\n"); 5593 } 5594 5595 return; 5596 5597 reschedule_task: 5598 be_schedule_err_detection(adapter, resched_delay); 5599 } 5600 5601 static void be_log_sfp_info(struct be_adapter *adapter) 5602 { 5603 int status; 5604 5605 status = be_cmd_query_sfp_info(adapter); 5606 if (!status) { 5607 dev_err(&adapter->pdev->dev, 5608 "Port %c: %s Vendor: %s part no: %s", 5609 adapter->port_name, 5610 be_misconfig_evt_port_state[adapter->phy_state], 5611 adapter->phy.vendor_name, 5612 adapter->phy.vendor_pn); 5613 } 5614 adapter->flags &= ~BE_FLAGS_PHY_MISCONFIGURED; 5615 } 5616 5617 static void be_worker(struct work_struct *work) 5618 { 5619 struct be_adapter *adapter = 5620 container_of(work, struct be_adapter, work.work); 5621 struct be_rx_obj *rxo; 5622 int i; 5623 5624 if (be_physfn(adapter) && 5625 MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0) 5626 be_cmd_get_die_temperature(adapter); 5627 5628 /* when interrupts are not yet enabled, just reap any pending 5629 * mcc completions 5630 */ 5631 if (!netif_running(adapter->netdev)) { 5632 local_bh_disable(); 5633 be_process_mcc(adapter); 5634 local_bh_enable(); 5635 goto reschedule; 5636 } 5637 5638 if (!adapter->stats_cmd_sent) { 5639 if (lancer_chip(adapter)) 5640 lancer_cmd_get_pport_stats(adapter, 5641 &adapter->stats_cmd); 5642 else 5643 be_cmd_get_stats(adapter, &adapter->stats_cmd); 5644 } 5645 5646 for_all_rx_queues(adapter, rxo, i) { 5647 /* Replenish RX-queues starved due to memory 5648 * allocation failures. 5649 */ 5650 if (rxo->rx_post_starved) 5651 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST); 5652 } 5653 5654 /* EQ-delay update for Skyhawk is done while notifying EQ */ 5655 if (!skyhawk_chip(adapter)) 5656 be_eqd_update(adapter, false); 5657 5658 if (adapter->flags & BE_FLAGS_PHY_MISCONFIGURED) 5659 be_log_sfp_info(adapter); 5660 5661 reschedule: 5662 adapter->work_counter++; 5663 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000)); 5664 } 5665 5666 static void be_unmap_pci_bars(struct be_adapter *adapter) 5667 { 5668 if (adapter->csr) 5669 pci_iounmap(adapter->pdev, adapter->csr); 5670 if (adapter->db) 5671 pci_iounmap(adapter->pdev, adapter->db); 5672 if (adapter->pcicfg && adapter->pcicfg_mapped) 5673 pci_iounmap(adapter->pdev, adapter->pcicfg); 5674 } 5675 5676 static int db_bar(struct be_adapter *adapter) 5677 { 5678 if (lancer_chip(adapter) || be_virtfn(adapter)) 5679 return 0; 5680 else 5681 return 4; 5682 } 5683 5684 static int be_roce_map_pci_bars(struct be_adapter *adapter) 5685 { 5686 if (skyhawk_chip(adapter)) { 5687 adapter->roce_db.size = 4096; 5688 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev, 5689 db_bar(adapter)); 5690 adapter->roce_db.total_size = pci_resource_len(adapter->pdev, 5691 db_bar(adapter)); 5692 } 5693 return 0; 5694 } 5695 5696 static int be_map_pci_bars(struct be_adapter *adapter) 5697 { 5698 struct pci_dev *pdev = adapter->pdev; 5699 u8 __iomem *addr; 5700 u32 sli_intf; 5701 5702 pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf); 5703 adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >> 5704 SLI_INTF_FAMILY_SHIFT; 5705 adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0; 5706 5707 if (BEx_chip(adapter) && be_physfn(adapter)) { 5708 adapter->csr = pci_iomap(pdev, 2, 0); 5709 if (!adapter->csr) 5710 return -ENOMEM; 5711 } 5712 5713 addr = pci_iomap(pdev, db_bar(adapter), 0); 5714 if (!addr) 5715 goto pci_map_err; 5716 adapter->db = addr; 5717 5718 if (skyhawk_chip(adapter) || BEx_chip(adapter)) { 5719 if (be_physfn(adapter)) { 5720 /* PCICFG is the 2nd BAR in BE2 */ 5721 addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0); 5722 if (!addr) 5723 goto pci_map_err; 5724 adapter->pcicfg = addr; 5725 adapter->pcicfg_mapped = true; 5726 } else { 5727 adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET; 5728 adapter->pcicfg_mapped = false; 5729 } 5730 } 5731 5732 be_roce_map_pci_bars(adapter); 5733 return 0; 5734 5735 pci_map_err: 5736 dev_err(&pdev->dev, "Error in mapping PCI BARs\n"); 5737 be_unmap_pci_bars(adapter); 5738 return -ENOMEM; 5739 } 5740 5741 static void be_drv_cleanup(struct be_adapter *adapter) 5742 { 5743 struct be_dma_mem *mem = &adapter->mbox_mem_alloced; 5744 struct device *dev = &adapter->pdev->dev; 5745 5746 if (mem->va) 5747 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5748 5749 mem = &adapter->rx_filter; 5750 if (mem->va) 5751 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5752 5753 mem = &adapter->stats_cmd; 5754 if (mem->va) 5755 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5756 } 5757 5758 /* Allocate and initialize various fields in be_adapter struct */ 5759 static int be_drv_init(struct be_adapter *adapter) 5760 { 5761 struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced; 5762 struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem; 5763 struct be_dma_mem *rx_filter = &adapter->rx_filter; 5764 struct be_dma_mem *stats_cmd = &adapter->stats_cmd; 5765 struct device *dev = &adapter->pdev->dev; 5766 int status = 0; 5767 5768 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 5769 mbox_mem_alloc->va = dma_zalloc_coherent(dev, mbox_mem_alloc->size, 5770 &mbox_mem_alloc->dma, 5771 GFP_KERNEL); 5772 if (!mbox_mem_alloc->va) 5773 return -ENOMEM; 5774 5775 mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 5776 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 5777 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 5778 5779 rx_filter->size = sizeof(struct be_cmd_req_rx_filter); 5780 rx_filter->va = dma_zalloc_coherent(dev, rx_filter->size, 5781 &rx_filter->dma, GFP_KERNEL); 5782 if (!rx_filter->va) { 5783 status = -ENOMEM; 5784 goto free_mbox; 5785 } 5786 5787 if (lancer_chip(adapter)) 5788 stats_cmd->size = sizeof(struct lancer_cmd_req_pport_stats); 5789 else if (BE2_chip(adapter)) 5790 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v0); 5791 else if (BE3_chip(adapter)) 5792 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v1); 5793 else 5794 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v2); 5795 stats_cmd->va = dma_zalloc_coherent(dev, stats_cmd->size, 5796 &stats_cmd->dma, GFP_KERNEL); 5797 if (!stats_cmd->va) { 5798 status = -ENOMEM; 5799 goto free_rx_filter; 5800 } 5801 5802 mutex_init(&adapter->mbox_lock); 5803 mutex_init(&adapter->mcc_lock); 5804 mutex_init(&adapter->rx_filter_lock); 5805 spin_lock_init(&adapter->mcc_cq_lock); 5806 init_completion(&adapter->et_cmd_compl); 5807 5808 pci_save_state(adapter->pdev); 5809 5810 INIT_DELAYED_WORK(&adapter->work, be_worker); 5811 5812 adapter->error_recovery.recovery_state = ERR_RECOVERY_ST_NONE; 5813 adapter->error_recovery.resched_delay = 0; 5814 INIT_DELAYED_WORK(&adapter->error_recovery.err_detection_work, 5815 be_err_detection_task); 5816 5817 adapter->rx_fc = true; 5818 adapter->tx_fc = true; 5819 5820 /* Must be a power of 2 or else MODULO will BUG_ON */ 5821 adapter->be_get_temp_freq = 64; 5822 5823 INIT_LIST_HEAD(&adapter->vxlan_port_list); 5824 return 0; 5825 5826 free_rx_filter: 5827 dma_free_coherent(dev, rx_filter->size, rx_filter->va, rx_filter->dma); 5828 free_mbox: 5829 dma_free_coherent(dev, mbox_mem_alloc->size, mbox_mem_alloc->va, 5830 mbox_mem_alloc->dma); 5831 return status; 5832 } 5833 5834 static void be_remove(struct pci_dev *pdev) 5835 { 5836 struct be_adapter *adapter = pci_get_drvdata(pdev); 5837 5838 if (!adapter) 5839 return; 5840 5841 be_roce_dev_remove(adapter); 5842 be_intr_set(adapter, false); 5843 5844 be_cancel_err_detection(adapter); 5845 5846 unregister_netdev(adapter->netdev); 5847 5848 be_clear(adapter); 5849 5850 if (!pci_vfs_assigned(adapter->pdev)) 5851 be_cmd_reset_function(adapter); 5852 5853 /* tell fw we're done with firing cmds */ 5854 be_cmd_fw_clean(adapter); 5855 5856 be_unmap_pci_bars(adapter); 5857 be_drv_cleanup(adapter); 5858 5859 pci_disable_pcie_error_reporting(pdev); 5860 5861 pci_release_regions(pdev); 5862 pci_disable_device(pdev); 5863 5864 free_netdev(adapter->netdev); 5865 } 5866 5867 static ssize_t be_hwmon_show_temp(struct device *dev, 5868 struct device_attribute *dev_attr, 5869 char *buf) 5870 { 5871 struct be_adapter *adapter = dev_get_drvdata(dev); 5872 5873 /* Unit: millidegree Celsius */ 5874 if (adapter->hwmon_info.be_on_die_temp == BE_INVALID_DIE_TEMP) 5875 return -EIO; 5876 else 5877 return sprintf(buf, "%u\n", 5878 adapter->hwmon_info.be_on_die_temp * 1000); 5879 } 5880 5881 static SENSOR_DEVICE_ATTR(temp1_input, 0444, 5882 be_hwmon_show_temp, NULL, 1); 5883 5884 static struct attribute *be_hwmon_attrs[] = { 5885 &sensor_dev_attr_temp1_input.dev_attr.attr, 5886 NULL 5887 }; 5888 5889 ATTRIBUTE_GROUPS(be_hwmon); 5890 5891 static char *mc_name(struct be_adapter *adapter) 5892 { 5893 char *str = ""; /* default */ 5894 5895 switch (adapter->mc_type) { 5896 case UMC: 5897 str = "UMC"; 5898 break; 5899 case FLEX10: 5900 str = "FLEX10"; 5901 break; 5902 case vNIC1: 5903 str = "vNIC-1"; 5904 break; 5905 case nPAR: 5906 str = "nPAR"; 5907 break; 5908 case UFP: 5909 str = "UFP"; 5910 break; 5911 case vNIC2: 5912 str = "vNIC-2"; 5913 break; 5914 default: 5915 str = ""; 5916 } 5917 5918 return str; 5919 } 5920 5921 static inline char *func_name(struct be_adapter *adapter) 5922 { 5923 return be_physfn(adapter) ? "PF" : "VF"; 5924 } 5925 5926 static inline char *nic_name(struct pci_dev *pdev) 5927 { 5928 switch (pdev->device) { 5929 case OC_DEVICE_ID1: 5930 return OC_NAME; 5931 case OC_DEVICE_ID2: 5932 return OC_NAME_BE; 5933 case OC_DEVICE_ID3: 5934 case OC_DEVICE_ID4: 5935 return OC_NAME_LANCER; 5936 case BE_DEVICE_ID2: 5937 return BE3_NAME; 5938 case OC_DEVICE_ID5: 5939 case OC_DEVICE_ID6: 5940 return OC_NAME_SH; 5941 default: 5942 return BE_NAME; 5943 } 5944 } 5945 5946 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id) 5947 { 5948 struct be_adapter *adapter; 5949 struct net_device *netdev; 5950 int status = 0; 5951 5952 dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER); 5953 5954 status = pci_enable_device(pdev); 5955 if (status) 5956 goto do_none; 5957 5958 status = pci_request_regions(pdev, DRV_NAME); 5959 if (status) 5960 goto disable_dev; 5961 pci_set_master(pdev); 5962 5963 netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS); 5964 if (!netdev) { 5965 status = -ENOMEM; 5966 goto rel_reg; 5967 } 5968 adapter = netdev_priv(netdev); 5969 adapter->pdev = pdev; 5970 pci_set_drvdata(pdev, adapter); 5971 adapter->netdev = netdev; 5972 SET_NETDEV_DEV(netdev, &pdev->dev); 5973 5974 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 5975 if (!status) { 5976 netdev->features |= NETIF_F_HIGHDMA; 5977 } else { 5978 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 5979 if (status) { 5980 dev_err(&pdev->dev, "Could not set PCI DMA Mask\n"); 5981 goto free_netdev; 5982 } 5983 } 5984 5985 status = pci_enable_pcie_error_reporting(pdev); 5986 if (!status) 5987 dev_info(&pdev->dev, "PCIe error reporting enabled\n"); 5988 5989 status = be_map_pci_bars(adapter); 5990 if (status) 5991 goto free_netdev; 5992 5993 status = be_drv_init(adapter); 5994 if (status) 5995 goto unmap_bars; 5996 5997 status = be_setup(adapter); 5998 if (status) 5999 goto drv_cleanup; 6000 6001 be_netdev_init(netdev); 6002 status = register_netdev(netdev); 6003 if (status != 0) 6004 goto unsetup; 6005 6006 be_roce_dev_add(adapter); 6007 6008 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 6009 adapter->error_recovery.probe_time = jiffies; 6010 6011 /* On Die temperature not supported for VF. */ 6012 if (be_physfn(adapter) && IS_ENABLED(CONFIG_BE2NET_HWMON)) { 6013 adapter->hwmon_info.hwmon_dev = 6014 devm_hwmon_device_register_with_groups(&pdev->dev, 6015 DRV_NAME, 6016 adapter, 6017 be_hwmon_groups); 6018 adapter->hwmon_info.be_on_die_temp = BE_INVALID_DIE_TEMP; 6019 } 6020 6021 dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev), 6022 func_name(adapter), mc_name(adapter), adapter->port_name); 6023 6024 return 0; 6025 6026 unsetup: 6027 be_clear(adapter); 6028 drv_cleanup: 6029 be_drv_cleanup(adapter); 6030 unmap_bars: 6031 be_unmap_pci_bars(adapter); 6032 free_netdev: 6033 free_netdev(netdev); 6034 rel_reg: 6035 pci_release_regions(pdev); 6036 disable_dev: 6037 pci_disable_device(pdev); 6038 do_none: 6039 dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev)); 6040 return status; 6041 } 6042 6043 static int be_suspend(struct pci_dev *pdev, pm_message_t state) 6044 { 6045 struct be_adapter *adapter = pci_get_drvdata(pdev); 6046 6047 be_intr_set(adapter, false); 6048 be_cancel_err_detection(adapter); 6049 6050 be_cleanup(adapter); 6051 6052 pci_save_state(pdev); 6053 pci_disable_device(pdev); 6054 pci_set_power_state(pdev, pci_choose_state(pdev, state)); 6055 return 0; 6056 } 6057 6058 static int be_pci_resume(struct pci_dev *pdev) 6059 { 6060 struct be_adapter *adapter = pci_get_drvdata(pdev); 6061 int status = 0; 6062 6063 status = pci_enable_device(pdev); 6064 if (status) 6065 return status; 6066 6067 pci_restore_state(pdev); 6068 6069 status = be_resume(adapter); 6070 if (status) 6071 return status; 6072 6073 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 6074 6075 return 0; 6076 } 6077 6078 /* 6079 * An FLR will stop BE from DMAing any data. 6080 */ 6081 static void be_shutdown(struct pci_dev *pdev) 6082 { 6083 struct be_adapter *adapter = pci_get_drvdata(pdev); 6084 6085 if (!adapter) 6086 return; 6087 6088 be_roce_dev_shutdown(adapter); 6089 cancel_delayed_work_sync(&adapter->work); 6090 be_cancel_err_detection(adapter); 6091 6092 netif_device_detach(adapter->netdev); 6093 6094 be_cmd_reset_function(adapter); 6095 6096 pci_disable_device(pdev); 6097 } 6098 6099 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev, 6100 pci_channel_state_t state) 6101 { 6102 struct be_adapter *adapter = pci_get_drvdata(pdev); 6103 6104 dev_err(&adapter->pdev->dev, "EEH error detected\n"); 6105 6106 be_roce_dev_remove(adapter); 6107 6108 if (!be_check_error(adapter, BE_ERROR_EEH)) { 6109 be_set_error(adapter, BE_ERROR_EEH); 6110 6111 be_cancel_err_detection(adapter); 6112 6113 be_cleanup(adapter); 6114 } 6115 6116 if (state == pci_channel_io_perm_failure) 6117 return PCI_ERS_RESULT_DISCONNECT; 6118 6119 pci_disable_device(pdev); 6120 6121 /* The error could cause the FW to trigger a flash debug dump. 6122 * Resetting the card while flash dump is in progress 6123 * can cause it not to recover; wait for it to finish. 6124 * Wait only for first function as it is needed only once per 6125 * adapter. 6126 */ 6127 if (pdev->devfn == 0) 6128 ssleep(30); 6129 6130 return PCI_ERS_RESULT_NEED_RESET; 6131 } 6132 6133 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev) 6134 { 6135 struct be_adapter *adapter = pci_get_drvdata(pdev); 6136 int status; 6137 6138 dev_info(&adapter->pdev->dev, "EEH reset\n"); 6139 6140 status = pci_enable_device(pdev); 6141 if (status) 6142 return PCI_ERS_RESULT_DISCONNECT; 6143 6144 pci_set_master(pdev); 6145 pci_restore_state(pdev); 6146 6147 /* Check if card is ok and fw is ready */ 6148 dev_info(&adapter->pdev->dev, 6149 "Waiting for FW to be ready after EEH reset\n"); 6150 status = be_fw_wait_ready(adapter); 6151 if (status) 6152 return PCI_ERS_RESULT_DISCONNECT; 6153 6154 be_clear_error(adapter, BE_CLEAR_ALL); 6155 return PCI_ERS_RESULT_RECOVERED; 6156 } 6157 6158 static void be_eeh_resume(struct pci_dev *pdev) 6159 { 6160 int status = 0; 6161 struct be_adapter *adapter = pci_get_drvdata(pdev); 6162 6163 dev_info(&adapter->pdev->dev, "EEH resume\n"); 6164 6165 pci_save_state(pdev); 6166 6167 status = be_resume(adapter); 6168 if (status) 6169 goto err; 6170 6171 be_roce_dev_add(adapter); 6172 6173 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 6174 return; 6175 err: 6176 dev_err(&adapter->pdev->dev, "EEH resume failed\n"); 6177 } 6178 6179 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) 6180 { 6181 struct be_adapter *adapter = pci_get_drvdata(pdev); 6182 struct be_resources vft_res = {0}; 6183 int status; 6184 6185 if (!num_vfs) 6186 be_vf_clear(adapter); 6187 6188 adapter->num_vfs = num_vfs; 6189 6190 if (adapter->num_vfs == 0 && pci_vfs_assigned(pdev)) { 6191 dev_warn(&pdev->dev, 6192 "Cannot disable VFs while they are assigned\n"); 6193 return -EBUSY; 6194 } 6195 6196 /* When the HW is in SRIOV capable configuration, the PF-pool resources 6197 * are equally distributed across the max-number of VFs. The user may 6198 * request only a subset of the max-vfs to be enabled. 6199 * Based on num_vfs, redistribute the resources across num_vfs so that 6200 * each VF will have access to more number of resources. 6201 * This facility is not available in BE3 FW. 6202 * Also, this is done by FW in Lancer chip. 6203 */ 6204 if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) { 6205 be_calculate_vf_res(adapter, adapter->num_vfs, 6206 &vft_res); 6207 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 6208 adapter->num_vfs, &vft_res); 6209 if (status) 6210 dev_err(&pdev->dev, 6211 "Failed to optimize SR-IOV resources\n"); 6212 } 6213 6214 status = be_get_resources(adapter); 6215 if (status) 6216 return be_cmd_status(status); 6217 6218 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 6219 rtnl_lock(); 6220 status = be_update_queues(adapter); 6221 rtnl_unlock(); 6222 if (status) 6223 return be_cmd_status(status); 6224 6225 if (adapter->num_vfs) 6226 status = be_vf_setup(adapter); 6227 6228 if (!status) 6229 return adapter->num_vfs; 6230 6231 return 0; 6232 } 6233 6234 static const struct pci_error_handlers be_eeh_handlers = { 6235 .error_detected = be_eeh_err_detected, 6236 .slot_reset = be_eeh_reset, 6237 .resume = be_eeh_resume, 6238 }; 6239 6240 static struct pci_driver be_driver = { 6241 .name = DRV_NAME, 6242 .id_table = be_dev_ids, 6243 .probe = be_probe, 6244 .remove = be_remove, 6245 .suspend = be_suspend, 6246 .resume = be_pci_resume, 6247 .shutdown = be_shutdown, 6248 .sriov_configure = be_pci_sriov_configure, 6249 .err_handler = &be_eeh_handlers 6250 }; 6251 6252 static int __init be_init_module(void) 6253 { 6254 int status; 6255 6256 if (rx_frag_size != 8192 && rx_frag_size != 4096 && 6257 rx_frag_size != 2048) { 6258 printk(KERN_WARNING DRV_NAME 6259 " : Module param rx_frag_size must be 2048/4096/8192." 6260 " Using 2048\n"); 6261 rx_frag_size = 2048; 6262 } 6263 6264 if (num_vfs > 0) { 6265 pr_info(DRV_NAME " : Module param num_vfs is obsolete."); 6266 pr_info(DRV_NAME " : Use sysfs method to enable VFs\n"); 6267 } 6268 6269 be_wq = create_singlethread_workqueue("be_wq"); 6270 if (!be_wq) { 6271 pr_warn(DRV_NAME "workqueue creation failed\n"); 6272 return -1; 6273 } 6274 6275 be_err_recovery_workq = 6276 create_singlethread_workqueue("be_err_recover"); 6277 if (!be_err_recovery_workq) 6278 pr_warn(DRV_NAME "Could not create error recovery workqueue\n"); 6279 6280 status = pci_register_driver(&be_driver); 6281 if (status) { 6282 destroy_workqueue(be_wq); 6283 be_destroy_err_recovery_workq(); 6284 } 6285 return status; 6286 } 6287 module_init(be_init_module); 6288 6289 static void __exit be_exit_module(void) 6290 { 6291 pci_unregister_driver(&be_driver); 6292 6293 be_destroy_err_recovery_workq(); 6294 6295 if (be_wq) 6296 destroy_workqueue(be_wq); 6297 } 6298 module_exit(be_exit_module); 6299