1 /* 2 * Copyright (C) 2005 - 2016 Broadcom 3 * All rights reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License version 2 7 * as published by the Free Software Foundation. The full GNU General 8 * Public License is included in this distribution in the file called COPYING. 9 * 10 * Contact Information: 11 * linux-drivers@emulex.com 12 * 13 * Emulex 14 * 3333 Susan Street 15 * Costa Mesa, CA 92626 16 */ 17 18 #include <linux/prefetch.h> 19 #include <linux/module.h> 20 #include "be.h" 21 #include "be_cmds.h" 22 #include <asm/div64.h> 23 #include <linux/aer.h> 24 #include <linux/if_bridge.h> 25 #include <net/busy_poll.h> 26 #include <net/vxlan.h> 27 28 MODULE_VERSION(DRV_VER); 29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER); 30 MODULE_AUTHOR("Emulex Corporation"); 31 MODULE_LICENSE("GPL"); 32 33 /* num_vfs module param is obsolete. 34 * Use sysfs method to enable/disable VFs. 35 */ 36 static unsigned int num_vfs; 37 module_param(num_vfs, uint, S_IRUGO); 38 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize"); 39 40 static ushort rx_frag_size = 2048; 41 module_param(rx_frag_size, ushort, S_IRUGO); 42 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data."); 43 44 /* Per-module error detection/recovery workq shared across all functions. 45 * Each function schedules its own work request on this shared workq. 46 */ 47 struct workqueue_struct *be_err_recovery_workq; 48 49 static const struct pci_device_id be_dev_ids[] = { 50 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 51 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 52 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 53 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 54 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)}, 55 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)}, 56 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)}, 57 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)}, 58 { 0 } 59 }; 60 MODULE_DEVICE_TABLE(pci, be_dev_ids); 61 62 /* Workqueue used by all functions for defering cmd calls to the adapter */ 63 struct workqueue_struct *be_wq; 64 65 /* UE Status Low CSR */ 66 static const char * const ue_status_low_desc[] = { 67 "CEV", 68 "CTX", 69 "DBUF", 70 "ERX", 71 "Host", 72 "MPU", 73 "NDMA", 74 "PTC ", 75 "RDMA ", 76 "RXF ", 77 "RXIPS ", 78 "RXULP0 ", 79 "RXULP1 ", 80 "RXULP2 ", 81 "TIM ", 82 "TPOST ", 83 "TPRE ", 84 "TXIPS ", 85 "TXULP0 ", 86 "TXULP1 ", 87 "UC ", 88 "WDMA ", 89 "TXULP2 ", 90 "HOST1 ", 91 "P0_OB_LINK ", 92 "P1_OB_LINK ", 93 "HOST_GPIO ", 94 "MBOX ", 95 "ERX2 ", 96 "SPARE ", 97 "JTAG ", 98 "MPU_INTPEND " 99 }; 100 101 /* UE Status High CSR */ 102 static const char * const ue_status_hi_desc[] = { 103 "LPCMEMHOST", 104 "MGMT_MAC", 105 "PCS0ONLINE", 106 "MPU_IRAM", 107 "PCS1ONLINE", 108 "PCTL0", 109 "PCTL1", 110 "PMEM", 111 "RR", 112 "TXPB", 113 "RXPP", 114 "XAUI", 115 "TXP", 116 "ARM", 117 "IPC", 118 "HOST2", 119 "HOST3", 120 "HOST4", 121 "HOST5", 122 "HOST6", 123 "HOST7", 124 "ECRC", 125 "Poison TLP", 126 "NETC", 127 "PERIPH", 128 "LLTXULP", 129 "D2P", 130 "RCON", 131 "LDMA", 132 "LLTXP", 133 "LLTXPB", 134 "Unknown" 135 }; 136 137 #define BE_VF_IF_EN_FLAGS (BE_IF_FLAGS_UNTAGGED | \ 138 BE_IF_FLAGS_BROADCAST | \ 139 BE_IF_FLAGS_MULTICAST | \ 140 BE_IF_FLAGS_PASS_L3L4_ERRORS) 141 142 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q) 143 { 144 struct be_dma_mem *mem = &q->dma_mem; 145 146 if (mem->va) { 147 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 148 mem->dma); 149 mem->va = NULL; 150 } 151 } 152 153 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q, 154 u16 len, u16 entry_size) 155 { 156 struct be_dma_mem *mem = &q->dma_mem; 157 158 memset(q, 0, sizeof(*q)); 159 q->len = len; 160 q->entry_size = entry_size; 161 mem->size = len * entry_size; 162 mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma, 163 GFP_KERNEL); 164 if (!mem->va) 165 return -ENOMEM; 166 return 0; 167 } 168 169 static void be_reg_intr_set(struct be_adapter *adapter, bool enable) 170 { 171 u32 reg, enabled; 172 173 pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, 174 ®); 175 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 176 177 if (!enabled && enable) 178 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 179 else if (enabled && !enable) 180 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 181 else 182 return; 183 184 pci_write_config_dword(adapter->pdev, 185 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg); 186 } 187 188 static void be_intr_set(struct be_adapter *adapter, bool enable) 189 { 190 int status = 0; 191 192 /* On lancer interrupts can't be controlled via this register */ 193 if (lancer_chip(adapter)) 194 return; 195 196 if (be_check_error(adapter, BE_ERROR_EEH)) 197 return; 198 199 status = be_cmd_intr_set(adapter, enable); 200 if (status) 201 be_reg_intr_set(adapter, enable); 202 } 203 204 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted) 205 { 206 u32 val = 0; 207 208 if (be_check_error(adapter, BE_ERROR_HW)) 209 return; 210 211 val |= qid & DB_RQ_RING_ID_MASK; 212 val |= posted << DB_RQ_NUM_POSTED_SHIFT; 213 214 wmb(); 215 iowrite32(val, adapter->db + DB_RQ_OFFSET); 216 } 217 218 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo, 219 u16 posted) 220 { 221 u32 val = 0; 222 223 if (be_check_error(adapter, BE_ERROR_HW)) 224 return; 225 226 val |= txo->q.id & DB_TXULP_RING_ID_MASK; 227 val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT; 228 229 wmb(); 230 iowrite32(val, adapter->db + txo->db_offset); 231 } 232 233 static void be_eq_notify(struct be_adapter *adapter, u16 qid, 234 bool arm, bool clear_int, u16 num_popped, 235 u32 eq_delay_mult_enc) 236 { 237 u32 val = 0; 238 239 val |= qid & DB_EQ_RING_ID_MASK; 240 val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT); 241 242 if (be_check_error(adapter, BE_ERROR_HW)) 243 return; 244 245 if (arm) 246 val |= 1 << DB_EQ_REARM_SHIFT; 247 if (clear_int) 248 val |= 1 << DB_EQ_CLR_SHIFT; 249 val |= 1 << DB_EQ_EVNT_SHIFT; 250 val |= num_popped << DB_EQ_NUM_POPPED_SHIFT; 251 val |= eq_delay_mult_enc << DB_EQ_R2I_DLY_SHIFT; 252 iowrite32(val, adapter->db + DB_EQ_OFFSET); 253 } 254 255 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped) 256 { 257 u32 val = 0; 258 259 val |= qid & DB_CQ_RING_ID_MASK; 260 val |= ((qid & DB_CQ_RING_ID_EXT_MASK) << 261 DB_CQ_RING_ID_EXT_MASK_SHIFT); 262 263 if (be_check_error(adapter, BE_ERROR_HW)) 264 return; 265 266 if (arm) 267 val |= 1 << DB_CQ_REARM_SHIFT; 268 val |= num_popped << DB_CQ_NUM_POPPED_SHIFT; 269 iowrite32(val, adapter->db + DB_CQ_OFFSET); 270 } 271 272 static int be_dev_mac_add(struct be_adapter *adapter, u8 *mac) 273 { 274 int i; 275 276 /* Check if mac has already been added as part of uc-list */ 277 for (i = 0; i < adapter->uc_macs; i++) { 278 if (ether_addr_equal((u8 *)&adapter->uc_list[i * ETH_ALEN], 279 mac)) { 280 /* mac already added, skip addition */ 281 adapter->pmac_id[0] = adapter->pmac_id[i + 1]; 282 return 0; 283 } 284 } 285 286 return be_cmd_pmac_add(adapter, mac, adapter->if_handle, 287 &adapter->pmac_id[0], 0); 288 } 289 290 static void be_dev_mac_del(struct be_adapter *adapter, int pmac_id) 291 { 292 int i; 293 294 /* Skip deletion if the programmed mac is 295 * being used in uc-list 296 */ 297 for (i = 0; i < adapter->uc_macs; i++) { 298 if (adapter->pmac_id[i + 1] == pmac_id) 299 return; 300 } 301 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0); 302 } 303 304 static int be_mac_addr_set(struct net_device *netdev, void *p) 305 { 306 struct be_adapter *adapter = netdev_priv(netdev); 307 struct device *dev = &adapter->pdev->dev; 308 struct sockaddr *addr = p; 309 int status; 310 u8 mac[ETH_ALEN]; 311 u32 old_pmac_id = adapter->pmac_id[0]; 312 313 if (!is_valid_ether_addr(addr->sa_data)) 314 return -EADDRNOTAVAIL; 315 316 /* Proceed further only if, User provided MAC is different 317 * from active MAC 318 */ 319 if (ether_addr_equal(addr->sa_data, adapter->dev_mac)) 320 return 0; 321 322 /* if device is not running, copy MAC to netdev->dev_addr */ 323 if (!netif_running(netdev)) 324 goto done; 325 326 /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT 327 * privilege or if PF did not provision the new MAC address. 328 * On BE3, this cmd will always fail if the VF doesn't have the 329 * FILTMGMT privilege. This failure is OK, only if the PF programmed 330 * the MAC for the VF. 331 */ 332 mutex_lock(&adapter->rx_filter_lock); 333 status = be_dev_mac_add(adapter, (u8 *)addr->sa_data); 334 if (!status) { 335 336 /* Delete the old programmed MAC. This call may fail if the 337 * old MAC was already deleted by the PF driver. 338 */ 339 if (adapter->pmac_id[0] != old_pmac_id) 340 be_dev_mac_del(adapter, old_pmac_id); 341 } 342 343 mutex_unlock(&adapter->rx_filter_lock); 344 /* Decide if the new MAC is successfully activated only after 345 * querying the FW 346 */ 347 status = be_cmd_get_active_mac(adapter, adapter->pmac_id[0], mac, 348 adapter->if_handle, true, 0); 349 if (status) 350 goto err; 351 352 /* The MAC change did not happen, either due to lack of privilege 353 * or PF didn't pre-provision. 354 */ 355 if (!ether_addr_equal(addr->sa_data, mac)) { 356 status = -EPERM; 357 goto err; 358 } 359 done: 360 ether_addr_copy(adapter->dev_mac, addr->sa_data); 361 ether_addr_copy(netdev->dev_addr, addr->sa_data); 362 dev_info(dev, "MAC address changed to %pM\n", addr->sa_data); 363 return 0; 364 err: 365 dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data); 366 return status; 367 } 368 369 /* BE2 supports only v0 cmd */ 370 static void *hw_stats_from_cmd(struct be_adapter *adapter) 371 { 372 if (BE2_chip(adapter)) { 373 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va; 374 375 return &cmd->hw_stats; 376 } else if (BE3_chip(adapter)) { 377 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va; 378 379 return &cmd->hw_stats; 380 } else { 381 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va; 382 383 return &cmd->hw_stats; 384 } 385 } 386 387 /* BE2 supports only v0 cmd */ 388 static void *be_erx_stats_from_cmd(struct be_adapter *adapter) 389 { 390 if (BE2_chip(adapter)) { 391 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 392 393 return &hw_stats->erx; 394 } else if (BE3_chip(adapter)) { 395 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 396 397 return &hw_stats->erx; 398 } else { 399 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 400 401 return &hw_stats->erx; 402 } 403 } 404 405 static void populate_be_v0_stats(struct be_adapter *adapter) 406 { 407 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 408 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 409 struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf; 410 struct be_port_rxf_stats_v0 *port_stats = 411 &rxf_stats->port[adapter->port_num]; 412 struct be_drv_stats *drvs = &adapter->drv_stats; 413 414 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 415 drvs->rx_pause_frames = port_stats->rx_pause_frames; 416 drvs->rx_crc_errors = port_stats->rx_crc_errors; 417 drvs->rx_control_frames = port_stats->rx_control_frames; 418 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 419 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 420 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 421 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 422 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 423 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 424 drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow; 425 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 426 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 427 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 428 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 429 drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow; 430 drvs->rx_dropped_header_too_small = 431 port_stats->rx_dropped_header_too_small; 432 drvs->rx_address_filtered = 433 port_stats->rx_address_filtered + 434 port_stats->rx_vlan_filtered; 435 drvs->rx_alignment_symbol_errors = 436 port_stats->rx_alignment_symbol_errors; 437 438 drvs->tx_pauseframes = port_stats->tx_pauseframes; 439 drvs->tx_controlframes = port_stats->tx_controlframes; 440 441 if (adapter->port_num) 442 drvs->jabber_events = rxf_stats->port1_jabber_events; 443 else 444 drvs->jabber_events = rxf_stats->port0_jabber_events; 445 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 446 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 447 drvs->forwarded_packets = rxf_stats->forwarded_packets; 448 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 449 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 450 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 451 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 452 } 453 454 static void populate_be_v1_stats(struct be_adapter *adapter) 455 { 456 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 457 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 458 struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf; 459 struct be_port_rxf_stats_v1 *port_stats = 460 &rxf_stats->port[adapter->port_num]; 461 struct be_drv_stats *drvs = &adapter->drv_stats; 462 463 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 464 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 465 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 466 drvs->rx_pause_frames = port_stats->rx_pause_frames; 467 drvs->rx_crc_errors = port_stats->rx_crc_errors; 468 drvs->rx_control_frames = port_stats->rx_control_frames; 469 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 470 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 471 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 472 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 473 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 474 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 475 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 476 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 477 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 478 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 479 drvs->rx_dropped_header_too_small = 480 port_stats->rx_dropped_header_too_small; 481 drvs->rx_input_fifo_overflow_drop = 482 port_stats->rx_input_fifo_overflow_drop; 483 drvs->rx_address_filtered = port_stats->rx_address_filtered; 484 drvs->rx_alignment_symbol_errors = 485 port_stats->rx_alignment_symbol_errors; 486 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 487 drvs->tx_pauseframes = port_stats->tx_pauseframes; 488 drvs->tx_controlframes = port_stats->tx_controlframes; 489 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 490 drvs->jabber_events = port_stats->jabber_events; 491 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 492 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 493 drvs->forwarded_packets = rxf_stats->forwarded_packets; 494 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 495 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 496 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 497 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 498 } 499 500 static void populate_be_v2_stats(struct be_adapter *adapter) 501 { 502 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 503 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 504 struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf; 505 struct be_port_rxf_stats_v2 *port_stats = 506 &rxf_stats->port[adapter->port_num]; 507 struct be_drv_stats *drvs = &adapter->drv_stats; 508 509 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 510 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 511 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 512 drvs->rx_pause_frames = port_stats->rx_pause_frames; 513 drvs->rx_crc_errors = port_stats->rx_crc_errors; 514 drvs->rx_control_frames = port_stats->rx_control_frames; 515 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 516 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 517 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 518 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 519 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 520 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 521 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 522 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 523 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 524 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 525 drvs->rx_dropped_header_too_small = 526 port_stats->rx_dropped_header_too_small; 527 drvs->rx_input_fifo_overflow_drop = 528 port_stats->rx_input_fifo_overflow_drop; 529 drvs->rx_address_filtered = port_stats->rx_address_filtered; 530 drvs->rx_alignment_symbol_errors = 531 port_stats->rx_alignment_symbol_errors; 532 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 533 drvs->tx_pauseframes = port_stats->tx_pauseframes; 534 drvs->tx_controlframes = port_stats->tx_controlframes; 535 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 536 drvs->jabber_events = port_stats->jabber_events; 537 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 538 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 539 drvs->forwarded_packets = rxf_stats->forwarded_packets; 540 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 541 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 542 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 543 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 544 if (be_roce_supported(adapter)) { 545 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd; 546 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd; 547 drvs->rx_roce_frames = port_stats->roce_frames_received; 548 drvs->roce_drops_crc = port_stats->roce_drops_crc; 549 drvs->roce_drops_payload_len = 550 port_stats->roce_drops_payload_len; 551 } 552 } 553 554 static void populate_lancer_stats(struct be_adapter *adapter) 555 { 556 struct be_drv_stats *drvs = &adapter->drv_stats; 557 struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter); 558 559 be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats)); 560 drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo; 561 drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo; 562 drvs->rx_control_frames = pport_stats->rx_control_frames_lo; 563 drvs->rx_in_range_errors = pport_stats->rx_in_range_errors; 564 drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo; 565 drvs->rx_dropped_runt = pport_stats->rx_dropped_runt; 566 drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors; 567 drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors; 568 drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors; 569 drvs->rx_dropped_tcp_length = 570 pport_stats->rx_dropped_invalid_tcp_length; 571 drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small; 572 drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short; 573 drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors; 574 drvs->rx_dropped_header_too_small = 575 pport_stats->rx_dropped_header_too_small; 576 drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 577 drvs->rx_address_filtered = 578 pport_stats->rx_address_filtered + 579 pport_stats->rx_vlan_filtered; 580 drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo; 581 drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 582 drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo; 583 drvs->tx_controlframes = pport_stats->tx_control_frames_lo; 584 drvs->jabber_events = pport_stats->rx_jabbers; 585 drvs->forwarded_packets = pport_stats->num_forwards_lo; 586 drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo; 587 drvs->rx_drops_too_many_frags = 588 pport_stats->rx_drops_too_many_frags_lo; 589 } 590 591 static void accumulate_16bit_val(u32 *acc, u16 val) 592 { 593 #define lo(x) (x & 0xFFFF) 594 #define hi(x) (x & 0xFFFF0000) 595 bool wrapped = val < lo(*acc); 596 u32 newacc = hi(*acc) + val; 597 598 if (wrapped) 599 newacc += 65536; 600 ACCESS_ONCE(*acc) = newacc; 601 } 602 603 static void populate_erx_stats(struct be_adapter *adapter, 604 struct be_rx_obj *rxo, u32 erx_stat) 605 { 606 if (!BEx_chip(adapter)) 607 rx_stats(rxo)->rx_drops_no_frags = erx_stat; 608 else 609 /* below erx HW counter can actually wrap around after 610 * 65535. Driver accumulates a 32-bit value 611 */ 612 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags, 613 (u16)erx_stat); 614 } 615 616 void be_parse_stats(struct be_adapter *adapter) 617 { 618 struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter); 619 struct be_rx_obj *rxo; 620 int i; 621 u32 erx_stat; 622 623 if (lancer_chip(adapter)) { 624 populate_lancer_stats(adapter); 625 } else { 626 if (BE2_chip(adapter)) 627 populate_be_v0_stats(adapter); 628 else if (BE3_chip(adapter)) 629 /* for BE3 */ 630 populate_be_v1_stats(adapter); 631 else 632 populate_be_v2_stats(adapter); 633 634 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */ 635 for_all_rx_queues(adapter, rxo, i) { 636 erx_stat = erx->rx_drops_no_fragments[rxo->q.id]; 637 populate_erx_stats(adapter, rxo, erx_stat); 638 } 639 } 640 } 641 642 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev, 643 struct rtnl_link_stats64 *stats) 644 { 645 struct be_adapter *adapter = netdev_priv(netdev); 646 struct be_drv_stats *drvs = &adapter->drv_stats; 647 struct be_rx_obj *rxo; 648 struct be_tx_obj *txo; 649 u64 pkts, bytes; 650 unsigned int start; 651 int i; 652 653 for_all_rx_queues(adapter, rxo, i) { 654 const struct be_rx_stats *rx_stats = rx_stats(rxo); 655 656 do { 657 start = u64_stats_fetch_begin_irq(&rx_stats->sync); 658 pkts = rx_stats(rxo)->rx_pkts; 659 bytes = rx_stats(rxo)->rx_bytes; 660 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start)); 661 stats->rx_packets += pkts; 662 stats->rx_bytes += bytes; 663 stats->multicast += rx_stats(rxo)->rx_mcast_pkts; 664 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs + 665 rx_stats(rxo)->rx_drops_no_frags; 666 } 667 668 for_all_tx_queues(adapter, txo, i) { 669 const struct be_tx_stats *tx_stats = tx_stats(txo); 670 671 do { 672 start = u64_stats_fetch_begin_irq(&tx_stats->sync); 673 pkts = tx_stats(txo)->tx_pkts; 674 bytes = tx_stats(txo)->tx_bytes; 675 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start)); 676 stats->tx_packets += pkts; 677 stats->tx_bytes += bytes; 678 } 679 680 /* bad pkts received */ 681 stats->rx_errors = drvs->rx_crc_errors + 682 drvs->rx_alignment_symbol_errors + 683 drvs->rx_in_range_errors + 684 drvs->rx_out_range_errors + 685 drvs->rx_frame_too_long + 686 drvs->rx_dropped_too_small + 687 drvs->rx_dropped_too_short + 688 drvs->rx_dropped_header_too_small + 689 drvs->rx_dropped_tcp_length + 690 drvs->rx_dropped_runt; 691 692 /* detailed rx errors */ 693 stats->rx_length_errors = drvs->rx_in_range_errors + 694 drvs->rx_out_range_errors + 695 drvs->rx_frame_too_long; 696 697 stats->rx_crc_errors = drvs->rx_crc_errors; 698 699 /* frame alignment errors */ 700 stats->rx_frame_errors = drvs->rx_alignment_symbol_errors; 701 702 /* receiver fifo overrun */ 703 /* drops_no_pbuf is no per i/f, it's per BE card */ 704 stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop + 705 drvs->rx_input_fifo_overflow_drop + 706 drvs->rx_drops_no_pbuf; 707 return stats; 708 } 709 710 void be_link_status_update(struct be_adapter *adapter, u8 link_status) 711 { 712 struct net_device *netdev = adapter->netdev; 713 714 if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) { 715 netif_carrier_off(netdev); 716 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT; 717 } 718 719 if (link_status) 720 netif_carrier_on(netdev); 721 else 722 netif_carrier_off(netdev); 723 724 netdev_info(netdev, "Link is %s\n", link_status ? "Up" : "Down"); 725 } 726 727 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb) 728 { 729 struct be_tx_stats *stats = tx_stats(txo); 730 u64 tx_pkts = skb_shinfo(skb)->gso_segs ? : 1; 731 732 u64_stats_update_begin(&stats->sync); 733 stats->tx_reqs++; 734 stats->tx_bytes += skb->len; 735 stats->tx_pkts += tx_pkts; 736 if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL) 737 stats->tx_vxlan_offload_pkts += tx_pkts; 738 u64_stats_update_end(&stats->sync); 739 } 740 741 /* Returns number of WRBs needed for the skb */ 742 static u32 skb_wrb_cnt(struct sk_buff *skb) 743 { 744 /* +1 for the header wrb */ 745 return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags; 746 } 747 748 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len) 749 { 750 wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr)); 751 wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr)); 752 wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK); 753 wrb->rsvd0 = 0; 754 } 755 756 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb 757 * to avoid the swap and shift/mask operations in wrb_fill(). 758 */ 759 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb) 760 { 761 wrb->frag_pa_hi = 0; 762 wrb->frag_pa_lo = 0; 763 wrb->frag_len = 0; 764 wrb->rsvd0 = 0; 765 } 766 767 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter, 768 struct sk_buff *skb) 769 { 770 u8 vlan_prio; 771 u16 vlan_tag; 772 773 vlan_tag = skb_vlan_tag_get(skb); 774 vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 775 /* If vlan priority provided by OS is NOT in available bmap */ 776 if (!(adapter->vlan_prio_bmap & (1 << vlan_prio))) 777 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) | 778 adapter->recommended_prio_bits; 779 780 return vlan_tag; 781 } 782 783 /* Used only for IP tunnel packets */ 784 static u16 skb_inner_ip_proto(struct sk_buff *skb) 785 { 786 return (inner_ip_hdr(skb)->version == 4) ? 787 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr; 788 } 789 790 static u16 skb_ip_proto(struct sk_buff *skb) 791 { 792 return (ip_hdr(skb)->version == 4) ? 793 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr; 794 } 795 796 static inline bool be_is_txq_full(struct be_tx_obj *txo) 797 { 798 return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len; 799 } 800 801 static inline bool be_can_txq_wake(struct be_tx_obj *txo) 802 { 803 return atomic_read(&txo->q.used) < txo->q.len / 2; 804 } 805 806 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo) 807 { 808 return atomic_read(&txo->q.used) > txo->pend_wrb_cnt; 809 } 810 811 static void be_get_wrb_params_from_skb(struct be_adapter *adapter, 812 struct sk_buff *skb, 813 struct be_wrb_params *wrb_params) 814 { 815 u16 proto; 816 817 if (skb_is_gso(skb)) { 818 BE_WRB_F_SET(wrb_params->features, LSO, 1); 819 wrb_params->lso_mss = skb_shinfo(skb)->gso_size; 820 if (skb_is_gso_v6(skb) && !lancer_chip(adapter)) 821 BE_WRB_F_SET(wrb_params->features, LSO6, 1); 822 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 823 if (skb->encapsulation) { 824 BE_WRB_F_SET(wrb_params->features, IPCS, 1); 825 proto = skb_inner_ip_proto(skb); 826 } else { 827 proto = skb_ip_proto(skb); 828 } 829 if (proto == IPPROTO_TCP) 830 BE_WRB_F_SET(wrb_params->features, TCPCS, 1); 831 else if (proto == IPPROTO_UDP) 832 BE_WRB_F_SET(wrb_params->features, UDPCS, 1); 833 } 834 835 if (skb_vlan_tag_present(skb)) { 836 BE_WRB_F_SET(wrb_params->features, VLAN, 1); 837 wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb); 838 } 839 840 BE_WRB_F_SET(wrb_params->features, CRC, 1); 841 } 842 843 static void wrb_fill_hdr(struct be_adapter *adapter, 844 struct be_eth_hdr_wrb *hdr, 845 struct be_wrb_params *wrb_params, 846 struct sk_buff *skb) 847 { 848 memset(hdr, 0, sizeof(*hdr)); 849 850 SET_TX_WRB_HDR_BITS(crc, hdr, 851 BE_WRB_F_GET(wrb_params->features, CRC)); 852 SET_TX_WRB_HDR_BITS(ipcs, hdr, 853 BE_WRB_F_GET(wrb_params->features, IPCS)); 854 SET_TX_WRB_HDR_BITS(tcpcs, hdr, 855 BE_WRB_F_GET(wrb_params->features, TCPCS)); 856 SET_TX_WRB_HDR_BITS(udpcs, hdr, 857 BE_WRB_F_GET(wrb_params->features, UDPCS)); 858 859 SET_TX_WRB_HDR_BITS(lso, hdr, 860 BE_WRB_F_GET(wrb_params->features, LSO)); 861 SET_TX_WRB_HDR_BITS(lso6, hdr, 862 BE_WRB_F_GET(wrb_params->features, LSO6)); 863 SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss); 864 865 /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this 866 * hack is not needed, the evt bit is set while ringing DB. 867 */ 868 SET_TX_WRB_HDR_BITS(event, hdr, 869 BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW)); 870 SET_TX_WRB_HDR_BITS(vlan, hdr, 871 BE_WRB_F_GET(wrb_params->features, VLAN)); 872 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag); 873 874 SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb)); 875 SET_TX_WRB_HDR_BITS(len, hdr, skb->len); 876 SET_TX_WRB_HDR_BITS(mgmt, hdr, 877 BE_WRB_F_GET(wrb_params->features, OS2BMC)); 878 } 879 880 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb, 881 bool unmap_single) 882 { 883 dma_addr_t dma; 884 u32 frag_len = le32_to_cpu(wrb->frag_len); 885 886 887 dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 | 888 (u64)le32_to_cpu(wrb->frag_pa_lo); 889 if (frag_len) { 890 if (unmap_single) 891 dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE); 892 else 893 dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE); 894 } 895 } 896 897 /* Grab a WRB header for xmit */ 898 static u32 be_tx_get_wrb_hdr(struct be_tx_obj *txo) 899 { 900 u32 head = txo->q.head; 901 902 queue_head_inc(&txo->q); 903 return head; 904 } 905 906 /* Set up the WRB header for xmit */ 907 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter, 908 struct be_tx_obj *txo, 909 struct be_wrb_params *wrb_params, 910 struct sk_buff *skb, u16 head) 911 { 912 u32 num_frags = skb_wrb_cnt(skb); 913 struct be_queue_info *txq = &txo->q; 914 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head); 915 916 wrb_fill_hdr(adapter, hdr, wrb_params, skb); 917 be_dws_cpu_to_le(hdr, sizeof(*hdr)); 918 919 BUG_ON(txo->sent_skb_list[head]); 920 txo->sent_skb_list[head] = skb; 921 txo->last_req_hdr = head; 922 atomic_add(num_frags, &txq->used); 923 txo->last_req_wrb_cnt = num_frags; 924 txo->pend_wrb_cnt += num_frags; 925 } 926 927 /* Setup a WRB fragment (buffer descriptor) for xmit */ 928 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr, 929 int len) 930 { 931 struct be_eth_wrb *wrb; 932 struct be_queue_info *txq = &txo->q; 933 934 wrb = queue_head_node(txq); 935 wrb_fill(wrb, busaddr, len); 936 queue_head_inc(txq); 937 } 938 939 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine 940 * was invoked. The producer index is restored to the previous packet and the 941 * WRBs of the current packet are unmapped. Invoked to handle tx setup errors. 942 */ 943 static void be_xmit_restore(struct be_adapter *adapter, 944 struct be_tx_obj *txo, u32 head, bool map_single, 945 u32 copied) 946 { 947 struct device *dev; 948 struct be_eth_wrb *wrb; 949 struct be_queue_info *txq = &txo->q; 950 951 dev = &adapter->pdev->dev; 952 txq->head = head; 953 954 /* skip the first wrb (hdr); it's not mapped */ 955 queue_head_inc(txq); 956 while (copied) { 957 wrb = queue_head_node(txq); 958 unmap_tx_frag(dev, wrb, map_single); 959 map_single = false; 960 copied -= le32_to_cpu(wrb->frag_len); 961 queue_head_inc(txq); 962 } 963 964 txq->head = head; 965 } 966 967 /* Enqueue the given packet for transmit. This routine allocates WRBs for the 968 * packet, dma maps the packet buffers and sets up the WRBs. Returns the number 969 * of WRBs used up by the packet. 970 */ 971 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo, 972 struct sk_buff *skb, 973 struct be_wrb_params *wrb_params) 974 { 975 u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb); 976 struct device *dev = &adapter->pdev->dev; 977 struct be_queue_info *txq = &txo->q; 978 bool map_single = false; 979 u32 head = txq->head; 980 dma_addr_t busaddr; 981 int len; 982 983 head = be_tx_get_wrb_hdr(txo); 984 985 if (skb->len > skb->data_len) { 986 len = skb_headlen(skb); 987 988 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE); 989 if (dma_mapping_error(dev, busaddr)) 990 goto dma_err; 991 map_single = true; 992 be_tx_setup_wrb_frag(txo, busaddr, len); 993 copied += len; 994 } 995 996 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 997 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; 998 len = skb_frag_size(frag); 999 1000 busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE); 1001 if (dma_mapping_error(dev, busaddr)) 1002 goto dma_err; 1003 be_tx_setup_wrb_frag(txo, busaddr, len); 1004 copied += len; 1005 } 1006 1007 be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head); 1008 1009 be_tx_stats_update(txo, skb); 1010 return wrb_cnt; 1011 1012 dma_err: 1013 adapter->drv_stats.dma_map_errors++; 1014 be_xmit_restore(adapter, txo, head, map_single, copied); 1015 return 0; 1016 } 1017 1018 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter) 1019 { 1020 return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD; 1021 } 1022 1023 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter, 1024 struct sk_buff *skb, 1025 struct be_wrb_params 1026 *wrb_params) 1027 { 1028 u16 vlan_tag = 0; 1029 1030 skb = skb_share_check(skb, GFP_ATOMIC); 1031 if (unlikely(!skb)) 1032 return skb; 1033 1034 if (skb_vlan_tag_present(skb)) 1035 vlan_tag = be_get_tx_vlan_tag(adapter, skb); 1036 1037 if (qnq_async_evt_rcvd(adapter) && adapter->pvid) { 1038 if (!vlan_tag) 1039 vlan_tag = adapter->pvid; 1040 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to 1041 * skip VLAN insertion 1042 */ 1043 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1044 } 1045 1046 if (vlan_tag) { 1047 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 1048 vlan_tag); 1049 if (unlikely(!skb)) 1050 return skb; 1051 skb->vlan_tci = 0; 1052 } 1053 1054 /* Insert the outer VLAN, if any */ 1055 if (adapter->qnq_vid) { 1056 vlan_tag = adapter->qnq_vid; 1057 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 1058 vlan_tag); 1059 if (unlikely(!skb)) 1060 return skb; 1061 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1062 } 1063 1064 return skb; 1065 } 1066 1067 static bool be_ipv6_exthdr_check(struct sk_buff *skb) 1068 { 1069 struct ethhdr *eh = (struct ethhdr *)skb->data; 1070 u16 offset = ETH_HLEN; 1071 1072 if (eh->h_proto == htons(ETH_P_IPV6)) { 1073 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset); 1074 1075 offset += sizeof(struct ipv6hdr); 1076 if (ip6h->nexthdr != NEXTHDR_TCP && 1077 ip6h->nexthdr != NEXTHDR_UDP) { 1078 struct ipv6_opt_hdr *ehdr = 1079 (struct ipv6_opt_hdr *)(skb->data + offset); 1080 1081 /* offending pkt: 2nd byte following IPv6 hdr is 0xff */ 1082 if (ehdr->hdrlen == 0xff) 1083 return true; 1084 } 1085 } 1086 return false; 1087 } 1088 1089 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb) 1090 { 1091 return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid; 1092 } 1093 1094 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb) 1095 { 1096 return BE3_chip(adapter) && be_ipv6_exthdr_check(skb); 1097 } 1098 1099 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter, 1100 struct sk_buff *skb, 1101 struct be_wrb_params 1102 *wrb_params) 1103 { 1104 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 1105 unsigned int eth_hdr_len; 1106 struct iphdr *ip; 1107 1108 /* For padded packets, BE HW modifies tot_len field in IP header 1109 * incorrecly when VLAN tag is inserted by HW. 1110 * For padded packets, Lancer computes incorrect checksum. 1111 */ 1112 eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ? 1113 VLAN_ETH_HLEN : ETH_HLEN; 1114 if (skb->len <= 60 && 1115 (lancer_chip(adapter) || skb_vlan_tag_present(skb)) && 1116 is_ipv4_pkt(skb)) { 1117 ip = (struct iphdr *)ip_hdr(skb); 1118 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len)); 1119 } 1120 1121 /* If vlan tag is already inlined in the packet, skip HW VLAN 1122 * tagging in pvid-tagging mode 1123 */ 1124 if (be_pvid_tagging_enabled(adapter) && 1125 veh->h_vlan_proto == htons(ETH_P_8021Q)) 1126 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1127 1128 /* HW has a bug wherein it will calculate CSUM for VLAN 1129 * pkts even though it is disabled. 1130 * Manually insert VLAN in pkt. 1131 */ 1132 if (skb->ip_summed != CHECKSUM_PARTIAL && 1133 skb_vlan_tag_present(skb)) { 1134 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1135 if (unlikely(!skb)) 1136 goto err; 1137 } 1138 1139 /* HW may lockup when VLAN HW tagging is requested on 1140 * certain ipv6 packets. Drop such pkts if the HW workaround to 1141 * skip HW tagging is not enabled by FW. 1142 */ 1143 if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) && 1144 (adapter->pvid || adapter->qnq_vid) && 1145 !qnq_async_evt_rcvd(adapter))) 1146 goto tx_drop; 1147 1148 /* Manual VLAN tag insertion to prevent: 1149 * ASIC lockup when the ASIC inserts VLAN tag into 1150 * certain ipv6 packets. Insert VLAN tags in driver, 1151 * and set event, completion, vlan bits accordingly 1152 * in the Tx WRB. 1153 */ 1154 if (be_ipv6_tx_stall_chk(adapter, skb) && 1155 be_vlan_tag_tx_chk(adapter, skb)) { 1156 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1157 if (unlikely(!skb)) 1158 goto err; 1159 } 1160 1161 return skb; 1162 tx_drop: 1163 dev_kfree_skb_any(skb); 1164 err: 1165 return NULL; 1166 } 1167 1168 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter, 1169 struct sk_buff *skb, 1170 struct be_wrb_params *wrb_params) 1171 { 1172 int err; 1173 1174 /* Lancer, SH and BE3 in SRIOV mode have a bug wherein 1175 * packets that are 32b or less may cause a transmit stall 1176 * on that port. The workaround is to pad such packets 1177 * (len <= 32 bytes) to a minimum length of 36b. 1178 */ 1179 if (skb->len <= 32) { 1180 if (skb_put_padto(skb, 36)) 1181 return NULL; 1182 } 1183 1184 if (BEx_chip(adapter) || lancer_chip(adapter)) { 1185 skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params); 1186 if (!skb) 1187 return NULL; 1188 } 1189 1190 /* The stack can send us skbs with length greater than 1191 * what the HW can handle. Trim the extra bytes. 1192 */ 1193 WARN_ON_ONCE(skb->len > BE_MAX_GSO_SIZE); 1194 err = pskb_trim(skb, BE_MAX_GSO_SIZE); 1195 WARN_ON(err); 1196 1197 return skb; 1198 } 1199 1200 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo) 1201 { 1202 struct be_queue_info *txq = &txo->q; 1203 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr); 1204 1205 /* Mark the last request eventable if it hasn't been marked already */ 1206 if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT))) 1207 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL); 1208 1209 /* compose a dummy wrb if there are odd set of wrbs to notify */ 1210 if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) { 1211 wrb_fill_dummy(queue_head_node(txq)); 1212 queue_head_inc(txq); 1213 atomic_inc(&txq->used); 1214 txo->pend_wrb_cnt++; 1215 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK << 1216 TX_HDR_WRB_NUM_SHIFT); 1217 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) << 1218 TX_HDR_WRB_NUM_SHIFT); 1219 } 1220 be_txq_notify(adapter, txo, txo->pend_wrb_cnt); 1221 txo->pend_wrb_cnt = 0; 1222 } 1223 1224 /* OS2BMC related */ 1225 1226 #define DHCP_CLIENT_PORT 68 1227 #define DHCP_SERVER_PORT 67 1228 #define NET_BIOS_PORT1 137 1229 #define NET_BIOS_PORT2 138 1230 #define DHCPV6_RAS_PORT 547 1231 1232 #define is_mc_allowed_on_bmc(adapter, eh) \ 1233 (!is_multicast_filt_enabled(adapter) && \ 1234 is_multicast_ether_addr(eh->h_dest) && \ 1235 !is_broadcast_ether_addr(eh->h_dest)) 1236 1237 #define is_bc_allowed_on_bmc(adapter, eh) \ 1238 (!is_broadcast_filt_enabled(adapter) && \ 1239 is_broadcast_ether_addr(eh->h_dest)) 1240 1241 #define is_arp_allowed_on_bmc(adapter, skb) \ 1242 (is_arp(skb) && is_arp_filt_enabled(adapter)) 1243 1244 #define is_broadcast_packet(eh, adapter) \ 1245 (is_multicast_ether_addr(eh->h_dest) && \ 1246 !compare_ether_addr(eh->h_dest, adapter->netdev->broadcast)) 1247 1248 #define is_arp(skb) (skb->protocol == htons(ETH_P_ARP)) 1249 1250 #define is_arp_filt_enabled(adapter) \ 1251 (adapter->bmc_filt_mask & (BMC_FILT_BROADCAST_ARP)) 1252 1253 #define is_dhcp_client_filt_enabled(adapter) \ 1254 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_CLIENT) 1255 1256 #define is_dhcp_srvr_filt_enabled(adapter) \ 1257 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_SERVER) 1258 1259 #define is_nbios_filt_enabled(adapter) \ 1260 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_NET_BIOS) 1261 1262 #define is_ipv6_na_filt_enabled(adapter) \ 1263 (adapter->bmc_filt_mask & \ 1264 BMC_FILT_MULTICAST_IPV6_NEIGH_ADVER) 1265 1266 #define is_ipv6_ra_filt_enabled(adapter) \ 1267 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RA) 1268 1269 #define is_ipv6_ras_filt_enabled(adapter) \ 1270 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RAS) 1271 1272 #define is_broadcast_filt_enabled(adapter) \ 1273 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST) 1274 1275 #define is_multicast_filt_enabled(adapter) \ 1276 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST) 1277 1278 static bool be_send_pkt_to_bmc(struct be_adapter *adapter, 1279 struct sk_buff **skb) 1280 { 1281 struct ethhdr *eh = (struct ethhdr *)(*skb)->data; 1282 bool os2bmc = false; 1283 1284 if (!be_is_os2bmc_enabled(adapter)) 1285 goto done; 1286 1287 if (!is_multicast_ether_addr(eh->h_dest)) 1288 goto done; 1289 1290 if (is_mc_allowed_on_bmc(adapter, eh) || 1291 is_bc_allowed_on_bmc(adapter, eh) || 1292 is_arp_allowed_on_bmc(adapter, (*skb))) { 1293 os2bmc = true; 1294 goto done; 1295 } 1296 1297 if ((*skb)->protocol == htons(ETH_P_IPV6)) { 1298 struct ipv6hdr *hdr = ipv6_hdr((*skb)); 1299 u8 nexthdr = hdr->nexthdr; 1300 1301 if (nexthdr == IPPROTO_ICMPV6) { 1302 struct icmp6hdr *icmp6 = icmp6_hdr((*skb)); 1303 1304 switch (icmp6->icmp6_type) { 1305 case NDISC_ROUTER_ADVERTISEMENT: 1306 os2bmc = is_ipv6_ra_filt_enabled(adapter); 1307 goto done; 1308 case NDISC_NEIGHBOUR_ADVERTISEMENT: 1309 os2bmc = is_ipv6_na_filt_enabled(adapter); 1310 goto done; 1311 default: 1312 break; 1313 } 1314 } 1315 } 1316 1317 if (is_udp_pkt((*skb))) { 1318 struct udphdr *udp = udp_hdr((*skb)); 1319 1320 switch (ntohs(udp->dest)) { 1321 case DHCP_CLIENT_PORT: 1322 os2bmc = is_dhcp_client_filt_enabled(adapter); 1323 goto done; 1324 case DHCP_SERVER_PORT: 1325 os2bmc = is_dhcp_srvr_filt_enabled(adapter); 1326 goto done; 1327 case NET_BIOS_PORT1: 1328 case NET_BIOS_PORT2: 1329 os2bmc = is_nbios_filt_enabled(adapter); 1330 goto done; 1331 case DHCPV6_RAS_PORT: 1332 os2bmc = is_ipv6_ras_filt_enabled(adapter); 1333 goto done; 1334 default: 1335 break; 1336 } 1337 } 1338 done: 1339 /* For packets over a vlan, which are destined 1340 * to BMC, asic expects the vlan to be inline in the packet. 1341 */ 1342 if (os2bmc) 1343 *skb = be_insert_vlan_in_pkt(adapter, *skb, NULL); 1344 1345 return os2bmc; 1346 } 1347 1348 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev) 1349 { 1350 struct be_adapter *adapter = netdev_priv(netdev); 1351 u16 q_idx = skb_get_queue_mapping(skb); 1352 struct be_tx_obj *txo = &adapter->tx_obj[q_idx]; 1353 struct be_wrb_params wrb_params = { 0 }; 1354 bool flush = !skb->xmit_more; 1355 u16 wrb_cnt; 1356 1357 skb = be_xmit_workarounds(adapter, skb, &wrb_params); 1358 if (unlikely(!skb)) 1359 goto drop; 1360 1361 be_get_wrb_params_from_skb(adapter, skb, &wrb_params); 1362 1363 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); 1364 if (unlikely(!wrb_cnt)) { 1365 dev_kfree_skb_any(skb); 1366 goto drop; 1367 } 1368 1369 /* if os2bmc is enabled and if the pkt is destined to bmc, 1370 * enqueue the pkt a 2nd time with mgmt bit set. 1371 */ 1372 if (be_send_pkt_to_bmc(adapter, &skb)) { 1373 BE_WRB_F_SET(wrb_params.features, OS2BMC, 1); 1374 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); 1375 if (unlikely(!wrb_cnt)) 1376 goto drop; 1377 else 1378 skb_get(skb); 1379 } 1380 1381 if (be_is_txq_full(txo)) { 1382 netif_stop_subqueue(netdev, q_idx); 1383 tx_stats(txo)->tx_stops++; 1384 } 1385 1386 if (flush || __netif_subqueue_stopped(netdev, q_idx)) 1387 be_xmit_flush(adapter, txo); 1388 1389 return NETDEV_TX_OK; 1390 drop: 1391 tx_stats(txo)->tx_drv_drops++; 1392 /* Flush the already enqueued tx requests */ 1393 if (flush && txo->pend_wrb_cnt) 1394 be_xmit_flush(adapter, txo); 1395 1396 return NETDEV_TX_OK; 1397 } 1398 1399 static int be_change_mtu(struct net_device *netdev, int new_mtu) 1400 { 1401 struct be_adapter *adapter = netdev_priv(netdev); 1402 struct device *dev = &adapter->pdev->dev; 1403 1404 if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) { 1405 dev_info(dev, "MTU must be between %d and %d bytes\n", 1406 BE_MIN_MTU, BE_MAX_MTU); 1407 return -EINVAL; 1408 } 1409 1410 dev_info(dev, "MTU changed from %d to %d bytes\n", 1411 netdev->mtu, new_mtu); 1412 netdev->mtu = new_mtu; 1413 return 0; 1414 } 1415 1416 static inline bool be_in_all_promisc(struct be_adapter *adapter) 1417 { 1418 return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) == 1419 BE_IF_FLAGS_ALL_PROMISCUOUS; 1420 } 1421 1422 static int be_set_vlan_promisc(struct be_adapter *adapter) 1423 { 1424 struct device *dev = &adapter->pdev->dev; 1425 int status; 1426 1427 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) 1428 return 0; 1429 1430 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON); 1431 if (!status) { 1432 dev_info(dev, "Enabled VLAN promiscuous mode\n"); 1433 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS; 1434 } else { 1435 dev_err(dev, "Failed to enable VLAN promiscuous mode\n"); 1436 } 1437 return status; 1438 } 1439 1440 static int be_clear_vlan_promisc(struct be_adapter *adapter) 1441 { 1442 struct device *dev = &adapter->pdev->dev; 1443 int status; 1444 1445 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF); 1446 if (!status) { 1447 dev_info(dev, "Disabling VLAN promiscuous mode\n"); 1448 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 1449 } 1450 return status; 1451 } 1452 1453 /* 1454 * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE. 1455 * If the user configures more, place BE in vlan promiscuous mode. 1456 */ 1457 static int be_vid_config(struct be_adapter *adapter) 1458 { 1459 struct device *dev = &adapter->pdev->dev; 1460 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1461 u16 num = 0, i = 0; 1462 int status = 0; 1463 1464 /* No need to change the VLAN state if the I/F is in promiscuous */ 1465 if (adapter->netdev->flags & IFF_PROMISC) 1466 return 0; 1467 1468 if (adapter->vlans_added > be_max_vlans(adapter)) 1469 return be_set_vlan_promisc(adapter); 1470 1471 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 1472 status = be_clear_vlan_promisc(adapter); 1473 if (status) 1474 return status; 1475 } 1476 /* Construct VLAN Table to give to HW */ 1477 for_each_set_bit(i, adapter->vids, VLAN_N_VID) 1478 vids[num++] = cpu_to_le16(i); 1479 1480 status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0); 1481 if (status) { 1482 dev_err(dev, "Setting HW VLAN filtering failed\n"); 1483 /* Set to VLAN promisc mode as setting VLAN filter failed */ 1484 if (addl_status(status) == MCC_ADDL_STATUS_INSUFFICIENT_VLANS || 1485 addl_status(status) == 1486 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES) 1487 return be_set_vlan_promisc(adapter); 1488 } 1489 return status; 1490 } 1491 1492 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 1493 { 1494 struct be_adapter *adapter = netdev_priv(netdev); 1495 int status = 0; 1496 1497 mutex_lock(&adapter->rx_filter_lock); 1498 1499 /* Packets with VID 0 are always received by Lancer by default */ 1500 if (lancer_chip(adapter) && vid == 0) 1501 goto done; 1502 1503 if (test_bit(vid, adapter->vids)) 1504 goto done; 1505 1506 set_bit(vid, adapter->vids); 1507 adapter->vlans_added++; 1508 1509 status = be_vid_config(adapter); 1510 done: 1511 mutex_unlock(&adapter->rx_filter_lock); 1512 return status; 1513 } 1514 1515 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid) 1516 { 1517 struct be_adapter *adapter = netdev_priv(netdev); 1518 int status = 0; 1519 1520 mutex_lock(&adapter->rx_filter_lock); 1521 1522 /* Packets with VID 0 are always received by Lancer by default */ 1523 if (lancer_chip(adapter) && vid == 0) 1524 goto done; 1525 1526 if (!test_bit(vid, adapter->vids)) 1527 goto done; 1528 1529 clear_bit(vid, adapter->vids); 1530 adapter->vlans_added--; 1531 1532 status = be_vid_config(adapter); 1533 done: 1534 mutex_unlock(&adapter->rx_filter_lock); 1535 return status; 1536 } 1537 1538 static void be_set_all_promisc(struct be_adapter *adapter) 1539 { 1540 be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON); 1541 adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS; 1542 } 1543 1544 static void be_set_mc_promisc(struct be_adapter *adapter) 1545 { 1546 int status; 1547 1548 if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) 1549 return; 1550 1551 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON); 1552 if (!status) 1553 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS; 1554 } 1555 1556 static void be_set_uc_promisc(struct be_adapter *adapter) 1557 { 1558 int status; 1559 1560 if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) 1561 return; 1562 1563 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, ON); 1564 if (!status) 1565 adapter->if_flags |= BE_IF_FLAGS_PROMISCUOUS; 1566 } 1567 1568 static void be_clear_uc_promisc(struct be_adapter *adapter) 1569 { 1570 int status; 1571 1572 if (!(adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS)) 1573 return; 1574 1575 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, OFF); 1576 if (!status) 1577 adapter->if_flags &= ~BE_IF_FLAGS_PROMISCUOUS; 1578 } 1579 1580 /* The below 2 functions are the callback args for __dev_mc_sync/dev_uc_sync(). 1581 * We use a single callback function for both sync and unsync. We really don't 1582 * add/remove addresses through this callback. But, we use it to detect changes 1583 * to the uc/mc lists. The entire uc/mc list is programmed in be_set_rx_mode(). 1584 */ 1585 static int be_uc_list_update(struct net_device *netdev, 1586 const unsigned char *addr) 1587 { 1588 struct be_adapter *adapter = netdev_priv(netdev); 1589 1590 adapter->update_uc_list = true; 1591 return 0; 1592 } 1593 1594 static int be_mc_list_update(struct net_device *netdev, 1595 const unsigned char *addr) 1596 { 1597 struct be_adapter *adapter = netdev_priv(netdev); 1598 1599 adapter->update_mc_list = true; 1600 return 0; 1601 } 1602 1603 static void be_set_mc_list(struct be_adapter *adapter) 1604 { 1605 struct net_device *netdev = adapter->netdev; 1606 struct netdev_hw_addr *ha; 1607 bool mc_promisc = false; 1608 int status; 1609 1610 netif_addr_lock_bh(netdev); 1611 __dev_mc_sync(netdev, be_mc_list_update, be_mc_list_update); 1612 1613 if (netdev->flags & IFF_PROMISC) { 1614 adapter->update_mc_list = false; 1615 } else if (netdev->flags & IFF_ALLMULTI || 1616 netdev_mc_count(netdev) > be_max_mc(adapter)) { 1617 /* Enable multicast promisc if num configured exceeds 1618 * what we support 1619 */ 1620 mc_promisc = true; 1621 adapter->update_mc_list = false; 1622 } else if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) { 1623 /* Update mc-list unconditionally if the iface was previously 1624 * in mc-promisc mode and now is out of that mode. 1625 */ 1626 adapter->update_mc_list = true; 1627 } 1628 1629 if (adapter->update_mc_list) { 1630 int i = 0; 1631 1632 /* cache the mc-list in adapter */ 1633 netdev_for_each_mc_addr(ha, netdev) { 1634 ether_addr_copy(adapter->mc_list[i].mac, ha->addr); 1635 i++; 1636 } 1637 adapter->mc_count = netdev_mc_count(netdev); 1638 } 1639 netif_addr_unlock_bh(netdev); 1640 1641 if (mc_promisc) { 1642 be_set_mc_promisc(adapter); 1643 } else if (adapter->update_mc_list) { 1644 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON); 1645 if (!status) 1646 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS; 1647 else 1648 be_set_mc_promisc(adapter); 1649 1650 adapter->update_mc_list = false; 1651 } 1652 } 1653 1654 static void be_clear_mc_list(struct be_adapter *adapter) 1655 { 1656 struct net_device *netdev = adapter->netdev; 1657 1658 __dev_mc_unsync(netdev, NULL); 1659 be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, OFF); 1660 adapter->mc_count = 0; 1661 } 1662 1663 static int be_uc_mac_add(struct be_adapter *adapter, int uc_idx) 1664 { 1665 if (ether_addr_equal((u8 *)&adapter->uc_list[uc_idx * ETH_ALEN], 1666 adapter->dev_mac)) { 1667 adapter->pmac_id[uc_idx + 1] = adapter->pmac_id[0]; 1668 return 0; 1669 } 1670 1671 return be_cmd_pmac_add(adapter, 1672 (u8 *)&adapter->uc_list[uc_idx * ETH_ALEN], 1673 adapter->if_handle, 1674 &adapter->pmac_id[uc_idx + 1], 0); 1675 } 1676 1677 static void be_uc_mac_del(struct be_adapter *adapter, int pmac_id) 1678 { 1679 if (pmac_id == adapter->pmac_id[0]) 1680 return; 1681 1682 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0); 1683 } 1684 1685 static void be_set_uc_list(struct be_adapter *adapter) 1686 { 1687 struct net_device *netdev = adapter->netdev; 1688 struct netdev_hw_addr *ha; 1689 bool uc_promisc = false; 1690 int curr_uc_macs = 0, i; 1691 1692 netif_addr_lock_bh(netdev); 1693 __dev_uc_sync(netdev, be_uc_list_update, be_uc_list_update); 1694 1695 if (netdev->flags & IFF_PROMISC) { 1696 adapter->update_uc_list = false; 1697 } else if (netdev_uc_count(netdev) > (be_max_uc(adapter) - 1)) { 1698 uc_promisc = true; 1699 adapter->update_uc_list = false; 1700 } else if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) { 1701 /* Update uc-list unconditionally if the iface was previously 1702 * in uc-promisc mode and now is out of that mode. 1703 */ 1704 adapter->update_uc_list = true; 1705 } 1706 1707 if (adapter->update_uc_list) { 1708 i = 1; /* First slot is claimed by the Primary MAC */ 1709 1710 /* cache the uc-list in adapter array */ 1711 netdev_for_each_uc_addr(ha, netdev) { 1712 ether_addr_copy(adapter->uc_list[i].mac, ha->addr); 1713 i++; 1714 } 1715 curr_uc_macs = netdev_uc_count(netdev); 1716 } 1717 netif_addr_unlock_bh(netdev); 1718 1719 if (uc_promisc) { 1720 be_set_uc_promisc(adapter); 1721 } else if (adapter->update_uc_list) { 1722 be_clear_uc_promisc(adapter); 1723 1724 for (i = 0; i < adapter->uc_macs; i++) 1725 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]); 1726 1727 for (i = 0; i < curr_uc_macs; i++) 1728 be_uc_mac_add(adapter, i); 1729 adapter->uc_macs = curr_uc_macs; 1730 adapter->update_uc_list = false; 1731 } 1732 } 1733 1734 static void be_clear_uc_list(struct be_adapter *adapter) 1735 { 1736 struct net_device *netdev = adapter->netdev; 1737 int i; 1738 1739 __dev_uc_unsync(netdev, NULL); 1740 for (i = 0; i < adapter->uc_macs; i++) 1741 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]); 1742 1743 adapter->uc_macs = 0; 1744 } 1745 1746 static void __be_set_rx_mode(struct be_adapter *adapter) 1747 { 1748 struct net_device *netdev = adapter->netdev; 1749 1750 mutex_lock(&adapter->rx_filter_lock); 1751 1752 if (netdev->flags & IFF_PROMISC) { 1753 if (!be_in_all_promisc(adapter)) 1754 be_set_all_promisc(adapter); 1755 } else if (be_in_all_promisc(adapter)) { 1756 /* We need to re-program the vlan-list or clear 1757 * vlan-promisc mode (if needed) when the interface 1758 * comes out of promisc mode. 1759 */ 1760 be_vid_config(adapter); 1761 } 1762 1763 be_set_uc_list(adapter); 1764 be_set_mc_list(adapter); 1765 1766 mutex_unlock(&adapter->rx_filter_lock); 1767 } 1768 1769 static void be_work_set_rx_mode(struct work_struct *work) 1770 { 1771 struct be_cmd_work *cmd_work = 1772 container_of(work, struct be_cmd_work, work); 1773 1774 __be_set_rx_mode(cmd_work->adapter); 1775 kfree(cmd_work); 1776 } 1777 1778 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 1779 { 1780 struct be_adapter *adapter = netdev_priv(netdev); 1781 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1782 int status; 1783 1784 if (!sriov_enabled(adapter)) 1785 return -EPERM; 1786 1787 if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs) 1788 return -EINVAL; 1789 1790 /* Proceed further only if user provided MAC is different 1791 * from active MAC 1792 */ 1793 if (ether_addr_equal(mac, vf_cfg->mac_addr)) 1794 return 0; 1795 1796 if (BEx_chip(adapter)) { 1797 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id, 1798 vf + 1); 1799 1800 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle, 1801 &vf_cfg->pmac_id, vf + 1); 1802 } else { 1803 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 1804 vf + 1); 1805 } 1806 1807 if (status) { 1808 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x", 1809 mac, vf, status); 1810 return be_cmd_status(status); 1811 } 1812 1813 ether_addr_copy(vf_cfg->mac_addr, mac); 1814 1815 return 0; 1816 } 1817 1818 static int be_get_vf_config(struct net_device *netdev, int vf, 1819 struct ifla_vf_info *vi) 1820 { 1821 struct be_adapter *adapter = netdev_priv(netdev); 1822 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1823 1824 if (!sriov_enabled(adapter)) 1825 return -EPERM; 1826 1827 if (vf >= adapter->num_vfs) 1828 return -EINVAL; 1829 1830 vi->vf = vf; 1831 vi->max_tx_rate = vf_cfg->tx_rate; 1832 vi->min_tx_rate = 0; 1833 vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK; 1834 vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT; 1835 memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN); 1836 vi->linkstate = adapter->vf_cfg[vf].plink_tracking; 1837 vi->spoofchk = adapter->vf_cfg[vf].spoofchk; 1838 1839 return 0; 1840 } 1841 1842 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan) 1843 { 1844 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1845 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1846 int vf_if_id = vf_cfg->if_handle; 1847 int status; 1848 1849 /* Enable Transparent VLAN Tagging */ 1850 status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0, 0); 1851 if (status) 1852 return status; 1853 1854 /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */ 1855 vids[0] = 0; 1856 status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1); 1857 if (!status) 1858 dev_info(&adapter->pdev->dev, 1859 "Cleared guest VLANs on VF%d", vf); 1860 1861 /* After TVT is enabled, disallow VFs to program VLAN filters */ 1862 if (vf_cfg->privileges & BE_PRIV_FILTMGMT) { 1863 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges & 1864 ~BE_PRIV_FILTMGMT, vf + 1); 1865 if (!status) 1866 vf_cfg->privileges &= ~BE_PRIV_FILTMGMT; 1867 } 1868 return 0; 1869 } 1870 1871 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf) 1872 { 1873 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1874 struct device *dev = &adapter->pdev->dev; 1875 int status; 1876 1877 /* Reset Transparent VLAN Tagging. */ 1878 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1, 1879 vf_cfg->if_handle, 0, 0); 1880 if (status) 1881 return status; 1882 1883 /* Allow VFs to program VLAN filtering */ 1884 if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 1885 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges | 1886 BE_PRIV_FILTMGMT, vf + 1); 1887 if (!status) { 1888 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 1889 dev_info(dev, "VF%d: FILTMGMT priv enabled", vf); 1890 } 1891 } 1892 1893 dev_info(dev, 1894 "Disable/re-enable i/f in VM to clear Transparent VLAN tag"); 1895 return 0; 1896 } 1897 1898 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos) 1899 { 1900 struct be_adapter *adapter = netdev_priv(netdev); 1901 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1902 int status; 1903 1904 if (!sriov_enabled(adapter)) 1905 return -EPERM; 1906 1907 if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7) 1908 return -EINVAL; 1909 1910 if (vlan || qos) { 1911 vlan |= qos << VLAN_PRIO_SHIFT; 1912 status = be_set_vf_tvt(adapter, vf, vlan); 1913 } else { 1914 status = be_clear_vf_tvt(adapter, vf); 1915 } 1916 1917 if (status) { 1918 dev_err(&adapter->pdev->dev, 1919 "VLAN %d config on VF %d failed : %#x\n", vlan, vf, 1920 status); 1921 return be_cmd_status(status); 1922 } 1923 1924 vf_cfg->vlan_tag = vlan; 1925 return 0; 1926 } 1927 1928 static int be_set_vf_tx_rate(struct net_device *netdev, int vf, 1929 int min_tx_rate, int max_tx_rate) 1930 { 1931 struct be_adapter *adapter = netdev_priv(netdev); 1932 struct device *dev = &adapter->pdev->dev; 1933 int percent_rate, status = 0; 1934 u16 link_speed = 0; 1935 u8 link_status; 1936 1937 if (!sriov_enabled(adapter)) 1938 return -EPERM; 1939 1940 if (vf >= adapter->num_vfs) 1941 return -EINVAL; 1942 1943 if (min_tx_rate) 1944 return -EINVAL; 1945 1946 if (!max_tx_rate) 1947 goto config_qos; 1948 1949 status = be_cmd_link_status_query(adapter, &link_speed, 1950 &link_status, 0); 1951 if (status) 1952 goto err; 1953 1954 if (!link_status) { 1955 dev_err(dev, "TX-rate setting not allowed when link is down\n"); 1956 status = -ENETDOWN; 1957 goto err; 1958 } 1959 1960 if (max_tx_rate < 100 || max_tx_rate > link_speed) { 1961 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n", 1962 link_speed); 1963 status = -EINVAL; 1964 goto err; 1965 } 1966 1967 /* On Skyhawk the QOS setting must be done only as a % value */ 1968 percent_rate = link_speed / 100; 1969 if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) { 1970 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n", 1971 percent_rate); 1972 status = -EINVAL; 1973 goto err; 1974 } 1975 1976 config_qos: 1977 status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1); 1978 if (status) 1979 goto err; 1980 1981 adapter->vf_cfg[vf].tx_rate = max_tx_rate; 1982 return 0; 1983 1984 err: 1985 dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n", 1986 max_tx_rate, vf); 1987 return be_cmd_status(status); 1988 } 1989 1990 static int be_set_vf_link_state(struct net_device *netdev, int vf, 1991 int link_state) 1992 { 1993 struct be_adapter *adapter = netdev_priv(netdev); 1994 int status; 1995 1996 if (!sriov_enabled(adapter)) 1997 return -EPERM; 1998 1999 if (vf >= adapter->num_vfs) 2000 return -EINVAL; 2001 2002 status = be_cmd_set_logical_link_config(adapter, link_state, vf+1); 2003 if (status) { 2004 dev_err(&adapter->pdev->dev, 2005 "Link state change on VF %d failed: %#x\n", vf, status); 2006 return be_cmd_status(status); 2007 } 2008 2009 adapter->vf_cfg[vf].plink_tracking = link_state; 2010 2011 return 0; 2012 } 2013 2014 static int be_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable) 2015 { 2016 struct be_adapter *adapter = netdev_priv(netdev); 2017 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 2018 u8 spoofchk; 2019 int status; 2020 2021 if (!sriov_enabled(adapter)) 2022 return -EPERM; 2023 2024 if (vf >= adapter->num_vfs) 2025 return -EINVAL; 2026 2027 if (BEx_chip(adapter)) 2028 return -EOPNOTSUPP; 2029 2030 if (enable == vf_cfg->spoofchk) 2031 return 0; 2032 2033 spoofchk = enable ? ENABLE_MAC_SPOOFCHK : DISABLE_MAC_SPOOFCHK; 2034 2035 status = be_cmd_set_hsw_config(adapter, 0, vf + 1, vf_cfg->if_handle, 2036 0, spoofchk); 2037 if (status) { 2038 dev_err(&adapter->pdev->dev, 2039 "Spoofchk change on VF %d failed: %#x\n", vf, status); 2040 return be_cmd_status(status); 2041 } 2042 2043 vf_cfg->spoofchk = enable; 2044 return 0; 2045 } 2046 2047 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts, 2048 ulong now) 2049 { 2050 aic->rx_pkts_prev = rx_pkts; 2051 aic->tx_reqs_prev = tx_pkts; 2052 aic->jiffies = now; 2053 } 2054 2055 static int be_get_new_eqd(struct be_eq_obj *eqo) 2056 { 2057 struct be_adapter *adapter = eqo->adapter; 2058 int eqd, start; 2059 struct be_aic_obj *aic; 2060 struct be_rx_obj *rxo; 2061 struct be_tx_obj *txo; 2062 u64 rx_pkts = 0, tx_pkts = 0; 2063 ulong now; 2064 u32 pps, delta; 2065 int i; 2066 2067 aic = &adapter->aic_obj[eqo->idx]; 2068 if (!aic->enable) { 2069 if (aic->jiffies) 2070 aic->jiffies = 0; 2071 eqd = aic->et_eqd; 2072 return eqd; 2073 } 2074 2075 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 2076 do { 2077 start = u64_stats_fetch_begin_irq(&rxo->stats.sync); 2078 rx_pkts += rxo->stats.rx_pkts; 2079 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start)); 2080 } 2081 2082 for_all_tx_queues_on_eq(adapter, eqo, txo, i) { 2083 do { 2084 start = u64_stats_fetch_begin_irq(&txo->stats.sync); 2085 tx_pkts += txo->stats.tx_reqs; 2086 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start)); 2087 } 2088 2089 /* Skip, if wrapped around or first calculation */ 2090 now = jiffies; 2091 if (!aic->jiffies || time_before(now, aic->jiffies) || 2092 rx_pkts < aic->rx_pkts_prev || 2093 tx_pkts < aic->tx_reqs_prev) { 2094 be_aic_update(aic, rx_pkts, tx_pkts, now); 2095 return aic->prev_eqd; 2096 } 2097 2098 delta = jiffies_to_msecs(now - aic->jiffies); 2099 if (delta == 0) 2100 return aic->prev_eqd; 2101 2102 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) + 2103 (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta); 2104 eqd = (pps / 15000) << 2; 2105 2106 if (eqd < 8) 2107 eqd = 0; 2108 eqd = min_t(u32, eqd, aic->max_eqd); 2109 eqd = max_t(u32, eqd, aic->min_eqd); 2110 2111 be_aic_update(aic, rx_pkts, tx_pkts, now); 2112 2113 return eqd; 2114 } 2115 2116 /* For Skyhawk-R only */ 2117 static u32 be_get_eq_delay_mult_enc(struct be_eq_obj *eqo) 2118 { 2119 struct be_adapter *adapter = eqo->adapter; 2120 struct be_aic_obj *aic = &adapter->aic_obj[eqo->idx]; 2121 ulong now = jiffies; 2122 int eqd; 2123 u32 mult_enc; 2124 2125 if (!aic->enable) 2126 return 0; 2127 2128 if (jiffies_to_msecs(now - aic->jiffies) < 1) 2129 eqd = aic->prev_eqd; 2130 else 2131 eqd = be_get_new_eqd(eqo); 2132 2133 if (eqd > 100) 2134 mult_enc = R2I_DLY_ENC_1; 2135 else if (eqd > 60) 2136 mult_enc = R2I_DLY_ENC_2; 2137 else if (eqd > 20) 2138 mult_enc = R2I_DLY_ENC_3; 2139 else 2140 mult_enc = R2I_DLY_ENC_0; 2141 2142 aic->prev_eqd = eqd; 2143 2144 return mult_enc; 2145 } 2146 2147 void be_eqd_update(struct be_adapter *adapter, bool force_update) 2148 { 2149 struct be_set_eqd set_eqd[MAX_EVT_QS]; 2150 struct be_aic_obj *aic; 2151 struct be_eq_obj *eqo; 2152 int i, num = 0, eqd; 2153 2154 for_all_evt_queues(adapter, eqo, i) { 2155 aic = &adapter->aic_obj[eqo->idx]; 2156 eqd = be_get_new_eqd(eqo); 2157 if (force_update || eqd != aic->prev_eqd) { 2158 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2159 set_eqd[num].eq_id = eqo->q.id; 2160 aic->prev_eqd = eqd; 2161 num++; 2162 } 2163 } 2164 2165 if (num) 2166 be_cmd_modify_eqd(adapter, set_eqd, num); 2167 } 2168 2169 static void be_rx_stats_update(struct be_rx_obj *rxo, 2170 struct be_rx_compl_info *rxcp) 2171 { 2172 struct be_rx_stats *stats = rx_stats(rxo); 2173 2174 u64_stats_update_begin(&stats->sync); 2175 stats->rx_compl++; 2176 stats->rx_bytes += rxcp->pkt_size; 2177 stats->rx_pkts++; 2178 if (rxcp->tunneled) 2179 stats->rx_vxlan_offload_pkts++; 2180 if (rxcp->pkt_type == BE_MULTICAST_PACKET) 2181 stats->rx_mcast_pkts++; 2182 if (rxcp->err) 2183 stats->rx_compl_err++; 2184 u64_stats_update_end(&stats->sync); 2185 } 2186 2187 static inline bool csum_passed(struct be_rx_compl_info *rxcp) 2188 { 2189 /* L4 checksum is not reliable for non TCP/UDP packets. 2190 * Also ignore ipcksm for ipv6 pkts 2191 */ 2192 return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum && 2193 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err; 2194 } 2195 2196 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo) 2197 { 2198 struct be_adapter *adapter = rxo->adapter; 2199 struct be_rx_page_info *rx_page_info; 2200 struct be_queue_info *rxq = &rxo->q; 2201 u32 frag_idx = rxq->tail; 2202 2203 rx_page_info = &rxo->page_info_tbl[frag_idx]; 2204 BUG_ON(!rx_page_info->page); 2205 2206 if (rx_page_info->last_frag) { 2207 dma_unmap_page(&adapter->pdev->dev, 2208 dma_unmap_addr(rx_page_info, bus), 2209 adapter->big_page_size, DMA_FROM_DEVICE); 2210 rx_page_info->last_frag = false; 2211 } else { 2212 dma_sync_single_for_cpu(&adapter->pdev->dev, 2213 dma_unmap_addr(rx_page_info, bus), 2214 rx_frag_size, DMA_FROM_DEVICE); 2215 } 2216 2217 queue_tail_inc(rxq); 2218 atomic_dec(&rxq->used); 2219 return rx_page_info; 2220 } 2221 2222 /* Throwaway the data in the Rx completion */ 2223 static void be_rx_compl_discard(struct be_rx_obj *rxo, 2224 struct be_rx_compl_info *rxcp) 2225 { 2226 struct be_rx_page_info *page_info; 2227 u16 i, num_rcvd = rxcp->num_rcvd; 2228 2229 for (i = 0; i < num_rcvd; i++) { 2230 page_info = get_rx_page_info(rxo); 2231 put_page(page_info->page); 2232 memset(page_info, 0, sizeof(*page_info)); 2233 } 2234 } 2235 2236 /* 2237 * skb_fill_rx_data forms a complete skb for an ether frame 2238 * indicated by rxcp. 2239 */ 2240 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb, 2241 struct be_rx_compl_info *rxcp) 2242 { 2243 struct be_rx_page_info *page_info; 2244 u16 i, j; 2245 u16 hdr_len, curr_frag_len, remaining; 2246 u8 *start; 2247 2248 page_info = get_rx_page_info(rxo); 2249 start = page_address(page_info->page) + page_info->page_offset; 2250 prefetch(start); 2251 2252 /* Copy data in the first descriptor of this completion */ 2253 curr_frag_len = min(rxcp->pkt_size, rx_frag_size); 2254 2255 skb->len = curr_frag_len; 2256 if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */ 2257 memcpy(skb->data, start, curr_frag_len); 2258 /* Complete packet has now been moved to data */ 2259 put_page(page_info->page); 2260 skb->data_len = 0; 2261 skb->tail += curr_frag_len; 2262 } else { 2263 hdr_len = ETH_HLEN; 2264 memcpy(skb->data, start, hdr_len); 2265 skb_shinfo(skb)->nr_frags = 1; 2266 skb_frag_set_page(skb, 0, page_info->page); 2267 skb_shinfo(skb)->frags[0].page_offset = 2268 page_info->page_offset + hdr_len; 2269 skb_frag_size_set(&skb_shinfo(skb)->frags[0], 2270 curr_frag_len - hdr_len); 2271 skb->data_len = curr_frag_len - hdr_len; 2272 skb->truesize += rx_frag_size; 2273 skb->tail += hdr_len; 2274 } 2275 page_info->page = NULL; 2276 2277 if (rxcp->pkt_size <= rx_frag_size) { 2278 BUG_ON(rxcp->num_rcvd != 1); 2279 return; 2280 } 2281 2282 /* More frags present for this completion */ 2283 remaining = rxcp->pkt_size - curr_frag_len; 2284 for (i = 1, j = 0; i < rxcp->num_rcvd; i++) { 2285 page_info = get_rx_page_info(rxo); 2286 curr_frag_len = min(remaining, rx_frag_size); 2287 2288 /* Coalesce all frags from the same physical page in one slot */ 2289 if (page_info->page_offset == 0) { 2290 /* Fresh page */ 2291 j++; 2292 skb_frag_set_page(skb, j, page_info->page); 2293 skb_shinfo(skb)->frags[j].page_offset = 2294 page_info->page_offset; 2295 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 2296 skb_shinfo(skb)->nr_frags++; 2297 } else { 2298 put_page(page_info->page); 2299 } 2300 2301 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 2302 skb->len += curr_frag_len; 2303 skb->data_len += curr_frag_len; 2304 skb->truesize += rx_frag_size; 2305 remaining -= curr_frag_len; 2306 page_info->page = NULL; 2307 } 2308 BUG_ON(j > MAX_SKB_FRAGS); 2309 } 2310 2311 /* Process the RX completion indicated by rxcp when GRO is disabled */ 2312 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi, 2313 struct be_rx_compl_info *rxcp) 2314 { 2315 struct be_adapter *adapter = rxo->adapter; 2316 struct net_device *netdev = adapter->netdev; 2317 struct sk_buff *skb; 2318 2319 skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE); 2320 if (unlikely(!skb)) { 2321 rx_stats(rxo)->rx_drops_no_skbs++; 2322 be_rx_compl_discard(rxo, rxcp); 2323 return; 2324 } 2325 2326 skb_fill_rx_data(rxo, skb, rxcp); 2327 2328 if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp))) 2329 skb->ip_summed = CHECKSUM_UNNECESSARY; 2330 else 2331 skb_checksum_none_assert(skb); 2332 2333 skb->protocol = eth_type_trans(skb, netdev); 2334 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 2335 if (netdev->features & NETIF_F_RXHASH) 2336 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 2337 2338 skb->csum_level = rxcp->tunneled; 2339 skb_mark_napi_id(skb, napi); 2340 2341 if (rxcp->vlanf) 2342 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 2343 2344 netif_receive_skb(skb); 2345 } 2346 2347 /* Process the RX completion indicated by rxcp when GRO is enabled */ 2348 static void be_rx_compl_process_gro(struct be_rx_obj *rxo, 2349 struct napi_struct *napi, 2350 struct be_rx_compl_info *rxcp) 2351 { 2352 struct be_adapter *adapter = rxo->adapter; 2353 struct be_rx_page_info *page_info; 2354 struct sk_buff *skb = NULL; 2355 u16 remaining, curr_frag_len; 2356 u16 i, j; 2357 2358 skb = napi_get_frags(napi); 2359 if (!skb) { 2360 be_rx_compl_discard(rxo, rxcp); 2361 return; 2362 } 2363 2364 remaining = rxcp->pkt_size; 2365 for (i = 0, j = -1; i < rxcp->num_rcvd; i++) { 2366 page_info = get_rx_page_info(rxo); 2367 2368 curr_frag_len = min(remaining, rx_frag_size); 2369 2370 /* Coalesce all frags from the same physical page in one slot */ 2371 if (i == 0 || page_info->page_offset == 0) { 2372 /* First frag or Fresh page */ 2373 j++; 2374 skb_frag_set_page(skb, j, page_info->page); 2375 skb_shinfo(skb)->frags[j].page_offset = 2376 page_info->page_offset; 2377 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 2378 } else { 2379 put_page(page_info->page); 2380 } 2381 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 2382 skb->truesize += rx_frag_size; 2383 remaining -= curr_frag_len; 2384 memset(page_info, 0, sizeof(*page_info)); 2385 } 2386 BUG_ON(j > MAX_SKB_FRAGS); 2387 2388 skb_shinfo(skb)->nr_frags = j + 1; 2389 skb->len = rxcp->pkt_size; 2390 skb->data_len = rxcp->pkt_size; 2391 skb->ip_summed = CHECKSUM_UNNECESSARY; 2392 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 2393 if (adapter->netdev->features & NETIF_F_RXHASH) 2394 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 2395 2396 skb->csum_level = rxcp->tunneled; 2397 2398 if (rxcp->vlanf) 2399 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 2400 2401 napi_gro_frags(napi); 2402 } 2403 2404 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl, 2405 struct be_rx_compl_info *rxcp) 2406 { 2407 rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl); 2408 rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl); 2409 rxcp->err = GET_RX_COMPL_V1_BITS(err, compl); 2410 rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl); 2411 rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl); 2412 rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl); 2413 rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl); 2414 rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl); 2415 rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl); 2416 rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl); 2417 rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl); 2418 if (rxcp->vlanf) { 2419 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl); 2420 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl); 2421 } 2422 rxcp->port = GET_RX_COMPL_V1_BITS(port, compl); 2423 rxcp->tunneled = 2424 GET_RX_COMPL_V1_BITS(tunneled, compl); 2425 } 2426 2427 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl, 2428 struct be_rx_compl_info *rxcp) 2429 { 2430 rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl); 2431 rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl); 2432 rxcp->err = GET_RX_COMPL_V0_BITS(err, compl); 2433 rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl); 2434 rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl); 2435 rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl); 2436 rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl); 2437 rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl); 2438 rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl); 2439 rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl); 2440 rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl); 2441 if (rxcp->vlanf) { 2442 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl); 2443 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl); 2444 } 2445 rxcp->port = GET_RX_COMPL_V0_BITS(port, compl); 2446 rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl); 2447 } 2448 2449 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo) 2450 { 2451 struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq); 2452 struct be_rx_compl_info *rxcp = &rxo->rxcp; 2453 struct be_adapter *adapter = rxo->adapter; 2454 2455 /* For checking the valid bit it is Ok to use either definition as the 2456 * valid bit is at the same position in both v0 and v1 Rx compl */ 2457 if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0) 2458 return NULL; 2459 2460 rmb(); 2461 be_dws_le_to_cpu(compl, sizeof(*compl)); 2462 2463 if (adapter->be3_native) 2464 be_parse_rx_compl_v1(compl, rxcp); 2465 else 2466 be_parse_rx_compl_v0(compl, rxcp); 2467 2468 if (rxcp->ip_frag) 2469 rxcp->l4_csum = 0; 2470 2471 if (rxcp->vlanf) { 2472 /* In QNQ modes, if qnq bit is not set, then the packet was 2473 * tagged only with the transparent outer vlan-tag and must 2474 * not be treated as a vlan packet by host 2475 */ 2476 if (be_is_qnq_mode(adapter) && !rxcp->qnq) 2477 rxcp->vlanf = 0; 2478 2479 if (!lancer_chip(adapter)) 2480 rxcp->vlan_tag = swab16(rxcp->vlan_tag); 2481 2482 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) && 2483 !test_bit(rxcp->vlan_tag, adapter->vids)) 2484 rxcp->vlanf = 0; 2485 } 2486 2487 /* As the compl has been parsed, reset it; we wont touch it again */ 2488 compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0; 2489 2490 queue_tail_inc(&rxo->cq); 2491 return rxcp; 2492 } 2493 2494 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp) 2495 { 2496 u32 order = get_order(size); 2497 2498 if (order > 0) 2499 gfp |= __GFP_COMP; 2500 return alloc_pages(gfp, order); 2501 } 2502 2503 /* 2504 * Allocate a page, split it to fragments of size rx_frag_size and post as 2505 * receive buffers to BE 2506 */ 2507 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed) 2508 { 2509 struct be_adapter *adapter = rxo->adapter; 2510 struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL; 2511 struct be_queue_info *rxq = &rxo->q; 2512 struct page *pagep = NULL; 2513 struct device *dev = &adapter->pdev->dev; 2514 struct be_eth_rx_d *rxd; 2515 u64 page_dmaaddr = 0, frag_dmaaddr; 2516 u32 posted, page_offset = 0, notify = 0; 2517 2518 page_info = &rxo->page_info_tbl[rxq->head]; 2519 for (posted = 0; posted < frags_needed && !page_info->page; posted++) { 2520 if (!pagep) { 2521 pagep = be_alloc_pages(adapter->big_page_size, gfp); 2522 if (unlikely(!pagep)) { 2523 rx_stats(rxo)->rx_post_fail++; 2524 break; 2525 } 2526 page_dmaaddr = dma_map_page(dev, pagep, 0, 2527 adapter->big_page_size, 2528 DMA_FROM_DEVICE); 2529 if (dma_mapping_error(dev, page_dmaaddr)) { 2530 put_page(pagep); 2531 pagep = NULL; 2532 adapter->drv_stats.dma_map_errors++; 2533 break; 2534 } 2535 page_offset = 0; 2536 } else { 2537 get_page(pagep); 2538 page_offset += rx_frag_size; 2539 } 2540 page_info->page_offset = page_offset; 2541 page_info->page = pagep; 2542 2543 rxd = queue_head_node(rxq); 2544 frag_dmaaddr = page_dmaaddr + page_info->page_offset; 2545 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF); 2546 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr)); 2547 2548 /* Any space left in the current big page for another frag? */ 2549 if ((page_offset + rx_frag_size + rx_frag_size) > 2550 adapter->big_page_size) { 2551 pagep = NULL; 2552 page_info->last_frag = true; 2553 dma_unmap_addr_set(page_info, bus, page_dmaaddr); 2554 } else { 2555 dma_unmap_addr_set(page_info, bus, frag_dmaaddr); 2556 } 2557 2558 prev_page_info = page_info; 2559 queue_head_inc(rxq); 2560 page_info = &rxo->page_info_tbl[rxq->head]; 2561 } 2562 2563 /* Mark the last frag of a page when we break out of the above loop 2564 * with no more slots available in the RXQ 2565 */ 2566 if (pagep) { 2567 prev_page_info->last_frag = true; 2568 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr); 2569 } 2570 2571 if (posted) { 2572 atomic_add(posted, &rxq->used); 2573 if (rxo->rx_post_starved) 2574 rxo->rx_post_starved = false; 2575 do { 2576 notify = min(MAX_NUM_POST_ERX_DB, posted); 2577 be_rxq_notify(adapter, rxq->id, notify); 2578 posted -= notify; 2579 } while (posted); 2580 } else if (atomic_read(&rxq->used) == 0) { 2581 /* Let be_worker replenish when memory is available */ 2582 rxo->rx_post_starved = true; 2583 } 2584 } 2585 2586 static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo) 2587 { 2588 struct be_queue_info *tx_cq = &txo->cq; 2589 struct be_tx_compl_info *txcp = &txo->txcp; 2590 struct be_eth_tx_compl *compl = queue_tail_node(tx_cq); 2591 2592 if (compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0) 2593 return NULL; 2594 2595 /* Ensure load ordering of valid bit dword and other dwords below */ 2596 rmb(); 2597 be_dws_le_to_cpu(compl, sizeof(*compl)); 2598 2599 txcp->status = GET_TX_COMPL_BITS(status, compl); 2600 txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl); 2601 2602 compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0; 2603 queue_tail_inc(tx_cq); 2604 return txcp; 2605 } 2606 2607 static u16 be_tx_compl_process(struct be_adapter *adapter, 2608 struct be_tx_obj *txo, u16 last_index) 2609 { 2610 struct sk_buff **sent_skbs = txo->sent_skb_list; 2611 struct be_queue_info *txq = &txo->q; 2612 struct sk_buff *skb = NULL; 2613 bool unmap_skb_hdr = false; 2614 struct be_eth_wrb *wrb; 2615 u16 num_wrbs = 0; 2616 u32 frag_index; 2617 2618 do { 2619 if (sent_skbs[txq->tail]) { 2620 /* Free skb from prev req */ 2621 if (skb) 2622 dev_consume_skb_any(skb); 2623 skb = sent_skbs[txq->tail]; 2624 sent_skbs[txq->tail] = NULL; 2625 queue_tail_inc(txq); /* skip hdr wrb */ 2626 num_wrbs++; 2627 unmap_skb_hdr = true; 2628 } 2629 wrb = queue_tail_node(txq); 2630 frag_index = txq->tail; 2631 unmap_tx_frag(&adapter->pdev->dev, wrb, 2632 (unmap_skb_hdr && skb_headlen(skb))); 2633 unmap_skb_hdr = false; 2634 queue_tail_inc(txq); 2635 num_wrbs++; 2636 } while (frag_index != last_index); 2637 dev_consume_skb_any(skb); 2638 2639 return num_wrbs; 2640 } 2641 2642 /* Return the number of events in the event queue */ 2643 static inline int events_get(struct be_eq_obj *eqo) 2644 { 2645 struct be_eq_entry *eqe; 2646 int num = 0; 2647 2648 do { 2649 eqe = queue_tail_node(&eqo->q); 2650 if (eqe->evt == 0) 2651 break; 2652 2653 rmb(); 2654 eqe->evt = 0; 2655 num++; 2656 queue_tail_inc(&eqo->q); 2657 } while (true); 2658 2659 return num; 2660 } 2661 2662 /* Leaves the EQ is disarmed state */ 2663 static void be_eq_clean(struct be_eq_obj *eqo) 2664 { 2665 int num = events_get(eqo); 2666 2667 be_eq_notify(eqo->adapter, eqo->q.id, false, true, num, 0); 2668 } 2669 2670 /* Free posted rx buffers that were not used */ 2671 static void be_rxq_clean(struct be_rx_obj *rxo) 2672 { 2673 struct be_queue_info *rxq = &rxo->q; 2674 struct be_rx_page_info *page_info; 2675 2676 while (atomic_read(&rxq->used) > 0) { 2677 page_info = get_rx_page_info(rxo); 2678 put_page(page_info->page); 2679 memset(page_info, 0, sizeof(*page_info)); 2680 } 2681 BUG_ON(atomic_read(&rxq->used)); 2682 rxq->tail = 0; 2683 rxq->head = 0; 2684 } 2685 2686 static void be_rx_cq_clean(struct be_rx_obj *rxo) 2687 { 2688 struct be_queue_info *rx_cq = &rxo->cq; 2689 struct be_rx_compl_info *rxcp; 2690 struct be_adapter *adapter = rxo->adapter; 2691 int flush_wait = 0; 2692 2693 /* Consume pending rx completions. 2694 * Wait for the flush completion (identified by zero num_rcvd) 2695 * to arrive. Notify CQ even when there are no more CQ entries 2696 * for HW to flush partially coalesced CQ entries. 2697 * In Lancer, there is no need to wait for flush compl. 2698 */ 2699 for (;;) { 2700 rxcp = be_rx_compl_get(rxo); 2701 if (!rxcp) { 2702 if (lancer_chip(adapter)) 2703 break; 2704 2705 if (flush_wait++ > 50 || 2706 be_check_error(adapter, 2707 BE_ERROR_HW)) { 2708 dev_warn(&adapter->pdev->dev, 2709 "did not receive flush compl\n"); 2710 break; 2711 } 2712 be_cq_notify(adapter, rx_cq->id, true, 0); 2713 mdelay(1); 2714 } else { 2715 be_rx_compl_discard(rxo, rxcp); 2716 be_cq_notify(adapter, rx_cq->id, false, 1); 2717 if (rxcp->num_rcvd == 0) 2718 break; 2719 } 2720 } 2721 2722 /* After cleanup, leave the CQ in unarmed state */ 2723 be_cq_notify(adapter, rx_cq->id, false, 0); 2724 } 2725 2726 static void be_tx_compl_clean(struct be_adapter *adapter) 2727 { 2728 struct device *dev = &adapter->pdev->dev; 2729 u16 cmpl = 0, timeo = 0, num_wrbs = 0; 2730 struct be_tx_compl_info *txcp; 2731 struct be_queue_info *txq; 2732 u32 end_idx, notified_idx; 2733 struct be_tx_obj *txo; 2734 int i, pending_txqs; 2735 2736 /* Stop polling for compls when HW has been silent for 10ms */ 2737 do { 2738 pending_txqs = adapter->num_tx_qs; 2739 2740 for_all_tx_queues(adapter, txo, i) { 2741 cmpl = 0; 2742 num_wrbs = 0; 2743 txq = &txo->q; 2744 while ((txcp = be_tx_compl_get(txo))) { 2745 num_wrbs += 2746 be_tx_compl_process(adapter, txo, 2747 txcp->end_index); 2748 cmpl++; 2749 } 2750 if (cmpl) { 2751 be_cq_notify(adapter, txo->cq.id, false, cmpl); 2752 atomic_sub(num_wrbs, &txq->used); 2753 timeo = 0; 2754 } 2755 if (!be_is_tx_compl_pending(txo)) 2756 pending_txqs--; 2757 } 2758 2759 if (pending_txqs == 0 || ++timeo > 10 || 2760 be_check_error(adapter, BE_ERROR_HW)) 2761 break; 2762 2763 mdelay(1); 2764 } while (true); 2765 2766 /* Free enqueued TX that was never notified to HW */ 2767 for_all_tx_queues(adapter, txo, i) { 2768 txq = &txo->q; 2769 2770 if (atomic_read(&txq->used)) { 2771 dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n", 2772 i, atomic_read(&txq->used)); 2773 notified_idx = txq->tail; 2774 end_idx = txq->tail; 2775 index_adv(&end_idx, atomic_read(&txq->used) - 1, 2776 txq->len); 2777 /* Use the tx-compl process logic to handle requests 2778 * that were not sent to the HW. 2779 */ 2780 num_wrbs = be_tx_compl_process(adapter, txo, end_idx); 2781 atomic_sub(num_wrbs, &txq->used); 2782 BUG_ON(atomic_read(&txq->used)); 2783 txo->pend_wrb_cnt = 0; 2784 /* Since hw was never notified of these requests, 2785 * reset TXQ indices 2786 */ 2787 txq->head = notified_idx; 2788 txq->tail = notified_idx; 2789 } 2790 } 2791 } 2792 2793 static void be_evt_queues_destroy(struct be_adapter *adapter) 2794 { 2795 struct be_eq_obj *eqo; 2796 int i; 2797 2798 for_all_evt_queues(adapter, eqo, i) { 2799 if (eqo->q.created) { 2800 be_eq_clean(eqo); 2801 be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ); 2802 napi_hash_del(&eqo->napi); 2803 netif_napi_del(&eqo->napi); 2804 free_cpumask_var(eqo->affinity_mask); 2805 } 2806 be_queue_free(adapter, &eqo->q); 2807 } 2808 } 2809 2810 static int be_evt_queues_create(struct be_adapter *adapter) 2811 { 2812 struct be_queue_info *eq; 2813 struct be_eq_obj *eqo; 2814 struct be_aic_obj *aic; 2815 int i, rc; 2816 2817 /* need enough EQs to service both RX and TX queues */ 2818 adapter->num_evt_qs = min_t(u16, num_irqs(adapter), 2819 max(adapter->cfg_num_rx_irqs, 2820 adapter->cfg_num_tx_irqs)); 2821 2822 for_all_evt_queues(adapter, eqo, i) { 2823 int numa_node = dev_to_node(&adapter->pdev->dev); 2824 2825 aic = &adapter->aic_obj[i]; 2826 eqo->adapter = adapter; 2827 eqo->idx = i; 2828 aic->max_eqd = BE_MAX_EQD; 2829 aic->enable = true; 2830 2831 eq = &eqo->q; 2832 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN, 2833 sizeof(struct be_eq_entry)); 2834 if (rc) 2835 return rc; 2836 2837 rc = be_cmd_eq_create(adapter, eqo); 2838 if (rc) 2839 return rc; 2840 2841 if (!zalloc_cpumask_var(&eqo->affinity_mask, GFP_KERNEL)) 2842 return -ENOMEM; 2843 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 2844 eqo->affinity_mask); 2845 netif_napi_add(adapter->netdev, &eqo->napi, be_poll, 2846 BE_NAPI_WEIGHT); 2847 } 2848 return 0; 2849 } 2850 2851 static void be_mcc_queues_destroy(struct be_adapter *adapter) 2852 { 2853 struct be_queue_info *q; 2854 2855 q = &adapter->mcc_obj.q; 2856 if (q->created) 2857 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ); 2858 be_queue_free(adapter, q); 2859 2860 q = &adapter->mcc_obj.cq; 2861 if (q->created) 2862 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2863 be_queue_free(adapter, q); 2864 } 2865 2866 /* Must be called only after TX qs are created as MCC shares TX EQ */ 2867 static int be_mcc_queues_create(struct be_adapter *adapter) 2868 { 2869 struct be_queue_info *q, *cq; 2870 2871 cq = &adapter->mcc_obj.cq; 2872 if (be_queue_alloc(adapter, cq, MCC_CQ_LEN, 2873 sizeof(struct be_mcc_compl))) 2874 goto err; 2875 2876 /* Use the default EQ for MCC completions */ 2877 if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0)) 2878 goto mcc_cq_free; 2879 2880 q = &adapter->mcc_obj.q; 2881 if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 2882 goto mcc_cq_destroy; 2883 2884 if (be_cmd_mccq_create(adapter, q, cq)) 2885 goto mcc_q_free; 2886 2887 return 0; 2888 2889 mcc_q_free: 2890 be_queue_free(adapter, q); 2891 mcc_cq_destroy: 2892 be_cmd_q_destroy(adapter, cq, QTYPE_CQ); 2893 mcc_cq_free: 2894 be_queue_free(adapter, cq); 2895 err: 2896 return -1; 2897 } 2898 2899 static void be_tx_queues_destroy(struct be_adapter *adapter) 2900 { 2901 struct be_queue_info *q; 2902 struct be_tx_obj *txo; 2903 u8 i; 2904 2905 for_all_tx_queues(adapter, txo, i) { 2906 q = &txo->q; 2907 if (q->created) 2908 be_cmd_q_destroy(adapter, q, QTYPE_TXQ); 2909 be_queue_free(adapter, q); 2910 2911 q = &txo->cq; 2912 if (q->created) 2913 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2914 be_queue_free(adapter, q); 2915 } 2916 } 2917 2918 static int be_tx_qs_create(struct be_adapter *adapter) 2919 { 2920 struct be_queue_info *cq; 2921 struct be_tx_obj *txo; 2922 struct be_eq_obj *eqo; 2923 int status, i; 2924 2925 adapter->num_tx_qs = min(adapter->num_evt_qs, adapter->cfg_num_tx_irqs); 2926 2927 for_all_tx_queues(adapter, txo, i) { 2928 cq = &txo->cq; 2929 status = be_queue_alloc(adapter, cq, TX_CQ_LEN, 2930 sizeof(struct be_eth_tx_compl)); 2931 if (status) 2932 return status; 2933 2934 u64_stats_init(&txo->stats.sync); 2935 u64_stats_init(&txo->stats.sync_compl); 2936 2937 /* If num_evt_qs is less than num_tx_qs, then more than 2938 * one txq share an eq 2939 */ 2940 eqo = &adapter->eq_obj[i % adapter->num_evt_qs]; 2941 status = be_cmd_cq_create(adapter, cq, &eqo->q, false, 3); 2942 if (status) 2943 return status; 2944 2945 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN, 2946 sizeof(struct be_eth_wrb)); 2947 if (status) 2948 return status; 2949 2950 status = be_cmd_txq_create(adapter, txo); 2951 if (status) 2952 return status; 2953 2954 netif_set_xps_queue(adapter->netdev, eqo->affinity_mask, 2955 eqo->idx); 2956 } 2957 2958 dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n", 2959 adapter->num_tx_qs); 2960 return 0; 2961 } 2962 2963 static void be_rx_cqs_destroy(struct be_adapter *adapter) 2964 { 2965 struct be_queue_info *q; 2966 struct be_rx_obj *rxo; 2967 int i; 2968 2969 for_all_rx_queues(adapter, rxo, i) { 2970 q = &rxo->cq; 2971 if (q->created) 2972 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2973 be_queue_free(adapter, q); 2974 } 2975 } 2976 2977 static int be_rx_cqs_create(struct be_adapter *adapter) 2978 { 2979 struct be_queue_info *eq, *cq; 2980 struct be_rx_obj *rxo; 2981 int rc, i; 2982 2983 adapter->num_rss_qs = 2984 min(adapter->num_evt_qs, adapter->cfg_num_rx_irqs); 2985 2986 /* We'll use RSS only if atleast 2 RSS rings are supported. */ 2987 if (adapter->num_rss_qs < 2) 2988 adapter->num_rss_qs = 0; 2989 2990 adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq; 2991 2992 /* When the interface is not capable of RSS rings (and there is no 2993 * need to create a default RXQ) we'll still need one RXQ 2994 */ 2995 if (adapter->num_rx_qs == 0) 2996 adapter->num_rx_qs = 1; 2997 2998 adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE; 2999 for_all_rx_queues(adapter, rxo, i) { 3000 rxo->adapter = adapter; 3001 cq = &rxo->cq; 3002 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN, 3003 sizeof(struct be_eth_rx_compl)); 3004 if (rc) 3005 return rc; 3006 3007 u64_stats_init(&rxo->stats.sync); 3008 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; 3009 rc = be_cmd_cq_create(adapter, cq, eq, false, 3); 3010 if (rc) 3011 return rc; 3012 } 3013 3014 dev_info(&adapter->pdev->dev, 3015 "created %d RX queue(s)\n", adapter->num_rx_qs); 3016 return 0; 3017 } 3018 3019 static irqreturn_t be_intx(int irq, void *dev) 3020 { 3021 struct be_eq_obj *eqo = dev; 3022 struct be_adapter *adapter = eqo->adapter; 3023 int num_evts = 0; 3024 3025 /* IRQ is not expected when NAPI is scheduled as the EQ 3026 * will not be armed. 3027 * But, this can happen on Lancer INTx where it takes 3028 * a while to de-assert INTx or in BE2 where occasionaly 3029 * an interrupt may be raised even when EQ is unarmed. 3030 * If NAPI is already scheduled, then counting & notifying 3031 * events will orphan them. 3032 */ 3033 if (napi_schedule_prep(&eqo->napi)) { 3034 num_evts = events_get(eqo); 3035 __napi_schedule(&eqo->napi); 3036 if (num_evts) 3037 eqo->spurious_intr = 0; 3038 } 3039 be_eq_notify(adapter, eqo->q.id, false, true, num_evts, 0); 3040 3041 /* Return IRQ_HANDLED only for the the first spurious intr 3042 * after a valid intr to stop the kernel from branding 3043 * this irq as a bad one! 3044 */ 3045 if (num_evts || eqo->spurious_intr++ == 0) 3046 return IRQ_HANDLED; 3047 else 3048 return IRQ_NONE; 3049 } 3050 3051 static irqreturn_t be_msix(int irq, void *dev) 3052 { 3053 struct be_eq_obj *eqo = dev; 3054 3055 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0); 3056 napi_schedule(&eqo->napi); 3057 return IRQ_HANDLED; 3058 } 3059 3060 static inline bool do_gro(struct be_rx_compl_info *rxcp) 3061 { 3062 return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false; 3063 } 3064 3065 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi, 3066 int budget, int polling) 3067 { 3068 struct be_adapter *adapter = rxo->adapter; 3069 struct be_queue_info *rx_cq = &rxo->cq; 3070 struct be_rx_compl_info *rxcp; 3071 u32 work_done; 3072 u32 frags_consumed = 0; 3073 3074 for (work_done = 0; work_done < budget; work_done++) { 3075 rxcp = be_rx_compl_get(rxo); 3076 if (!rxcp) 3077 break; 3078 3079 /* Is it a flush compl that has no data */ 3080 if (unlikely(rxcp->num_rcvd == 0)) 3081 goto loop_continue; 3082 3083 /* Discard compl with partial DMA Lancer B0 */ 3084 if (unlikely(!rxcp->pkt_size)) { 3085 be_rx_compl_discard(rxo, rxcp); 3086 goto loop_continue; 3087 } 3088 3089 /* On BE drop pkts that arrive due to imperfect filtering in 3090 * promiscuous mode on some skews 3091 */ 3092 if (unlikely(rxcp->port != adapter->port_num && 3093 !lancer_chip(adapter))) { 3094 be_rx_compl_discard(rxo, rxcp); 3095 goto loop_continue; 3096 } 3097 3098 /* Don't do gro when we're busy_polling */ 3099 if (do_gro(rxcp) && polling != BUSY_POLLING) 3100 be_rx_compl_process_gro(rxo, napi, rxcp); 3101 else 3102 be_rx_compl_process(rxo, napi, rxcp); 3103 3104 loop_continue: 3105 frags_consumed += rxcp->num_rcvd; 3106 be_rx_stats_update(rxo, rxcp); 3107 } 3108 3109 if (work_done) { 3110 be_cq_notify(adapter, rx_cq->id, true, work_done); 3111 3112 /* When an rx-obj gets into post_starved state, just 3113 * let be_worker do the posting. 3114 */ 3115 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM && 3116 !rxo->rx_post_starved) 3117 be_post_rx_frags(rxo, GFP_ATOMIC, 3118 max_t(u32, MAX_RX_POST, 3119 frags_consumed)); 3120 } 3121 3122 return work_done; 3123 } 3124 3125 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status) 3126 { 3127 switch (status) { 3128 case BE_TX_COMP_HDR_PARSE_ERR: 3129 tx_stats(txo)->tx_hdr_parse_err++; 3130 break; 3131 case BE_TX_COMP_NDMA_ERR: 3132 tx_stats(txo)->tx_dma_err++; 3133 break; 3134 case BE_TX_COMP_ACL_ERR: 3135 tx_stats(txo)->tx_spoof_check_err++; 3136 break; 3137 } 3138 } 3139 3140 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status) 3141 { 3142 switch (status) { 3143 case LANCER_TX_COMP_LSO_ERR: 3144 tx_stats(txo)->tx_tso_err++; 3145 break; 3146 case LANCER_TX_COMP_HSW_DROP_MAC_ERR: 3147 case LANCER_TX_COMP_HSW_DROP_VLAN_ERR: 3148 tx_stats(txo)->tx_spoof_check_err++; 3149 break; 3150 case LANCER_TX_COMP_QINQ_ERR: 3151 tx_stats(txo)->tx_qinq_err++; 3152 break; 3153 case LANCER_TX_COMP_PARITY_ERR: 3154 tx_stats(txo)->tx_internal_parity_err++; 3155 break; 3156 case LANCER_TX_COMP_DMA_ERR: 3157 tx_stats(txo)->tx_dma_err++; 3158 break; 3159 } 3160 } 3161 3162 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo, 3163 int idx) 3164 { 3165 int num_wrbs = 0, work_done = 0; 3166 struct be_tx_compl_info *txcp; 3167 3168 while ((txcp = be_tx_compl_get(txo))) { 3169 num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index); 3170 work_done++; 3171 3172 if (txcp->status) { 3173 if (lancer_chip(adapter)) 3174 lancer_update_tx_err(txo, txcp->status); 3175 else 3176 be_update_tx_err(txo, txcp->status); 3177 } 3178 } 3179 3180 if (work_done) { 3181 be_cq_notify(adapter, txo->cq.id, true, work_done); 3182 atomic_sub(num_wrbs, &txo->q.used); 3183 3184 /* As Tx wrbs have been freed up, wake up netdev queue 3185 * if it was stopped due to lack of tx wrbs. */ 3186 if (__netif_subqueue_stopped(adapter->netdev, idx) && 3187 be_can_txq_wake(txo)) { 3188 netif_wake_subqueue(adapter->netdev, idx); 3189 } 3190 3191 u64_stats_update_begin(&tx_stats(txo)->sync_compl); 3192 tx_stats(txo)->tx_compl += work_done; 3193 u64_stats_update_end(&tx_stats(txo)->sync_compl); 3194 } 3195 } 3196 3197 #ifdef CONFIG_NET_RX_BUSY_POLL 3198 static inline bool be_lock_napi(struct be_eq_obj *eqo) 3199 { 3200 bool status = true; 3201 3202 spin_lock(&eqo->lock); /* BH is already disabled */ 3203 if (eqo->state & BE_EQ_LOCKED) { 3204 WARN_ON(eqo->state & BE_EQ_NAPI); 3205 eqo->state |= BE_EQ_NAPI_YIELD; 3206 status = false; 3207 } else { 3208 eqo->state = BE_EQ_NAPI; 3209 } 3210 spin_unlock(&eqo->lock); 3211 return status; 3212 } 3213 3214 static inline void be_unlock_napi(struct be_eq_obj *eqo) 3215 { 3216 spin_lock(&eqo->lock); /* BH is already disabled */ 3217 3218 WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD)); 3219 eqo->state = BE_EQ_IDLE; 3220 3221 spin_unlock(&eqo->lock); 3222 } 3223 3224 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo) 3225 { 3226 bool status = true; 3227 3228 spin_lock_bh(&eqo->lock); 3229 if (eqo->state & BE_EQ_LOCKED) { 3230 eqo->state |= BE_EQ_POLL_YIELD; 3231 status = false; 3232 } else { 3233 eqo->state |= BE_EQ_POLL; 3234 } 3235 spin_unlock_bh(&eqo->lock); 3236 return status; 3237 } 3238 3239 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo) 3240 { 3241 spin_lock_bh(&eqo->lock); 3242 3243 WARN_ON(eqo->state & (BE_EQ_NAPI)); 3244 eqo->state = BE_EQ_IDLE; 3245 3246 spin_unlock_bh(&eqo->lock); 3247 } 3248 3249 static inline void be_enable_busy_poll(struct be_eq_obj *eqo) 3250 { 3251 spin_lock_init(&eqo->lock); 3252 eqo->state = BE_EQ_IDLE; 3253 } 3254 3255 static inline void be_disable_busy_poll(struct be_eq_obj *eqo) 3256 { 3257 local_bh_disable(); 3258 3259 /* It's enough to just acquire napi lock on the eqo to stop 3260 * be_busy_poll() from processing any queueus. 3261 */ 3262 while (!be_lock_napi(eqo)) 3263 mdelay(1); 3264 3265 local_bh_enable(); 3266 } 3267 3268 #else /* CONFIG_NET_RX_BUSY_POLL */ 3269 3270 static inline bool be_lock_napi(struct be_eq_obj *eqo) 3271 { 3272 return true; 3273 } 3274 3275 static inline void be_unlock_napi(struct be_eq_obj *eqo) 3276 { 3277 } 3278 3279 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo) 3280 { 3281 return false; 3282 } 3283 3284 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo) 3285 { 3286 } 3287 3288 static inline void be_enable_busy_poll(struct be_eq_obj *eqo) 3289 { 3290 } 3291 3292 static inline void be_disable_busy_poll(struct be_eq_obj *eqo) 3293 { 3294 } 3295 #endif /* CONFIG_NET_RX_BUSY_POLL */ 3296 3297 int be_poll(struct napi_struct *napi, int budget) 3298 { 3299 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 3300 struct be_adapter *adapter = eqo->adapter; 3301 int max_work = 0, work, i, num_evts; 3302 struct be_rx_obj *rxo; 3303 struct be_tx_obj *txo; 3304 u32 mult_enc = 0; 3305 3306 num_evts = events_get(eqo); 3307 3308 for_all_tx_queues_on_eq(adapter, eqo, txo, i) 3309 be_process_tx(adapter, txo, i); 3310 3311 if (be_lock_napi(eqo)) { 3312 /* This loop will iterate twice for EQ0 in which 3313 * completions of the last RXQ (default one) are also processed 3314 * For other EQs the loop iterates only once 3315 */ 3316 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 3317 work = be_process_rx(rxo, napi, budget, NAPI_POLLING); 3318 max_work = max(work, max_work); 3319 } 3320 be_unlock_napi(eqo); 3321 } else { 3322 max_work = budget; 3323 } 3324 3325 if (is_mcc_eqo(eqo)) 3326 be_process_mcc(adapter); 3327 3328 if (max_work < budget) { 3329 napi_complete(napi); 3330 3331 /* Skyhawk EQ_DB has a provision to set the rearm to interrupt 3332 * delay via a delay multiplier encoding value 3333 */ 3334 if (skyhawk_chip(adapter)) 3335 mult_enc = be_get_eq_delay_mult_enc(eqo); 3336 3337 be_eq_notify(adapter, eqo->q.id, true, false, num_evts, 3338 mult_enc); 3339 } else { 3340 /* As we'll continue in polling mode, count and clear events */ 3341 be_eq_notify(adapter, eqo->q.id, false, false, num_evts, 0); 3342 } 3343 return max_work; 3344 } 3345 3346 #ifdef CONFIG_NET_RX_BUSY_POLL 3347 static int be_busy_poll(struct napi_struct *napi) 3348 { 3349 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 3350 struct be_adapter *adapter = eqo->adapter; 3351 struct be_rx_obj *rxo; 3352 int i, work = 0; 3353 3354 if (!be_lock_busy_poll(eqo)) 3355 return LL_FLUSH_BUSY; 3356 3357 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 3358 work = be_process_rx(rxo, napi, 4, BUSY_POLLING); 3359 if (work) 3360 break; 3361 } 3362 3363 be_unlock_busy_poll(eqo); 3364 return work; 3365 } 3366 #endif 3367 3368 void be_detect_error(struct be_adapter *adapter) 3369 { 3370 u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0; 3371 u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 3372 u32 i; 3373 struct device *dev = &adapter->pdev->dev; 3374 3375 if (be_check_error(adapter, BE_ERROR_HW)) 3376 return; 3377 3378 if (lancer_chip(adapter)) { 3379 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET); 3380 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 3381 be_set_error(adapter, BE_ERROR_UE); 3382 sliport_err1 = ioread32(adapter->db + 3383 SLIPORT_ERROR1_OFFSET); 3384 sliport_err2 = ioread32(adapter->db + 3385 SLIPORT_ERROR2_OFFSET); 3386 /* Do not log error messages if its a FW reset */ 3387 if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 && 3388 sliport_err2 == SLIPORT_ERROR_FW_RESET2) { 3389 dev_info(dev, "Firmware update in progress\n"); 3390 } else { 3391 dev_err(dev, "Error detected in the card\n"); 3392 dev_err(dev, "ERR: sliport status 0x%x\n", 3393 sliport_status); 3394 dev_err(dev, "ERR: sliport error1 0x%x\n", 3395 sliport_err1); 3396 dev_err(dev, "ERR: sliport error2 0x%x\n", 3397 sliport_err2); 3398 } 3399 } 3400 } else { 3401 ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW); 3402 ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH); 3403 ue_lo_mask = ioread32(adapter->pcicfg + 3404 PCICFG_UE_STATUS_LOW_MASK); 3405 ue_hi_mask = ioread32(adapter->pcicfg + 3406 PCICFG_UE_STATUS_HI_MASK); 3407 3408 ue_lo = (ue_lo & ~ue_lo_mask); 3409 ue_hi = (ue_hi & ~ue_hi_mask); 3410 3411 /* On certain platforms BE hardware can indicate spurious UEs. 3412 * Allow HW to stop working completely in case of a real UE. 3413 * Hence not setting the hw_error for UE detection. 3414 */ 3415 3416 if (ue_lo || ue_hi) { 3417 dev_err(dev, "Error detected in the adapter"); 3418 if (skyhawk_chip(adapter)) 3419 be_set_error(adapter, BE_ERROR_UE); 3420 3421 for (i = 0; ue_lo; ue_lo >>= 1, i++) { 3422 if (ue_lo & 1) 3423 dev_err(dev, "UE: %s bit set\n", 3424 ue_status_low_desc[i]); 3425 } 3426 for (i = 0; ue_hi; ue_hi >>= 1, i++) { 3427 if (ue_hi & 1) 3428 dev_err(dev, "UE: %s bit set\n", 3429 ue_status_hi_desc[i]); 3430 } 3431 } 3432 } 3433 } 3434 3435 static void be_msix_disable(struct be_adapter *adapter) 3436 { 3437 if (msix_enabled(adapter)) { 3438 pci_disable_msix(adapter->pdev); 3439 adapter->num_msix_vec = 0; 3440 adapter->num_msix_roce_vec = 0; 3441 } 3442 } 3443 3444 static int be_msix_enable(struct be_adapter *adapter) 3445 { 3446 unsigned int i, max_roce_eqs; 3447 struct device *dev = &adapter->pdev->dev; 3448 int num_vec; 3449 3450 /* If RoCE is supported, program the max number of vectors that 3451 * could be used for NIC and RoCE, else, just program the number 3452 * we'll use initially. 3453 */ 3454 if (be_roce_supported(adapter)) { 3455 max_roce_eqs = 3456 be_max_func_eqs(adapter) - be_max_nic_eqs(adapter); 3457 max_roce_eqs = min(max_roce_eqs, num_online_cpus()); 3458 num_vec = be_max_any_irqs(adapter) + max_roce_eqs; 3459 } else { 3460 num_vec = max(adapter->cfg_num_rx_irqs, 3461 adapter->cfg_num_tx_irqs); 3462 } 3463 3464 for (i = 0; i < num_vec; i++) 3465 adapter->msix_entries[i].entry = i; 3466 3467 num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 3468 MIN_MSIX_VECTORS, num_vec); 3469 if (num_vec < 0) 3470 goto fail; 3471 3472 if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) { 3473 adapter->num_msix_roce_vec = num_vec / 2; 3474 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n", 3475 adapter->num_msix_roce_vec); 3476 } 3477 3478 adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec; 3479 3480 dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n", 3481 adapter->num_msix_vec); 3482 return 0; 3483 3484 fail: 3485 dev_warn(dev, "MSIx enable failed\n"); 3486 3487 /* INTx is not supported in VFs, so fail probe if enable_msix fails */ 3488 if (be_virtfn(adapter)) 3489 return num_vec; 3490 return 0; 3491 } 3492 3493 static inline int be_msix_vec_get(struct be_adapter *adapter, 3494 struct be_eq_obj *eqo) 3495 { 3496 return adapter->msix_entries[eqo->msix_idx].vector; 3497 } 3498 3499 static int be_msix_register(struct be_adapter *adapter) 3500 { 3501 struct net_device *netdev = adapter->netdev; 3502 struct be_eq_obj *eqo; 3503 int status, i, vec; 3504 3505 for_all_evt_queues(adapter, eqo, i) { 3506 sprintf(eqo->desc, "%s-q%d", netdev->name, i); 3507 vec = be_msix_vec_get(adapter, eqo); 3508 status = request_irq(vec, be_msix, 0, eqo->desc, eqo); 3509 if (status) 3510 goto err_msix; 3511 3512 irq_set_affinity_hint(vec, eqo->affinity_mask); 3513 } 3514 3515 return 0; 3516 err_msix: 3517 for (i--; i >= 0; i--) { 3518 eqo = &adapter->eq_obj[i]; 3519 free_irq(be_msix_vec_get(adapter, eqo), eqo); 3520 } 3521 dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n", 3522 status); 3523 be_msix_disable(adapter); 3524 return status; 3525 } 3526 3527 static int be_irq_register(struct be_adapter *adapter) 3528 { 3529 struct net_device *netdev = adapter->netdev; 3530 int status; 3531 3532 if (msix_enabled(adapter)) { 3533 status = be_msix_register(adapter); 3534 if (status == 0) 3535 goto done; 3536 /* INTx is not supported for VF */ 3537 if (be_virtfn(adapter)) 3538 return status; 3539 } 3540 3541 /* INTx: only the first EQ is used */ 3542 netdev->irq = adapter->pdev->irq; 3543 status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name, 3544 &adapter->eq_obj[0]); 3545 if (status) { 3546 dev_err(&adapter->pdev->dev, 3547 "INTx request IRQ failed - err %d\n", status); 3548 return status; 3549 } 3550 done: 3551 adapter->isr_registered = true; 3552 return 0; 3553 } 3554 3555 static void be_irq_unregister(struct be_adapter *adapter) 3556 { 3557 struct net_device *netdev = adapter->netdev; 3558 struct be_eq_obj *eqo; 3559 int i, vec; 3560 3561 if (!adapter->isr_registered) 3562 return; 3563 3564 /* INTx */ 3565 if (!msix_enabled(adapter)) { 3566 free_irq(netdev->irq, &adapter->eq_obj[0]); 3567 goto done; 3568 } 3569 3570 /* MSIx */ 3571 for_all_evt_queues(adapter, eqo, i) { 3572 vec = be_msix_vec_get(adapter, eqo); 3573 irq_set_affinity_hint(vec, NULL); 3574 free_irq(vec, eqo); 3575 } 3576 3577 done: 3578 adapter->isr_registered = false; 3579 } 3580 3581 static void be_rx_qs_destroy(struct be_adapter *adapter) 3582 { 3583 struct rss_info *rss = &adapter->rss_info; 3584 struct be_queue_info *q; 3585 struct be_rx_obj *rxo; 3586 int i; 3587 3588 for_all_rx_queues(adapter, rxo, i) { 3589 q = &rxo->q; 3590 if (q->created) { 3591 /* If RXQs are destroyed while in an "out of buffer" 3592 * state, there is a possibility of an HW stall on 3593 * Lancer. So, post 64 buffers to each queue to relieve 3594 * the "out of buffer" condition. 3595 * Make sure there's space in the RXQ before posting. 3596 */ 3597 if (lancer_chip(adapter)) { 3598 be_rx_cq_clean(rxo); 3599 if (atomic_read(&q->used) == 0) 3600 be_post_rx_frags(rxo, GFP_KERNEL, 3601 MAX_RX_POST); 3602 } 3603 3604 be_cmd_rxq_destroy(adapter, q); 3605 be_rx_cq_clean(rxo); 3606 be_rxq_clean(rxo); 3607 } 3608 be_queue_free(adapter, q); 3609 } 3610 3611 if (rss->rss_flags) { 3612 rss->rss_flags = RSS_ENABLE_NONE; 3613 be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3614 128, rss->rss_hkey); 3615 } 3616 } 3617 3618 static void be_disable_if_filters(struct be_adapter *adapter) 3619 { 3620 be_dev_mac_del(adapter, adapter->pmac_id[0]); 3621 be_clear_uc_list(adapter); 3622 be_clear_mc_list(adapter); 3623 3624 /* The IFACE flags are enabled in the open path and cleared 3625 * in the close path. When a VF gets detached from the host and 3626 * assigned to a VM the following happens: 3627 * - VF's IFACE flags get cleared in the detach path 3628 * - IFACE create is issued by the VF in the attach path 3629 * Due to a bug in the BE3/Skyhawk-R FW 3630 * (Lancer FW doesn't have the bug), the IFACE capability flags 3631 * specified along with the IFACE create cmd issued by a VF are not 3632 * honoured by FW. As a consequence, if a *new* driver 3633 * (that enables/disables IFACE flags in open/close) 3634 * is loaded in the host and an *old* driver is * used by a VM/VF, 3635 * the IFACE gets created *without* the needed flags. 3636 * To avoid this, disable RX-filter flags only for Lancer. 3637 */ 3638 if (lancer_chip(adapter)) { 3639 be_cmd_rx_filter(adapter, BE_IF_ALL_FILT_FLAGS, OFF); 3640 adapter->if_flags &= ~BE_IF_ALL_FILT_FLAGS; 3641 } 3642 } 3643 3644 static int be_close(struct net_device *netdev) 3645 { 3646 struct be_adapter *adapter = netdev_priv(netdev); 3647 struct be_eq_obj *eqo; 3648 int i; 3649 3650 /* This protection is needed as be_close() may be called even when the 3651 * adapter is in cleared state (after eeh perm failure) 3652 */ 3653 if (!(adapter->flags & BE_FLAGS_SETUP_DONE)) 3654 return 0; 3655 3656 /* Before attempting cleanup ensure all the pending cmds in the 3657 * config_wq have finished execution 3658 */ 3659 flush_workqueue(be_wq); 3660 3661 be_disable_if_filters(adapter); 3662 3663 if (adapter->flags & BE_FLAGS_NAPI_ENABLED) { 3664 for_all_evt_queues(adapter, eqo, i) { 3665 napi_disable(&eqo->napi); 3666 be_disable_busy_poll(eqo); 3667 } 3668 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED; 3669 } 3670 3671 be_async_mcc_disable(adapter); 3672 3673 /* Wait for all pending tx completions to arrive so that 3674 * all tx skbs are freed. 3675 */ 3676 netif_tx_disable(netdev); 3677 be_tx_compl_clean(adapter); 3678 3679 be_rx_qs_destroy(adapter); 3680 3681 for_all_evt_queues(adapter, eqo, i) { 3682 if (msix_enabled(adapter)) 3683 synchronize_irq(be_msix_vec_get(adapter, eqo)); 3684 else 3685 synchronize_irq(netdev->irq); 3686 be_eq_clean(eqo); 3687 } 3688 3689 be_irq_unregister(adapter); 3690 3691 return 0; 3692 } 3693 3694 static int be_rx_qs_create(struct be_adapter *adapter) 3695 { 3696 struct rss_info *rss = &adapter->rss_info; 3697 u8 rss_key[RSS_HASH_KEY_LEN]; 3698 struct be_rx_obj *rxo; 3699 int rc, i, j; 3700 3701 for_all_rx_queues(adapter, rxo, i) { 3702 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN, 3703 sizeof(struct be_eth_rx_d)); 3704 if (rc) 3705 return rc; 3706 } 3707 3708 if (adapter->need_def_rxq || !adapter->num_rss_qs) { 3709 rxo = default_rxo(adapter); 3710 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3711 rx_frag_size, adapter->if_handle, 3712 false, &rxo->rss_id); 3713 if (rc) 3714 return rc; 3715 } 3716 3717 for_all_rss_queues(adapter, rxo, i) { 3718 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3719 rx_frag_size, adapter->if_handle, 3720 true, &rxo->rss_id); 3721 if (rc) 3722 return rc; 3723 } 3724 3725 if (be_multi_rxq(adapter)) { 3726 for (j = 0; j < RSS_INDIR_TABLE_LEN; j += adapter->num_rss_qs) { 3727 for_all_rss_queues(adapter, rxo, i) { 3728 if ((j + i) >= RSS_INDIR_TABLE_LEN) 3729 break; 3730 rss->rsstable[j + i] = rxo->rss_id; 3731 rss->rss_queue[j + i] = i; 3732 } 3733 } 3734 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 | 3735 RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6; 3736 3737 if (!BEx_chip(adapter)) 3738 rss->rss_flags |= RSS_ENABLE_UDP_IPV4 | 3739 RSS_ENABLE_UDP_IPV6; 3740 3741 netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN); 3742 rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3743 RSS_INDIR_TABLE_LEN, rss_key); 3744 if (rc) { 3745 rss->rss_flags = RSS_ENABLE_NONE; 3746 return rc; 3747 } 3748 3749 memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN); 3750 } else { 3751 /* Disable RSS, if only default RX Q is created */ 3752 rss->rss_flags = RSS_ENABLE_NONE; 3753 } 3754 3755 3756 /* Post 1 less than RXQ-len to avoid head being equal to tail, 3757 * which is a queue empty condition 3758 */ 3759 for_all_rx_queues(adapter, rxo, i) 3760 be_post_rx_frags(rxo, GFP_KERNEL, RX_Q_LEN - 1); 3761 3762 return 0; 3763 } 3764 3765 static int be_enable_if_filters(struct be_adapter *adapter) 3766 { 3767 int status; 3768 3769 status = be_cmd_rx_filter(adapter, BE_IF_FILT_FLAGS_BASIC, ON); 3770 if (status) 3771 return status; 3772 3773 /* For BE3 VFs, the PF programs the initial MAC address */ 3774 if (!(BEx_chip(adapter) && be_virtfn(adapter))) { 3775 status = be_dev_mac_add(adapter, adapter->netdev->dev_addr); 3776 if (status) 3777 return status; 3778 ether_addr_copy(adapter->dev_mac, adapter->netdev->dev_addr); 3779 } 3780 3781 if (adapter->vlans_added) 3782 be_vid_config(adapter); 3783 3784 __be_set_rx_mode(adapter); 3785 3786 return 0; 3787 } 3788 3789 static int be_open(struct net_device *netdev) 3790 { 3791 struct be_adapter *adapter = netdev_priv(netdev); 3792 struct be_eq_obj *eqo; 3793 struct be_rx_obj *rxo; 3794 struct be_tx_obj *txo; 3795 u8 link_status; 3796 int status, i; 3797 3798 status = be_rx_qs_create(adapter); 3799 if (status) 3800 goto err; 3801 3802 status = be_enable_if_filters(adapter); 3803 if (status) 3804 goto err; 3805 3806 status = be_irq_register(adapter); 3807 if (status) 3808 goto err; 3809 3810 for_all_rx_queues(adapter, rxo, i) 3811 be_cq_notify(adapter, rxo->cq.id, true, 0); 3812 3813 for_all_tx_queues(adapter, txo, i) 3814 be_cq_notify(adapter, txo->cq.id, true, 0); 3815 3816 be_async_mcc_enable(adapter); 3817 3818 for_all_evt_queues(adapter, eqo, i) { 3819 napi_enable(&eqo->napi); 3820 be_enable_busy_poll(eqo); 3821 be_eq_notify(adapter, eqo->q.id, true, true, 0, 0); 3822 } 3823 adapter->flags |= BE_FLAGS_NAPI_ENABLED; 3824 3825 status = be_cmd_link_status_query(adapter, NULL, &link_status, 0); 3826 if (!status) 3827 be_link_status_update(adapter, link_status); 3828 3829 netif_tx_start_all_queues(netdev); 3830 if (skyhawk_chip(adapter)) 3831 udp_tunnel_get_rx_info(netdev); 3832 3833 return 0; 3834 err: 3835 be_close(adapter->netdev); 3836 return -EIO; 3837 } 3838 3839 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac) 3840 { 3841 u32 addr; 3842 3843 addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0); 3844 3845 mac[5] = (u8)(addr & 0xFF); 3846 mac[4] = (u8)((addr >> 8) & 0xFF); 3847 mac[3] = (u8)((addr >> 16) & 0xFF); 3848 /* Use the OUI from the current MAC address */ 3849 memcpy(mac, adapter->netdev->dev_addr, 3); 3850 } 3851 3852 /* 3853 * Generate a seed MAC address from the PF MAC Address using jhash. 3854 * MAC Address for VFs are assigned incrementally starting from the seed. 3855 * These addresses are programmed in the ASIC by the PF and the VF driver 3856 * queries for the MAC address during its probe. 3857 */ 3858 static int be_vf_eth_addr_config(struct be_adapter *adapter) 3859 { 3860 u32 vf; 3861 int status = 0; 3862 u8 mac[ETH_ALEN]; 3863 struct be_vf_cfg *vf_cfg; 3864 3865 be_vf_eth_addr_generate(adapter, mac); 3866 3867 for_all_vfs(adapter, vf_cfg, vf) { 3868 if (BEx_chip(adapter)) 3869 status = be_cmd_pmac_add(adapter, mac, 3870 vf_cfg->if_handle, 3871 &vf_cfg->pmac_id, vf + 1); 3872 else 3873 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 3874 vf + 1); 3875 3876 if (status) 3877 dev_err(&adapter->pdev->dev, 3878 "Mac address assignment failed for VF %d\n", 3879 vf); 3880 else 3881 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3882 3883 mac[5] += 1; 3884 } 3885 return status; 3886 } 3887 3888 static int be_vfs_mac_query(struct be_adapter *adapter) 3889 { 3890 int status, vf; 3891 u8 mac[ETH_ALEN]; 3892 struct be_vf_cfg *vf_cfg; 3893 3894 for_all_vfs(adapter, vf_cfg, vf) { 3895 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id, 3896 mac, vf_cfg->if_handle, 3897 false, vf+1); 3898 if (status) 3899 return status; 3900 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3901 } 3902 return 0; 3903 } 3904 3905 static void be_vf_clear(struct be_adapter *adapter) 3906 { 3907 struct be_vf_cfg *vf_cfg; 3908 u32 vf; 3909 3910 if (pci_vfs_assigned(adapter->pdev)) { 3911 dev_warn(&adapter->pdev->dev, 3912 "VFs are assigned to VMs: not disabling VFs\n"); 3913 goto done; 3914 } 3915 3916 pci_disable_sriov(adapter->pdev); 3917 3918 for_all_vfs(adapter, vf_cfg, vf) { 3919 if (BEx_chip(adapter)) 3920 be_cmd_pmac_del(adapter, vf_cfg->if_handle, 3921 vf_cfg->pmac_id, vf + 1); 3922 else 3923 be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle, 3924 vf + 1); 3925 3926 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1); 3927 } 3928 3929 if (BE3_chip(adapter)) 3930 be_cmd_set_hsw_config(adapter, 0, 0, 3931 adapter->if_handle, 3932 PORT_FWD_TYPE_PASSTHRU, 0); 3933 done: 3934 kfree(adapter->vf_cfg); 3935 adapter->num_vfs = 0; 3936 adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED; 3937 } 3938 3939 static void be_clear_queues(struct be_adapter *adapter) 3940 { 3941 be_mcc_queues_destroy(adapter); 3942 be_rx_cqs_destroy(adapter); 3943 be_tx_queues_destroy(adapter); 3944 be_evt_queues_destroy(adapter); 3945 } 3946 3947 static void be_cancel_worker(struct be_adapter *adapter) 3948 { 3949 if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) { 3950 cancel_delayed_work_sync(&adapter->work); 3951 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED; 3952 } 3953 } 3954 3955 static void be_cancel_err_detection(struct be_adapter *adapter) 3956 { 3957 struct be_error_recovery *err_rec = &adapter->error_recovery; 3958 3959 if (!be_err_recovery_workq) 3960 return; 3961 3962 if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) { 3963 cancel_delayed_work_sync(&err_rec->err_detection_work); 3964 adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED; 3965 } 3966 } 3967 3968 static void be_disable_vxlan_offloads(struct be_adapter *adapter) 3969 { 3970 struct net_device *netdev = adapter->netdev; 3971 3972 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) 3973 be_cmd_manage_iface(adapter, adapter->if_handle, 3974 OP_CONVERT_TUNNEL_TO_NORMAL); 3975 3976 if (adapter->vxlan_port) 3977 be_cmd_set_vxlan_port(adapter, 0); 3978 3979 adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS; 3980 adapter->vxlan_port = 0; 3981 3982 netdev->hw_enc_features = 0; 3983 netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL); 3984 netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL); 3985 } 3986 3987 static void be_calculate_vf_res(struct be_adapter *adapter, u16 num_vfs, 3988 struct be_resources *vft_res) 3989 { 3990 struct be_resources res = adapter->pool_res; 3991 u32 vf_if_cap_flags = res.vf_if_cap_flags; 3992 struct be_resources res_mod = {0}; 3993 u16 num_vf_qs = 1; 3994 3995 /* Distribute the queue resources among the PF and it's VFs */ 3996 if (num_vfs) { 3997 /* Divide the rx queues evenly among the VFs and the PF, capped 3998 * at VF-EQ-count. Any remainder queues belong to the PF. 3999 */ 4000 num_vf_qs = min(SH_VF_MAX_NIC_EQS, 4001 res.max_rss_qs / (num_vfs + 1)); 4002 4003 /* Skyhawk-R chip supports only MAX_PORT_RSS_TABLES 4004 * RSS Tables per port. Provide RSS on VFs, only if number of 4005 * VFs requested is less than it's PF Pool's RSS Tables limit. 4006 */ 4007 if (num_vfs >= be_max_pf_pool_rss_tables(adapter)) 4008 num_vf_qs = 1; 4009 } 4010 4011 /* Resource with fields set to all '1's by GET_PROFILE_CONFIG cmd, 4012 * which are modifiable using SET_PROFILE_CONFIG cmd. 4013 */ 4014 be_cmd_get_profile_config(adapter, &res_mod, NULL, ACTIVE_PROFILE_TYPE, 4015 RESOURCE_MODIFIABLE, 0); 4016 4017 /* If RSS IFACE capability flags are modifiable for a VF, set the 4018 * capability flag as valid and set RSS and DEFQ_RSS IFACE flags if 4019 * more than 1 RSSQ is available for a VF. 4020 * Otherwise, provision only 1 queue pair for VF. 4021 */ 4022 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_RSS) { 4023 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT); 4024 if (num_vf_qs > 1) { 4025 vf_if_cap_flags |= BE_IF_FLAGS_RSS; 4026 if (res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS) 4027 vf_if_cap_flags |= BE_IF_FLAGS_DEFQ_RSS; 4028 } else { 4029 vf_if_cap_flags &= ~(BE_IF_FLAGS_RSS | 4030 BE_IF_FLAGS_DEFQ_RSS); 4031 } 4032 } else { 4033 num_vf_qs = 1; 4034 } 4035 4036 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 4037 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT); 4038 vf_if_cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 4039 } 4040 4041 vft_res->vf_if_cap_flags = vf_if_cap_flags; 4042 vft_res->max_rx_qs = num_vf_qs; 4043 vft_res->max_rss_qs = num_vf_qs; 4044 vft_res->max_tx_qs = res.max_tx_qs / (num_vfs + 1); 4045 vft_res->max_cq_count = res.max_cq_count / (num_vfs + 1); 4046 4047 /* Distribute unicast MACs, VLANs, IFACE count and MCCQ count equally 4048 * among the PF and it's VFs, if the fields are changeable 4049 */ 4050 if (res_mod.max_uc_mac == FIELD_MODIFIABLE) 4051 vft_res->max_uc_mac = res.max_uc_mac / (num_vfs + 1); 4052 4053 if (res_mod.max_vlans == FIELD_MODIFIABLE) 4054 vft_res->max_vlans = res.max_vlans / (num_vfs + 1); 4055 4056 if (res_mod.max_iface_count == FIELD_MODIFIABLE) 4057 vft_res->max_iface_count = res.max_iface_count / (num_vfs + 1); 4058 4059 if (res_mod.max_mcc_count == FIELD_MODIFIABLE) 4060 vft_res->max_mcc_count = res.max_mcc_count / (num_vfs + 1); 4061 } 4062 4063 static void be_if_destroy(struct be_adapter *adapter) 4064 { 4065 be_cmd_if_destroy(adapter, adapter->if_handle, 0); 4066 4067 kfree(adapter->pmac_id); 4068 adapter->pmac_id = NULL; 4069 4070 kfree(adapter->mc_list); 4071 adapter->mc_list = NULL; 4072 4073 kfree(adapter->uc_list); 4074 adapter->uc_list = NULL; 4075 } 4076 4077 static int be_clear(struct be_adapter *adapter) 4078 { 4079 struct pci_dev *pdev = adapter->pdev; 4080 struct be_resources vft_res = {0}; 4081 4082 be_cancel_worker(adapter); 4083 4084 flush_workqueue(be_wq); 4085 4086 if (sriov_enabled(adapter)) 4087 be_vf_clear(adapter); 4088 4089 /* Re-configure FW to distribute resources evenly across max-supported 4090 * number of VFs, only when VFs are not already enabled. 4091 */ 4092 if (skyhawk_chip(adapter) && be_physfn(adapter) && 4093 !pci_vfs_assigned(pdev)) { 4094 be_calculate_vf_res(adapter, 4095 pci_sriov_get_totalvfs(pdev), 4096 &vft_res); 4097 be_cmd_set_sriov_config(adapter, adapter->pool_res, 4098 pci_sriov_get_totalvfs(pdev), 4099 &vft_res); 4100 } 4101 4102 be_disable_vxlan_offloads(adapter); 4103 4104 be_if_destroy(adapter); 4105 4106 be_clear_queues(adapter); 4107 4108 be_msix_disable(adapter); 4109 adapter->flags &= ~BE_FLAGS_SETUP_DONE; 4110 return 0; 4111 } 4112 4113 static int be_vfs_if_create(struct be_adapter *adapter) 4114 { 4115 struct be_resources res = {0}; 4116 u32 cap_flags, en_flags, vf; 4117 struct be_vf_cfg *vf_cfg; 4118 int status; 4119 4120 /* If a FW profile exists, then cap_flags are updated */ 4121 cap_flags = BE_VF_IF_EN_FLAGS; 4122 4123 for_all_vfs(adapter, vf_cfg, vf) { 4124 if (!BE3_chip(adapter)) { 4125 status = be_cmd_get_profile_config(adapter, &res, NULL, 4126 ACTIVE_PROFILE_TYPE, 4127 RESOURCE_LIMITS, 4128 vf + 1); 4129 if (!status) { 4130 cap_flags = res.if_cap_flags; 4131 /* Prevent VFs from enabling VLAN promiscuous 4132 * mode 4133 */ 4134 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 4135 } 4136 } 4137 4138 /* PF should enable IF flags during proxy if_create call */ 4139 en_flags = cap_flags & BE_VF_IF_EN_FLAGS; 4140 status = be_cmd_if_create(adapter, cap_flags, en_flags, 4141 &vf_cfg->if_handle, vf + 1); 4142 if (status) 4143 return status; 4144 } 4145 4146 return 0; 4147 } 4148 4149 static int be_vf_setup_init(struct be_adapter *adapter) 4150 { 4151 struct be_vf_cfg *vf_cfg; 4152 int vf; 4153 4154 adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg), 4155 GFP_KERNEL); 4156 if (!adapter->vf_cfg) 4157 return -ENOMEM; 4158 4159 for_all_vfs(adapter, vf_cfg, vf) { 4160 vf_cfg->if_handle = -1; 4161 vf_cfg->pmac_id = -1; 4162 } 4163 return 0; 4164 } 4165 4166 static int be_vf_setup(struct be_adapter *adapter) 4167 { 4168 struct device *dev = &adapter->pdev->dev; 4169 struct be_vf_cfg *vf_cfg; 4170 int status, old_vfs, vf; 4171 bool spoofchk; 4172 4173 old_vfs = pci_num_vf(adapter->pdev); 4174 4175 status = be_vf_setup_init(adapter); 4176 if (status) 4177 goto err; 4178 4179 if (old_vfs) { 4180 for_all_vfs(adapter, vf_cfg, vf) { 4181 status = be_cmd_get_if_id(adapter, vf_cfg, vf); 4182 if (status) 4183 goto err; 4184 } 4185 4186 status = be_vfs_mac_query(adapter); 4187 if (status) 4188 goto err; 4189 } else { 4190 status = be_vfs_if_create(adapter); 4191 if (status) 4192 goto err; 4193 4194 status = be_vf_eth_addr_config(adapter); 4195 if (status) 4196 goto err; 4197 } 4198 4199 for_all_vfs(adapter, vf_cfg, vf) { 4200 /* Allow VFs to programs MAC/VLAN filters */ 4201 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges, 4202 vf + 1); 4203 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 4204 status = be_cmd_set_fn_privileges(adapter, 4205 vf_cfg->privileges | 4206 BE_PRIV_FILTMGMT, 4207 vf + 1); 4208 if (!status) { 4209 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 4210 dev_info(dev, "VF%d has FILTMGMT privilege\n", 4211 vf); 4212 } 4213 } 4214 4215 /* Allow full available bandwidth */ 4216 if (!old_vfs) 4217 be_cmd_config_qos(adapter, 0, 0, vf + 1); 4218 4219 status = be_cmd_get_hsw_config(adapter, NULL, vf + 1, 4220 vf_cfg->if_handle, NULL, 4221 &spoofchk); 4222 if (!status) 4223 vf_cfg->spoofchk = spoofchk; 4224 4225 if (!old_vfs) { 4226 be_cmd_enable_vf(adapter, vf + 1); 4227 be_cmd_set_logical_link_config(adapter, 4228 IFLA_VF_LINK_STATE_AUTO, 4229 vf+1); 4230 } 4231 } 4232 4233 if (!old_vfs) { 4234 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs); 4235 if (status) { 4236 dev_err(dev, "SRIOV enable failed\n"); 4237 adapter->num_vfs = 0; 4238 goto err; 4239 } 4240 } 4241 4242 if (BE3_chip(adapter)) { 4243 /* On BE3, enable VEB only when SRIOV is enabled */ 4244 status = be_cmd_set_hsw_config(adapter, 0, 0, 4245 adapter->if_handle, 4246 PORT_FWD_TYPE_VEB, 0); 4247 if (status) 4248 goto err; 4249 } 4250 4251 adapter->flags |= BE_FLAGS_SRIOV_ENABLED; 4252 return 0; 4253 err: 4254 dev_err(dev, "VF setup failed\n"); 4255 be_vf_clear(adapter); 4256 return status; 4257 } 4258 4259 /* Converting function_mode bits on BE3 to SH mc_type enums */ 4260 4261 static u8 be_convert_mc_type(u32 function_mode) 4262 { 4263 if (function_mode & VNIC_MODE && function_mode & QNQ_MODE) 4264 return vNIC1; 4265 else if (function_mode & QNQ_MODE) 4266 return FLEX10; 4267 else if (function_mode & VNIC_MODE) 4268 return vNIC2; 4269 else if (function_mode & UMC_ENABLED) 4270 return UMC; 4271 else 4272 return MC_NONE; 4273 } 4274 4275 /* On BE2/BE3 FW does not suggest the supported limits */ 4276 static void BEx_get_resources(struct be_adapter *adapter, 4277 struct be_resources *res) 4278 { 4279 bool use_sriov = adapter->num_vfs ? 1 : 0; 4280 4281 if (be_physfn(adapter)) 4282 res->max_uc_mac = BE_UC_PMAC_COUNT; 4283 else 4284 res->max_uc_mac = BE_VF_UC_PMAC_COUNT; 4285 4286 adapter->mc_type = be_convert_mc_type(adapter->function_mode); 4287 4288 if (be_is_mc(adapter)) { 4289 /* Assuming that there are 4 channels per port, 4290 * when multi-channel is enabled 4291 */ 4292 if (be_is_qnq_mode(adapter)) 4293 res->max_vlans = BE_NUM_VLANS_SUPPORTED/8; 4294 else 4295 /* In a non-qnq multichannel mode, the pvid 4296 * takes up one vlan entry 4297 */ 4298 res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1; 4299 } else { 4300 res->max_vlans = BE_NUM_VLANS_SUPPORTED; 4301 } 4302 4303 res->max_mcast_mac = BE_MAX_MC; 4304 4305 /* 1) For BE3 1Gb ports, FW does not support multiple TXQs 4306 * 2) Create multiple TX rings on a BE3-R multi-channel interface 4307 * *only* if it is RSS-capable. 4308 */ 4309 if (BE2_chip(adapter) || use_sriov || (adapter->port_num > 1) || 4310 be_virtfn(adapter) || 4311 (be_is_mc(adapter) && 4312 !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) { 4313 res->max_tx_qs = 1; 4314 } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) { 4315 struct be_resources super_nic_res = {0}; 4316 4317 /* On a SuperNIC profile, the driver needs to use the 4318 * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits 4319 */ 4320 be_cmd_get_profile_config(adapter, &super_nic_res, NULL, 4321 ACTIVE_PROFILE_TYPE, RESOURCE_LIMITS, 4322 0); 4323 /* Some old versions of BE3 FW don't report max_tx_qs value */ 4324 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS; 4325 } else { 4326 res->max_tx_qs = BE3_MAX_TX_QS; 4327 } 4328 4329 if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) && 4330 !use_sriov && be_physfn(adapter)) 4331 res->max_rss_qs = (adapter->be3_native) ? 4332 BE3_MAX_RSS_QS : BE2_MAX_RSS_QS; 4333 res->max_rx_qs = res->max_rss_qs + 1; 4334 4335 if (be_physfn(adapter)) 4336 res->max_evt_qs = (be_max_vfs(adapter) > 0) ? 4337 BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS; 4338 else 4339 res->max_evt_qs = 1; 4340 4341 res->if_cap_flags = BE_IF_CAP_FLAGS_WANT; 4342 res->if_cap_flags &= ~BE_IF_FLAGS_DEFQ_RSS; 4343 if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS)) 4344 res->if_cap_flags &= ~BE_IF_FLAGS_RSS; 4345 } 4346 4347 static void be_setup_init(struct be_adapter *adapter) 4348 { 4349 adapter->vlan_prio_bmap = 0xff; 4350 adapter->phy.link_speed = -1; 4351 adapter->if_handle = -1; 4352 adapter->be3_native = false; 4353 adapter->if_flags = 0; 4354 adapter->phy_state = BE_UNKNOWN_PHY_STATE; 4355 if (be_physfn(adapter)) 4356 adapter->cmd_privileges = MAX_PRIVILEGES; 4357 else 4358 adapter->cmd_privileges = MIN_PRIVILEGES; 4359 } 4360 4361 /* HW supports only MAX_PORT_RSS_TABLES RSS Policy Tables per port. 4362 * However, this HW limitation is not exposed to the host via any SLI cmd. 4363 * As a result, in the case of SRIOV and in particular multi-partition configs 4364 * the driver needs to calcuate a proportional share of RSS Tables per PF-pool 4365 * for distribution between the VFs. This self-imposed limit will determine the 4366 * no: of VFs for which RSS can be enabled. 4367 */ 4368 void be_calculate_pf_pool_rss_tables(struct be_adapter *adapter) 4369 { 4370 struct be_port_resources port_res = {0}; 4371 u8 rss_tables_on_port; 4372 u16 max_vfs = be_max_vfs(adapter); 4373 4374 be_cmd_get_profile_config(adapter, NULL, &port_res, SAVED_PROFILE_TYPE, 4375 RESOURCE_LIMITS, 0); 4376 4377 rss_tables_on_port = MAX_PORT_RSS_TABLES - port_res.nic_pfs; 4378 4379 /* Each PF Pool's RSS Tables limit = 4380 * PF's Max VFs / Total_Max_VFs on Port * RSS Tables on Port 4381 */ 4382 adapter->pool_res.max_rss_tables = 4383 max_vfs * rss_tables_on_port / port_res.max_vfs; 4384 } 4385 4386 static int be_get_sriov_config(struct be_adapter *adapter) 4387 { 4388 struct be_resources res = {0}; 4389 int max_vfs, old_vfs; 4390 4391 be_cmd_get_profile_config(adapter, &res, NULL, ACTIVE_PROFILE_TYPE, 4392 RESOURCE_LIMITS, 0); 4393 4394 /* Some old versions of BE3 FW don't report max_vfs value */ 4395 if (BE3_chip(adapter) && !res.max_vfs) { 4396 max_vfs = pci_sriov_get_totalvfs(adapter->pdev); 4397 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0; 4398 } 4399 4400 adapter->pool_res = res; 4401 4402 /* If during previous unload of the driver, the VFs were not disabled, 4403 * then we cannot rely on the PF POOL limits for the TotalVFs value. 4404 * Instead use the TotalVFs value stored in the pci-dev struct. 4405 */ 4406 old_vfs = pci_num_vf(adapter->pdev); 4407 if (old_vfs) { 4408 dev_info(&adapter->pdev->dev, "%d VFs are already enabled\n", 4409 old_vfs); 4410 4411 adapter->pool_res.max_vfs = 4412 pci_sriov_get_totalvfs(adapter->pdev); 4413 adapter->num_vfs = old_vfs; 4414 } 4415 4416 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) { 4417 be_calculate_pf_pool_rss_tables(adapter); 4418 dev_info(&adapter->pdev->dev, 4419 "RSS can be enabled for all VFs if num_vfs <= %d\n", 4420 be_max_pf_pool_rss_tables(adapter)); 4421 } 4422 return 0; 4423 } 4424 4425 static void be_alloc_sriov_res(struct be_adapter *adapter) 4426 { 4427 int old_vfs = pci_num_vf(adapter->pdev); 4428 struct be_resources vft_res = {0}; 4429 int status; 4430 4431 be_get_sriov_config(adapter); 4432 4433 if (!old_vfs) 4434 pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter)); 4435 4436 /* When the HW is in SRIOV capable configuration, the PF-pool 4437 * resources are given to PF during driver load, if there are no 4438 * old VFs. This facility is not available in BE3 FW. 4439 * Also, this is done by FW in Lancer chip. 4440 */ 4441 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) { 4442 be_calculate_vf_res(adapter, 0, &vft_res); 4443 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0, 4444 &vft_res); 4445 if (status) 4446 dev_err(&adapter->pdev->dev, 4447 "Failed to optimize SRIOV resources\n"); 4448 } 4449 } 4450 4451 static int be_get_resources(struct be_adapter *adapter) 4452 { 4453 struct device *dev = &adapter->pdev->dev; 4454 struct be_resources res = {0}; 4455 int status; 4456 4457 /* For Lancer, SH etc read per-function resource limits from FW. 4458 * GET_FUNC_CONFIG returns per function guaranteed limits. 4459 * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits 4460 */ 4461 if (BEx_chip(adapter)) { 4462 BEx_get_resources(adapter, &res); 4463 } else { 4464 status = be_cmd_get_func_config(adapter, &res); 4465 if (status) 4466 return status; 4467 4468 /* If a deafault RXQ must be created, we'll use up one RSSQ*/ 4469 if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs && 4470 !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS)) 4471 res.max_rss_qs -= 1; 4472 } 4473 4474 /* If RoCE is supported stash away half the EQs for RoCE */ 4475 res.max_nic_evt_qs = be_roce_supported(adapter) ? 4476 res.max_evt_qs / 2 : res.max_evt_qs; 4477 adapter->res = res; 4478 4479 /* If FW supports RSS default queue, then skip creating non-RSS 4480 * queue for non-IP traffic. 4481 */ 4482 adapter->need_def_rxq = (be_if_cap_flags(adapter) & 4483 BE_IF_FLAGS_DEFQ_RSS) ? 0 : 1; 4484 4485 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n", 4486 be_max_txqs(adapter), be_max_rxqs(adapter), 4487 be_max_rss(adapter), be_max_nic_eqs(adapter), 4488 be_max_vfs(adapter)); 4489 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n", 4490 be_max_uc(adapter), be_max_mc(adapter), 4491 be_max_vlans(adapter)); 4492 4493 /* Ensure RX and TX queues are created in pairs at init time */ 4494 adapter->cfg_num_rx_irqs = 4495 min_t(u16, netif_get_num_default_rss_queues(), 4496 be_max_qp_irqs(adapter)); 4497 adapter->cfg_num_tx_irqs = adapter->cfg_num_rx_irqs; 4498 return 0; 4499 } 4500 4501 static int be_get_config(struct be_adapter *adapter) 4502 { 4503 int status, level; 4504 u16 profile_id; 4505 4506 status = be_cmd_get_cntl_attributes(adapter); 4507 if (status) 4508 return status; 4509 4510 status = be_cmd_query_fw_cfg(adapter); 4511 if (status) 4512 return status; 4513 4514 if (!lancer_chip(adapter) && be_physfn(adapter)) 4515 be_cmd_get_fat_dump_len(adapter, &adapter->fat_dump_len); 4516 4517 if (BEx_chip(adapter)) { 4518 level = be_cmd_get_fw_log_level(adapter); 4519 adapter->msg_enable = 4520 level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0; 4521 } 4522 4523 be_cmd_get_acpi_wol_cap(adapter); 4524 pci_enable_wake(adapter->pdev, PCI_D3hot, adapter->wol_en); 4525 pci_enable_wake(adapter->pdev, PCI_D3cold, adapter->wol_en); 4526 4527 be_cmd_query_port_name(adapter); 4528 4529 if (be_physfn(adapter)) { 4530 status = be_cmd_get_active_profile(adapter, &profile_id); 4531 if (!status) 4532 dev_info(&adapter->pdev->dev, 4533 "Using profile 0x%x\n", profile_id); 4534 } 4535 4536 return 0; 4537 } 4538 4539 static int be_mac_setup(struct be_adapter *adapter) 4540 { 4541 u8 mac[ETH_ALEN]; 4542 int status; 4543 4544 if (is_zero_ether_addr(adapter->netdev->dev_addr)) { 4545 status = be_cmd_get_perm_mac(adapter, mac); 4546 if (status) 4547 return status; 4548 4549 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN); 4550 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN); 4551 } 4552 4553 return 0; 4554 } 4555 4556 static void be_schedule_worker(struct be_adapter *adapter) 4557 { 4558 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000)); 4559 adapter->flags |= BE_FLAGS_WORKER_SCHEDULED; 4560 } 4561 4562 static void be_destroy_err_recovery_workq(void) 4563 { 4564 if (!be_err_recovery_workq) 4565 return; 4566 4567 flush_workqueue(be_err_recovery_workq); 4568 destroy_workqueue(be_err_recovery_workq); 4569 be_err_recovery_workq = NULL; 4570 } 4571 4572 static void be_schedule_err_detection(struct be_adapter *adapter, u32 delay) 4573 { 4574 struct be_error_recovery *err_rec = &adapter->error_recovery; 4575 4576 if (!be_err_recovery_workq) 4577 return; 4578 4579 queue_delayed_work(be_err_recovery_workq, &err_rec->err_detection_work, 4580 msecs_to_jiffies(delay)); 4581 adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED; 4582 } 4583 4584 static int be_setup_queues(struct be_adapter *adapter) 4585 { 4586 struct net_device *netdev = adapter->netdev; 4587 int status; 4588 4589 status = be_evt_queues_create(adapter); 4590 if (status) 4591 goto err; 4592 4593 status = be_tx_qs_create(adapter); 4594 if (status) 4595 goto err; 4596 4597 status = be_rx_cqs_create(adapter); 4598 if (status) 4599 goto err; 4600 4601 status = be_mcc_queues_create(adapter); 4602 if (status) 4603 goto err; 4604 4605 status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs); 4606 if (status) 4607 goto err; 4608 4609 status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs); 4610 if (status) 4611 goto err; 4612 4613 return 0; 4614 err: 4615 dev_err(&adapter->pdev->dev, "queue_setup failed\n"); 4616 return status; 4617 } 4618 4619 static int be_if_create(struct be_adapter *adapter) 4620 { 4621 u32 en_flags = BE_IF_FLAGS_RSS | BE_IF_FLAGS_DEFQ_RSS; 4622 u32 cap_flags = be_if_cap_flags(adapter); 4623 int status; 4624 4625 /* alloc required memory for other filtering fields */ 4626 adapter->pmac_id = kcalloc(be_max_uc(adapter), 4627 sizeof(*adapter->pmac_id), GFP_KERNEL); 4628 if (!adapter->pmac_id) 4629 return -ENOMEM; 4630 4631 adapter->mc_list = kcalloc(be_max_mc(adapter), 4632 sizeof(*adapter->mc_list), GFP_KERNEL); 4633 if (!adapter->mc_list) 4634 return -ENOMEM; 4635 4636 adapter->uc_list = kcalloc(be_max_uc(adapter), 4637 sizeof(*adapter->uc_list), GFP_KERNEL); 4638 if (!adapter->uc_list) 4639 return -ENOMEM; 4640 4641 if (adapter->cfg_num_rx_irqs == 1) 4642 cap_flags &= ~(BE_IF_FLAGS_DEFQ_RSS | BE_IF_FLAGS_RSS); 4643 4644 en_flags &= cap_flags; 4645 /* will enable all the needed filter flags in be_open() */ 4646 status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags, 4647 &adapter->if_handle, 0); 4648 4649 if (status) 4650 return status; 4651 4652 return 0; 4653 } 4654 4655 int be_update_queues(struct be_adapter *adapter) 4656 { 4657 struct net_device *netdev = adapter->netdev; 4658 int status; 4659 4660 if (netif_running(netdev)) 4661 be_close(netdev); 4662 4663 be_cancel_worker(adapter); 4664 4665 /* If any vectors have been shared with RoCE we cannot re-program 4666 * the MSIx table. 4667 */ 4668 if (!adapter->num_msix_roce_vec) 4669 be_msix_disable(adapter); 4670 4671 be_clear_queues(adapter); 4672 status = be_cmd_if_destroy(adapter, adapter->if_handle, 0); 4673 if (status) 4674 return status; 4675 4676 if (!msix_enabled(adapter)) { 4677 status = be_msix_enable(adapter); 4678 if (status) 4679 return status; 4680 } 4681 4682 status = be_if_create(adapter); 4683 if (status) 4684 return status; 4685 4686 status = be_setup_queues(adapter); 4687 if (status) 4688 return status; 4689 4690 be_schedule_worker(adapter); 4691 4692 if (netif_running(netdev)) 4693 status = be_open(netdev); 4694 4695 return status; 4696 } 4697 4698 static inline int fw_major_num(const char *fw_ver) 4699 { 4700 int fw_major = 0, i; 4701 4702 i = sscanf(fw_ver, "%d.", &fw_major); 4703 if (i != 1) 4704 return 0; 4705 4706 return fw_major; 4707 } 4708 4709 /* If it is error recovery, FLR the PF 4710 * Else if any VFs are already enabled don't FLR the PF 4711 */ 4712 static bool be_reset_required(struct be_adapter *adapter) 4713 { 4714 if (be_error_recovering(adapter)) 4715 return true; 4716 else 4717 return pci_num_vf(adapter->pdev) == 0; 4718 } 4719 4720 /* Wait for the FW to be ready and perform the required initialization */ 4721 static int be_func_init(struct be_adapter *adapter) 4722 { 4723 int status; 4724 4725 status = be_fw_wait_ready(adapter); 4726 if (status) 4727 return status; 4728 4729 /* FW is now ready; clear errors to allow cmds/doorbell */ 4730 be_clear_error(adapter, BE_CLEAR_ALL); 4731 4732 if (be_reset_required(adapter)) { 4733 status = be_cmd_reset_function(adapter); 4734 if (status) 4735 return status; 4736 4737 /* Wait for interrupts to quiesce after an FLR */ 4738 msleep(100); 4739 } 4740 4741 /* Tell FW we're ready to fire cmds */ 4742 status = be_cmd_fw_init(adapter); 4743 if (status) 4744 return status; 4745 4746 /* Allow interrupts for other ULPs running on NIC function */ 4747 be_intr_set(adapter, true); 4748 4749 return 0; 4750 } 4751 4752 static int be_setup(struct be_adapter *adapter) 4753 { 4754 struct device *dev = &adapter->pdev->dev; 4755 int status; 4756 4757 status = be_func_init(adapter); 4758 if (status) 4759 return status; 4760 4761 be_setup_init(adapter); 4762 4763 if (!lancer_chip(adapter)) 4764 be_cmd_req_native_mode(adapter); 4765 4766 /* invoke this cmd first to get pf_num and vf_num which are needed 4767 * for issuing profile related cmds 4768 */ 4769 if (!BEx_chip(adapter)) { 4770 status = be_cmd_get_func_config(adapter, NULL); 4771 if (status) 4772 return status; 4773 } 4774 4775 status = be_get_config(adapter); 4776 if (status) 4777 goto err; 4778 4779 if (!BE2_chip(adapter) && be_physfn(adapter)) 4780 be_alloc_sriov_res(adapter); 4781 4782 status = be_get_resources(adapter); 4783 if (status) 4784 goto err; 4785 4786 status = be_msix_enable(adapter); 4787 if (status) 4788 goto err; 4789 4790 /* will enable all the needed filter flags in be_open() */ 4791 status = be_if_create(adapter); 4792 if (status) 4793 goto err; 4794 4795 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 4796 rtnl_lock(); 4797 status = be_setup_queues(adapter); 4798 rtnl_unlock(); 4799 if (status) 4800 goto err; 4801 4802 be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0); 4803 4804 status = be_mac_setup(adapter); 4805 if (status) 4806 goto err; 4807 4808 be_cmd_get_fw_ver(adapter); 4809 dev_info(dev, "FW version is %s\n", adapter->fw_ver); 4810 4811 if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) { 4812 dev_err(dev, "Firmware on card is old(%s), IRQs may not work", 4813 adapter->fw_ver); 4814 dev_err(dev, "Please upgrade firmware to version >= 4.0\n"); 4815 } 4816 4817 status = be_cmd_set_flow_control(adapter, adapter->tx_fc, 4818 adapter->rx_fc); 4819 if (status) 4820 be_cmd_get_flow_control(adapter, &adapter->tx_fc, 4821 &adapter->rx_fc); 4822 4823 dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n", 4824 adapter->tx_fc, adapter->rx_fc); 4825 4826 if (be_physfn(adapter)) 4827 be_cmd_set_logical_link_config(adapter, 4828 IFLA_VF_LINK_STATE_AUTO, 0); 4829 4830 /* BE3 EVB echoes broadcast/multicast packets back to PF's vport 4831 * confusing a linux bridge or OVS that it might be connected to. 4832 * Set the EVB to PASSTHRU mode which effectively disables the EVB 4833 * when SRIOV is not enabled. 4834 */ 4835 if (BE3_chip(adapter)) 4836 be_cmd_set_hsw_config(adapter, 0, 0, adapter->if_handle, 4837 PORT_FWD_TYPE_PASSTHRU, 0); 4838 4839 if (adapter->num_vfs) 4840 be_vf_setup(adapter); 4841 4842 status = be_cmd_get_phy_info(adapter); 4843 if (!status && be_pause_supported(adapter)) 4844 adapter->phy.fc_autoneg = 1; 4845 4846 if (be_physfn(adapter) && !lancer_chip(adapter)) 4847 be_cmd_set_features(adapter); 4848 4849 be_schedule_worker(adapter); 4850 adapter->flags |= BE_FLAGS_SETUP_DONE; 4851 return 0; 4852 err: 4853 be_clear(adapter); 4854 return status; 4855 } 4856 4857 #ifdef CONFIG_NET_POLL_CONTROLLER 4858 static void be_netpoll(struct net_device *netdev) 4859 { 4860 struct be_adapter *adapter = netdev_priv(netdev); 4861 struct be_eq_obj *eqo; 4862 int i; 4863 4864 for_all_evt_queues(adapter, eqo, i) { 4865 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0); 4866 napi_schedule(&eqo->napi); 4867 } 4868 } 4869 #endif 4870 4871 int be_load_fw(struct be_adapter *adapter, u8 *fw_file) 4872 { 4873 const struct firmware *fw; 4874 int status; 4875 4876 if (!netif_running(adapter->netdev)) { 4877 dev_err(&adapter->pdev->dev, 4878 "Firmware load not allowed (interface is down)\n"); 4879 return -ENETDOWN; 4880 } 4881 4882 status = request_firmware(&fw, fw_file, &adapter->pdev->dev); 4883 if (status) 4884 goto fw_exit; 4885 4886 dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file); 4887 4888 if (lancer_chip(adapter)) 4889 status = lancer_fw_download(adapter, fw); 4890 else 4891 status = be_fw_download(adapter, fw); 4892 4893 if (!status) 4894 be_cmd_get_fw_ver(adapter); 4895 4896 fw_exit: 4897 release_firmware(fw); 4898 return status; 4899 } 4900 4901 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4902 u16 flags) 4903 { 4904 struct be_adapter *adapter = netdev_priv(dev); 4905 struct nlattr *attr, *br_spec; 4906 int rem; 4907 int status = 0; 4908 u16 mode = 0; 4909 4910 if (!sriov_enabled(adapter)) 4911 return -EOPNOTSUPP; 4912 4913 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4914 if (!br_spec) 4915 return -EINVAL; 4916 4917 nla_for_each_nested(attr, br_spec, rem) { 4918 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4919 continue; 4920 4921 if (nla_len(attr) < sizeof(mode)) 4922 return -EINVAL; 4923 4924 mode = nla_get_u16(attr); 4925 if (BE3_chip(adapter) && mode == BRIDGE_MODE_VEPA) 4926 return -EOPNOTSUPP; 4927 4928 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 4929 return -EINVAL; 4930 4931 status = be_cmd_set_hsw_config(adapter, 0, 0, 4932 adapter->if_handle, 4933 mode == BRIDGE_MODE_VEPA ? 4934 PORT_FWD_TYPE_VEPA : 4935 PORT_FWD_TYPE_VEB, 0); 4936 if (status) 4937 goto err; 4938 4939 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n", 4940 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 4941 4942 return status; 4943 } 4944 err: 4945 dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n", 4946 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 4947 4948 return status; 4949 } 4950 4951 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 4952 struct net_device *dev, u32 filter_mask, 4953 int nlflags) 4954 { 4955 struct be_adapter *adapter = netdev_priv(dev); 4956 int status = 0; 4957 u8 hsw_mode; 4958 4959 /* BE and Lancer chips support VEB mode only */ 4960 if (BEx_chip(adapter) || lancer_chip(adapter)) { 4961 /* VEB is disabled in non-SR-IOV profiles on BE3/Lancer */ 4962 if (!pci_sriov_get_totalvfs(adapter->pdev)) 4963 return 0; 4964 hsw_mode = PORT_FWD_TYPE_VEB; 4965 } else { 4966 status = be_cmd_get_hsw_config(adapter, NULL, 0, 4967 adapter->if_handle, &hsw_mode, 4968 NULL); 4969 if (status) 4970 return 0; 4971 4972 if (hsw_mode == PORT_FWD_TYPE_PASSTHRU) 4973 return 0; 4974 } 4975 4976 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 4977 hsw_mode == PORT_FWD_TYPE_VEPA ? 4978 BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB, 4979 0, 0, nlflags, filter_mask, NULL); 4980 } 4981 4982 static struct be_cmd_work *be_alloc_work(struct be_adapter *adapter, 4983 void (*func)(struct work_struct *)) 4984 { 4985 struct be_cmd_work *work; 4986 4987 work = kzalloc(sizeof(*work), GFP_ATOMIC); 4988 if (!work) { 4989 dev_err(&adapter->pdev->dev, 4990 "be_work memory allocation failed\n"); 4991 return NULL; 4992 } 4993 4994 INIT_WORK(&work->work, func); 4995 work->adapter = adapter; 4996 return work; 4997 } 4998 4999 /* VxLAN offload Notes: 5000 * 5001 * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't 5002 * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload 5003 * is expected to work across all types of IP tunnels once exported. Skyhawk 5004 * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN 5005 * offloads in hw_enc_features only when a VxLAN port is added. If other (non 5006 * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for 5007 * those other tunnels are unexported on the fly through ndo_features_check(). 5008 * 5009 * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack 5010 * adds more than one port, disable offloads and don't re-enable them again 5011 * until after all the tunnels are removed. 5012 */ 5013 static void be_work_add_vxlan_port(struct work_struct *work) 5014 { 5015 struct be_cmd_work *cmd_work = 5016 container_of(work, struct be_cmd_work, work); 5017 struct be_adapter *adapter = cmd_work->adapter; 5018 struct net_device *netdev = adapter->netdev; 5019 struct device *dev = &adapter->pdev->dev; 5020 __be16 port = cmd_work->info.vxlan_port; 5021 int status; 5022 5023 if (adapter->vxlan_port == port && adapter->vxlan_port_count) { 5024 adapter->vxlan_port_aliases++; 5025 goto done; 5026 } 5027 5028 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) { 5029 dev_info(dev, 5030 "Only one UDP port supported for VxLAN offloads\n"); 5031 dev_info(dev, "Disabling VxLAN offloads\n"); 5032 adapter->vxlan_port_count++; 5033 goto err; 5034 } 5035 5036 if (adapter->vxlan_port_count++ >= 1) 5037 goto done; 5038 5039 status = be_cmd_manage_iface(adapter, adapter->if_handle, 5040 OP_CONVERT_NORMAL_TO_TUNNEL); 5041 if (status) { 5042 dev_warn(dev, "Failed to convert normal interface to tunnel\n"); 5043 goto err; 5044 } 5045 5046 status = be_cmd_set_vxlan_port(adapter, port); 5047 if (status) { 5048 dev_warn(dev, "Failed to add VxLAN port\n"); 5049 goto err; 5050 } 5051 adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS; 5052 adapter->vxlan_port = port; 5053 5054 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 5055 NETIF_F_TSO | NETIF_F_TSO6 | 5056 NETIF_F_GSO_UDP_TUNNEL; 5057 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL; 5058 netdev->features |= NETIF_F_GSO_UDP_TUNNEL; 5059 5060 dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n", 5061 be16_to_cpu(port)); 5062 goto done; 5063 err: 5064 be_disable_vxlan_offloads(adapter); 5065 done: 5066 kfree(cmd_work); 5067 } 5068 5069 static void be_work_del_vxlan_port(struct work_struct *work) 5070 { 5071 struct be_cmd_work *cmd_work = 5072 container_of(work, struct be_cmd_work, work); 5073 struct be_adapter *adapter = cmd_work->adapter; 5074 __be16 port = cmd_work->info.vxlan_port; 5075 5076 if (adapter->vxlan_port != port) 5077 goto done; 5078 5079 if (adapter->vxlan_port_aliases) { 5080 adapter->vxlan_port_aliases--; 5081 goto out; 5082 } 5083 5084 be_disable_vxlan_offloads(adapter); 5085 5086 dev_info(&adapter->pdev->dev, 5087 "Disabled VxLAN offloads for UDP port %d\n", 5088 be16_to_cpu(port)); 5089 done: 5090 adapter->vxlan_port_count--; 5091 out: 5092 kfree(cmd_work); 5093 } 5094 5095 static void be_cfg_vxlan_port(struct net_device *netdev, 5096 struct udp_tunnel_info *ti, 5097 void (*func)(struct work_struct *)) 5098 { 5099 struct be_adapter *adapter = netdev_priv(netdev); 5100 struct be_cmd_work *cmd_work; 5101 5102 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 5103 return; 5104 5105 if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter)) 5106 return; 5107 5108 cmd_work = be_alloc_work(adapter, func); 5109 if (cmd_work) { 5110 cmd_work->info.vxlan_port = ti->port; 5111 queue_work(be_wq, &cmd_work->work); 5112 } 5113 } 5114 5115 static void be_del_vxlan_port(struct net_device *netdev, 5116 struct udp_tunnel_info *ti) 5117 { 5118 be_cfg_vxlan_port(netdev, ti, be_work_del_vxlan_port); 5119 } 5120 5121 static void be_add_vxlan_port(struct net_device *netdev, 5122 struct udp_tunnel_info *ti) 5123 { 5124 be_cfg_vxlan_port(netdev, ti, be_work_add_vxlan_port); 5125 } 5126 5127 static netdev_features_t be_features_check(struct sk_buff *skb, 5128 struct net_device *dev, 5129 netdev_features_t features) 5130 { 5131 struct be_adapter *adapter = netdev_priv(dev); 5132 u8 l4_hdr = 0; 5133 5134 /* The code below restricts offload features for some tunneled packets. 5135 * Offload features for normal (non tunnel) packets are unchanged. 5136 */ 5137 if (!skb->encapsulation || 5138 !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)) 5139 return features; 5140 5141 /* It's an encapsulated packet and VxLAN offloads are enabled. We 5142 * should disable tunnel offload features if it's not a VxLAN packet, 5143 * as tunnel offloads have been enabled only for VxLAN. This is done to 5144 * allow other tunneled traffic like GRE work fine while VxLAN 5145 * offloads are configured in Skyhawk-R. 5146 */ 5147 switch (vlan_get_protocol(skb)) { 5148 case htons(ETH_P_IP): 5149 l4_hdr = ip_hdr(skb)->protocol; 5150 break; 5151 case htons(ETH_P_IPV6): 5152 l4_hdr = ipv6_hdr(skb)->nexthdr; 5153 break; 5154 default: 5155 return features; 5156 } 5157 5158 if (l4_hdr != IPPROTO_UDP || 5159 skb->inner_protocol_type != ENCAP_TYPE_ETHER || 5160 skb->inner_protocol != htons(ETH_P_TEB) || 5161 skb_inner_mac_header(skb) - skb_transport_header(skb) != 5162 sizeof(struct udphdr) + sizeof(struct vxlanhdr)) 5163 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 5164 5165 return features; 5166 } 5167 5168 static int be_get_phys_port_id(struct net_device *dev, 5169 struct netdev_phys_item_id *ppid) 5170 { 5171 int i, id_len = CNTL_SERIAL_NUM_WORDS * CNTL_SERIAL_NUM_WORD_SZ + 1; 5172 struct be_adapter *adapter = netdev_priv(dev); 5173 u8 *id; 5174 5175 if (MAX_PHYS_ITEM_ID_LEN < id_len) 5176 return -ENOSPC; 5177 5178 ppid->id[0] = adapter->hba_port_num + 1; 5179 id = &ppid->id[1]; 5180 for (i = CNTL_SERIAL_NUM_WORDS - 1; i >= 0; 5181 i--, id += CNTL_SERIAL_NUM_WORD_SZ) 5182 memcpy(id, &adapter->serial_num[i], CNTL_SERIAL_NUM_WORD_SZ); 5183 5184 ppid->id_len = id_len; 5185 5186 return 0; 5187 } 5188 5189 static void be_set_rx_mode(struct net_device *dev) 5190 { 5191 struct be_adapter *adapter = netdev_priv(dev); 5192 struct be_cmd_work *work; 5193 5194 work = be_alloc_work(adapter, be_work_set_rx_mode); 5195 if (work) 5196 queue_work(be_wq, &work->work); 5197 } 5198 5199 static const struct net_device_ops be_netdev_ops = { 5200 .ndo_open = be_open, 5201 .ndo_stop = be_close, 5202 .ndo_start_xmit = be_xmit, 5203 .ndo_set_rx_mode = be_set_rx_mode, 5204 .ndo_set_mac_address = be_mac_addr_set, 5205 .ndo_change_mtu = be_change_mtu, 5206 .ndo_get_stats64 = be_get_stats64, 5207 .ndo_validate_addr = eth_validate_addr, 5208 .ndo_vlan_rx_add_vid = be_vlan_add_vid, 5209 .ndo_vlan_rx_kill_vid = be_vlan_rem_vid, 5210 .ndo_set_vf_mac = be_set_vf_mac, 5211 .ndo_set_vf_vlan = be_set_vf_vlan, 5212 .ndo_set_vf_rate = be_set_vf_tx_rate, 5213 .ndo_get_vf_config = be_get_vf_config, 5214 .ndo_set_vf_link_state = be_set_vf_link_state, 5215 .ndo_set_vf_spoofchk = be_set_vf_spoofchk, 5216 #ifdef CONFIG_NET_POLL_CONTROLLER 5217 .ndo_poll_controller = be_netpoll, 5218 #endif 5219 .ndo_bridge_setlink = be_ndo_bridge_setlink, 5220 .ndo_bridge_getlink = be_ndo_bridge_getlink, 5221 #ifdef CONFIG_NET_RX_BUSY_POLL 5222 .ndo_busy_poll = be_busy_poll, 5223 #endif 5224 .ndo_udp_tunnel_add = be_add_vxlan_port, 5225 .ndo_udp_tunnel_del = be_del_vxlan_port, 5226 .ndo_features_check = be_features_check, 5227 .ndo_get_phys_port_id = be_get_phys_port_id, 5228 }; 5229 5230 static void be_netdev_init(struct net_device *netdev) 5231 { 5232 struct be_adapter *adapter = netdev_priv(netdev); 5233 5234 netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5235 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | 5236 NETIF_F_HW_VLAN_CTAG_TX; 5237 if ((be_if_cap_flags(adapter) & BE_IF_FLAGS_RSS)) 5238 netdev->hw_features |= NETIF_F_RXHASH; 5239 5240 netdev->features |= netdev->hw_features | 5241 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER; 5242 5243 netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5244 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 5245 5246 netdev->priv_flags |= IFF_UNICAST_FLT; 5247 5248 netdev->flags |= IFF_MULTICAST; 5249 5250 netif_set_gso_max_size(netdev, BE_MAX_GSO_SIZE - ETH_HLEN); 5251 5252 netdev->netdev_ops = &be_netdev_ops; 5253 5254 netdev->ethtool_ops = &be_ethtool_ops; 5255 } 5256 5257 static void be_cleanup(struct be_adapter *adapter) 5258 { 5259 struct net_device *netdev = adapter->netdev; 5260 5261 rtnl_lock(); 5262 netif_device_detach(netdev); 5263 if (netif_running(netdev)) 5264 be_close(netdev); 5265 rtnl_unlock(); 5266 5267 be_clear(adapter); 5268 } 5269 5270 static int be_resume(struct be_adapter *adapter) 5271 { 5272 struct net_device *netdev = adapter->netdev; 5273 int status; 5274 5275 status = be_setup(adapter); 5276 if (status) 5277 return status; 5278 5279 rtnl_lock(); 5280 if (netif_running(netdev)) 5281 status = be_open(netdev); 5282 rtnl_unlock(); 5283 5284 if (status) 5285 return status; 5286 5287 netif_device_attach(netdev); 5288 5289 return 0; 5290 } 5291 5292 static void be_soft_reset(struct be_adapter *adapter) 5293 { 5294 u32 val; 5295 5296 dev_info(&adapter->pdev->dev, "Initiating chip soft reset\n"); 5297 val = ioread32(adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET); 5298 val |= SLIPORT_SOFTRESET_SR_MASK; 5299 iowrite32(val, adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET); 5300 } 5301 5302 static bool be_err_is_recoverable(struct be_adapter *adapter) 5303 { 5304 struct be_error_recovery *err_rec = &adapter->error_recovery; 5305 unsigned long initial_idle_time = 5306 msecs_to_jiffies(ERR_RECOVERY_IDLE_TIME); 5307 unsigned long recovery_interval = 5308 msecs_to_jiffies(ERR_RECOVERY_INTERVAL); 5309 u16 ue_err_code; 5310 u32 val; 5311 5312 val = be_POST_stage_get(adapter); 5313 if ((val & POST_STAGE_RECOVERABLE_ERR) != POST_STAGE_RECOVERABLE_ERR) 5314 return false; 5315 ue_err_code = val & POST_ERR_RECOVERY_CODE_MASK; 5316 if (ue_err_code == 0) 5317 return false; 5318 5319 dev_err(&adapter->pdev->dev, "Recoverable HW error code: 0x%x\n", 5320 ue_err_code); 5321 5322 if (jiffies - err_rec->probe_time <= initial_idle_time) { 5323 dev_err(&adapter->pdev->dev, 5324 "Cannot recover within %lu sec from driver load\n", 5325 jiffies_to_msecs(initial_idle_time) / MSEC_PER_SEC); 5326 return false; 5327 } 5328 5329 if (err_rec->last_recovery_time && 5330 (jiffies - err_rec->last_recovery_time <= recovery_interval)) { 5331 dev_err(&adapter->pdev->dev, 5332 "Cannot recover within %lu sec from last recovery\n", 5333 jiffies_to_msecs(recovery_interval) / MSEC_PER_SEC); 5334 return false; 5335 } 5336 5337 if (ue_err_code == err_rec->last_err_code) { 5338 dev_err(&adapter->pdev->dev, 5339 "Cannot recover from a consecutive TPE error\n"); 5340 return false; 5341 } 5342 5343 err_rec->last_recovery_time = jiffies; 5344 err_rec->last_err_code = ue_err_code; 5345 return true; 5346 } 5347 5348 static int be_tpe_recover(struct be_adapter *adapter) 5349 { 5350 struct be_error_recovery *err_rec = &adapter->error_recovery; 5351 int status = -EAGAIN; 5352 u32 val; 5353 5354 switch (err_rec->recovery_state) { 5355 case ERR_RECOVERY_ST_NONE: 5356 err_rec->recovery_state = ERR_RECOVERY_ST_DETECT; 5357 err_rec->resched_delay = ERR_RECOVERY_UE_DETECT_DURATION; 5358 break; 5359 5360 case ERR_RECOVERY_ST_DETECT: 5361 val = be_POST_stage_get(adapter); 5362 if ((val & POST_STAGE_RECOVERABLE_ERR) != 5363 POST_STAGE_RECOVERABLE_ERR) { 5364 dev_err(&adapter->pdev->dev, 5365 "Unrecoverable HW error detected: 0x%x\n", val); 5366 status = -EINVAL; 5367 err_rec->resched_delay = 0; 5368 break; 5369 } 5370 5371 dev_err(&adapter->pdev->dev, "Recoverable HW error detected\n"); 5372 5373 /* Only PF0 initiates Chip Soft Reset. But PF0 must wait UE2SR 5374 * milliseconds before it checks for final error status in 5375 * SLIPORT_SEMAPHORE to determine if recovery criteria is met. 5376 * If it does, then PF0 initiates a Soft Reset. 5377 */ 5378 if (adapter->pf_num == 0) { 5379 err_rec->recovery_state = ERR_RECOVERY_ST_RESET; 5380 err_rec->resched_delay = err_rec->ue_to_reset_time - 5381 ERR_RECOVERY_UE_DETECT_DURATION; 5382 break; 5383 } 5384 5385 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL; 5386 err_rec->resched_delay = err_rec->ue_to_poll_time - 5387 ERR_RECOVERY_UE_DETECT_DURATION; 5388 break; 5389 5390 case ERR_RECOVERY_ST_RESET: 5391 if (!be_err_is_recoverable(adapter)) { 5392 dev_err(&adapter->pdev->dev, 5393 "Failed to meet recovery criteria\n"); 5394 status = -EIO; 5395 err_rec->resched_delay = 0; 5396 break; 5397 } 5398 be_soft_reset(adapter); 5399 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL; 5400 err_rec->resched_delay = err_rec->ue_to_poll_time - 5401 err_rec->ue_to_reset_time; 5402 break; 5403 5404 case ERR_RECOVERY_ST_PRE_POLL: 5405 err_rec->recovery_state = ERR_RECOVERY_ST_REINIT; 5406 err_rec->resched_delay = 0; 5407 status = 0; /* done */ 5408 break; 5409 5410 default: 5411 status = -EINVAL; 5412 err_rec->resched_delay = 0; 5413 break; 5414 } 5415 5416 return status; 5417 } 5418 5419 static int be_err_recover(struct be_adapter *adapter) 5420 { 5421 int status; 5422 5423 if (!lancer_chip(adapter)) { 5424 if (!adapter->error_recovery.recovery_supported || 5425 adapter->priv_flags & BE_DISABLE_TPE_RECOVERY) 5426 return -EIO; 5427 status = be_tpe_recover(adapter); 5428 if (status) 5429 goto err; 5430 } 5431 5432 /* Wait for adapter to reach quiescent state before 5433 * destroying queues 5434 */ 5435 status = be_fw_wait_ready(adapter); 5436 if (status) 5437 goto err; 5438 5439 adapter->flags |= BE_FLAGS_TRY_RECOVERY; 5440 5441 be_cleanup(adapter); 5442 5443 status = be_resume(adapter); 5444 if (status) 5445 goto err; 5446 5447 adapter->flags &= ~BE_FLAGS_TRY_RECOVERY; 5448 5449 err: 5450 return status; 5451 } 5452 5453 static void be_err_detection_task(struct work_struct *work) 5454 { 5455 struct be_error_recovery *err_rec = 5456 container_of(work, struct be_error_recovery, 5457 err_detection_work.work); 5458 struct be_adapter *adapter = 5459 container_of(err_rec, struct be_adapter, 5460 error_recovery); 5461 u32 resched_delay = ERR_RECOVERY_DETECTION_DELAY; 5462 struct device *dev = &adapter->pdev->dev; 5463 int recovery_status; 5464 5465 be_detect_error(adapter); 5466 if (!be_check_error(adapter, BE_ERROR_HW)) 5467 goto reschedule_task; 5468 5469 recovery_status = be_err_recover(adapter); 5470 if (!recovery_status) { 5471 err_rec->recovery_retries = 0; 5472 err_rec->recovery_state = ERR_RECOVERY_ST_NONE; 5473 dev_info(dev, "Adapter recovery successful\n"); 5474 goto reschedule_task; 5475 } else if (!lancer_chip(adapter) && err_rec->resched_delay) { 5476 /* BEx/SH recovery state machine */ 5477 if (adapter->pf_num == 0 && 5478 err_rec->recovery_state > ERR_RECOVERY_ST_DETECT) 5479 dev_err(&adapter->pdev->dev, 5480 "Adapter recovery in progress\n"); 5481 resched_delay = err_rec->resched_delay; 5482 goto reschedule_task; 5483 } else if (lancer_chip(adapter) && be_virtfn(adapter)) { 5484 /* For VFs, check if PF have allocated resources 5485 * every second. 5486 */ 5487 dev_err(dev, "Re-trying adapter recovery\n"); 5488 goto reschedule_task; 5489 } else if (lancer_chip(adapter) && err_rec->recovery_retries++ < 5490 ERR_RECOVERY_MAX_RETRY_COUNT) { 5491 /* In case of another error during recovery, it takes 30 sec 5492 * for adapter to come out of error. Retry error recovery after 5493 * this time interval. 5494 */ 5495 dev_err(&adapter->pdev->dev, "Re-trying adapter recovery\n"); 5496 resched_delay = ERR_RECOVERY_RETRY_DELAY; 5497 goto reschedule_task; 5498 } else { 5499 dev_err(dev, "Adapter recovery failed\n"); 5500 dev_err(dev, "Please reboot server to recover\n"); 5501 } 5502 5503 return; 5504 5505 reschedule_task: 5506 be_schedule_err_detection(adapter, resched_delay); 5507 } 5508 5509 static void be_log_sfp_info(struct be_adapter *adapter) 5510 { 5511 int status; 5512 5513 status = be_cmd_query_sfp_info(adapter); 5514 if (!status) { 5515 dev_err(&adapter->pdev->dev, 5516 "Port %c: %s Vendor: %s part no: %s", 5517 adapter->port_name, 5518 be_misconfig_evt_port_state[adapter->phy_state], 5519 adapter->phy.vendor_name, 5520 adapter->phy.vendor_pn); 5521 } 5522 adapter->flags &= ~BE_FLAGS_PHY_MISCONFIGURED; 5523 } 5524 5525 static void be_worker(struct work_struct *work) 5526 { 5527 struct be_adapter *adapter = 5528 container_of(work, struct be_adapter, work.work); 5529 struct be_rx_obj *rxo; 5530 int i; 5531 5532 if (be_physfn(adapter) && 5533 MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0) 5534 be_cmd_get_die_temperature(adapter); 5535 5536 /* when interrupts are not yet enabled, just reap any pending 5537 * mcc completions 5538 */ 5539 if (!netif_running(adapter->netdev)) { 5540 local_bh_disable(); 5541 be_process_mcc(adapter); 5542 local_bh_enable(); 5543 goto reschedule; 5544 } 5545 5546 if (!adapter->stats_cmd_sent) { 5547 if (lancer_chip(adapter)) 5548 lancer_cmd_get_pport_stats(adapter, 5549 &adapter->stats_cmd); 5550 else 5551 be_cmd_get_stats(adapter, &adapter->stats_cmd); 5552 } 5553 5554 for_all_rx_queues(adapter, rxo, i) { 5555 /* Replenish RX-queues starved due to memory 5556 * allocation failures. 5557 */ 5558 if (rxo->rx_post_starved) 5559 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST); 5560 } 5561 5562 /* EQ-delay update for Skyhawk is done while notifying EQ */ 5563 if (!skyhawk_chip(adapter)) 5564 be_eqd_update(adapter, false); 5565 5566 if (adapter->flags & BE_FLAGS_PHY_MISCONFIGURED) 5567 be_log_sfp_info(adapter); 5568 5569 reschedule: 5570 adapter->work_counter++; 5571 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000)); 5572 } 5573 5574 static void be_unmap_pci_bars(struct be_adapter *adapter) 5575 { 5576 if (adapter->csr) 5577 pci_iounmap(adapter->pdev, adapter->csr); 5578 if (adapter->db) 5579 pci_iounmap(adapter->pdev, adapter->db); 5580 if (adapter->pcicfg && adapter->pcicfg_mapped) 5581 pci_iounmap(adapter->pdev, adapter->pcicfg); 5582 } 5583 5584 static int db_bar(struct be_adapter *adapter) 5585 { 5586 if (lancer_chip(adapter) || be_virtfn(adapter)) 5587 return 0; 5588 else 5589 return 4; 5590 } 5591 5592 static int be_roce_map_pci_bars(struct be_adapter *adapter) 5593 { 5594 if (skyhawk_chip(adapter)) { 5595 adapter->roce_db.size = 4096; 5596 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev, 5597 db_bar(adapter)); 5598 adapter->roce_db.total_size = pci_resource_len(adapter->pdev, 5599 db_bar(adapter)); 5600 } 5601 return 0; 5602 } 5603 5604 static int be_map_pci_bars(struct be_adapter *adapter) 5605 { 5606 struct pci_dev *pdev = adapter->pdev; 5607 u8 __iomem *addr; 5608 u32 sli_intf; 5609 5610 pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf); 5611 adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >> 5612 SLI_INTF_FAMILY_SHIFT; 5613 adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0; 5614 5615 if (BEx_chip(adapter) && be_physfn(adapter)) { 5616 adapter->csr = pci_iomap(pdev, 2, 0); 5617 if (!adapter->csr) 5618 return -ENOMEM; 5619 } 5620 5621 addr = pci_iomap(pdev, db_bar(adapter), 0); 5622 if (!addr) 5623 goto pci_map_err; 5624 adapter->db = addr; 5625 5626 if (skyhawk_chip(adapter) || BEx_chip(adapter)) { 5627 if (be_physfn(adapter)) { 5628 /* PCICFG is the 2nd BAR in BE2 */ 5629 addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0); 5630 if (!addr) 5631 goto pci_map_err; 5632 adapter->pcicfg = addr; 5633 adapter->pcicfg_mapped = true; 5634 } else { 5635 adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET; 5636 adapter->pcicfg_mapped = false; 5637 } 5638 } 5639 5640 be_roce_map_pci_bars(adapter); 5641 return 0; 5642 5643 pci_map_err: 5644 dev_err(&pdev->dev, "Error in mapping PCI BARs\n"); 5645 be_unmap_pci_bars(adapter); 5646 return -ENOMEM; 5647 } 5648 5649 static void be_drv_cleanup(struct be_adapter *adapter) 5650 { 5651 struct be_dma_mem *mem = &adapter->mbox_mem_alloced; 5652 struct device *dev = &adapter->pdev->dev; 5653 5654 if (mem->va) 5655 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5656 5657 mem = &adapter->rx_filter; 5658 if (mem->va) 5659 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5660 5661 mem = &adapter->stats_cmd; 5662 if (mem->va) 5663 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5664 } 5665 5666 /* Allocate and initialize various fields in be_adapter struct */ 5667 static int be_drv_init(struct be_adapter *adapter) 5668 { 5669 struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced; 5670 struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem; 5671 struct be_dma_mem *rx_filter = &adapter->rx_filter; 5672 struct be_dma_mem *stats_cmd = &adapter->stats_cmd; 5673 struct device *dev = &adapter->pdev->dev; 5674 int status = 0; 5675 5676 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 5677 mbox_mem_alloc->va = dma_zalloc_coherent(dev, mbox_mem_alloc->size, 5678 &mbox_mem_alloc->dma, 5679 GFP_KERNEL); 5680 if (!mbox_mem_alloc->va) 5681 return -ENOMEM; 5682 5683 mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 5684 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 5685 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 5686 5687 rx_filter->size = sizeof(struct be_cmd_req_rx_filter); 5688 rx_filter->va = dma_zalloc_coherent(dev, rx_filter->size, 5689 &rx_filter->dma, GFP_KERNEL); 5690 if (!rx_filter->va) { 5691 status = -ENOMEM; 5692 goto free_mbox; 5693 } 5694 5695 if (lancer_chip(adapter)) 5696 stats_cmd->size = sizeof(struct lancer_cmd_req_pport_stats); 5697 else if (BE2_chip(adapter)) 5698 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v0); 5699 else if (BE3_chip(adapter)) 5700 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v1); 5701 else 5702 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v2); 5703 stats_cmd->va = dma_zalloc_coherent(dev, stats_cmd->size, 5704 &stats_cmd->dma, GFP_KERNEL); 5705 if (!stats_cmd->va) { 5706 status = -ENOMEM; 5707 goto free_rx_filter; 5708 } 5709 5710 mutex_init(&adapter->mbox_lock); 5711 mutex_init(&adapter->mcc_lock); 5712 mutex_init(&adapter->rx_filter_lock); 5713 spin_lock_init(&adapter->mcc_cq_lock); 5714 init_completion(&adapter->et_cmd_compl); 5715 5716 pci_save_state(adapter->pdev); 5717 5718 INIT_DELAYED_WORK(&adapter->work, be_worker); 5719 5720 adapter->error_recovery.recovery_state = ERR_RECOVERY_ST_NONE; 5721 adapter->error_recovery.resched_delay = 0; 5722 INIT_DELAYED_WORK(&adapter->error_recovery.err_detection_work, 5723 be_err_detection_task); 5724 5725 adapter->rx_fc = true; 5726 adapter->tx_fc = true; 5727 5728 /* Must be a power of 2 or else MODULO will BUG_ON */ 5729 adapter->be_get_temp_freq = 64; 5730 5731 return 0; 5732 5733 free_rx_filter: 5734 dma_free_coherent(dev, rx_filter->size, rx_filter->va, rx_filter->dma); 5735 free_mbox: 5736 dma_free_coherent(dev, mbox_mem_alloc->size, mbox_mem_alloc->va, 5737 mbox_mem_alloc->dma); 5738 return status; 5739 } 5740 5741 static void be_remove(struct pci_dev *pdev) 5742 { 5743 struct be_adapter *adapter = pci_get_drvdata(pdev); 5744 5745 if (!adapter) 5746 return; 5747 5748 be_roce_dev_remove(adapter); 5749 be_intr_set(adapter, false); 5750 5751 be_cancel_err_detection(adapter); 5752 5753 unregister_netdev(adapter->netdev); 5754 5755 be_clear(adapter); 5756 5757 if (!pci_vfs_assigned(adapter->pdev)) 5758 be_cmd_reset_function(adapter); 5759 5760 /* tell fw we're done with firing cmds */ 5761 be_cmd_fw_clean(adapter); 5762 5763 be_unmap_pci_bars(adapter); 5764 be_drv_cleanup(adapter); 5765 5766 pci_disable_pcie_error_reporting(pdev); 5767 5768 pci_release_regions(pdev); 5769 pci_disable_device(pdev); 5770 5771 free_netdev(adapter->netdev); 5772 } 5773 5774 static ssize_t be_hwmon_show_temp(struct device *dev, 5775 struct device_attribute *dev_attr, 5776 char *buf) 5777 { 5778 struct be_adapter *adapter = dev_get_drvdata(dev); 5779 5780 /* Unit: millidegree Celsius */ 5781 if (adapter->hwmon_info.be_on_die_temp == BE_INVALID_DIE_TEMP) 5782 return -EIO; 5783 else 5784 return sprintf(buf, "%u\n", 5785 adapter->hwmon_info.be_on_die_temp * 1000); 5786 } 5787 5788 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, 5789 be_hwmon_show_temp, NULL, 1); 5790 5791 static struct attribute *be_hwmon_attrs[] = { 5792 &sensor_dev_attr_temp1_input.dev_attr.attr, 5793 NULL 5794 }; 5795 5796 ATTRIBUTE_GROUPS(be_hwmon); 5797 5798 static char *mc_name(struct be_adapter *adapter) 5799 { 5800 char *str = ""; /* default */ 5801 5802 switch (adapter->mc_type) { 5803 case UMC: 5804 str = "UMC"; 5805 break; 5806 case FLEX10: 5807 str = "FLEX10"; 5808 break; 5809 case vNIC1: 5810 str = "vNIC-1"; 5811 break; 5812 case nPAR: 5813 str = "nPAR"; 5814 break; 5815 case UFP: 5816 str = "UFP"; 5817 break; 5818 case vNIC2: 5819 str = "vNIC-2"; 5820 break; 5821 default: 5822 str = ""; 5823 } 5824 5825 return str; 5826 } 5827 5828 static inline char *func_name(struct be_adapter *adapter) 5829 { 5830 return be_physfn(adapter) ? "PF" : "VF"; 5831 } 5832 5833 static inline char *nic_name(struct pci_dev *pdev) 5834 { 5835 switch (pdev->device) { 5836 case OC_DEVICE_ID1: 5837 return OC_NAME; 5838 case OC_DEVICE_ID2: 5839 return OC_NAME_BE; 5840 case OC_DEVICE_ID3: 5841 case OC_DEVICE_ID4: 5842 return OC_NAME_LANCER; 5843 case BE_DEVICE_ID2: 5844 return BE3_NAME; 5845 case OC_DEVICE_ID5: 5846 case OC_DEVICE_ID6: 5847 return OC_NAME_SH; 5848 default: 5849 return BE_NAME; 5850 } 5851 } 5852 5853 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id) 5854 { 5855 struct be_adapter *adapter; 5856 struct net_device *netdev; 5857 int status = 0; 5858 5859 dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER); 5860 5861 status = pci_enable_device(pdev); 5862 if (status) 5863 goto do_none; 5864 5865 status = pci_request_regions(pdev, DRV_NAME); 5866 if (status) 5867 goto disable_dev; 5868 pci_set_master(pdev); 5869 5870 netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS); 5871 if (!netdev) { 5872 status = -ENOMEM; 5873 goto rel_reg; 5874 } 5875 adapter = netdev_priv(netdev); 5876 adapter->pdev = pdev; 5877 pci_set_drvdata(pdev, adapter); 5878 adapter->netdev = netdev; 5879 SET_NETDEV_DEV(netdev, &pdev->dev); 5880 5881 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 5882 if (!status) { 5883 netdev->features |= NETIF_F_HIGHDMA; 5884 } else { 5885 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 5886 if (status) { 5887 dev_err(&pdev->dev, "Could not set PCI DMA Mask\n"); 5888 goto free_netdev; 5889 } 5890 } 5891 5892 status = pci_enable_pcie_error_reporting(pdev); 5893 if (!status) 5894 dev_info(&pdev->dev, "PCIe error reporting enabled\n"); 5895 5896 status = be_map_pci_bars(adapter); 5897 if (status) 5898 goto free_netdev; 5899 5900 status = be_drv_init(adapter); 5901 if (status) 5902 goto unmap_bars; 5903 5904 status = be_setup(adapter); 5905 if (status) 5906 goto drv_cleanup; 5907 5908 be_netdev_init(netdev); 5909 status = register_netdev(netdev); 5910 if (status != 0) 5911 goto unsetup; 5912 5913 be_roce_dev_add(adapter); 5914 5915 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 5916 adapter->error_recovery.probe_time = jiffies; 5917 5918 /* On Die temperature not supported for VF. */ 5919 if (be_physfn(adapter) && IS_ENABLED(CONFIG_BE2NET_HWMON)) { 5920 adapter->hwmon_info.hwmon_dev = 5921 devm_hwmon_device_register_with_groups(&pdev->dev, 5922 DRV_NAME, 5923 adapter, 5924 be_hwmon_groups); 5925 adapter->hwmon_info.be_on_die_temp = BE_INVALID_DIE_TEMP; 5926 } 5927 5928 dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev), 5929 func_name(adapter), mc_name(adapter), adapter->port_name); 5930 5931 return 0; 5932 5933 unsetup: 5934 be_clear(adapter); 5935 drv_cleanup: 5936 be_drv_cleanup(adapter); 5937 unmap_bars: 5938 be_unmap_pci_bars(adapter); 5939 free_netdev: 5940 free_netdev(netdev); 5941 rel_reg: 5942 pci_release_regions(pdev); 5943 disable_dev: 5944 pci_disable_device(pdev); 5945 do_none: 5946 dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev)); 5947 return status; 5948 } 5949 5950 static int be_suspend(struct pci_dev *pdev, pm_message_t state) 5951 { 5952 struct be_adapter *adapter = pci_get_drvdata(pdev); 5953 5954 be_intr_set(adapter, false); 5955 be_cancel_err_detection(adapter); 5956 5957 be_cleanup(adapter); 5958 5959 pci_save_state(pdev); 5960 pci_disable_device(pdev); 5961 pci_set_power_state(pdev, pci_choose_state(pdev, state)); 5962 return 0; 5963 } 5964 5965 static int be_pci_resume(struct pci_dev *pdev) 5966 { 5967 struct be_adapter *adapter = pci_get_drvdata(pdev); 5968 int status = 0; 5969 5970 status = pci_enable_device(pdev); 5971 if (status) 5972 return status; 5973 5974 pci_restore_state(pdev); 5975 5976 status = be_resume(adapter); 5977 if (status) 5978 return status; 5979 5980 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 5981 5982 return 0; 5983 } 5984 5985 /* 5986 * An FLR will stop BE from DMAing any data. 5987 */ 5988 static void be_shutdown(struct pci_dev *pdev) 5989 { 5990 struct be_adapter *adapter = pci_get_drvdata(pdev); 5991 5992 if (!adapter) 5993 return; 5994 5995 be_roce_dev_shutdown(adapter); 5996 cancel_delayed_work_sync(&adapter->work); 5997 be_cancel_err_detection(adapter); 5998 5999 netif_device_detach(adapter->netdev); 6000 6001 be_cmd_reset_function(adapter); 6002 6003 pci_disable_device(pdev); 6004 } 6005 6006 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev, 6007 pci_channel_state_t state) 6008 { 6009 struct be_adapter *adapter = pci_get_drvdata(pdev); 6010 6011 dev_err(&adapter->pdev->dev, "EEH error detected\n"); 6012 6013 be_roce_dev_remove(adapter); 6014 6015 if (!be_check_error(adapter, BE_ERROR_EEH)) { 6016 be_set_error(adapter, BE_ERROR_EEH); 6017 6018 be_cancel_err_detection(adapter); 6019 6020 be_cleanup(adapter); 6021 } 6022 6023 if (state == pci_channel_io_perm_failure) 6024 return PCI_ERS_RESULT_DISCONNECT; 6025 6026 pci_disable_device(pdev); 6027 6028 /* The error could cause the FW to trigger a flash debug dump. 6029 * Resetting the card while flash dump is in progress 6030 * can cause it not to recover; wait for it to finish. 6031 * Wait only for first function as it is needed only once per 6032 * adapter. 6033 */ 6034 if (pdev->devfn == 0) 6035 ssleep(30); 6036 6037 return PCI_ERS_RESULT_NEED_RESET; 6038 } 6039 6040 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev) 6041 { 6042 struct be_adapter *adapter = pci_get_drvdata(pdev); 6043 int status; 6044 6045 dev_info(&adapter->pdev->dev, "EEH reset\n"); 6046 6047 status = pci_enable_device(pdev); 6048 if (status) 6049 return PCI_ERS_RESULT_DISCONNECT; 6050 6051 pci_set_master(pdev); 6052 pci_restore_state(pdev); 6053 6054 /* Check if card is ok and fw is ready */ 6055 dev_info(&adapter->pdev->dev, 6056 "Waiting for FW to be ready after EEH reset\n"); 6057 status = be_fw_wait_ready(adapter); 6058 if (status) 6059 return PCI_ERS_RESULT_DISCONNECT; 6060 6061 pci_cleanup_aer_uncorrect_error_status(pdev); 6062 be_clear_error(adapter, BE_CLEAR_ALL); 6063 return PCI_ERS_RESULT_RECOVERED; 6064 } 6065 6066 static void be_eeh_resume(struct pci_dev *pdev) 6067 { 6068 int status = 0; 6069 struct be_adapter *adapter = pci_get_drvdata(pdev); 6070 6071 dev_info(&adapter->pdev->dev, "EEH resume\n"); 6072 6073 pci_save_state(pdev); 6074 6075 status = be_resume(adapter); 6076 if (status) 6077 goto err; 6078 6079 be_roce_dev_add(adapter); 6080 6081 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 6082 return; 6083 err: 6084 dev_err(&adapter->pdev->dev, "EEH resume failed\n"); 6085 } 6086 6087 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) 6088 { 6089 struct be_adapter *adapter = pci_get_drvdata(pdev); 6090 struct be_resources vft_res = {0}; 6091 int status; 6092 6093 if (!num_vfs) 6094 be_vf_clear(adapter); 6095 6096 adapter->num_vfs = num_vfs; 6097 6098 if (adapter->num_vfs == 0 && pci_vfs_assigned(pdev)) { 6099 dev_warn(&pdev->dev, 6100 "Cannot disable VFs while they are assigned\n"); 6101 return -EBUSY; 6102 } 6103 6104 /* When the HW is in SRIOV capable configuration, the PF-pool resources 6105 * are equally distributed across the max-number of VFs. The user may 6106 * request only a subset of the max-vfs to be enabled. 6107 * Based on num_vfs, redistribute the resources across num_vfs so that 6108 * each VF will have access to more number of resources. 6109 * This facility is not available in BE3 FW. 6110 * Also, this is done by FW in Lancer chip. 6111 */ 6112 if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) { 6113 be_calculate_vf_res(adapter, adapter->num_vfs, 6114 &vft_res); 6115 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 6116 adapter->num_vfs, &vft_res); 6117 if (status) 6118 dev_err(&pdev->dev, 6119 "Failed to optimize SR-IOV resources\n"); 6120 } 6121 6122 status = be_get_resources(adapter); 6123 if (status) 6124 return be_cmd_status(status); 6125 6126 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 6127 rtnl_lock(); 6128 status = be_update_queues(adapter); 6129 rtnl_unlock(); 6130 if (status) 6131 return be_cmd_status(status); 6132 6133 if (adapter->num_vfs) 6134 status = be_vf_setup(adapter); 6135 6136 if (!status) 6137 return adapter->num_vfs; 6138 6139 return 0; 6140 } 6141 6142 static const struct pci_error_handlers be_eeh_handlers = { 6143 .error_detected = be_eeh_err_detected, 6144 .slot_reset = be_eeh_reset, 6145 .resume = be_eeh_resume, 6146 }; 6147 6148 static struct pci_driver be_driver = { 6149 .name = DRV_NAME, 6150 .id_table = be_dev_ids, 6151 .probe = be_probe, 6152 .remove = be_remove, 6153 .suspend = be_suspend, 6154 .resume = be_pci_resume, 6155 .shutdown = be_shutdown, 6156 .sriov_configure = be_pci_sriov_configure, 6157 .err_handler = &be_eeh_handlers 6158 }; 6159 6160 static int __init be_init_module(void) 6161 { 6162 int status; 6163 6164 if (rx_frag_size != 8192 && rx_frag_size != 4096 && 6165 rx_frag_size != 2048) { 6166 printk(KERN_WARNING DRV_NAME 6167 " : Module param rx_frag_size must be 2048/4096/8192." 6168 " Using 2048\n"); 6169 rx_frag_size = 2048; 6170 } 6171 6172 if (num_vfs > 0) { 6173 pr_info(DRV_NAME " : Module param num_vfs is obsolete."); 6174 pr_info(DRV_NAME " : Use sysfs method to enable VFs\n"); 6175 } 6176 6177 be_wq = create_singlethread_workqueue("be_wq"); 6178 if (!be_wq) { 6179 pr_warn(DRV_NAME "workqueue creation failed\n"); 6180 return -1; 6181 } 6182 6183 be_err_recovery_workq = 6184 create_singlethread_workqueue("be_err_recover"); 6185 if (!be_err_recovery_workq) 6186 pr_warn(DRV_NAME "Could not create error recovery workqueue\n"); 6187 6188 status = pci_register_driver(&be_driver); 6189 if (status) { 6190 destroy_workqueue(be_wq); 6191 be_destroy_err_recovery_workq(); 6192 } 6193 return status; 6194 } 6195 module_init(be_init_module); 6196 6197 static void __exit be_exit_module(void) 6198 { 6199 pci_unregister_driver(&be_driver); 6200 6201 be_destroy_err_recovery_workq(); 6202 6203 if (be_wq) 6204 destroy_workqueue(be_wq); 6205 } 6206 module_exit(be_exit_module); 6207