1 /* 2 * Copyright (C) 2005 - 2016 Broadcom 3 * All rights reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License version 2 7 * as published by the Free Software Foundation. The full GNU General 8 * Public License is included in this distribution in the file called COPYING. 9 * 10 * Contact Information: 11 * linux-drivers@emulex.com 12 * 13 * Emulex 14 * 3333 Susan Street 15 * Costa Mesa, CA 92626 16 */ 17 18 #include <linux/prefetch.h> 19 #include <linux/module.h> 20 #include "be.h" 21 #include "be_cmds.h" 22 #include <asm/div64.h> 23 #include <linux/aer.h> 24 #include <linux/if_bridge.h> 25 #include <net/busy_poll.h> 26 #include <net/vxlan.h> 27 28 MODULE_VERSION(DRV_VER); 29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER); 30 MODULE_AUTHOR("Emulex Corporation"); 31 MODULE_LICENSE("GPL"); 32 33 /* num_vfs module param is obsolete. 34 * Use sysfs method to enable/disable VFs. 35 */ 36 static unsigned int num_vfs; 37 module_param(num_vfs, uint, S_IRUGO); 38 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize"); 39 40 static ushort rx_frag_size = 2048; 41 module_param(rx_frag_size, ushort, S_IRUGO); 42 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data."); 43 44 /* Per-module error detection/recovery workq shared across all functions. 45 * Each function schedules its own work request on this shared workq. 46 */ 47 static struct workqueue_struct *be_err_recovery_workq; 48 49 static const struct pci_device_id be_dev_ids[] = { 50 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 51 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 52 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 53 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 54 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)}, 55 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)}, 56 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)}, 57 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)}, 58 { 0 } 59 }; 60 MODULE_DEVICE_TABLE(pci, be_dev_ids); 61 62 /* Workqueue used by all functions for defering cmd calls to the adapter */ 63 static struct workqueue_struct *be_wq; 64 65 /* UE Status Low CSR */ 66 static const char * const ue_status_low_desc[] = { 67 "CEV", 68 "CTX", 69 "DBUF", 70 "ERX", 71 "Host", 72 "MPU", 73 "NDMA", 74 "PTC ", 75 "RDMA ", 76 "RXF ", 77 "RXIPS ", 78 "RXULP0 ", 79 "RXULP1 ", 80 "RXULP2 ", 81 "TIM ", 82 "TPOST ", 83 "TPRE ", 84 "TXIPS ", 85 "TXULP0 ", 86 "TXULP1 ", 87 "UC ", 88 "WDMA ", 89 "TXULP2 ", 90 "HOST1 ", 91 "P0_OB_LINK ", 92 "P1_OB_LINK ", 93 "HOST_GPIO ", 94 "MBOX ", 95 "ERX2 ", 96 "SPARE ", 97 "JTAG ", 98 "MPU_INTPEND " 99 }; 100 101 /* UE Status High CSR */ 102 static const char * const ue_status_hi_desc[] = { 103 "LPCMEMHOST", 104 "MGMT_MAC", 105 "PCS0ONLINE", 106 "MPU_IRAM", 107 "PCS1ONLINE", 108 "PCTL0", 109 "PCTL1", 110 "PMEM", 111 "RR", 112 "TXPB", 113 "RXPP", 114 "XAUI", 115 "TXP", 116 "ARM", 117 "IPC", 118 "HOST2", 119 "HOST3", 120 "HOST4", 121 "HOST5", 122 "HOST6", 123 "HOST7", 124 "ECRC", 125 "Poison TLP", 126 "NETC", 127 "PERIPH", 128 "LLTXULP", 129 "D2P", 130 "RCON", 131 "LDMA", 132 "LLTXP", 133 "LLTXPB", 134 "Unknown" 135 }; 136 137 #define BE_VF_IF_EN_FLAGS (BE_IF_FLAGS_UNTAGGED | \ 138 BE_IF_FLAGS_BROADCAST | \ 139 BE_IF_FLAGS_MULTICAST | \ 140 BE_IF_FLAGS_PASS_L3L4_ERRORS) 141 142 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q) 143 { 144 struct be_dma_mem *mem = &q->dma_mem; 145 146 if (mem->va) { 147 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 148 mem->dma); 149 mem->va = NULL; 150 } 151 } 152 153 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q, 154 u16 len, u16 entry_size) 155 { 156 struct be_dma_mem *mem = &q->dma_mem; 157 158 memset(q, 0, sizeof(*q)); 159 q->len = len; 160 q->entry_size = entry_size; 161 mem->size = len * entry_size; 162 mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma, 163 GFP_KERNEL); 164 if (!mem->va) 165 return -ENOMEM; 166 return 0; 167 } 168 169 static void be_reg_intr_set(struct be_adapter *adapter, bool enable) 170 { 171 u32 reg, enabled; 172 173 pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, 174 ®); 175 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 176 177 if (!enabled && enable) 178 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 179 else if (enabled && !enable) 180 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 181 else 182 return; 183 184 pci_write_config_dword(adapter->pdev, 185 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg); 186 } 187 188 static void be_intr_set(struct be_adapter *adapter, bool enable) 189 { 190 int status = 0; 191 192 /* On lancer interrupts can't be controlled via this register */ 193 if (lancer_chip(adapter)) 194 return; 195 196 if (be_check_error(adapter, BE_ERROR_EEH)) 197 return; 198 199 status = be_cmd_intr_set(adapter, enable); 200 if (status) 201 be_reg_intr_set(adapter, enable); 202 } 203 204 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted) 205 { 206 u32 val = 0; 207 208 if (be_check_error(adapter, BE_ERROR_HW)) 209 return; 210 211 val |= qid & DB_RQ_RING_ID_MASK; 212 val |= posted << DB_RQ_NUM_POSTED_SHIFT; 213 214 wmb(); 215 iowrite32(val, adapter->db + DB_RQ_OFFSET); 216 } 217 218 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo, 219 u16 posted) 220 { 221 u32 val = 0; 222 223 if (be_check_error(adapter, BE_ERROR_HW)) 224 return; 225 226 val |= txo->q.id & DB_TXULP_RING_ID_MASK; 227 val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT; 228 229 wmb(); 230 iowrite32(val, adapter->db + txo->db_offset); 231 } 232 233 static void be_eq_notify(struct be_adapter *adapter, u16 qid, 234 bool arm, bool clear_int, u16 num_popped, 235 u32 eq_delay_mult_enc) 236 { 237 u32 val = 0; 238 239 val |= qid & DB_EQ_RING_ID_MASK; 240 val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT); 241 242 if (be_check_error(adapter, BE_ERROR_HW)) 243 return; 244 245 if (arm) 246 val |= 1 << DB_EQ_REARM_SHIFT; 247 if (clear_int) 248 val |= 1 << DB_EQ_CLR_SHIFT; 249 val |= 1 << DB_EQ_EVNT_SHIFT; 250 val |= num_popped << DB_EQ_NUM_POPPED_SHIFT; 251 val |= eq_delay_mult_enc << DB_EQ_R2I_DLY_SHIFT; 252 iowrite32(val, adapter->db + DB_EQ_OFFSET); 253 } 254 255 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped) 256 { 257 u32 val = 0; 258 259 val |= qid & DB_CQ_RING_ID_MASK; 260 val |= ((qid & DB_CQ_RING_ID_EXT_MASK) << 261 DB_CQ_RING_ID_EXT_MASK_SHIFT); 262 263 if (be_check_error(adapter, BE_ERROR_HW)) 264 return; 265 266 if (arm) 267 val |= 1 << DB_CQ_REARM_SHIFT; 268 val |= num_popped << DB_CQ_NUM_POPPED_SHIFT; 269 iowrite32(val, adapter->db + DB_CQ_OFFSET); 270 } 271 272 static int be_dev_mac_add(struct be_adapter *adapter, u8 *mac) 273 { 274 int i; 275 276 /* Check if mac has already been added as part of uc-list */ 277 for (i = 0; i < adapter->uc_macs; i++) { 278 if (ether_addr_equal((u8 *)&adapter->uc_list[i * ETH_ALEN], 279 mac)) { 280 /* mac already added, skip addition */ 281 adapter->pmac_id[0] = adapter->pmac_id[i + 1]; 282 return 0; 283 } 284 } 285 286 return be_cmd_pmac_add(adapter, mac, adapter->if_handle, 287 &adapter->pmac_id[0], 0); 288 } 289 290 static void be_dev_mac_del(struct be_adapter *adapter, int pmac_id) 291 { 292 int i; 293 294 /* Skip deletion if the programmed mac is 295 * being used in uc-list 296 */ 297 for (i = 0; i < adapter->uc_macs; i++) { 298 if (adapter->pmac_id[i + 1] == pmac_id) 299 return; 300 } 301 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0); 302 } 303 304 static int be_mac_addr_set(struct net_device *netdev, void *p) 305 { 306 struct be_adapter *adapter = netdev_priv(netdev); 307 struct device *dev = &adapter->pdev->dev; 308 struct sockaddr *addr = p; 309 int status; 310 u8 mac[ETH_ALEN]; 311 u32 old_pmac_id = adapter->pmac_id[0]; 312 313 if (!is_valid_ether_addr(addr->sa_data)) 314 return -EADDRNOTAVAIL; 315 316 /* Proceed further only if, User provided MAC is different 317 * from active MAC 318 */ 319 if (ether_addr_equal(addr->sa_data, adapter->dev_mac)) 320 return 0; 321 322 /* if device is not running, copy MAC to netdev->dev_addr */ 323 if (!netif_running(netdev)) 324 goto done; 325 326 /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT 327 * privilege or if PF did not provision the new MAC address. 328 * On BE3, this cmd will always fail if the VF doesn't have the 329 * FILTMGMT privilege. This failure is OK, only if the PF programmed 330 * the MAC for the VF. 331 */ 332 mutex_lock(&adapter->rx_filter_lock); 333 status = be_dev_mac_add(adapter, (u8 *)addr->sa_data); 334 if (!status) { 335 336 /* Delete the old programmed MAC. This call may fail if the 337 * old MAC was already deleted by the PF driver. 338 */ 339 if (adapter->pmac_id[0] != old_pmac_id) 340 be_dev_mac_del(adapter, old_pmac_id); 341 } 342 343 mutex_unlock(&adapter->rx_filter_lock); 344 /* Decide if the new MAC is successfully activated only after 345 * querying the FW 346 */ 347 status = be_cmd_get_active_mac(adapter, adapter->pmac_id[0], mac, 348 adapter->if_handle, true, 0); 349 if (status) 350 goto err; 351 352 /* The MAC change did not happen, either due to lack of privilege 353 * or PF didn't pre-provision. 354 */ 355 if (!ether_addr_equal(addr->sa_data, mac)) { 356 status = -EPERM; 357 goto err; 358 } 359 done: 360 ether_addr_copy(adapter->dev_mac, addr->sa_data); 361 ether_addr_copy(netdev->dev_addr, addr->sa_data); 362 dev_info(dev, "MAC address changed to %pM\n", addr->sa_data); 363 return 0; 364 err: 365 dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data); 366 return status; 367 } 368 369 /* BE2 supports only v0 cmd */ 370 static void *hw_stats_from_cmd(struct be_adapter *adapter) 371 { 372 if (BE2_chip(adapter)) { 373 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va; 374 375 return &cmd->hw_stats; 376 } else if (BE3_chip(adapter)) { 377 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va; 378 379 return &cmd->hw_stats; 380 } else { 381 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va; 382 383 return &cmd->hw_stats; 384 } 385 } 386 387 /* BE2 supports only v0 cmd */ 388 static void *be_erx_stats_from_cmd(struct be_adapter *adapter) 389 { 390 if (BE2_chip(adapter)) { 391 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 392 393 return &hw_stats->erx; 394 } else if (BE3_chip(adapter)) { 395 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 396 397 return &hw_stats->erx; 398 } else { 399 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 400 401 return &hw_stats->erx; 402 } 403 } 404 405 static void populate_be_v0_stats(struct be_adapter *adapter) 406 { 407 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 408 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 409 struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf; 410 struct be_port_rxf_stats_v0 *port_stats = 411 &rxf_stats->port[adapter->port_num]; 412 struct be_drv_stats *drvs = &adapter->drv_stats; 413 414 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 415 drvs->rx_pause_frames = port_stats->rx_pause_frames; 416 drvs->rx_crc_errors = port_stats->rx_crc_errors; 417 drvs->rx_control_frames = port_stats->rx_control_frames; 418 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 419 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 420 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 421 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 422 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 423 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 424 drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow; 425 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 426 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 427 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 428 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 429 drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow; 430 drvs->rx_dropped_header_too_small = 431 port_stats->rx_dropped_header_too_small; 432 drvs->rx_address_filtered = 433 port_stats->rx_address_filtered + 434 port_stats->rx_vlan_filtered; 435 drvs->rx_alignment_symbol_errors = 436 port_stats->rx_alignment_symbol_errors; 437 438 drvs->tx_pauseframes = port_stats->tx_pauseframes; 439 drvs->tx_controlframes = port_stats->tx_controlframes; 440 441 if (adapter->port_num) 442 drvs->jabber_events = rxf_stats->port1_jabber_events; 443 else 444 drvs->jabber_events = rxf_stats->port0_jabber_events; 445 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 446 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 447 drvs->forwarded_packets = rxf_stats->forwarded_packets; 448 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 449 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 450 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 451 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 452 } 453 454 static void populate_be_v1_stats(struct be_adapter *adapter) 455 { 456 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 457 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 458 struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf; 459 struct be_port_rxf_stats_v1 *port_stats = 460 &rxf_stats->port[adapter->port_num]; 461 struct be_drv_stats *drvs = &adapter->drv_stats; 462 463 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 464 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 465 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 466 drvs->rx_pause_frames = port_stats->rx_pause_frames; 467 drvs->rx_crc_errors = port_stats->rx_crc_errors; 468 drvs->rx_control_frames = port_stats->rx_control_frames; 469 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 470 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 471 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 472 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 473 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 474 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 475 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 476 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 477 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 478 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 479 drvs->rx_dropped_header_too_small = 480 port_stats->rx_dropped_header_too_small; 481 drvs->rx_input_fifo_overflow_drop = 482 port_stats->rx_input_fifo_overflow_drop; 483 drvs->rx_address_filtered = port_stats->rx_address_filtered; 484 drvs->rx_alignment_symbol_errors = 485 port_stats->rx_alignment_symbol_errors; 486 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 487 drvs->tx_pauseframes = port_stats->tx_pauseframes; 488 drvs->tx_controlframes = port_stats->tx_controlframes; 489 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 490 drvs->jabber_events = port_stats->jabber_events; 491 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 492 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 493 drvs->forwarded_packets = rxf_stats->forwarded_packets; 494 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 495 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 496 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 497 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 498 } 499 500 static void populate_be_v2_stats(struct be_adapter *adapter) 501 { 502 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 503 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 504 struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf; 505 struct be_port_rxf_stats_v2 *port_stats = 506 &rxf_stats->port[adapter->port_num]; 507 struct be_drv_stats *drvs = &adapter->drv_stats; 508 509 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 510 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 511 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 512 drvs->rx_pause_frames = port_stats->rx_pause_frames; 513 drvs->rx_crc_errors = port_stats->rx_crc_errors; 514 drvs->rx_control_frames = port_stats->rx_control_frames; 515 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 516 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 517 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 518 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 519 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 520 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 521 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 522 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 523 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 524 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 525 drvs->rx_dropped_header_too_small = 526 port_stats->rx_dropped_header_too_small; 527 drvs->rx_input_fifo_overflow_drop = 528 port_stats->rx_input_fifo_overflow_drop; 529 drvs->rx_address_filtered = port_stats->rx_address_filtered; 530 drvs->rx_alignment_symbol_errors = 531 port_stats->rx_alignment_symbol_errors; 532 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 533 drvs->tx_pauseframes = port_stats->tx_pauseframes; 534 drvs->tx_controlframes = port_stats->tx_controlframes; 535 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 536 drvs->jabber_events = port_stats->jabber_events; 537 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 538 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 539 drvs->forwarded_packets = rxf_stats->forwarded_packets; 540 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 541 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 542 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 543 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 544 if (be_roce_supported(adapter)) { 545 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd; 546 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd; 547 drvs->rx_roce_frames = port_stats->roce_frames_received; 548 drvs->roce_drops_crc = port_stats->roce_drops_crc; 549 drvs->roce_drops_payload_len = 550 port_stats->roce_drops_payload_len; 551 } 552 } 553 554 static void populate_lancer_stats(struct be_adapter *adapter) 555 { 556 struct be_drv_stats *drvs = &adapter->drv_stats; 557 struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter); 558 559 be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats)); 560 drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo; 561 drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo; 562 drvs->rx_control_frames = pport_stats->rx_control_frames_lo; 563 drvs->rx_in_range_errors = pport_stats->rx_in_range_errors; 564 drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo; 565 drvs->rx_dropped_runt = pport_stats->rx_dropped_runt; 566 drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors; 567 drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors; 568 drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors; 569 drvs->rx_dropped_tcp_length = 570 pport_stats->rx_dropped_invalid_tcp_length; 571 drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small; 572 drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short; 573 drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors; 574 drvs->rx_dropped_header_too_small = 575 pport_stats->rx_dropped_header_too_small; 576 drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 577 drvs->rx_address_filtered = 578 pport_stats->rx_address_filtered + 579 pport_stats->rx_vlan_filtered; 580 drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo; 581 drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 582 drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo; 583 drvs->tx_controlframes = pport_stats->tx_control_frames_lo; 584 drvs->jabber_events = pport_stats->rx_jabbers; 585 drvs->forwarded_packets = pport_stats->num_forwards_lo; 586 drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo; 587 drvs->rx_drops_too_many_frags = 588 pport_stats->rx_drops_too_many_frags_lo; 589 } 590 591 static void accumulate_16bit_val(u32 *acc, u16 val) 592 { 593 #define lo(x) (x & 0xFFFF) 594 #define hi(x) (x & 0xFFFF0000) 595 bool wrapped = val < lo(*acc); 596 u32 newacc = hi(*acc) + val; 597 598 if (wrapped) 599 newacc += 65536; 600 ACCESS_ONCE(*acc) = newacc; 601 } 602 603 static void populate_erx_stats(struct be_adapter *adapter, 604 struct be_rx_obj *rxo, u32 erx_stat) 605 { 606 if (!BEx_chip(adapter)) 607 rx_stats(rxo)->rx_drops_no_frags = erx_stat; 608 else 609 /* below erx HW counter can actually wrap around after 610 * 65535. Driver accumulates a 32-bit value 611 */ 612 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags, 613 (u16)erx_stat); 614 } 615 616 void be_parse_stats(struct be_adapter *adapter) 617 { 618 struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter); 619 struct be_rx_obj *rxo; 620 int i; 621 u32 erx_stat; 622 623 if (lancer_chip(adapter)) { 624 populate_lancer_stats(adapter); 625 } else { 626 if (BE2_chip(adapter)) 627 populate_be_v0_stats(adapter); 628 else if (BE3_chip(adapter)) 629 /* for BE3 */ 630 populate_be_v1_stats(adapter); 631 else 632 populate_be_v2_stats(adapter); 633 634 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */ 635 for_all_rx_queues(adapter, rxo, i) { 636 erx_stat = erx->rx_drops_no_fragments[rxo->q.id]; 637 populate_erx_stats(adapter, rxo, erx_stat); 638 } 639 } 640 } 641 642 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev, 643 struct rtnl_link_stats64 *stats) 644 { 645 struct be_adapter *adapter = netdev_priv(netdev); 646 struct be_drv_stats *drvs = &adapter->drv_stats; 647 struct be_rx_obj *rxo; 648 struct be_tx_obj *txo; 649 u64 pkts, bytes; 650 unsigned int start; 651 int i; 652 653 for_all_rx_queues(adapter, rxo, i) { 654 const struct be_rx_stats *rx_stats = rx_stats(rxo); 655 656 do { 657 start = u64_stats_fetch_begin_irq(&rx_stats->sync); 658 pkts = rx_stats(rxo)->rx_pkts; 659 bytes = rx_stats(rxo)->rx_bytes; 660 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start)); 661 stats->rx_packets += pkts; 662 stats->rx_bytes += bytes; 663 stats->multicast += rx_stats(rxo)->rx_mcast_pkts; 664 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs + 665 rx_stats(rxo)->rx_drops_no_frags; 666 } 667 668 for_all_tx_queues(adapter, txo, i) { 669 const struct be_tx_stats *tx_stats = tx_stats(txo); 670 671 do { 672 start = u64_stats_fetch_begin_irq(&tx_stats->sync); 673 pkts = tx_stats(txo)->tx_pkts; 674 bytes = tx_stats(txo)->tx_bytes; 675 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start)); 676 stats->tx_packets += pkts; 677 stats->tx_bytes += bytes; 678 } 679 680 /* bad pkts received */ 681 stats->rx_errors = drvs->rx_crc_errors + 682 drvs->rx_alignment_symbol_errors + 683 drvs->rx_in_range_errors + 684 drvs->rx_out_range_errors + 685 drvs->rx_frame_too_long + 686 drvs->rx_dropped_too_small + 687 drvs->rx_dropped_too_short + 688 drvs->rx_dropped_header_too_small + 689 drvs->rx_dropped_tcp_length + 690 drvs->rx_dropped_runt; 691 692 /* detailed rx errors */ 693 stats->rx_length_errors = drvs->rx_in_range_errors + 694 drvs->rx_out_range_errors + 695 drvs->rx_frame_too_long; 696 697 stats->rx_crc_errors = drvs->rx_crc_errors; 698 699 /* frame alignment errors */ 700 stats->rx_frame_errors = drvs->rx_alignment_symbol_errors; 701 702 /* receiver fifo overrun */ 703 /* drops_no_pbuf is no per i/f, it's per BE card */ 704 stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop + 705 drvs->rx_input_fifo_overflow_drop + 706 drvs->rx_drops_no_pbuf; 707 return stats; 708 } 709 710 void be_link_status_update(struct be_adapter *adapter, u8 link_status) 711 { 712 struct net_device *netdev = adapter->netdev; 713 714 if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) { 715 netif_carrier_off(netdev); 716 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT; 717 } 718 719 if (link_status) 720 netif_carrier_on(netdev); 721 else 722 netif_carrier_off(netdev); 723 724 netdev_info(netdev, "Link is %s\n", link_status ? "Up" : "Down"); 725 } 726 727 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb) 728 { 729 struct be_tx_stats *stats = tx_stats(txo); 730 u64 tx_pkts = skb_shinfo(skb)->gso_segs ? : 1; 731 732 u64_stats_update_begin(&stats->sync); 733 stats->tx_reqs++; 734 stats->tx_bytes += skb->len; 735 stats->tx_pkts += tx_pkts; 736 if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL) 737 stats->tx_vxlan_offload_pkts += tx_pkts; 738 u64_stats_update_end(&stats->sync); 739 } 740 741 /* Returns number of WRBs needed for the skb */ 742 static u32 skb_wrb_cnt(struct sk_buff *skb) 743 { 744 /* +1 for the header wrb */ 745 return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags; 746 } 747 748 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len) 749 { 750 wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr)); 751 wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr)); 752 wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK); 753 wrb->rsvd0 = 0; 754 } 755 756 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb 757 * to avoid the swap and shift/mask operations in wrb_fill(). 758 */ 759 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb) 760 { 761 wrb->frag_pa_hi = 0; 762 wrb->frag_pa_lo = 0; 763 wrb->frag_len = 0; 764 wrb->rsvd0 = 0; 765 } 766 767 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter, 768 struct sk_buff *skb) 769 { 770 u8 vlan_prio; 771 u16 vlan_tag; 772 773 vlan_tag = skb_vlan_tag_get(skb); 774 vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 775 /* If vlan priority provided by OS is NOT in available bmap */ 776 if (!(adapter->vlan_prio_bmap & (1 << vlan_prio))) 777 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) | 778 adapter->recommended_prio_bits; 779 780 return vlan_tag; 781 } 782 783 /* Used only for IP tunnel packets */ 784 static u16 skb_inner_ip_proto(struct sk_buff *skb) 785 { 786 return (inner_ip_hdr(skb)->version == 4) ? 787 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr; 788 } 789 790 static u16 skb_ip_proto(struct sk_buff *skb) 791 { 792 return (ip_hdr(skb)->version == 4) ? 793 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr; 794 } 795 796 static inline bool be_is_txq_full(struct be_tx_obj *txo) 797 { 798 return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len; 799 } 800 801 static inline bool be_can_txq_wake(struct be_tx_obj *txo) 802 { 803 return atomic_read(&txo->q.used) < txo->q.len / 2; 804 } 805 806 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo) 807 { 808 return atomic_read(&txo->q.used) > txo->pend_wrb_cnt; 809 } 810 811 static void be_get_wrb_params_from_skb(struct be_adapter *adapter, 812 struct sk_buff *skb, 813 struct be_wrb_params *wrb_params) 814 { 815 u16 proto; 816 817 if (skb_is_gso(skb)) { 818 BE_WRB_F_SET(wrb_params->features, LSO, 1); 819 wrb_params->lso_mss = skb_shinfo(skb)->gso_size; 820 if (skb_is_gso_v6(skb) && !lancer_chip(adapter)) 821 BE_WRB_F_SET(wrb_params->features, LSO6, 1); 822 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 823 if (skb->encapsulation) { 824 BE_WRB_F_SET(wrb_params->features, IPCS, 1); 825 proto = skb_inner_ip_proto(skb); 826 } else { 827 proto = skb_ip_proto(skb); 828 } 829 if (proto == IPPROTO_TCP) 830 BE_WRB_F_SET(wrb_params->features, TCPCS, 1); 831 else if (proto == IPPROTO_UDP) 832 BE_WRB_F_SET(wrb_params->features, UDPCS, 1); 833 } 834 835 if (skb_vlan_tag_present(skb)) { 836 BE_WRB_F_SET(wrb_params->features, VLAN, 1); 837 wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb); 838 } 839 840 BE_WRB_F_SET(wrb_params->features, CRC, 1); 841 } 842 843 static void wrb_fill_hdr(struct be_adapter *adapter, 844 struct be_eth_hdr_wrb *hdr, 845 struct be_wrb_params *wrb_params, 846 struct sk_buff *skb) 847 { 848 memset(hdr, 0, sizeof(*hdr)); 849 850 SET_TX_WRB_HDR_BITS(crc, hdr, 851 BE_WRB_F_GET(wrb_params->features, CRC)); 852 SET_TX_WRB_HDR_BITS(ipcs, hdr, 853 BE_WRB_F_GET(wrb_params->features, IPCS)); 854 SET_TX_WRB_HDR_BITS(tcpcs, hdr, 855 BE_WRB_F_GET(wrb_params->features, TCPCS)); 856 SET_TX_WRB_HDR_BITS(udpcs, hdr, 857 BE_WRB_F_GET(wrb_params->features, UDPCS)); 858 859 SET_TX_WRB_HDR_BITS(lso, hdr, 860 BE_WRB_F_GET(wrb_params->features, LSO)); 861 SET_TX_WRB_HDR_BITS(lso6, hdr, 862 BE_WRB_F_GET(wrb_params->features, LSO6)); 863 SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss); 864 865 /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this 866 * hack is not needed, the evt bit is set while ringing DB. 867 */ 868 SET_TX_WRB_HDR_BITS(event, hdr, 869 BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW)); 870 SET_TX_WRB_HDR_BITS(vlan, hdr, 871 BE_WRB_F_GET(wrb_params->features, VLAN)); 872 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag); 873 874 SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb)); 875 SET_TX_WRB_HDR_BITS(len, hdr, skb->len); 876 SET_TX_WRB_HDR_BITS(mgmt, hdr, 877 BE_WRB_F_GET(wrb_params->features, OS2BMC)); 878 } 879 880 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb, 881 bool unmap_single) 882 { 883 dma_addr_t dma; 884 u32 frag_len = le32_to_cpu(wrb->frag_len); 885 886 887 dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 | 888 (u64)le32_to_cpu(wrb->frag_pa_lo); 889 if (frag_len) { 890 if (unmap_single) 891 dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE); 892 else 893 dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE); 894 } 895 } 896 897 /* Grab a WRB header for xmit */ 898 static u32 be_tx_get_wrb_hdr(struct be_tx_obj *txo) 899 { 900 u32 head = txo->q.head; 901 902 queue_head_inc(&txo->q); 903 return head; 904 } 905 906 /* Set up the WRB header for xmit */ 907 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter, 908 struct be_tx_obj *txo, 909 struct be_wrb_params *wrb_params, 910 struct sk_buff *skb, u16 head) 911 { 912 u32 num_frags = skb_wrb_cnt(skb); 913 struct be_queue_info *txq = &txo->q; 914 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head); 915 916 wrb_fill_hdr(adapter, hdr, wrb_params, skb); 917 be_dws_cpu_to_le(hdr, sizeof(*hdr)); 918 919 BUG_ON(txo->sent_skb_list[head]); 920 txo->sent_skb_list[head] = skb; 921 txo->last_req_hdr = head; 922 atomic_add(num_frags, &txq->used); 923 txo->last_req_wrb_cnt = num_frags; 924 txo->pend_wrb_cnt += num_frags; 925 } 926 927 /* Setup a WRB fragment (buffer descriptor) for xmit */ 928 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr, 929 int len) 930 { 931 struct be_eth_wrb *wrb; 932 struct be_queue_info *txq = &txo->q; 933 934 wrb = queue_head_node(txq); 935 wrb_fill(wrb, busaddr, len); 936 queue_head_inc(txq); 937 } 938 939 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine 940 * was invoked. The producer index is restored to the previous packet and the 941 * WRBs of the current packet are unmapped. Invoked to handle tx setup errors. 942 */ 943 static void be_xmit_restore(struct be_adapter *adapter, 944 struct be_tx_obj *txo, u32 head, bool map_single, 945 u32 copied) 946 { 947 struct device *dev; 948 struct be_eth_wrb *wrb; 949 struct be_queue_info *txq = &txo->q; 950 951 dev = &adapter->pdev->dev; 952 txq->head = head; 953 954 /* skip the first wrb (hdr); it's not mapped */ 955 queue_head_inc(txq); 956 while (copied) { 957 wrb = queue_head_node(txq); 958 unmap_tx_frag(dev, wrb, map_single); 959 map_single = false; 960 copied -= le32_to_cpu(wrb->frag_len); 961 queue_head_inc(txq); 962 } 963 964 txq->head = head; 965 } 966 967 /* Enqueue the given packet for transmit. This routine allocates WRBs for the 968 * packet, dma maps the packet buffers and sets up the WRBs. Returns the number 969 * of WRBs used up by the packet. 970 */ 971 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo, 972 struct sk_buff *skb, 973 struct be_wrb_params *wrb_params) 974 { 975 u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb); 976 struct device *dev = &adapter->pdev->dev; 977 struct be_queue_info *txq = &txo->q; 978 bool map_single = false; 979 u32 head = txq->head; 980 dma_addr_t busaddr; 981 int len; 982 983 head = be_tx_get_wrb_hdr(txo); 984 985 if (skb->len > skb->data_len) { 986 len = skb_headlen(skb); 987 988 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE); 989 if (dma_mapping_error(dev, busaddr)) 990 goto dma_err; 991 map_single = true; 992 be_tx_setup_wrb_frag(txo, busaddr, len); 993 copied += len; 994 } 995 996 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 997 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; 998 len = skb_frag_size(frag); 999 1000 busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE); 1001 if (dma_mapping_error(dev, busaddr)) 1002 goto dma_err; 1003 be_tx_setup_wrb_frag(txo, busaddr, len); 1004 copied += len; 1005 } 1006 1007 be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head); 1008 1009 be_tx_stats_update(txo, skb); 1010 return wrb_cnt; 1011 1012 dma_err: 1013 adapter->drv_stats.dma_map_errors++; 1014 be_xmit_restore(adapter, txo, head, map_single, copied); 1015 return 0; 1016 } 1017 1018 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter) 1019 { 1020 return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD; 1021 } 1022 1023 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter, 1024 struct sk_buff *skb, 1025 struct be_wrb_params 1026 *wrb_params) 1027 { 1028 u16 vlan_tag = 0; 1029 1030 skb = skb_share_check(skb, GFP_ATOMIC); 1031 if (unlikely(!skb)) 1032 return skb; 1033 1034 if (skb_vlan_tag_present(skb)) 1035 vlan_tag = be_get_tx_vlan_tag(adapter, skb); 1036 1037 if (qnq_async_evt_rcvd(adapter) && adapter->pvid) { 1038 if (!vlan_tag) 1039 vlan_tag = adapter->pvid; 1040 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to 1041 * skip VLAN insertion 1042 */ 1043 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1044 } 1045 1046 if (vlan_tag) { 1047 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 1048 vlan_tag); 1049 if (unlikely(!skb)) 1050 return skb; 1051 skb->vlan_tci = 0; 1052 } 1053 1054 /* Insert the outer VLAN, if any */ 1055 if (adapter->qnq_vid) { 1056 vlan_tag = adapter->qnq_vid; 1057 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 1058 vlan_tag); 1059 if (unlikely(!skb)) 1060 return skb; 1061 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1062 } 1063 1064 return skb; 1065 } 1066 1067 static bool be_ipv6_exthdr_check(struct sk_buff *skb) 1068 { 1069 struct ethhdr *eh = (struct ethhdr *)skb->data; 1070 u16 offset = ETH_HLEN; 1071 1072 if (eh->h_proto == htons(ETH_P_IPV6)) { 1073 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset); 1074 1075 offset += sizeof(struct ipv6hdr); 1076 if (ip6h->nexthdr != NEXTHDR_TCP && 1077 ip6h->nexthdr != NEXTHDR_UDP) { 1078 struct ipv6_opt_hdr *ehdr = 1079 (struct ipv6_opt_hdr *)(skb->data + offset); 1080 1081 /* offending pkt: 2nd byte following IPv6 hdr is 0xff */ 1082 if (ehdr->hdrlen == 0xff) 1083 return true; 1084 } 1085 } 1086 return false; 1087 } 1088 1089 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb) 1090 { 1091 return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid; 1092 } 1093 1094 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb) 1095 { 1096 return BE3_chip(adapter) && be_ipv6_exthdr_check(skb); 1097 } 1098 1099 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter, 1100 struct sk_buff *skb, 1101 struct be_wrb_params 1102 *wrb_params) 1103 { 1104 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 1105 unsigned int eth_hdr_len; 1106 struct iphdr *ip; 1107 1108 /* For padded packets, BE HW modifies tot_len field in IP header 1109 * incorrecly when VLAN tag is inserted by HW. 1110 * For padded packets, Lancer computes incorrect checksum. 1111 */ 1112 eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ? 1113 VLAN_ETH_HLEN : ETH_HLEN; 1114 if (skb->len <= 60 && 1115 (lancer_chip(adapter) || skb_vlan_tag_present(skb)) && 1116 is_ipv4_pkt(skb)) { 1117 ip = (struct iphdr *)ip_hdr(skb); 1118 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len)); 1119 } 1120 1121 /* If vlan tag is already inlined in the packet, skip HW VLAN 1122 * tagging in pvid-tagging mode 1123 */ 1124 if (be_pvid_tagging_enabled(adapter) && 1125 veh->h_vlan_proto == htons(ETH_P_8021Q)) 1126 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1127 1128 /* HW has a bug wherein it will calculate CSUM for VLAN 1129 * pkts even though it is disabled. 1130 * Manually insert VLAN in pkt. 1131 */ 1132 if (skb->ip_summed != CHECKSUM_PARTIAL && 1133 skb_vlan_tag_present(skb)) { 1134 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1135 if (unlikely(!skb)) 1136 goto err; 1137 } 1138 1139 /* HW may lockup when VLAN HW tagging is requested on 1140 * certain ipv6 packets. Drop such pkts if the HW workaround to 1141 * skip HW tagging is not enabled by FW. 1142 */ 1143 if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) && 1144 (adapter->pvid || adapter->qnq_vid) && 1145 !qnq_async_evt_rcvd(adapter))) 1146 goto tx_drop; 1147 1148 /* Manual VLAN tag insertion to prevent: 1149 * ASIC lockup when the ASIC inserts VLAN tag into 1150 * certain ipv6 packets. Insert VLAN tags in driver, 1151 * and set event, completion, vlan bits accordingly 1152 * in the Tx WRB. 1153 */ 1154 if (be_ipv6_tx_stall_chk(adapter, skb) && 1155 be_vlan_tag_tx_chk(adapter, skb)) { 1156 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1157 if (unlikely(!skb)) 1158 goto err; 1159 } 1160 1161 return skb; 1162 tx_drop: 1163 dev_kfree_skb_any(skb); 1164 err: 1165 return NULL; 1166 } 1167 1168 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter, 1169 struct sk_buff *skb, 1170 struct be_wrb_params *wrb_params) 1171 { 1172 int err; 1173 1174 /* Lancer, SH and BE3 in SRIOV mode have a bug wherein 1175 * packets that are 32b or less may cause a transmit stall 1176 * on that port. The workaround is to pad such packets 1177 * (len <= 32 bytes) to a minimum length of 36b. 1178 */ 1179 if (skb->len <= 32) { 1180 if (skb_put_padto(skb, 36)) 1181 return NULL; 1182 } 1183 1184 if (BEx_chip(adapter) || lancer_chip(adapter)) { 1185 skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params); 1186 if (!skb) 1187 return NULL; 1188 } 1189 1190 /* The stack can send us skbs with length greater than 1191 * what the HW can handle. Trim the extra bytes. 1192 */ 1193 WARN_ON_ONCE(skb->len > BE_MAX_GSO_SIZE); 1194 err = pskb_trim(skb, BE_MAX_GSO_SIZE); 1195 WARN_ON(err); 1196 1197 return skb; 1198 } 1199 1200 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo) 1201 { 1202 struct be_queue_info *txq = &txo->q; 1203 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr); 1204 1205 /* Mark the last request eventable if it hasn't been marked already */ 1206 if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT))) 1207 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL); 1208 1209 /* compose a dummy wrb if there are odd set of wrbs to notify */ 1210 if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) { 1211 wrb_fill_dummy(queue_head_node(txq)); 1212 queue_head_inc(txq); 1213 atomic_inc(&txq->used); 1214 txo->pend_wrb_cnt++; 1215 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK << 1216 TX_HDR_WRB_NUM_SHIFT); 1217 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) << 1218 TX_HDR_WRB_NUM_SHIFT); 1219 } 1220 be_txq_notify(adapter, txo, txo->pend_wrb_cnt); 1221 txo->pend_wrb_cnt = 0; 1222 } 1223 1224 /* OS2BMC related */ 1225 1226 #define DHCP_CLIENT_PORT 68 1227 #define DHCP_SERVER_PORT 67 1228 #define NET_BIOS_PORT1 137 1229 #define NET_BIOS_PORT2 138 1230 #define DHCPV6_RAS_PORT 547 1231 1232 #define is_mc_allowed_on_bmc(adapter, eh) \ 1233 (!is_multicast_filt_enabled(adapter) && \ 1234 is_multicast_ether_addr(eh->h_dest) && \ 1235 !is_broadcast_ether_addr(eh->h_dest)) 1236 1237 #define is_bc_allowed_on_bmc(adapter, eh) \ 1238 (!is_broadcast_filt_enabled(adapter) && \ 1239 is_broadcast_ether_addr(eh->h_dest)) 1240 1241 #define is_arp_allowed_on_bmc(adapter, skb) \ 1242 (is_arp(skb) && is_arp_filt_enabled(adapter)) 1243 1244 #define is_broadcast_packet(eh, adapter) \ 1245 (is_multicast_ether_addr(eh->h_dest) && \ 1246 !compare_ether_addr(eh->h_dest, adapter->netdev->broadcast)) 1247 1248 #define is_arp(skb) (skb->protocol == htons(ETH_P_ARP)) 1249 1250 #define is_arp_filt_enabled(adapter) \ 1251 (adapter->bmc_filt_mask & (BMC_FILT_BROADCAST_ARP)) 1252 1253 #define is_dhcp_client_filt_enabled(adapter) \ 1254 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_CLIENT) 1255 1256 #define is_dhcp_srvr_filt_enabled(adapter) \ 1257 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_SERVER) 1258 1259 #define is_nbios_filt_enabled(adapter) \ 1260 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_NET_BIOS) 1261 1262 #define is_ipv6_na_filt_enabled(adapter) \ 1263 (adapter->bmc_filt_mask & \ 1264 BMC_FILT_MULTICAST_IPV6_NEIGH_ADVER) 1265 1266 #define is_ipv6_ra_filt_enabled(adapter) \ 1267 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RA) 1268 1269 #define is_ipv6_ras_filt_enabled(adapter) \ 1270 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RAS) 1271 1272 #define is_broadcast_filt_enabled(adapter) \ 1273 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST) 1274 1275 #define is_multicast_filt_enabled(adapter) \ 1276 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST) 1277 1278 static bool be_send_pkt_to_bmc(struct be_adapter *adapter, 1279 struct sk_buff **skb) 1280 { 1281 struct ethhdr *eh = (struct ethhdr *)(*skb)->data; 1282 bool os2bmc = false; 1283 1284 if (!be_is_os2bmc_enabled(adapter)) 1285 goto done; 1286 1287 if (!is_multicast_ether_addr(eh->h_dest)) 1288 goto done; 1289 1290 if (is_mc_allowed_on_bmc(adapter, eh) || 1291 is_bc_allowed_on_bmc(adapter, eh) || 1292 is_arp_allowed_on_bmc(adapter, (*skb))) { 1293 os2bmc = true; 1294 goto done; 1295 } 1296 1297 if ((*skb)->protocol == htons(ETH_P_IPV6)) { 1298 struct ipv6hdr *hdr = ipv6_hdr((*skb)); 1299 u8 nexthdr = hdr->nexthdr; 1300 1301 if (nexthdr == IPPROTO_ICMPV6) { 1302 struct icmp6hdr *icmp6 = icmp6_hdr((*skb)); 1303 1304 switch (icmp6->icmp6_type) { 1305 case NDISC_ROUTER_ADVERTISEMENT: 1306 os2bmc = is_ipv6_ra_filt_enabled(adapter); 1307 goto done; 1308 case NDISC_NEIGHBOUR_ADVERTISEMENT: 1309 os2bmc = is_ipv6_na_filt_enabled(adapter); 1310 goto done; 1311 default: 1312 break; 1313 } 1314 } 1315 } 1316 1317 if (is_udp_pkt((*skb))) { 1318 struct udphdr *udp = udp_hdr((*skb)); 1319 1320 switch (ntohs(udp->dest)) { 1321 case DHCP_CLIENT_PORT: 1322 os2bmc = is_dhcp_client_filt_enabled(adapter); 1323 goto done; 1324 case DHCP_SERVER_PORT: 1325 os2bmc = is_dhcp_srvr_filt_enabled(adapter); 1326 goto done; 1327 case NET_BIOS_PORT1: 1328 case NET_BIOS_PORT2: 1329 os2bmc = is_nbios_filt_enabled(adapter); 1330 goto done; 1331 case DHCPV6_RAS_PORT: 1332 os2bmc = is_ipv6_ras_filt_enabled(adapter); 1333 goto done; 1334 default: 1335 break; 1336 } 1337 } 1338 done: 1339 /* For packets over a vlan, which are destined 1340 * to BMC, asic expects the vlan to be inline in the packet. 1341 */ 1342 if (os2bmc) 1343 *skb = be_insert_vlan_in_pkt(adapter, *skb, NULL); 1344 1345 return os2bmc; 1346 } 1347 1348 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev) 1349 { 1350 struct be_adapter *adapter = netdev_priv(netdev); 1351 u16 q_idx = skb_get_queue_mapping(skb); 1352 struct be_tx_obj *txo = &adapter->tx_obj[q_idx]; 1353 struct be_wrb_params wrb_params = { 0 }; 1354 bool flush = !skb->xmit_more; 1355 u16 wrb_cnt; 1356 1357 skb = be_xmit_workarounds(adapter, skb, &wrb_params); 1358 if (unlikely(!skb)) 1359 goto drop; 1360 1361 be_get_wrb_params_from_skb(adapter, skb, &wrb_params); 1362 1363 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); 1364 if (unlikely(!wrb_cnt)) { 1365 dev_kfree_skb_any(skb); 1366 goto drop; 1367 } 1368 1369 /* if os2bmc is enabled and if the pkt is destined to bmc, 1370 * enqueue the pkt a 2nd time with mgmt bit set. 1371 */ 1372 if (be_send_pkt_to_bmc(adapter, &skb)) { 1373 BE_WRB_F_SET(wrb_params.features, OS2BMC, 1); 1374 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); 1375 if (unlikely(!wrb_cnt)) 1376 goto drop; 1377 else 1378 skb_get(skb); 1379 } 1380 1381 if (be_is_txq_full(txo)) { 1382 netif_stop_subqueue(netdev, q_idx); 1383 tx_stats(txo)->tx_stops++; 1384 } 1385 1386 if (flush || __netif_subqueue_stopped(netdev, q_idx)) 1387 be_xmit_flush(adapter, txo); 1388 1389 return NETDEV_TX_OK; 1390 drop: 1391 tx_stats(txo)->tx_drv_drops++; 1392 /* Flush the already enqueued tx requests */ 1393 if (flush && txo->pend_wrb_cnt) 1394 be_xmit_flush(adapter, txo); 1395 1396 return NETDEV_TX_OK; 1397 } 1398 1399 static int be_change_mtu(struct net_device *netdev, int new_mtu) 1400 { 1401 struct be_adapter *adapter = netdev_priv(netdev); 1402 struct device *dev = &adapter->pdev->dev; 1403 1404 if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) { 1405 dev_info(dev, "MTU must be between %d and %d bytes\n", 1406 BE_MIN_MTU, BE_MAX_MTU); 1407 return -EINVAL; 1408 } 1409 1410 dev_info(dev, "MTU changed from %d to %d bytes\n", 1411 netdev->mtu, new_mtu); 1412 netdev->mtu = new_mtu; 1413 return 0; 1414 } 1415 1416 static inline bool be_in_all_promisc(struct be_adapter *adapter) 1417 { 1418 return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) == 1419 BE_IF_FLAGS_ALL_PROMISCUOUS; 1420 } 1421 1422 static int be_set_vlan_promisc(struct be_adapter *adapter) 1423 { 1424 struct device *dev = &adapter->pdev->dev; 1425 int status; 1426 1427 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) 1428 return 0; 1429 1430 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON); 1431 if (!status) { 1432 dev_info(dev, "Enabled VLAN promiscuous mode\n"); 1433 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS; 1434 } else { 1435 dev_err(dev, "Failed to enable VLAN promiscuous mode\n"); 1436 } 1437 return status; 1438 } 1439 1440 static int be_clear_vlan_promisc(struct be_adapter *adapter) 1441 { 1442 struct device *dev = &adapter->pdev->dev; 1443 int status; 1444 1445 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF); 1446 if (!status) { 1447 dev_info(dev, "Disabling VLAN promiscuous mode\n"); 1448 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 1449 } 1450 return status; 1451 } 1452 1453 /* 1454 * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE. 1455 * If the user configures more, place BE in vlan promiscuous mode. 1456 */ 1457 static int be_vid_config(struct be_adapter *adapter) 1458 { 1459 struct device *dev = &adapter->pdev->dev; 1460 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1461 u16 num = 0, i = 0; 1462 int status = 0; 1463 1464 /* No need to change the VLAN state if the I/F is in promiscuous */ 1465 if (adapter->netdev->flags & IFF_PROMISC) 1466 return 0; 1467 1468 if (adapter->vlans_added > be_max_vlans(adapter)) 1469 return be_set_vlan_promisc(adapter); 1470 1471 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 1472 status = be_clear_vlan_promisc(adapter); 1473 if (status) 1474 return status; 1475 } 1476 /* Construct VLAN Table to give to HW */ 1477 for_each_set_bit(i, adapter->vids, VLAN_N_VID) 1478 vids[num++] = cpu_to_le16(i); 1479 1480 status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0); 1481 if (status) { 1482 dev_err(dev, "Setting HW VLAN filtering failed\n"); 1483 /* Set to VLAN promisc mode as setting VLAN filter failed */ 1484 if (addl_status(status) == MCC_ADDL_STATUS_INSUFFICIENT_VLANS || 1485 addl_status(status) == 1486 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES) 1487 return be_set_vlan_promisc(adapter); 1488 } 1489 return status; 1490 } 1491 1492 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 1493 { 1494 struct be_adapter *adapter = netdev_priv(netdev); 1495 int status = 0; 1496 1497 mutex_lock(&adapter->rx_filter_lock); 1498 1499 /* Packets with VID 0 are always received by Lancer by default */ 1500 if (lancer_chip(adapter) && vid == 0) 1501 goto done; 1502 1503 if (test_bit(vid, adapter->vids)) 1504 goto done; 1505 1506 set_bit(vid, adapter->vids); 1507 adapter->vlans_added++; 1508 1509 status = be_vid_config(adapter); 1510 done: 1511 mutex_unlock(&adapter->rx_filter_lock); 1512 return status; 1513 } 1514 1515 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid) 1516 { 1517 struct be_adapter *adapter = netdev_priv(netdev); 1518 int status = 0; 1519 1520 mutex_lock(&adapter->rx_filter_lock); 1521 1522 /* Packets with VID 0 are always received by Lancer by default */ 1523 if (lancer_chip(adapter) && vid == 0) 1524 goto done; 1525 1526 if (!test_bit(vid, adapter->vids)) 1527 goto done; 1528 1529 clear_bit(vid, adapter->vids); 1530 adapter->vlans_added--; 1531 1532 status = be_vid_config(adapter); 1533 done: 1534 mutex_unlock(&adapter->rx_filter_lock); 1535 return status; 1536 } 1537 1538 static void be_set_all_promisc(struct be_adapter *adapter) 1539 { 1540 be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON); 1541 adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS; 1542 } 1543 1544 static void be_set_mc_promisc(struct be_adapter *adapter) 1545 { 1546 int status; 1547 1548 if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) 1549 return; 1550 1551 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON); 1552 if (!status) 1553 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS; 1554 } 1555 1556 static void be_set_uc_promisc(struct be_adapter *adapter) 1557 { 1558 int status; 1559 1560 if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) 1561 return; 1562 1563 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, ON); 1564 if (!status) 1565 adapter->if_flags |= BE_IF_FLAGS_PROMISCUOUS; 1566 } 1567 1568 static void be_clear_uc_promisc(struct be_adapter *adapter) 1569 { 1570 int status; 1571 1572 if (!(adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS)) 1573 return; 1574 1575 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, OFF); 1576 if (!status) 1577 adapter->if_flags &= ~BE_IF_FLAGS_PROMISCUOUS; 1578 } 1579 1580 /* The below 2 functions are the callback args for __dev_mc_sync/dev_uc_sync(). 1581 * We use a single callback function for both sync and unsync. We really don't 1582 * add/remove addresses through this callback. But, we use it to detect changes 1583 * to the uc/mc lists. The entire uc/mc list is programmed in be_set_rx_mode(). 1584 */ 1585 static int be_uc_list_update(struct net_device *netdev, 1586 const unsigned char *addr) 1587 { 1588 struct be_adapter *adapter = netdev_priv(netdev); 1589 1590 adapter->update_uc_list = true; 1591 return 0; 1592 } 1593 1594 static int be_mc_list_update(struct net_device *netdev, 1595 const unsigned char *addr) 1596 { 1597 struct be_adapter *adapter = netdev_priv(netdev); 1598 1599 adapter->update_mc_list = true; 1600 return 0; 1601 } 1602 1603 static void be_set_mc_list(struct be_adapter *adapter) 1604 { 1605 struct net_device *netdev = adapter->netdev; 1606 struct netdev_hw_addr *ha; 1607 bool mc_promisc = false; 1608 int status; 1609 1610 netif_addr_lock_bh(netdev); 1611 __dev_mc_sync(netdev, be_mc_list_update, be_mc_list_update); 1612 1613 if (netdev->flags & IFF_PROMISC) { 1614 adapter->update_mc_list = false; 1615 } else if (netdev->flags & IFF_ALLMULTI || 1616 netdev_mc_count(netdev) > be_max_mc(adapter)) { 1617 /* Enable multicast promisc if num configured exceeds 1618 * what we support 1619 */ 1620 mc_promisc = true; 1621 adapter->update_mc_list = false; 1622 } else if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) { 1623 /* Update mc-list unconditionally if the iface was previously 1624 * in mc-promisc mode and now is out of that mode. 1625 */ 1626 adapter->update_mc_list = true; 1627 } 1628 1629 if (adapter->update_mc_list) { 1630 int i = 0; 1631 1632 /* cache the mc-list in adapter */ 1633 netdev_for_each_mc_addr(ha, netdev) { 1634 ether_addr_copy(adapter->mc_list[i].mac, ha->addr); 1635 i++; 1636 } 1637 adapter->mc_count = netdev_mc_count(netdev); 1638 } 1639 netif_addr_unlock_bh(netdev); 1640 1641 if (mc_promisc) { 1642 be_set_mc_promisc(adapter); 1643 } else if (adapter->update_mc_list) { 1644 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON); 1645 if (!status) 1646 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS; 1647 else 1648 be_set_mc_promisc(adapter); 1649 1650 adapter->update_mc_list = false; 1651 } 1652 } 1653 1654 static void be_clear_mc_list(struct be_adapter *adapter) 1655 { 1656 struct net_device *netdev = adapter->netdev; 1657 1658 __dev_mc_unsync(netdev, NULL); 1659 be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, OFF); 1660 adapter->mc_count = 0; 1661 } 1662 1663 static int be_uc_mac_add(struct be_adapter *adapter, int uc_idx) 1664 { 1665 if (ether_addr_equal((u8 *)&adapter->uc_list[uc_idx * ETH_ALEN], 1666 adapter->dev_mac)) { 1667 adapter->pmac_id[uc_idx + 1] = adapter->pmac_id[0]; 1668 return 0; 1669 } 1670 1671 return be_cmd_pmac_add(adapter, 1672 (u8 *)&adapter->uc_list[uc_idx * ETH_ALEN], 1673 adapter->if_handle, 1674 &adapter->pmac_id[uc_idx + 1], 0); 1675 } 1676 1677 static void be_uc_mac_del(struct be_adapter *adapter, int pmac_id) 1678 { 1679 if (pmac_id == adapter->pmac_id[0]) 1680 return; 1681 1682 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0); 1683 } 1684 1685 static void be_set_uc_list(struct be_adapter *adapter) 1686 { 1687 struct net_device *netdev = adapter->netdev; 1688 struct netdev_hw_addr *ha; 1689 bool uc_promisc = false; 1690 int curr_uc_macs = 0, i; 1691 1692 netif_addr_lock_bh(netdev); 1693 __dev_uc_sync(netdev, be_uc_list_update, be_uc_list_update); 1694 1695 if (netdev->flags & IFF_PROMISC) { 1696 adapter->update_uc_list = false; 1697 } else if (netdev_uc_count(netdev) > (be_max_uc(adapter) - 1)) { 1698 uc_promisc = true; 1699 adapter->update_uc_list = false; 1700 } else if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) { 1701 /* Update uc-list unconditionally if the iface was previously 1702 * in uc-promisc mode and now is out of that mode. 1703 */ 1704 adapter->update_uc_list = true; 1705 } 1706 1707 if (adapter->update_uc_list) { 1708 i = 1; /* First slot is claimed by the Primary MAC */ 1709 1710 /* cache the uc-list in adapter array */ 1711 netdev_for_each_uc_addr(ha, netdev) { 1712 ether_addr_copy(adapter->uc_list[i].mac, ha->addr); 1713 i++; 1714 } 1715 curr_uc_macs = netdev_uc_count(netdev); 1716 } 1717 netif_addr_unlock_bh(netdev); 1718 1719 if (uc_promisc) { 1720 be_set_uc_promisc(adapter); 1721 } else if (adapter->update_uc_list) { 1722 be_clear_uc_promisc(adapter); 1723 1724 for (i = 0; i < adapter->uc_macs; i++) 1725 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]); 1726 1727 for (i = 0; i < curr_uc_macs; i++) 1728 be_uc_mac_add(adapter, i); 1729 adapter->uc_macs = curr_uc_macs; 1730 adapter->update_uc_list = false; 1731 } 1732 } 1733 1734 static void be_clear_uc_list(struct be_adapter *adapter) 1735 { 1736 struct net_device *netdev = adapter->netdev; 1737 int i; 1738 1739 __dev_uc_unsync(netdev, NULL); 1740 for (i = 0; i < adapter->uc_macs; i++) 1741 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]); 1742 1743 adapter->uc_macs = 0; 1744 } 1745 1746 static void __be_set_rx_mode(struct be_adapter *adapter) 1747 { 1748 struct net_device *netdev = adapter->netdev; 1749 1750 mutex_lock(&adapter->rx_filter_lock); 1751 1752 if (netdev->flags & IFF_PROMISC) { 1753 if (!be_in_all_promisc(adapter)) 1754 be_set_all_promisc(adapter); 1755 } else if (be_in_all_promisc(adapter)) { 1756 /* We need to re-program the vlan-list or clear 1757 * vlan-promisc mode (if needed) when the interface 1758 * comes out of promisc mode. 1759 */ 1760 be_vid_config(adapter); 1761 } 1762 1763 be_set_uc_list(adapter); 1764 be_set_mc_list(adapter); 1765 1766 mutex_unlock(&adapter->rx_filter_lock); 1767 } 1768 1769 static void be_work_set_rx_mode(struct work_struct *work) 1770 { 1771 struct be_cmd_work *cmd_work = 1772 container_of(work, struct be_cmd_work, work); 1773 1774 __be_set_rx_mode(cmd_work->adapter); 1775 kfree(cmd_work); 1776 } 1777 1778 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 1779 { 1780 struct be_adapter *adapter = netdev_priv(netdev); 1781 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1782 int status; 1783 1784 if (!sriov_enabled(adapter)) 1785 return -EPERM; 1786 1787 if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs) 1788 return -EINVAL; 1789 1790 /* Proceed further only if user provided MAC is different 1791 * from active MAC 1792 */ 1793 if (ether_addr_equal(mac, vf_cfg->mac_addr)) 1794 return 0; 1795 1796 if (BEx_chip(adapter)) { 1797 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id, 1798 vf + 1); 1799 1800 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle, 1801 &vf_cfg->pmac_id, vf + 1); 1802 } else { 1803 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 1804 vf + 1); 1805 } 1806 1807 if (status) { 1808 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x", 1809 mac, vf, status); 1810 return be_cmd_status(status); 1811 } 1812 1813 ether_addr_copy(vf_cfg->mac_addr, mac); 1814 1815 return 0; 1816 } 1817 1818 static int be_get_vf_config(struct net_device *netdev, int vf, 1819 struct ifla_vf_info *vi) 1820 { 1821 struct be_adapter *adapter = netdev_priv(netdev); 1822 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1823 1824 if (!sriov_enabled(adapter)) 1825 return -EPERM; 1826 1827 if (vf >= adapter->num_vfs) 1828 return -EINVAL; 1829 1830 vi->vf = vf; 1831 vi->max_tx_rate = vf_cfg->tx_rate; 1832 vi->min_tx_rate = 0; 1833 vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK; 1834 vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT; 1835 memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN); 1836 vi->linkstate = adapter->vf_cfg[vf].plink_tracking; 1837 vi->spoofchk = adapter->vf_cfg[vf].spoofchk; 1838 1839 return 0; 1840 } 1841 1842 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan) 1843 { 1844 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1845 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1846 int vf_if_id = vf_cfg->if_handle; 1847 int status; 1848 1849 /* Enable Transparent VLAN Tagging */ 1850 status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0, 0); 1851 if (status) 1852 return status; 1853 1854 /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */ 1855 vids[0] = 0; 1856 status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1); 1857 if (!status) 1858 dev_info(&adapter->pdev->dev, 1859 "Cleared guest VLANs on VF%d", vf); 1860 1861 /* After TVT is enabled, disallow VFs to program VLAN filters */ 1862 if (vf_cfg->privileges & BE_PRIV_FILTMGMT) { 1863 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges & 1864 ~BE_PRIV_FILTMGMT, vf + 1); 1865 if (!status) 1866 vf_cfg->privileges &= ~BE_PRIV_FILTMGMT; 1867 } 1868 return 0; 1869 } 1870 1871 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf) 1872 { 1873 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1874 struct device *dev = &adapter->pdev->dev; 1875 int status; 1876 1877 /* Reset Transparent VLAN Tagging. */ 1878 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1, 1879 vf_cfg->if_handle, 0, 0); 1880 if (status) 1881 return status; 1882 1883 /* Allow VFs to program VLAN filtering */ 1884 if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 1885 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges | 1886 BE_PRIV_FILTMGMT, vf + 1); 1887 if (!status) { 1888 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 1889 dev_info(dev, "VF%d: FILTMGMT priv enabled", vf); 1890 } 1891 } 1892 1893 dev_info(dev, 1894 "Disable/re-enable i/f in VM to clear Transparent VLAN tag"); 1895 return 0; 1896 } 1897 1898 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos, 1899 __be16 vlan_proto) 1900 { 1901 struct be_adapter *adapter = netdev_priv(netdev); 1902 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1903 int status; 1904 1905 if (!sriov_enabled(adapter)) 1906 return -EPERM; 1907 1908 if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7) 1909 return -EINVAL; 1910 1911 if (vlan_proto != htons(ETH_P_8021Q)) 1912 return -EPROTONOSUPPORT; 1913 1914 if (vlan || qos) { 1915 vlan |= qos << VLAN_PRIO_SHIFT; 1916 status = be_set_vf_tvt(adapter, vf, vlan); 1917 } else { 1918 status = be_clear_vf_tvt(adapter, vf); 1919 } 1920 1921 if (status) { 1922 dev_err(&adapter->pdev->dev, 1923 "VLAN %d config on VF %d failed : %#x\n", vlan, vf, 1924 status); 1925 return be_cmd_status(status); 1926 } 1927 1928 vf_cfg->vlan_tag = vlan; 1929 return 0; 1930 } 1931 1932 static int be_set_vf_tx_rate(struct net_device *netdev, int vf, 1933 int min_tx_rate, int max_tx_rate) 1934 { 1935 struct be_adapter *adapter = netdev_priv(netdev); 1936 struct device *dev = &adapter->pdev->dev; 1937 int percent_rate, status = 0; 1938 u16 link_speed = 0; 1939 u8 link_status; 1940 1941 if (!sriov_enabled(adapter)) 1942 return -EPERM; 1943 1944 if (vf >= adapter->num_vfs) 1945 return -EINVAL; 1946 1947 if (min_tx_rate) 1948 return -EINVAL; 1949 1950 if (!max_tx_rate) 1951 goto config_qos; 1952 1953 status = be_cmd_link_status_query(adapter, &link_speed, 1954 &link_status, 0); 1955 if (status) 1956 goto err; 1957 1958 if (!link_status) { 1959 dev_err(dev, "TX-rate setting not allowed when link is down\n"); 1960 status = -ENETDOWN; 1961 goto err; 1962 } 1963 1964 if (max_tx_rate < 100 || max_tx_rate > link_speed) { 1965 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n", 1966 link_speed); 1967 status = -EINVAL; 1968 goto err; 1969 } 1970 1971 /* On Skyhawk the QOS setting must be done only as a % value */ 1972 percent_rate = link_speed / 100; 1973 if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) { 1974 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n", 1975 percent_rate); 1976 status = -EINVAL; 1977 goto err; 1978 } 1979 1980 config_qos: 1981 status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1); 1982 if (status) 1983 goto err; 1984 1985 adapter->vf_cfg[vf].tx_rate = max_tx_rate; 1986 return 0; 1987 1988 err: 1989 dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n", 1990 max_tx_rate, vf); 1991 return be_cmd_status(status); 1992 } 1993 1994 static int be_set_vf_link_state(struct net_device *netdev, int vf, 1995 int link_state) 1996 { 1997 struct be_adapter *adapter = netdev_priv(netdev); 1998 int status; 1999 2000 if (!sriov_enabled(adapter)) 2001 return -EPERM; 2002 2003 if (vf >= adapter->num_vfs) 2004 return -EINVAL; 2005 2006 status = be_cmd_set_logical_link_config(adapter, link_state, vf+1); 2007 if (status) { 2008 dev_err(&adapter->pdev->dev, 2009 "Link state change on VF %d failed: %#x\n", vf, status); 2010 return be_cmd_status(status); 2011 } 2012 2013 adapter->vf_cfg[vf].plink_tracking = link_state; 2014 2015 return 0; 2016 } 2017 2018 static int be_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable) 2019 { 2020 struct be_adapter *adapter = netdev_priv(netdev); 2021 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 2022 u8 spoofchk; 2023 int status; 2024 2025 if (!sriov_enabled(adapter)) 2026 return -EPERM; 2027 2028 if (vf >= adapter->num_vfs) 2029 return -EINVAL; 2030 2031 if (BEx_chip(adapter)) 2032 return -EOPNOTSUPP; 2033 2034 if (enable == vf_cfg->spoofchk) 2035 return 0; 2036 2037 spoofchk = enable ? ENABLE_MAC_SPOOFCHK : DISABLE_MAC_SPOOFCHK; 2038 2039 status = be_cmd_set_hsw_config(adapter, 0, vf + 1, vf_cfg->if_handle, 2040 0, spoofchk); 2041 if (status) { 2042 dev_err(&adapter->pdev->dev, 2043 "Spoofchk change on VF %d failed: %#x\n", vf, status); 2044 return be_cmd_status(status); 2045 } 2046 2047 vf_cfg->spoofchk = enable; 2048 return 0; 2049 } 2050 2051 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts, 2052 ulong now) 2053 { 2054 aic->rx_pkts_prev = rx_pkts; 2055 aic->tx_reqs_prev = tx_pkts; 2056 aic->jiffies = now; 2057 } 2058 2059 static int be_get_new_eqd(struct be_eq_obj *eqo) 2060 { 2061 struct be_adapter *adapter = eqo->adapter; 2062 int eqd, start; 2063 struct be_aic_obj *aic; 2064 struct be_rx_obj *rxo; 2065 struct be_tx_obj *txo; 2066 u64 rx_pkts = 0, tx_pkts = 0; 2067 ulong now; 2068 u32 pps, delta; 2069 int i; 2070 2071 aic = &adapter->aic_obj[eqo->idx]; 2072 if (!aic->enable) { 2073 if (aic->jiffies) 2074 aic->jiffies = 0; 2075 eqd = aic->et_eqd; 2076 return eqd; 2077 } 2078 2079 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 2080 do { 2081 start = u64_stats_fetch_begin_irq(&rxo->stats.sync); 2082 rx_pkts += rxo->stats.rx_pkts; 2083 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start)); 2084 } 2085 2086 for_all_tx_queues_on_eq(adapter, eqo, txo, i) { 2087 do { 2088 start = u64_stats_fetch_begin_irq(&txo->stats.sync); 2089 tx_pkts += txo->stats.tx_reqs; 2090 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start)); 2091 } 2092 2093 /* Skip, if wrapped around or first calculation */ 2094 now = jiffies; 2095 if (!aic->jiffies || time_before(now, aic->jiffies) || 2096 rx_pkts < aic->rx_pkts_prev || 2097 tx_pkts < aic->tx_reqs_prev) { 2098 be_aic_update(aic, rx_pkts, tx_pkts, now); 2099 return aic->prev_eqd; 2100 } 2101 2102 delta = jiffies_to_msecs(now - aic->jiffies); 2103 if (delta == 0) 2104 return aic->prev_eqd; 2105 2106 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) + 2107 (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta); 2108 eqd = (pps / 15000) << 2; 2109 2110 if (eqd < 8) 2111 eqd = 0; 2112 eqd = min_t(u32, eqd, aic->max_eqd); 2113 eqd = max_t(u32, eqd, aic->min_eqd); 2114 2115 be_aic_update(aic, rx_pkts, tx_pkts, now); 2116 2117 return eqd; 2118 } 2119 2120 /* For Skyhawk-R only */ 2121 static u32 be_get_eq_delay_mult_enc(struct be_eq_obj *eqo) 2122 { 2123 struct be_adapter *adapter = eqo->adapter; 2124 struct be_aic_obj *aic = &adapter->aic_obj[eqo->idx]; 2125 ulong now = jiffies; 2126 int eqd; 2127 u32 mult_enc; 2128 2129 if (!aic->enable) 2130 return 0; 2131 2132 if (jiffies_to_msecs(now - aic->jiffies) < 1) 2133 eqd = aic->prev_eqd; 2134 else 2135 eqd = be_get_new_eqd(eqo); 2136 2137 if (eqd > 100) 2138 mult_enc = R2I_DLY_ENC_1; 2139 else if (eqd > 60) 2140 mult_enc = R2I_DLY_ENC_2; 2141 else if (eqd > 20) 2142 mult_enc = R2I_DLY_ENC_3; 2143 else 2144 mult_enc = R2I_DLY_ENC_0; 2145 2146 aic->prev_eqd = eqd; 2147 2148 return mult_enc; 2149 } 2150 2151 void be_eqd_update(struct be_adapter *adapter, bool force_update) 2152 { 2153 struct be_set_eqd set_eqd[MAX_EVT_QS]; 2154 struct be_aic_obj *aic; 2155 struct be_eq_obj *eqo; 2156 int i, num = 0, eqd; 2157 2158 for_all_evt_queues(adapter, eqo, i) { 2159 aic = &adapter->aic_obj[eqo->idx]; 2160 eqd = be_get_new_eqd(eqo); 2161 if (force_update || eqd != aic->prev_eqd) { 2162 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2163 set_eqd[num].eq_id = eqo->q.id; 2164 aic->prev_eqd = eqd; 2165 num++; 2166 } 2167 } 2168 2169 if (num) 2170 be_cmd_modify_eqd(adapter, set_eqd, num); 2171 } 2172 2173 static void be_rx_stats_update(struct be_rx_obj *rxo, 2174 struct be_rx_compl_info *rxcp) 2175 { 2176 struct be_rx_stats *stats = rx_stats(rxo); 2177 2178 u64_stats_update_begin(&stats->sync); 2179 stats->rx_compl++; 2180 stats->rx_bytes += rxcp->pkt_size; 2181 stats->rx_pkts++; 2182 if (rxcp->tunneled) 2183 stats->rx_vxlan_offload_pkts++; 2184 if (rxcp->pkt_type == BE_MULTICAST_PACKET) 2185 stats->rx_mcast_pkts++; 2186 if (rxcp->err) 2187 stats->rx_compl_err++; 2188 u64_stats_update_end(&stats->sync); 2189 } 2190 2191 static inline bool csum_passed(struct be_rx_compl_info *rxcp) 2192 { 2193 /* L4 checksum is not reliable for non TCP/UDP packets. 2194 * Also ignore ipcksm for ipv6 pkts 2195 */ 2196 return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum && 2197 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err; 2198 } 2199 2200 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo) 2201 { 2202 struct be_adapter *adapter = rxo->adapter; 2203 struct be_rx_page_info *rx_page_info; 2204 struct be_queue_info *rxq = &rxo->q; 2205 u32 frag_idx = rxq->tail; 2206 2207 rx_page_info = &rxo->page_info_tbl[frag_idx]; 2208 BUG_ON(!rx_page_info->page); 2209 2210 if (rx_page_info->last_frag) { 2211 dma_unmap_page(&adapter->pdev->dev, 2212 dma_unmap_addr(rx_page_info, bus), 2213 adapter->big_page_size, DMA_FROM_DEVICE); 2214 rx_page_info->last_frag = false; 2215 } else { 2216 dma_sync_single_for_cpu(&adapter->pdev->dev, 2217 dma_unmap_addr(rx_page_info, bus), 2218 rx_frag_size, DMA_FROM_DEVICE); 2219 } 2220 2221 queue_tail_inc(rxq); 2222 atomic_dec(&rxq->used); 2223 return rx_page_info; 2224 } 2225 2226 /* Throwaway the data in the Rx completion */ 2227 static void be_rx_compl_discard(struct be_rx_obj *rxo, 2228 struct be_rx_compl_info *rxcp) 2229 { 2230 struct be_rx_page_info *page_info; 2231 u16 i, num_rcvd = rxcp->num_rcvd; 2232 2233 for (i = 0; i < num_rcvd; i++) { 2234 page_info = get_rx_page_info(rxo); 2235 put_page(page_info->page); 2236 memset(page_info, 0, sizeof(*page_info)); 2237 } 2238 } 2239 2240 /* 2241 * skb_fill_rx_data forms a complete skb for an ether frame 2242 * indicated by rxcp. 2243 */ 2244 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb, 2245 struct be_rx_compl_info *rxcp) 2246 { 2247 struct be_rx_page_info *page_info; 2248 u16 i, j; 2249 u16 hdr_len, curr_frag_len, remaining; 2250 u8 *start; 2251 2252 page_info = get_rx_page_info(rxo); 2253 start = page_address(page_info->page) + page_info->page_offset; 2254 prefetch(start); 2255 2256 /* Copy data in the first descriptor of this completion */ 2257 curr_frag_len = min(rxcp->pkt_size, rx_frag_size); 2258 2259 skb->len = curr_frag_len; 2260 if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */ 2261 memcpy(skb->data, start, curr_frag_len); 2262 /* Complete packet has now been moved to data */ 2263 put_page(page_info->page); 2264 skb->data_len = 0; 2265 skb->tail += curr_frag_len; 2266 } else { 2267 hdr_len = ETH_HLEN; 2268 memcpy(skb->data, start, hdr_len); 2269 skb_shinfo(skb)->nr_frags = 1; 2270 skb_frag_set_page(skb, 0, page_info->page); 2271 skb_shinfo(skb)->frags[0].page_offset = 2272 page_info->page_offset + hdr_len; 2273 skb_frag_size_set(&skb_shinfo(skb)->frags[0], 2274 curr_frag_len - hdr_len); 2275 skb->data_len = curr_frag_len - hdr_len; 2276 skb->truesize += rx_frag_size; 2277 skb->tail += hdr_len; 2278 } 2279 page_info->page = NULL; 2280 2281 if (rxcp->pkt_size <= rx_frag_size) { 2282 BUG_ON(rxcp->num_rcvd != 1); 2283 return; 2284 } 2285 2286 /* More frags present for this completion */ 2287 remaining = rxcp->pkt_size - curr_frag_len; 2288 for (i = 1, j = 0; i < rxcp->num_rcvd; i++) { 2289 page_info = get_rx_page_info(rxo); 2290 curr_frag_len = min(remaining, rx_frag_size); 2291 2292 /* Coalesce all frags from the same physical page in one slot */ 2293 if (page_info->page_offset == 0) { 2294 /* Fresh page */ 2295 j++; 2296 skb_frag_set_page(skb, j, page_info->page); 2297 skb_shinfo(skb)->frags[j].page_offset = 2298 page_info->page_offset; 2299 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 2300 skb_shinfo(skb)->nr_frags++; 2301 } else { 2302 put_page(page_info->page); 2303 } 2304 2305 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 2306 skb->len += curr_frag_len; 2307 skb->data_len += curr_frag_len; 2308 skb->truesize += rx_frag_size; 2309 remaining -= curr_frag_len; 2310 page_info->page = NULL; 2311 } 2312 BUG_ON(j > MAX_SKB_FRAGS); 2313 } 2314 2315 /* Process the RX completion indicated by rxcp when GRO is disabled */ 2316 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi, 2317 struct be_rx_compl_info *rxcp) 2318 { 2319 struct be_adapter *adapter = rxo->adapter; 2320 struct net_device *netdev = adapter->netdev; 2321 struct sk_buff *skb; 2322 2323 skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE); 2324 if (unlikely(!skb)) { 2325 rx_stats(rxo)->rx_drops_no_skbs++; 2326 be_rx_compl_discard(rxo, rxcp); 2327 return; 2328 } 2329 2330 skb_fill_rx_data(rxo, skb, rxcp); 2331 2332 if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp))) 2333 skb->ip_summed = CHECKSUM_UNNECESSARY; 2334 else 2335 skb_checksum_none_assert(skb); 2336 2337 skb->protocol = eth_type_trans(skb, netdev); 2338 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 2339 if (netdev->features & NETIF_F_RXHASH) 2340 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 2341 2342 skb->csum_level = rxcp->tunneled; 2343 skb_mark_napi_id(skb, napi); 2344 2345 if (rxcp->vlanf) 2346 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 2347 2348 netif_receive_skb(skb); 2349 } 2350 2351 /* Process the RX completion indicated by rxcp when GRO is enabled */ 2352 static void be_rx_compl_process_gro(struct be_rx_obj *rxo, 2353 struct napi_struct *napi, 2354 struct be_rx_compl_info *rxcp) 2355 { 2356 struct be_adapter *adapter = rxo->adapter; 2357 struct be_rx_page_info *page_info; 2358 struct sk_buff *skb = NULL; 2359 u16 remaining, curr_frag_len; 2360 u16 i, j; 2361 2362 skb = napi_get_frags(napi); 2363 if (!skb) { 2364 be_rx_compl_discard(rxo, rxcp); 2365 return; 2366 } 2367 2368 remaining = rxcp->pkt_size; 2369 for (i = 0, j = -1; i < rxcp->num_rcvd; i++) { 2370 page_info = get_rx_page_info(rxo); 2371 2372 curr_frag_len = min(remaining, rx_frag_size); 2373 2374 /* Coalesce all frags from the same physical page in one slot */ 2375 if (i == 0 || page_info->page_offset == 0) { 2376 /* First frag or Fresh page */ 2377 j++; 2378 skb_frag_set_page(skb, j, page_info->page); 2379 skb_shinfo(skb)->frags[j].page_offset = 2380 page_info->page_offset; 2381 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 2382 } else { 2383 put_page(page_info->page); 2384 } 2385 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 2386 skb->truesize += rx_frag_size; 2387 remaining -= curr_frag_len; 2388 memset(page_info, 0, sizeof(*page_info)); 2389 } 2390 BUG_ON(j > MAX_SKB_FRAGS); 2391 2392 skb_shinfo(skb)->nr_frags = j + 1; 2393 skb->len = rxcp->pkt_size; 2394 skb->data_len = rxcp->pkt_size; 2395 skb->ip_summed = CHECKSUM_UNNECESSARY; 2396 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 2397 if (adapter->netdev->features & NETIF_F_RXHASH) 2398 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 2399 2400 skb->csum_level = rxcp->tunneled; 2401 2402 if (rxcp->vlanf) 2403 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 2404 2405 napi_gro_frags(napi); 2406 } 2407 2408 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl, 2409 struct be_rx_compl_info *rxcp) 2410 { 2411 rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl); 2412 rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl); 2413 rxcp->err = GET_RX_COMPL_V1_BITS(err, compl); 2414 rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl); 2415 rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl); 2416 rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl); 2417 rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl); 2418 rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl); 2419 rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl); 2420 rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl); 2421 rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl); 2422 if (rxcp->vlanf) { 2423 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl); 2424 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl); 2425 } 2426 rxcp->port = GET_RX_COMPL_V1_BITS(port, compl); 2427 rxcp->tunneled = 2428 GET_RX_COMPL_V1_BITS(tunneled, compl); 2429 } 2430 2431 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl, 2432 struct be_rx_compl_info *rxcp) 2433 { 2434 rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl); 2435 rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl); 2436 rxcp->err = GET_RX_COMPL_V0_BITS(err, compl); 2437 rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl); 2438 rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl); 2439 rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl); 2440 rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl); 2441 rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl); 2442 rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl); 2443 rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl); 2444 rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl); 2445 if (rxcp->vlanf) { 2446 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl); 2447 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl); 2448 } 2449 rxcp->port = GET_RX_COMPL_V0_BITS(port, compl); 2450 rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl); 2451 } 2452 2453 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo) 2454 { 2455 struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq); 2456 struct be_rx_compl_info *rxcp = &rxo->rxcp; 2457 struct be_adapter *adapter = rxo->adapter; 2458 2459 /* For checking the valid bit it is Ok to use either definition as the 2460 * valid bit is at the same position in both v0 and v1 Rx compl */ 2461 if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0) 2462 return NULL; 2463 2464 rmb(); 2465 be_dws_le_to_cpu(compl, sizeof(*compl)); 2466 2467 if (adapter->be3_native) 2468 be_parse_rx_compl_v1(compl, rxcp); 2469 else 2470 be_parse_rx_compl_v0(compl, rxcp); 2471 2472 if (rxcp->ip_frag) 2473 rxcp->l4_csum = 0; 2474 2475 if (rxcp->vlanf) { 2476 /* In QNQ modes, if qnq bit is not set, then the packet was 2477 * tagged only with the transparent outer vlan-tag and must 2478 * not be treated as a vlan packet by host 2479 */ 2480 if (be_is_qnq_mode(adapter) && !rxcp->qnq) 2481 rxcp->vlanf = 0; 2482 2483 if (!lancer_chip(adapter)) 2484 rxcp->vlan_tag = swab16(rxcp->vlan_tag); 2485 2486 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) && 2487 !test_bit(rxcp->vlan_tag, adapter->vids)) 2488 rxcp->vlanf = 0; 2489 } 2490 2491 /* As the compl has been parsed, reset it; we wont touch it again */ 2492 compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0; 2493 2494 queue_tail_inc(&rxo->cq); 2495 return rxcp; 2496 } 2497 2498 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp) 2499 { 2500 u32 order = get_order(size); 2501 2502 if (order > 0) 2503 gfp |= __GFP_COMP; 2504 return alloc_pages(gfp, order); 2505 } 2506 2507 /* 2508 * Allocate a page, split it to fragments of size rx_frag_size and post as 2509 * receive buffers to BE 2510 */ 2511 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed) 2512 { 2513 struct be_adapter *adapter = rxo->adapter; 2514 struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL; 2515 struct be_queue_info *rxq = &rxo->q; 2516 struct page *pagep = NULL; 2517 struct device *dev = &adapter->pdev->dev; 2518 struct be_eth_rx_d *rxd; 2519 u64 page_dmaaddr = 0, frag_dmaaddr; 2520 u32 posted, page_offset = 0, notify = 0; 2521 2522 page_info = &rxo->page_info_tbl[rxq->head]; 2523 for (posted = 0; posted < frags_needed && !page_info->page; posted++) { 2524 if (!pagep) { 2525 pagep = be_alloc_pages(adapter->big_page_size, gfp); 2526 if (unlikely(!pagep)) { 2527 rx_stats(rxo)->rx_post_fail++; 2528 break; 2529 } 2530 page_dmaaddr = dma_map_page(dev, pagep, 0, 2531 adapter->big_page_size, 2532 DMA_FROM_DEVICE); 2533 if (dma_mapping_error(dev, page_dmaaddr)) { 2534 put_page(pagep); 2535 pagep = NULL; 2536 adapter->drv_stats.dma_map_errors++; 2537 break; 2538 } 2539 page_offset = 0; 2540 } else { 2541 get_page(pagep); 2542 page_offset += rx_frag_size; 2543 } 2544 page_info->page_offset = page_offset; 2545 page_info->page = pagep; 2546 2547 rxd = queue_head_node(rxq); 2548 frag_dmaaddr = page_dmaaddr + page_info->page_offset; 2549 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF); 2550 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr)); 2551 2552 /* Any space left in the current big page for another frag? */ 2553 if ((page_offset + rx_frag_size + rx_frag_size) > 2554 adapter->big_page_size) { 2555 pagep = NULL; 2556 page_info->last_frag = true; 2557 dma_unmap_addr_set(page_info, bus, page_dmaaddr); 2558 } else { 2559 dma_unmap_addr_set(page_info, bus, frag_dmaaddr); 2560 } 2561 2562 prev_page_info = page_info; 2563 queue_head_inc(rxq); 2564 page_info = &rxo->page_info_tbl[rxq->head]; 2565 } 2566 2567 /* Mark the last frag of a page when we break out of the above loop 2568 * with no more slots available in the RXQ 2569 */ 2570 if (pagep) { 2571 prev_page_info->last_frag = true; 2572 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr); 2573 } 2574 2575 if (posted) { 2576 atomic_add(posted, &rxq->used); 2577 if (rxo->rx_post_starved) 2578 rxo->rx_post_starved = false; 2579 do { 2580 notify = min(MAX_NUM_POST_ERX_DB, posted); 2581 be_rxq_notify(adapter, rxq->id, notify); 2582 posted -= notify; 2583 } while (posted); 2584 } else if (atomic_read(&rxq->used) == 0) { 2585 /* Let be_worker replenish when memory is available */ 2586 rxo->rx_post_starved = true; 2587 } 2588 } 2589 2590 static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo) 2591 { 2592 struct be_queue_info *tx_cq = &txo->cq; 2593 struct be_tx_compl_info *txcp = &txo->txcp; 2594 struct be_eth_tx_compl *compl = queue_tail_node(tx_cq); 2595 2596 if (compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0) 2597 return NULL; 2598 2599 /* Ensure load ordering of valid bit dword and other dwords below */ 2600 rmb(); 2601 be_dws_le_to_cpu(compl, sizeof(*compl)); 2602 2603 txcp->status = GET_TX_COMPL_BITS(status, compl); 2604 txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl); 2605 2606 compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0; 2607 queue_tail_inc(tx_cq); 2608 return txcp; 2609 } 2610 2611 static u16 be_tx_compl_process(struct be_adapter *adapter, 2612 struct be_tx_obj *txo, u16 last_index) 2613 { 2614 struct sk_buff **sent_skbs = txo->sent_skb_list; 2615 struct be_queue_info *txq = &txo->q; 2616 struct sk_buff *skb = NULL; 2617 bool unmap_skb_hdr = false; 2618 struct be_eth_wrb *wrb; 2619 u16 num_wrbs = 0; 2620 u32 frag_index; 2621 2622 do { 2623 if (sent_skbs[txq->tail]) { 2624 /* Free skb from prev req */ 2625 if (skb) 2626 dev_consume_skb_any(skb); 2627 skb = sent_skbs[txq->tail]; 2628 sent_skbs[txq->tail] = NULL; 2629 queue_tail_inc(txq); /* skip hdr wrb */ 2630 num_wrbs++; 2631 unmap_skb_hdr = true; 2632 } 2633 wrb = queue_tail_node(txq); 2634 frag_index = txq->tail; 2635 unmap_tx_frag(&adapter->pdev->dev, wrb, 2636 (unmap_skb_hdr && skb_headlen(skb))); 2637 unmap_skb_hdr = false; 2638 queue_tail_inc(txq); 2639 num_wrbs++; 2640 } while (frag_index != last_index); 2641 dev_consume_skb_any(skb); 2642 2643 return num_wrbs; 2644 } 2645 2646 /* Return the number of events in the event queue */ 2647 static inline int events_get(struct be_eq_obj *eqo) 2648 { 2649 struct be_eq_entry *eqe; 2650 int num = 0; 2651 2652 do { 2653 eqe = queue_tail_node(&eqo->q); 2654 if (eqe->evt == 0) 2655 break; 2656 2657 rmb(); 2658 eqe->evt = 0; 2659 num++; 2660 queue_tail_inc(&eqo->q); 2661 } while (true); 2662 2663 return num; 2664 } 2665 2666 /* Leaves the EQ is disarmed state */ 2667 static void be_eq_clean(struct be_eq_obj *eqo) 2668 { 2669 int num = events_get(eqo); 2670 2671 be_eq_notify(eqo->adapter, eqo->q.id, false, true, num, 0); 2672 } 2673 2674 /* Free posted rx buffers that were not used */ 2675 static void be_rxq_clean(struct be_rx_obj *rxo) 2676 { 2677 struct be_queue_info *rxq = &rxo->q; 2678 struct be_rx_page_info *page_info; 2679 2680 while (atomic_read(&rxq->used) > 0) { 2681 page_info = get_rx_page_info(rxo); 2682 put_page(page_info->page); 2683 memset(page_info, 0, sizeof(*page_info)); 2684 } 2685 BUG_ON(atomic_read(&rxq->used)); 2686 rxq->tail = 0; 2687 rxq->head = 0; 2688 } 2689 2690 static void be_rx_cq_clean(struct be_rx_obj *rxo) 2691 { 2692 struct be_queue_info *rx_cq = &rxo->cq; 2693 struct be_rx_compl_info *rxcp; 2694 struct be_adapter *adapter = rxo->adapter; 2695 int flush_wait = 0; 2696 2697 /* Consume pending rx completions. 2698 * Wait for the flush completion (identified by zero num_rcvd) 2699 * to arrive. Notify CQ even when there are no more CQ entries 2700 * for HW to flush partially coalesced CQ entries. 2701 * In Lancer, there is no need to wait for flush compl. 2702 */ 2703 for (;;) { 2704 rxcp = be_rx_compl_get(rxo); 2705 if (!rxcp) { 2706 if (lancer_chip(adapter)) 2707 break; 2708 2709 if (flush_wait++ > 50 || 2710 be_check_error(adapter, 2711 BE_ERROR_HW)) { 2712 dev_warn(&adapter->pdev->dev, 2713 "did not receive flush compl\n"); 2714 break; 2715 } 2716 be_cq_notify(adapter, rx_cq->id, true, 0); 2717 mdelay(1); 2718 } else { 2719 be_rx_compl_discard(rxo, rxcp); 2720 be_cq_notify(adapter, rx_cq->id, false, 1); 2721 if (rxcp->num_rcvd == 0) 2722 break; 2723 } 2724 } 2725 2726 /* After cleanup, leave the CQ in unarmed state */ 2727 be_cq_notify(adapter, rx_cq->id, false, 0); 2728 } 2729 2730 static void be_tx_compl_clean(struct be_adapter *adapter) 2731 { 2732 struct device *dev = &adapter->pdev->dev; 2733 u16 cmpl = 0, timeo = 0, num_wrbs = 0; 2734 struct be_tx_compl_info *txcp; 2735 struct be_queue_info *txq; 2736 u32 end_idx, notified_idx; 2737 struct be_tx_obj *txo; 2738 int i, pending_txqs; 2739 2740 /* Stop polling for compls when HW has been silent for 10ms */ 2741 do { 2742 pending_txqs = adapter->num_tx_qs; 2743 2744 for_all_tx_queues(adapter, txo, i) { 2745 cmpl = 0; 2746 num_wrbs = 0; 2747 txq = &txo->q; 2748 while ((txcp = be_tx_compl_get(txo))) { 2749 num_wrbs += 2750 be_tx_compl_process(adapter, txo, 2751 txcp->end_index); 2752 cmpl++; 2753 } 2754 if (cmpl) { 2755 be_cq_notify(adapter, txo->cq.id, false, cmpl); 2756 atomic_sub(num_wrbs, &txq->used); 2757 timeo = 0; 2758 } 2759 if (!be_is_tx_compl_pending(txo)) 2760 pending_txqs--; 2761 } 2762 2763 if (pending_txqs == 0 || ++timeo > 10 || 2764 be_check_error(adapter, BE_ERROR_HW)) 2765 break; 2766 2767 mdelay(1); 2768 } while (true); 2769 2770 /* Free enqueued TX that was never notified to HW */ 2771 for_all_tx_queues(adapter, txo, i) { 2772 txq = &txo->q; 2773 2774 if (atomic_read(&txq->used)) { 2775 dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n", 2776 i, atomic_read(&txq->used)); 2777 notified_idx = txq->tail; 2778 end_idx = txq->tail; 2779 index_adv(&end_idx, atomic_read(&txq->used) - 1, 2780 txq->len); 2781 /* Use the tx-compl process logic to handle requests 2782 * that were not sent to the HW. 2783 */ 2784 num_wrbs = be_tx_compl_process(adapter, txo, end_idx); 2785 atomic_sub(num_wrbs, &txq->used); 2786 BUG_ON(atomic_read(&txq->used)); 2787 txo->pend_wrb_cnt = 0; 2788 /* Since hw was never notified of these requests, 2789 * reset TXQ indices 2790 */ 2791 txq->head = notified_idx; 2792 txq->tail = notified_idx; 2793 } 2794 } 2795 } 2796 2797 static void be_evt_queues_destroy(struct be_adapter *adapter) 2798 { 2799 struct be_eq_obj *eqo; 2800 int i; 2801 2802 for_all_evt_queues(adapter, eqo, i) { 2803 if (eqo->q.created) { 2804 be_eq_clean(eqo); 2805 be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ); 2806 napi_hash_del(&eqo->napi); 2807 netif_napi_del(&eqo->napi); 2808 free_cpumask_var(eqo->affinity_mask); 2809 } 2810 be_queue_free(adapter, &eqo->q); 2811 } 2812 } 2813 2814 static int be_evt_queues_create(struct be_adapter *adapter) 2815 { 2816 struct be_queue_info *eq; 2817 struct be_eq_obj *eqo; 2818 struct be_aic_obj *aic; 2819 int i, rc; 2820 2821 /* need enough EQs to service both RX and TX queues */ 2822 adapter->num_evt_qs = min_t(u16, num_irqs(adapter), 2823 max(adapter->cfg_num_rx_irqs, 2824 adapter->cfg_num_tx_irqs)); 2825 2826 for_all_evt_queues(adapter, eqo, i) { 2827 int numa_node = dev_to_node(&adapter->pdev->dev); 2828 2829 aic = &adapter->aic_obj[i]; 2830 eqo->adapter = adapter; 2831 eqo->idx = i; 2832 aic->max_eqd = BE_MAX_EQD; 2833 aic->enable = true; 2834 2835 eq = &eqo->q; 2836 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN, 2837 sizeof(struct be_eq_entry)); 2838 if (rc) 2839 return rc; 2840 2841 rc = be_cmd_eq_create(adapter, eqo); 2842 if (rc) 2843 return rc; 2844 2845 if (!zalloc_cpumask_var(&eqo->affinity_mask, GFP_KERNEL)) 2846 return -ENOMEM; 2847 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 2848 eqo->affinity_mask); 2849 netif_napi_add(adapter->netdev, &eqo->napi, be_poll, 2850 BE_NAPI_WEIGHT); 2851 } 2852 return 0; 2853 } 2854 2855 static void be_mcc_queues_destroy(struct be_adapter *adapter) 2856 { 2857 struct be_queue_info *q; 2858 2859 q = &adapter->mcc_obj.q; 2860 if (q->created) 2861 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ); 2862 be_queue_free(adapter, q); 2863 2864 q = &adapter->mcc_obj.cq; 2865 if (q->created) 2866 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2867 be_queue_free(adapter, q); 2868 } 2869 2870 /* Must be called only after TX qs are created as MCC shares TX EQ */ 2871 static int be_mcc_queues_create(struct be_adapter *adapter) 2872 { 2873 struct be_queue_info *q, *cq; 2874 2875 cq = &adapter->mcc_obj.cq; 2876 if (be_queue_alloc(adapter, cq, MCC_CQ_LEN, 2877 sizeof(struct be_mcc_compl))) 2878 goto err; 2879 2880 /* Use the default EQ for MCC completions */ 2881 if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0)) 2882 goto mcc_cq_free; 2883 2884 q = &adapter->mcc_obj.q; 2885 if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 2886 goto mcc_cq_destroy; 2887 2888 if (be_cmd_mccq_create(adapter, q, cq)) 2889 goto mcc_q_free; 2890 2891 return 0; 2892 2893 mcc_q_free: 2894 be_queue_free(adapter, q); 2895 mcc_cq_destroy: 2896 be_cmd_q_destroy(adapter, cq, QTYPE_CQ); 2897 mcc_cq_free: 2898 be_queue_free(adapter, cq); 2899 err: 2900 return -1; 2901 } 2902 2903 static void be_tx_queues_destroy(struct be_adapter *adapter) 2904 { 2905 struct be_queue_info *q; 2906 struct be_tx_obj *txo; 2907 u8 i; 2908 2909 for_all_tx_queues(adapter, txo, i) { 2910 q = &txo->q; 2911 if (q->created) 2912 be_cmd_q_destroy(adapter, q, QTYPE_TXQ); 2913 be_queue_free(adapter, q); 2914 2915 q = &txo->cq; 2916 if (q->created) 2917 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2918 be_queue_free(adapter, q); 2919 } 2920 } 2921 2922 static int be_tx_qs_create(struct be_adapter *adapter) 2923 { 2924 struct be_queue_info *cq; 2925 struct be_tx_obj *txo; 2926 struct be_eq_obj *eqo; 2927 int status, i; 2928 2929 adapter->num_tx_qs = min(adapter->num_evt_qs, adapter->cfg_num_tx_irqs); 2930 2931 for_all_tx_queues(adapter, txo, i) { 2932 cq = &txo->cq; 2933 status = be_queue_alloc(adapter, cq, TX_CQ_LEN, 2934 sizeof(struct be_eth_tx_compl)); 2935 if (status) 2936 return status; 2937 2938 u64_stats_init(&txo->stats.sync); 2939 u64_stats_init(&txo->stats.sync_compl); 2940 2941 /* If num_evt_qs is less than num_tx_qs, then more than 2942 * one txq share an eq 2943 */ 2944 eqo = &adapter->eq_obj[i % adapter->num_evt_qs]; 2945 status = be_cmd_cq_create(adapter, cq, &eqo->q, false, 3); 2946 if (status) 2947 return status; 2948 2949 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN, 2950 sizeof(struct be_eth_wrb)); 2951 if (status) 2952 return status; 2953 2954 status = be_cmd_txq_create(adapter, txo); 2955 if (status) 2956 return status; 2957 2958 netif_set_xps_queue(adapter->netdev, eqo->affinity_mask, 2959 eqo->idx); 2960 } 2961 2962 dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n", 2963 adapter->num_tx_qs); 2964 return 0; 2965 } 2966 2967 static void be_rx_cqs_destroy(struct be_adapter *adapter) 2968 { 2969 struct be_queue_info *q; 2970 struct be_rx_obj *rxo; 2971 int i; 2972 2973 for_all_rx_queues(adapter, rxo, i) { 2974 q = &rxo->cq; 2975 if (q->created) 2976 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2977 be_queue_free(adapter, q); 2978 } 2979 } 2980 2981 static int be_rx_cqs_create(struct be_adapter *adapter) 2982 { 2983 struct be_queue_info *eq, *cq; 2984 struct be_rx_obj *rxo; 2985 int rc, i; 2986 2987 adapter->num_rss_qs = 2988 min(adapter->num_evt_qs, adapter->cfg_num_rx_irqs); 2989 2990 /* We'll use RSS only if atleast 2 RSS rings are supported. */ 2991 if (adapter->num_rss_qs < 2) 2992 adapter->num_rss_qs = 0; 2993 2994 adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq; 2995 2996 /* When the interface is not capable of RSS rings (and there is no 2997 * need to create a default RXQ) we'll still need one RXQ 2998 */ 2999 if (adapter->num_rx_qs == 0) 3000 adapter->num_rx_qs = 1; 3001 3002 adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE; 3003 for_all_rx_queues(adapter, rxo, i) { 3004 rxo->adapter = adapter; 3005 cq = &rxo->cq; 3006 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN, 3007 sizeof(struct be_eth_rx_compl)); 3008 if (rc) 3009 return rc; 3010 3011 u64_stats_init(&rxo->stats.sync); 3012 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; 3013 rc = be_cmd_cq_create(adapter, cq, eq, false, 3); 3014 if (rc) 3015 return rc; 3016 } 3017 3018 dev_info(&adapter->pdev->dev, 3019 "created %d RX queue(s)\n", adapter->num_rx_qs); 3020 return 0; 3021 } 3022 3023 static irqreturn_t be_intx(int irq, void *dev) 3024 { 3025 struct be_eq_obj *eqo = dev; 3026 struct be_adapter *adapter = eqo->adapter; 3027 int num_evts = 0; 3028 3029 /* IRQ is not expected when NAPI is scheduled as the EQ 3030 * will not be armed. 3031 * But, this can happen on Lancer INTx where it takes 3032 * a while to de-assert INTx or in BE2 where occasionaly 3033 * an interrupt may be raised even when EQ is unarmed. 3034 * If NAPI is already scheduled, then counting & notifying 3035 * events will orphan them. 3036 */ 3037 if (napi_schedule_prep(&eqo->napi)) { 3038 num_evts = events_get(eqo); 3039 __napi_schedule(&eqo->napi); 3040 if (num_evts) 3041 eqo->spurious_intr = 0; 3042 } 3043 be_eq_notify(adapter, eqo->q.id, false, true, num_evts, 0); 3044 3045 /* Return IRQ_HANDLED only for the the first spurious intr 3046 * after a valid intr to stop the kernel from branding 3047 * this irq as a bad one! 3048 */ 3049 if (num_evts || eqo->spurious_intr++ == 0) 3050 return IRQ_HANDLED; 3051 else 3052 return IRQ_NONE; 3053 } 3054 3055 static irqreturn_t be_msix(int irq, void *dev) 3056 { 3057 struct be_eq_obj *eqo = dev; 3058 3059 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0); 3060 napi_schedule(&eqo->napi); 3061 return IRQ_HANDLED; 3062 } 3063 3064 static inline bool do_gro(struct be_rx_compl_info *rxcp) 3065 { 3066 return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false; 3067 } 3068 3069 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi, 3070 int budget, int polling) 3071 { 3072 struct be_adapter *adapter = rxo->adapter; 3073 struct be_queue_info *rx_cq = &rxo->cq; 3074 struct be_rx_compl_info *rxcp; 3075 u32 work_done; 3076 u32 frags_consumed = 0; 3077 3078 for (work_done = 0; work_done < budget; work_done++) { 3079 rxcp = be_rx_compl_get(rxo); 3080 if (!rxcp) 3081 break; 3082 3083 /* Is it a flush compl that has no data */ 3084 if (unlikely(rxcp->num_rcvd == 0)) 3085 goto loop_continue; 3086 3087 /* Discard compl with partial DMA Lancer B0 */ 3088 if (unlikely(!rxcp->pkt_size)) { 3089 be_rx_compl_discard(rxo, rxcp); 3090 goto loop_continue; 3091 } 3092 3093 /* On BE drop pkts that arrive due to imperfect filtering in 3094 * promiscuous mode on some skews 3095 */ 3096 if (unlikely(rxcp->port != adapter->port_num && 3097 !lancer_chip(adapter))) { 3098 be_rx_compl_discard(rxo, rxcp); 3099 goto loop_continue; 3100 } 3101 3102 /* Don't do gro when we're busy_polling */ 3103 if (do_gro(rxcp) && polling != BUSY_POLLING) 3104 be_rx_compl_process_gro(rxo, napi, rxcp); 3105 else 3106 be_rx_compl_process(rxo, napi, rxcp); 3107 3108 loop_continue: 3109 frags_consumed += rxcp->num_rcvd; 3110 be_rx_stats_update(rxo, rxcp); 3111 } 3112 3113 if (work_done) { 3114 be_cq_notify(adapter, rx_cq->id, true, work_done); 3115 3116 /* When an rx-obj gets into post_starved state, just 3117 * let be_worker do the posting. 3118 */ 3119 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM && 3120 !rxo->rx_post_starved) 3121 be_post_rx_frags(rxo, GFP_ATOMIC, 3122 max_t(u32, MAX_RX_POST, 3123 frags_consumed)); 3124 } 3125 3126 return work_done; 3127 } 3128 3129 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status) 3130 { 3131 switch (status) { 3132 case BE_TX_COMP_HDR_PARSE_ERR: 3133 tx_stats(txo)->tx_hdr_parse_err++; 3134 break; 3135 case BE_TX_COMP_NDMA_ERR: 3136 tx_stats(txo)->tx_dma_err++; 3137 break; 3138 case BE_TX_COMP_ACL_ERR: 3139 tx_stats(txo)->tx_spoof_check_err++; 3140 break; 3141 } 3142 } 3143 3144 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status) 3145 { 3146 switch (status) { 3147 case LANCER_TX_COMP_LSO_ERR: 3148 tx_stats(txo)->tx_tso_err++; 3149 break; 3150 case LANCER_TX_COMP_HSW_DROP_MAC_ERR: 3151 case LANCER_TX_COMP_HSW_DROP_VLAN_ERR: 3152 tx_stats(txo)->tx_spoof_check_err++; 3153 break; 3154 case LANCER_TX_COMP_QINQ_ERR: 3155 tx_stats(txo)->tx_qinq_err++; 3156 break; 3157 case LANCER_TX_COMP_PARITY_ERR: 3158 tx_stats(txo)->tx_internal_parity_err++; 3159 break; 3160 case LANCER_TX_COMP_DMA_ERR: 3161 tx_stats(txo)->tx_dma_err++; 3162 break; 3163 } 3164 } 3165 3166 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo, 3167 int idx) 3168 { 3169 int num_wrbs = 0, work_done = 0; 3170 struct be_tx_compl_info *txcp; 3171 3172 while ((txcp = be_tx_compl_get(txo))) { 3173 num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index); 3174 work_done++; 3175 3176 if (txcp->status) { 3177 if (lancer_chip(adapter)) 3178 lancer_update_tx_err(txo, txcp->status); 3179 else 3180 be_update_tx_err(txo, txcp->status); 3181 } 3182 } 3183 3184 if (work_done) { 3185 be_cq_notify(adapter, txo->cq.id, true, work_done); 3186 atomic_sub(num_wrbs, &txo->q.used); 3187 3188 /* As Tx wrbs have been freed up, wake up netdev queue 3189 * if it was stopped due to lack of tx wrbs. */ 3190 if (__netif_subqueue_stopped(adapter->netdev, idx) && 3191 be_can_txq_wake(txo)) { 3192 netif_wake_subqueue(adapter->netdev, idx); 3193 } 3194 3195 u64_stats_update_begin(&tx_stats(txo)->sync_compl); 3196 tx_stats(txo)->tx_compl += work_done; 3197 u64_stats_update_end(&tx_stats(txo)->sync_compl); 3198 } 3199 } 3200 3201 #ifdef CONFIG_NET_RX_BUSY_POLL 3202 static inline bool be_lock_napi(struct be_eq_obj *eqo) 3203 { 3204 bool status = true; 3205 3206 spin_lock(&eqo->lock); /* BH is already disabled */ 3207 if (eqo->state & BE_EQ_LOCKED) { 3208 WARN_ON(eqo->state & BE_EQ_NAPI); 3209 eqo->state |= BE_EQ_NAPI_YIELD; 3210 status = false; 3211 } else { 3212 eqo->state = BE_EQ_NAPI; 3213 } 3214 spin_unlock(&eqo->lock); 3215 return status; 3216 } 3217 3218 static inline void be_unlock_napi(struct be_eq_obj *eqo) 3219 { 3220 spin_lock(&eqo->lock); /* BH is already disabled */ 3221 3222 WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD)); 3223 eqo->state = BE_EQ_IDLE; 3224 3225 spin_unlock(&eqo->lock); 3226 } 3227 3228 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo) 3229 { 3230 bool status = true; 3231 3232 spin_lock_bh(&eqo->lock); 3233 if (eqo->state & BE_EQ_LOCKED) { 3234 eqo->state |= BE_EQ_POLL_YIELD; 3235 status = false; 3236 } else { 3237 eqo->state |= BE_EQ_POLL; 3238 } 3239 spin_unlock_bh(&eqo->lock); 3240 return status; 3241 } 3242 3243 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo) 3244 { 3245 spin_lock_bh(&eqo->lock); 3246 3247 WARN_ON(eqo->state & (BE_EQ_NAPI)); 3248 eqo->state = BE_EQ_IDLE; 3249 3250 spin_unlock_bh(&eqo->lock); 3251 } 3252 3253 static inline void be_enable_busy_poll(struct be_eq_obj *eqo) 3254 { 3255 spin_lock_init(&eqo->lock); 3256 eqo->state = BE_EQ_IDLE; 3257 } 3258 3259 static inline void be_disable_busy_poll(struct be_eq_obj *eqo) 3260 { 3261 local_bh_disable(); 3262 3263 /* It's enough to just acquire napi lock on the eqo to stop 3264 * be_busy_poll() from processing any queueus. 3265 */ 3266 while (!be_lock_napi(eqo)) 3267 mdelay(1); 3268 3269 local_bh_enable(); 3270 } 3271 3272 #else /* CONFIG_NET_RX_BUSY_POLL */ 3273 3274 static inline bool be_lock_napi(struct be_eq_obj *eqo) 3275 { 3276 return true; 3277 } 3278 3279 static inline void be_unlock_napi(struct be_eq_obj *eqo) 3280 { 3281 } 3282 3283 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo) 3284 { 3285 return false; 3286 } 3287 3288 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo) 3289 { 3290 } 3291 3292 static inline void be_enable_busy_poll(struct be_eq_obj *eqo) 3293 { 3294 } 3295 3296 static inline void be_disable_busy_poll(struct be_eq_obj *eqo) 3297 { 3298 } 3299 #endif /* CONFIG_NET_RX_BUSY_POLL */ 3300 3301 int be_poll(struct napi_struct *napi, int budget) 3302 { 3303 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 3304 struct be_adapter *adapter = eqo->adapter; 3305 int max_work = 0, work, i, num_evts; 3306 struct be_rx_obj *rxo; 3307 struct be_tx_obj *txo; 3308 u32 mult_enc = 0; 3309 3310 num_evts = events_get(eqo); 3311 3312 for_all_tx_queues_on_eq(adapter, eqo, txo, i) 3313 be_process_tx(adapter, txo, i); 3314 3315 if (be_lock_napi(eqo)) { 3316 /* This loop will iterate twice for EQ0 in which 3317 * completions of the last RXQ (default one) are also processed 3318 * For other EQs the loop iterates only once 3319 */ 3320 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 3321 work = be_process_rx(rxo, napi, budget, NAPI_POLLING); 3322 max_work = max(work, max_work); 3323 } 3324 be_unlock_napi(eqo); 3325 } else { 3326 max_work = budget; 3327 } 3328 3329 if (is_mcc_eqo(eqo)) 3330 be_process_mcc(adapter); 3331 3332 if (max_work < budget) { 3333 napi_complete(napi); 3334 3335 /* Skyhawk EQ_DB has a provision to set the rearm to interrupt 3336 * delay via a delay multiplier encoding value 3337 */ 3338 if (skyhawk_chip(adapter)) 3339 mult_enc = be_get_eq_delay_mult_enc(eqo); 3340 3341 be_eq_notify(adapter, eqo->q.id, true, false, num_evts, 3342 mult_enc); 3343 } else { 3344 /* As we'll continue in polling mode, count and clear events */ 3345 be_eq_notify(adapter, eqo->q.id, false, false, num_evts, 0); 3346 } 3347 return max_work; 3348 } 3349 3350 #ifdef CONFIG_NET_RX_BUSY_POLL 3351 static int be_busy_poll(struct napi_struct *napi) 3352 { 3353 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 3354 struct be_adapter *adapter = eqo->adapter; 3355 struct be_rx_obj *rxo; 3356 int i, work = 0; 3357 3358 if (!be_lock_busy_poll(eqo)) 3359 return LL_FLUSH_BUSY; 3360 3361 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 3362 work = be_process_rx(rxo, napi, 4, BUSY_POLLING); 3363 if (work) 3364 break; 3365 } 3366 3367 be_unlock_busy_poll(eqo); 3368 return work; 3369 } 3370 #endif 3371 3372 void be_detect_error(struct be_adapter *adapter) 3373 { 3374 u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0; 3375 u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 3376 u32 i; 3377 struct device *dev = &adapter->pdev->dev; 3378 3379 if (be_check_error(adapter, BE_ERROR_HW)) 3380 return; 3381 3382 if (lancer_chip(adapter)) { 3383 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET); 3384 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 3385 be_set_error(adapter, BE_ERROR_UE); 3386 sliport_err1 = ioread32(adapter->db + 3387 SLIPORT_ERROR1_OFFSET); 3388 sliport_err2 = ioread32(adapter->db + 3389 SLIPORT_ERROR2_OFFSET); 3390 /* Do not log error messages if its a FW reset */ 3391 if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 && 3392 sliport_err2 == SLIPORT_ERROR_FW_RESET2) { 3393 dev_info(dev, "Firmware update in progress\n"); 3394 } else { 3395 dev_err(dev, "Error detected in the card\n"); 3396 dev_err(dev, "ERR: sliport status 0x%x\n", 3397 sliport_status); 3398 dev_err(dev, "ERR: sliport error1 0x%x\n", 3399 sliport_err1); 3400 dev_err(dev, "ERR: sliport error2 0x%x\n", 3401 sliport_err2); 3402 } 3403 } 3404 } else { 3405 ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW); 3406 ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH); 3407 ue_lo_mask = ioread32(adapter->pcicfg + 3408 PCICFG_UE_STATUS_LOW_MASK); 3409 ue_hi_mask = ioread32(adapter->pcicfg + 3410 PCICFG_UE_STATUS_HI_MASK); 3411 3412 ue_lo = (ue_lo & ~ue_lo_mask); 3413 ue_hi = (ue_hi & ~ue_hi_mask); 3414 3415 /* On certain platforms BE hardware can indicate spurious UEs. 3416 * Allow HW to stop working completely in case of a real UE. 3417 * Hence not setting the hw_error for UE detection. 3418 */ 3419 3420 if (ue_lo || ue_hi) { 3421 dev_err(dev, "Error detected in the adapter"); 3422 if (skyhawk_chip(adapter)) 3423 be_set_error(adapter, BE_ERROR_UE); 3424 3425 for (i = 0; ue_lo; ue_lo >>= 1, i++) { 3426 if (ue_lo & 1) 3427 dev_err(dev, "UE: %s bit set\n", 3428 ue_status_low_desc[i]); 3429 } 3430 for (i = 0; ue_hi; ue_hi >>= 1, i++) { 3431 if (ue_hi & 1) 3432 dev_err(dev, "UE: %s bit set\n", 3433 ue_status_hi_desc[i]); 3434 } 3435 } 3436 } 3437 } 3438 3439 static void be_msix_disable(struct be_adapter *adapter) 3440 { 3441 if (msix_enabled(adapter)) { 3442 pci_disable_msix(adapter->pdev); 3443 adapter->num_msix_vec = 0; 3444 adapter->num_msix_roce_vec = 0; 3445 } 3446 } 3447 3448 static int be_msix_enable(struct be_adapter *adapter) 3449 { 3450 unsigned int i, max_roce_eqs; 3451 struct device *dev = &adapter->pdev->dev; 3452 int num_vec; 3453 3454 /* If RoCE is supported, program the max number of vectors that 3455 * could be used for NIC and RoCE, else, just program the number 3456 * we'll use initially. 3457 */ 3458 if (be_roce_supported(adapter)) { 3459 max_roce_eqs = 3460 be_max_func_eqs(adapter) - be_max_nic_eqs(adapter); 3461 max_roce_eqs = min(max_roce_eqs, num_online_cpus()); 3462 num_vec = be_max_any_irqs(adapter) + max_roce_eqs; 3463 } else { 3464 num_vec = max(adapter->cfg_num_rx_irqs, 3465 adapter->cfg_num_tx_irqs); 3466 } 3467 3468 for (i = 0; i < num_vec; i++) 3469 adapter->msix_entries[i].entry = i; 3470 3471 num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 3472 MIN_MSIX_VECTORS, num_vec); 3473 if (num_vec < 0) 3474 goto fail; 3475 3476 if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) { 3477 adapter->num_msix_roce_vec = num_vec / 2; 3478 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n", 3479 adapter->num_msix_roce_vec); 3480 } 3481 3482 adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec; 3483 3484 dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n", 3485 adapter->num_msix_vec); 3486 return 0; 3487 3488 fail: 3489 dev_warn(dev, "MSIx enable failed\n"); 3490 3491 /* INTx is not supported in VFs, so fail probe if enable_msix fails */ 3492 if (be_virtfn(adapter)) 3493 return num_vec; 3494 return 0; 3495 } 3496 3497 static inline int be_msix_vec_get(struct be_adapter *adapter, 3498 struct be_eq_obj *eqo) 3499 { 3500 return adapter->msix_entries[eqo->msix_idx].vector; 3501 } 3502 3503 static int be_msix_register(struct be_adapter *adapter) 3504 { 3505 struct net_device *netdev = adapter->netdev; 3506 struct be_eq_obj *eqo; 3507 int status, i, vec; 3508 3509 for_all_evt_queues(adapter, eqo, i) { 3510 sprintf(eqo->desc, "%s-q%d", netdev->name, i); 3511 vec = be_msix_vec_get(adapter, eqo); 3512 status = request_irq(vec, be_msix, 0, eqo->desc, eqo); 3513 if (status) 3514 goto err_msix; 3515 3516 irq_set_affinity_hint(vec, eqo->affinity_mask); 3517 } 3518 3519 return 0; 3520 err_msix: 3521 for (i--; i >= 0; i--) { 3522 eqo = &adapter->eq_obj[i]; 3523 free_irq(be_msix_vec_get(adapter, eqo), eqo); 3524 } 3525 dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n", 3526 status); 3527 be_msix_disable(adapter); 3528 return status; 3529 } 3530 3531 static int be_irq_register(struct be_adapter *adapter) 3532 { 3533 struct net_device *netdev = adapter->netdev; 3534 int status; 3535 3536 if (msix_enabled(adapter)) { 3537 status = be_msix_register(adapter); 3538 if (status == 0) 3539 goto done; 3540 /* INTx is not supported for VF */ 3541 if (be_virtfn(adapter)) 3542 return status; 3543 } 3544 3545 /* INTx: only the first EQ is used */ 3546 netdev->irq = adapter->pdev->irq; 3547 status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name, 3548 &adapter->eq_obj[0]); 3549 if (status) { 3550 dev_err(&adapter->pdev->dev, 3551 "INTx request IRQ failed - err %d\n", status); 3552 return status; 3553 } 3554 done: 3555 adapter->isr_registered = true; 3556 return 0; 3557 } 3558 3559 static void be_irq_unregister(struct be_adapter *adapter) 3560 { 3561 struct net_device *netdev = adapter->netdev; 3562 struct be_eq_obj *eqo; 3563 int i, vec; 3564 3565 if (!adapter->isr_registered) 3566 return; 3567 3568 /* INTx */ 3569 if (!msix_enabled(adapter)) { 3570 free_irq(netdev->irq, &adapter->eq_obj[0]); 3571 goto done; 3572 } 3573 3574 /* MSIx */ 3575 for_all_evt_queues(adapter, eqo, i) { 3576 vec = be_msix_vec_get(adapter, eqo); 3577 irq_set_affinity_hint(vec, NULL); 3578 free_irq(vec, eqo); 3579 } 3580 3581 done: 3582 adapter->isr_registered = false; 3583 } 3584 3585 static void be_rx_qs_destroy(struct be_adapter *adapter) 3586 { 3587 struct rss_info *rss = &adapter->rss_info; 3588 struct be_queue_info *q; 3589 struct be_rx_obj *rxo; 3590 int i; 3591 3592 for_all_rx_queues(adapter, rxo, i) { 3593 q = &rxo->q; 3594 if (q->created) { 3595 /* If RXQs are destroyed while in an "out of buffer" 3596 * state, there is a possibility of an HW stall on 3597 * Lancer. So, post 64 buffers to each queue to relieve 3598 * the "out of buffer" condition. 3599 * Make sure there's space in the RXQ before posting. 3600 */ 3601 if (lancer_chip(adapter)) { 3602 be_rx_cq_clean(rxo); 3603 if (atomic_read(&q->used) == 0) 3604 be_post_rx_frags(rxo, GFP_KERNEL, 3605 MAX_RX_POST); 3606 } 3607 3608 be_cmd_rxq_destroy(adapter, q); 3609 be_rx_cq_clean(rxo); 3610 be_rxq_clean(rxo); 3611 } 3612 be_queue_free(adapter, q); 3613 } 3614 3615 if (rss->rss_flags) { 3616 rss->rss_flags = RSS_ENABLE_NONE; 3617 be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3618 128, rss->rss_hkey); 3619 } 3620 } 3621 3622 static void be_disable_if_filters(struct be_adapter *adapter) 3623 { 3624 be_dev_mac_del(adapter, adapter->pmac_id[0]); 3625 be_clear_uc_list(adapter); 3626 be_clear_mc_list(adapter); 3627 3628 /* The IFACE flags are enabled in the open path and cleared 3629 * in the close path. When a VF gets detached from the host and 3630 * assigned to a VM the following happens: 3631 * - VF's IFACE flags get cleared in the detach path 3632 * - IFACE create is issued by the VF in the attach path 3633 * Due to a bug in the BE3/Skyhawk-R FW 3634 * (Lancer FW doesn't have the bug), the IFACE capability flags 3635 * specified along with the IFACE create cmd issued by a VF are not 3636 * honoured by FW. As a consequence, if a *new* driver 3637 * (that enables/disables IFACE flags in open/close) 3638 * is loaded in the host and an *old* driver is * used by a VM/VF, 3639 * the IFACE gets created *without* the needed flags. 3640 * To avoid this, disable RX-filter flags only for Lancer. 3641 */ 3642 if (lancer_chip(adapter)) { 3643 be_cmd_rx_filter(adapter, BE_IF_ALL_FILT_FLAGS, OFF); 3644 adapter->if_flags &= ~BE_IF_ALL_FILT_FLAGS; 3645 } 3646 } 3647 3648 static int be_close(struct net_device *netdev) 3649 { 3650 struct be_adapter *adapter = netdev_priv(netdev); 3651 struct be_eq_obj *eqo; 3652 int i; 3653 3654 /* This protection is needed as be_close() may be called even when the 3655 * adapter is in cleared state (after eeh perm failure) 3656 */ 3657 if (!(adapter->flags & BE_FLAGS_SETUP_DONE)) 3658 return 0; 3659 3660 /* Before attempting cleanup ensure all the pending cmds in the 3661 * config_wq have finished execution 3662 */ 3663 flush_workqueue(be_wq); 3664 3665 be_disable_if_filters(adapter); 3666 3667 if (adapter->flags & BE_FLAGS_NAPI_ENABLED) { 3668 for_all_evt_queues(adapter, eqo, i) { 3669 napi_disable(&eqo->napi); 3670 be_disable_busy_poll(eqo); 3671 } 3672 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED; 3673 } 3674 3675 be_async_mcc_disable(adapter); 3676 3677 /* Wait for all pending tx completions to arrive so that 3678 * all tx skbs are freed. 3679 */ 3680 netif_tx_disable(netdev); 3681 be_tx_compl_clean(adapter); 3682 3683 be_rx_qs_destroy(adapter); 3684 3685 for_all_evt_queues(adapter, eqo, i) { 3686 if (msix_enabled(adapter)) 3687 synchronize_irq(be_msix_vec_get(adapter, eqo)); 3688 else 3689 synchronize_irq(netdev->irq); 3690 be_eq_clean(eqo); 3691 } 3692 3693 be_irq_unregister(adapter); 3694 3695 return 0; 3696 } 3697 3698 static int be_rx_qs_create(struct be_adapter *adapter) 3699 { 3700 struct rss_info *rss = &adapter->rss_info; 3701 u8 rss_key[RSS_HASH_KEY_LEN]; 3702 struct be_rx_obj *rxo; 3703 int rc, i, j; 3704 3705 for_all_rx_queues(adapter, rxo, i) { 3706 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN, 3707 sizeof(struct be_eth_rx_d)); 3708 if (rc) 3709 return rc; 3710 } 3711 3712 if (adapter->need_def_rxq || !adapter->num_rss_qs) { 3713 rxo = default_rxo(adapter); 3714 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3715 rx_frag_size, adapter->if_handle, 3716 false, &rxo->rss_id); 3717 if (rc) 3718 return rc; 3719 } 3720 3721 for_all_rss_queues(adapter, rxo, i) { 3722 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3723 rx_frag_size, adapter->if_handle, 3724 true, &rxo->rss_id); 3725 if (rc) 3726 return rc; 3727 } 3728 3729 if (be_multi_rxq(adapter)) { 3730 for (j = 0; j < RSS_INDIR_TABLE_LEN; j += adapter->num_rss_qs) { 3731 for_all_rss_queues(adapter, rxo, i) { 3732 if ((j + i) >= RSS_INDIR_TABLE_LEN) 3733 break; 3734 rss->rsstable[j + i] = rxo->rss_id; 3735 rss->rss_queue[j + i] = i; 3736 } 3737 } 3738 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 | 3739 RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6; 3740 3741 if (!BEx_chip(adapter)) 3742 rss->rss_flags |= RSS_ENABLE_UDP_IPV4 | 3743 RSS_ENABLE_UDP_IPV6; 3744 3745 netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN); 3746 rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3747 RSS_INDIR_TABLE_LEN, rss_key); 3748 if (rc) { 3749 rss->rss_flags = RSS_ENABLE_NONE; 3750 return rc; 3751 } 3752 3753 memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN); 3754 } else { 3755 /* Disable RSS, if only default RX Q is created */ 3756 rss->rss_flags = RSS_ENABLE_NONE; 3757 } 3758 3759 3760 /* Post 1 less than RXQ-len to avoid head being equal to tail, 3761 * which is a queue empty condition 3762 */ 3763 for_all_rx_queues(adapter, rxo, i) 3764 be_post_rx_frags(rxo, GFP_KERNEL, RX_Q_LEN - 1); 3765 3766 return 0; 3767 } 3768 3769 static int be_enable_if_filters(struct be_adapter *adapter) 3770 { 3771 int status; 3772 3773 status = be_cmd_rx_filter(adapter, BE_IF_FILT_FLAGS_BASIC, ON); 3774 if (status) 3775 return status; 3776 3777 /* For BE3 VFs, the PF programs the initial MAC address */ 3778 if (!(BEx_chip(adapter) && be_virtfn(adapter))) { 3779 status = be_dev_mac_add(adapter, adapter->netdev->dev_addr); 3780 if (status) 3781 return status; 3782 ether_addr_copy(adapter->dev_mac, adapter->netdev->dev_addr); 3783 } 3784 3785 if (adapter->vlans_added) 3786 be_vid_config(adapter); 3787 3788 __be_set_rx_mode(adapter); 3789 3790 return 0; 3791 } 3792 3793 static int be_open(struct net_device *netdev) 3794 { 3795 struct be_adapter *adapter = netdev_priv(netdev); 3796 struct be_eq_obj *eqo; 3797 struct be_rx_obj *rxo; 3798 struct be_tx_obj *txo; 3799 u8 link_status; 3800 int status, i; 3801 3802 status = be_rx_qs_create(adapter); 3803 if (status) 3804 goto err; 3805 3806 status = be_enable_if_filters(adapter); 3807 if (status) 3808 goto err; 3809 3810 status = be_irq_register(adapter); 3811 if (status) 3812 goto err; 3813 3814 for_all_rx_queues(adapter, rxo, i) 3815 be_cq_notify(adapter, rxo->cq.id, true, 0); 3816 3817 for_all_tx_queues(adapter, txo, i) 3818 be_cq_notify(adapter, txo->cq.id, true, 0); 3819 3820 be_async_mcc_enable(adapter); 3821 3822 for_all_evt_queues(adapter, eqo, i) { 3823 napi_enable(&eqo->napi); 3824 be_enable_busy_poll(eqo); 3825 be_eq_notify(adapter, eqo->q.id, true, true, 0, 0); 3826 } 3827 adapter->flags |= BE_FLAGS_NAPI_ENABLED; 3828 3829 status = be_cmd_link_status_query(adapter, NULL, &link_status, 0); 3830 if (!status) 3831 be_link_status_update(adapter, link_status); 3832 3833 netif_tx_start_all_queues(netdev); 3834 if (skyhawk_chip(adapter)) 3835 udp_tunnel_get_rx_info(netdev); 3836 3837 return 0; 3838 err: 3839 be_close(adapter->netdev); 3840 return -EIO; 3841 } 3842 3843 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac) 3844 { 3845 u32 addr; 3846 3847 addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0); 3848 3849 mac[5] = (u8)(addr & 0xFF); 3850 mac[4] = (u8)((addr >> 8) & 0xFF); 3851 mac[3] = (u8)((addr >> 16) & 0xFF); 3852 /* Use the OUI from the current MAC address */ 3853 memcpy(mac, adapter->netdev->dev_addr, 3); 3854 } 3855 3856 /* 3857 * Generate a seed MAC address from the PF MAC Address using jhash. 3858 * MAC Address for VFs are assigned incrementally starting from the seed. 3859 * These addresses are programmed in the ASIC by the PF and the VF driver 3860 * queries for the MAC address during its probe. 3861 */ 3862 static int be_vf_eth_addr_config(struct be_adapter *adapter) 3863 { 3864 u32 vf; 3865 int status = 0; 3866 u8 mac[ETH_ALEN]; 3867 struct be_vf_cfg *vf_cfg; 3868 3869 be_vf_eth_addr_generate(adapter, mac); 3870 3871 for_all_vfs(adapter, vf_cfg, vf) { 3872 if (BEx_chip(adapter)) 3873 status = be_cmd_pmac_add(adapter, mac, 3874 vf_cfg->if_handle, 3875 &vf_cfg->pmac_id, vf + 1); 3876 else 3877 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 3878 vf + 1); 3879 3880 if (status) 3881 dev_err(&adapter->pdev->dev, 3882 "Mac address assignment failed for VF %d\n", 3883 vf); 3884 else 3885 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3886 3887 mac[5] += 1; 3888 } 3889 return status; 3890 } 3891 3892 static int be_vfs_mac_query(struct be_adapter *adapter) 3893 { 3894 int status, vf; 3895 u8 mac[ETH_ALEN]; 3896 struct be_vf_cfg *vf_cfg; 3897 3898 for_all_vfs(adapter, vf_cfg, vf) { 3899 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id, 3900 mac, vf_cfg->if_handle, 3901 false, vf+1); 3902 if (status) 3903 return status; 3904 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3905 } 3906 return 0; 3907 } 3908 3909 static void be_vf_clear(struct be_adapter *adapter) 3910 { 3911 struct be_vf_cfg *vf_cfg; 3912 u32 vf; 3913 3914 if (pci_vfs_assigned(adapter->pdev)) { 3915 dev_warn(&adapter->pdev->dev, 3916 "VFs are assigned to VMs: not disabling VFs\n"); 3917 goto done; 3918 } 3919 3920 pci_disable_sriov(adapter->pdev); 3921 3922 for_all_vfs(adapter, vf_cfg, vf) { 3923 if (BEx_chip(adapter)) 3924 be_cmd_pmac_del(adapter, vf_cfg->if_handle, 3925 vf_cfg->pmac_id, vf + 1); 3926 else 3927 be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle, 3928 vf + 1); 3929 3930 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1); 3931 } 3932 3933 if (BE3_chip(adapter)) 3934 be_cmd_set_hsw_config(adapter, 0, 0, 3935 adapter->if_handle, 3936 PORT_FWD_TYPE_PASSTHRU, 0); 3937 done: 3938 kfree(adapter->vf_cfg); 3939 adapter->num_vfs = 0; 3940 adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED; 3941 } 3942 3943 static void be_clear_queues(struct be_adapter *adapter) 3944 { 3945 be_mcc_queues_destroy(adapter); 3946 be_rx_cqs_destroy(adapter); 3947 be_tx_queues_destroy(adapter); 3948 be_evt_queues_destroy(adapter); 3949 } 3950 3951 static void be_cancel_worker(struct be_adapter *adapter) 3952 { 3953 if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) { 3954 cancel_delayed_work_sync(&adapter->work); 3955 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED; 3956 } 3957 } 3958 3959 static void be_cancel_err_detection(struct be_adapter *adapter) 3960 { 3961 struct be_error_recovery *err_rec = &adapter->error_recovery; 3962 3963 if (!be_err_recovery_workq) 3964 return; 3965 3966 if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) { 3967 cancel_delayed_work_sync(&err_rec->err_detection_work); 3968 adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED; 3969 } 3970 } 3971 3972 static void be_disable_vxlan_offloads(struct be_adapter *adapter) 3973 { 3974 struct net_device *netdev = adapter->netdev; 3975 3976 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) 3977 be_cmd_manage_iface(adapter, adapter->if_handle, 3978 OP_CONVERT_TUNNEL_TO_NORMAL); 3979 3980 if (adapter->vxlan_port) 3981 be_cmd_set_vxlan_port(adapter, 0); 3982 3983 adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS; 3984 adapter->vxlan_port = 0; 3985 3986 netdev->hw_enc_features = 0; 3987 netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL); 3988 netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL); 3989 } 3990 3991 static void be_calculate_vf_res(struct be_adapter *adapter, u16 num_vfs, 3992 struct be_resources *vft_res) 3993 { 3994 struct be_resources res = adapter->pool_res; 3995 u32 vf_if_cap_flags = res.vf_if_cap_flags; 3996 struct be_resources res_mod = {0}; 3997 u16 num_vf_qs = 1; 3998 3999 /* Distribute the queue resources among the PF and it's VFs */ 4000 if (num_vfs) { 4001 /* Divide the rx queues evenly among the VFs and the PF, capped 4002 * at VF-EQ-count. Any remainder queues belong to the PF. 4003 */ 4004 num_vf_qs = min(SH_VF_MAX_NIC_EQS, 4005 res.max_rss_qs / (num_vfs + 1)); 4006 4007 /* Skyhawk-R chip supports only MAX_PORT_RSS_TABLES 4008 * RSS Tables per port. Provide RSS on VFs, only if number of 4009 * VFs requested is less than it's PF Pool's RSS Tables limit. 4010 */ 4011 if (num_vfs >= be_max_pf_pool_rss_tables(adapter)) 4012 num_vf_qs = 1; 4013 } 4014 4015 /* Resource with fields set to all '1's by GET_PROFILE_CONFIG cmd, 4016 * which are modifiable using SET_PROFILE_CONFIG cmd. 4017 */ 4018 be_cmd_get_profile_config(adapter, &res_mod, NULL, ACTIVE_PROFILE_TYPE, 4019 RESOURCE_MODIFIABLE, 0); 4020 4021 /* If RSS IFACE capability flags are modifiable for a VF, set the 4022 * capability flag as valid and set RSS and DEFQ_RSS IFACE flags if 4023 * more than 1 RSSQ is available for a VF. 4024 * Otherwise, provision only 1 queue pair for VF. 4025 */ 4026 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_RSS) { 4027 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT); 4028 if (num_vf_qs > 1) { 4029 vf_if_cap_flags |= BE_IF_FLAGS_RSS; 4030 if (res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS) 4031 vf_if_cap_flags |= BE_IF_FLAGS_DEFQ_RSS; 4032 } else { 4033 vf_if_cap_flags &= ~(BE_IF_FLAGS_RSS | 4034 BE_IF_FLAGS_DEFQ_RSS); 4035 } 4036 } else { 4037 num_vf_qs = 1; 4038 } 4039 4040 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 4041 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT); 4042 vf_if_cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 4043 } 4044 4045 vft_res->vf_if_cap_flags = vf_if_cap_flags; 4046 vft_res->max_rx_qs = num_vf_qs; 4047 vft_res->max_rss_qs = num_vf_qs; 4048 vft_res->max_tx_qs = res.max_tx_qs / (num_vfs + 1); 4049 vft_res->max_cq_count = res.max_cq_count / (num_vfs + 1); 4050 4051 /* Distribute unicast MACs, VLANs, IFACE count and MCCQ count equally 4052 * among the PF and it's VFs, if the fields are changeable 4053 */ 4054 if (res_mod.max_uc_mac == FIELD_MODIFIABLE) 4055 vft_res->max_uc_mac = res.max_uc_mac / (num_vfs + 1); 4056 4057 if (res_mod.max_vlans == FIELD_MODIFIABLE) 4058 vft_res->max_vlans = res.max_vlans / (num_vfs + 1); 4059 4060 if (res_mod.max_iface_count == FIELD_MODIFIABLE) 4061 vft_res->max_iface_count = res.max_iface_count / (num_vfs + 1); 4062 4063 if (res_mod.max_mcc_count == FIELD_MODIFIABLE) 4064 vft_res->max_mcc_count = res.max_mcc_count / (num_vfs + 1); 4065 } 4066 4067 static void be_if_destroy(struct be_adapter *adapter) 4068 { 4069 be_cmd_if_destroy(adapter, adapter->if_handle, 0); 4070 4071 kfree(adapter->pmac_id); 4072 adapter->pmac_id = NULL; 4073 4074 kfree(adapter->mc_list); 4075 adapter->mc_list = NULL; 4076 4077 kfree(adapter->uc_list); 4078 adapter->uc_list = NULL; 4079 } 4080 4081 static int be_clear(struct be_adapter *adapter) 4082 { 4083 struct pci_dev *pdev = adapter->pdev; 4084 struct be_resources vft_res = {0}; 4085 4086 be_cancel_worker(adapter); 4087 4088 flush_workqueue(be_wq); 4089 4090 if (sriov_enabled(adapter)) 4091 be_vf_clear(adapter); 4092 4093 /* Re-configure FW to distribute resources evenly across max-supported 4094 * number of VFs, only when VFs are not already enabled. 4095 */ 4096 if (skyhawk_chip(adapter) && be_physfn(adapter) && 4097 !pci_vfs_assigned(pdev)) { 4098 be_calculate_vf_res(adapter, 4099 pci_sriov_get_totalvfs(pdev), 4100 &vft_res); 4101 be_cmd_set_sriov_config(adapter, adapter->pool_res, 4102 pci_sriov_get_totalvfs(pdev), 4103 &vft_res); 4104 } 4105 4106 be_disable_vxlan_offloads(adapter); 4107 4108 be_if_destroy(adapter); 4109 4110 be_clear_queues(adapter); 4111 4112 be_msix_disable(adapter); 4113 adapter->flags &= ~BE_FLAGS_SETUP_DONE; 4114 return 0; 4115 } 4116 4117 static int be_vfs_if_create(struct be_adapter *adapter) 4118 { 4119 struct be_resources res = {0}; 4120 u32 cap_flags, en_flags, vf; 4121 struct be_vf_cfg *vf_cfg; 4122 int status; 4123 4124 /* If a FW profile exists, then cap_flags are updated */ 4125 cap_flags = BE_VF_IF_EN_FLAGS; 4126 4127 for_all_vfs(adapter, vf_cfg, vf) { 4128 if (!BE3_chip(adapter)) { 4129 status = be_cmd_get_profile_config(adapter, &res, NULL, 4130 ACTIVE_PROFILE_TYPE, 4131 RESOURCE_LIMITS, 4132 vf + 1); 4133 if (!status) { 4134 cap_flags = res.if_cap_flags; 4135 /* Prevent VFs from enabling VLAN promiscuous 4136 * mode 4137 */ 4138 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 4139 } 4140 } 4141 4142 /* PF should enable IF flags during proxy if_create call */ 4143 en_flags = cap_flags & BE_VF_IF_EN_FLAGS; 4144 status = be_cmd_if_create(adapter, cap_flags, en_flags, 4145 &vf_cfg->if_handle, vf + 1); 4146 if (status) 4147 return status; 4148 } 4149 4150 return 0; 4151 } 4152 4153 static int be_vf_setup_init(struct be_adapter *adapter) 4154 { 4155 struct be_vf_cfg *vf_cfg; 4156 int vf; 4157 4158 adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg), 4159 GFP_KERNEL); 4160 if (!adapter->vf_cfg) 4161 return -ENOMEM; 4162 4163 for_all_vfs(adapter, vf_cfg, vf) { 4164 vf_cfg->if_handle = -1; 4165 vf_cfg->pmac_id = -1; 4166 } 4167 return 0; 4168 } 4169 4170 static int be_vf_setup(struct be_adapter *adapter) 4171 { 4172 struct device *dev = &adapter->pdev->dev; 4173 struct be_vf_cfg *vf_cfg; 4174 int status, old_vfs, vf; 4175 bool spoofchk; 4176 4177 old_vfs = pci_num_vf(adapter->pdev); 4178 4179 status = be_vf_setup_init(adapter); 4180 if (status) 4181 goto err; 4182 4183 if (old_vfs) { 4184 for_all_vfs(adapter, vf_cfg, vf) { 4185 status = be_cmd_get_if_id(adapter, vf_cfg, vf); 4186 if (status) 4187 goto err; 4188 } 4189 4190 status = be_vfs_mac_query(adapter); 4191 if (status) 4192 goto err; 4193 } else { 4194 status = be_vfs_if_create(adapter); 4195 if (status) 4196 goto err; 4197 4198 status = be_vf_eth_addr_config(adapter); 4199 if (status) 4200 goto err; 4201 } 4202 4203 for_all_vfs(adapter, vf_cfg, vf) { 4204 /* Allow VFs to programs MAC/VLAN filters */ 4205 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges, 4206 vf + 1); 4207 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 4208 status = be_cmd_set_fn_privileges(adapter, 4209 vf_cfg->privileges | 4210 BE_PRIV_FILTMGMT, 4211 vf + 1); 4212 if (!status) { 4213 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 4214 dev_info(dev, "VF%d has FILTMGMT privilege\n", 4215 vf); 4216 } 4217 } 4218 4219 /* Allow full available bandwidth */ 4220 if (!old_vfs) 4221 be_cmd_config_qos(adapter, 0, 0, vf + 1); 4222 4223 status = be_cmd_get_hsw_config(adapter, NULL, vf + 1, 4224 vf_cfg->if_handle, NULL, 4225 &spoofchk); 4226 if (!status) 4227 vf_cfg->spoofchk = spoofchk; 4228 4229 if (!old_vfs) { 4230 be_cmd_enable_vf(adapter, vf + 1); 4231 be_cmd_set_logical_link_config(adapter, 4232 IFLA_VF_LINK_STATE_AUTO, 4233 vf+1); 4234 } 4235 } 4236 4237 if (!old_vfs) { 4238 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs); 4239 if (status) { 4240 dev_err(dev, "SRIOV enable failed\n"); 4241 adapter->num_vfs = 0; 4242 goto err; 4243 } 4244 } 4245 4246 if (BE3_chip(adapter)) { 4247 /* On BE3, enable VEB only when SRIOV is enabled */ 4248 status = be_cmd_set_hsw_config(adapter, 0, 0, 4249 adapter->if_handle, 4250 PORT_FWD_TYPE_VEB, 0); 4251 if (status) 4252 goto err; 4253 } 4254 4255 adapter->flags |= BE_FLAGS_SRIOV_ENABLED; 4256 return 0; 4257 err: 4258 dev_err(dev, "VF setup failed\n"); 4259 be_vf_clear(adapter); 4260 return status; 4261 } 4262 4263 /* Converting function_mode bits on BE3 to SH mc_type enums */ 4264 4265 static u8 be_convert_mc_type(u32 function_mode) 4266 { 4267 if (function_mode & VNIC_MODE && function_mode & QNQ_MODE) 4268 return vNIC1; 4269 else if (function_mode & QNQ_MODE) 4270 return FLEX10; 4271 else if (function_mode & VNIC_MODE) 4272 return vNIC2; 4273 else if (function_mode & UMC_ENABLED) 4274 return UMC; 4275 else 4276 return MC_NONE; 4277 } 4278 4279 /* On BE2/BE3 FW does not suggest the supported limits */ 4280 static void BEx_get_resources(struct be_adapter *adapter, 4281 struct be_resources *res) 4282 { 4283 bool use_sriov = adapter->num_vfs ? 1 : 0; 4284 4285 if (be_physfn(adapter)) 4286 res->max_uc_mac = BE_UC_PMAC_COUNT; 4287 else 4288 res->max_uc_mac = BE_VF_UC_PMAC_COUNT; 4289 4290 adapter->mc_type = be_convert_mc_type(adapter->function_mode); 4291 4292 if (be_is_mc(adapter)) { 4293 /* Assuming that there are 4 channels per port, 4294 * when multi-channel is enabled 4295 */ 4296 if (be_is_qnq_mode(adapter)) 4297 res->max_vlans = BE_NUM_VLANS_SUPPORTED/8; 4298 else 4299 /* In a non-qnq multichannel mode, the pvid 4300 * takes up one vlan entry 4301 */ 4302 res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1; 4303 } else { 4304 res->max_vlans = BE_NUM_VLANS_SUPPORTED; 4305 } 4306 4307 res->max_mcast_mac = BE_MAX_MC; 4308 4309 /* 1) For BE3 1Gb ports, FW does not support multiple TXQs 4310 * 2) Create multiple TX rings on a BE3-R multi-channel interface 4311 * *only* if it is RSS-capable. 4312 */ 4313 if (BE2_chip(adapter) || use_sriov || (adapter->port_num > 1) || 4314 be_virtfn(adapter) || 4315 (be_is_mc(adapter) && 4316 !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) { 4317 res->max_tx_qs = 1; 4318 } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) { 4319 struct be_resources super_nic_res = {0}; 4320 4321 /* On a SuperNIC profile, the driver needs to use the 4322 * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits 4323 */ 4324 be_cmd_get_profile_config(adapter, &super_nic_res, NULL, 4325 ACTIVE_PROFILE_TYPE, RESOURCE_LIMITS, 4326 0); 4327 /* Some old versions of BE3 FW don't report max_tx_qs value */ 4328 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS; 4329 } else { 4330 res->max_tx_qs = BE3_MAX_TX_QS; 4331 } 4332 4333 if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) && 4334 !use_sriov && be_physfn(adapter)) 4335 res->max_rss_qs = (adapter->be3_native) ? 4336 BE3_MAX_RSS_QS : BE2_MAX_RSS_QS; 4337 res->max_rx_qs = res->max_rss_qs + 1; 4338 4339 if (be_physfn(adapter)) 4340 res->max_evt_qs = (be_max_vfs(adapter) > 0) ? 4341 BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS; 4342 else 4343 res->max_evt_qs = 1; 4344 4345 res->if_cap_flags = BE_IF_CAP_FLAGS_WANT; 4346 res->if_cap_flags &= ~BE_IF_FLAGS_DEFQ_RSS; 4347 if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS)) 4348 res->if_cap_flags &= ~BE_IF_FLAGS_RSS; 4349 } 4350 4351 static void be_setup_init(struct be_adapter *adapter) 4352 { 4353 adapter->vlan_prio_bmap = 0xff; 4354 adapter->phy.link_speed = -1; 4355 adapter->if_handle = -1; 4356 adapter->be3_native = false; 4357 adapter->if_flags = 0; 4358 adapter->phy_state = BE_UNKNOWN_PHY_STATE; 4359 if (be_physfn(adapter)) 4360 adapter->cmd_privileges = MAX_PRIVILEGES; 4361 else 4362 adapter->cmd_privileges = MIN_PRIVILEGES; 4363 } 4364 4365 /* HW supports only MAX_PORT_RSS_TABLES RSS Policy Tables per port. 4366 * However, this HW limitation is not exposed to the host via any SLI cmd. 4367 * As a result, in the case of SRIOV and in particular multi-partition configs 4368 * the driver needs to calcuate a proportional share of RSS Tables per PF-pool 4369 * for distribution between the VFs. This self-imposed limit will determine the 4370 * no: of VFs for which RSS can be enabled. 4371 */ 4372 static void be_calculate_pf_pool_rss_tables(struct be_adapter *adapter) 4373 { 4374 struct be_port_resources port_res = {0}; 4375 u8 rss_tables_on_port; 4376 u16 max_vfs = be_max_vfs(adapter); 4377 4378 be_cmd_get_profile_config(adapter, NULL, &port_res, SAVED_PROFILE_TYPE, 4379 RESOURCE_LIMITS, 0); 4380 4381 rss_tables_on_port = MAX_PORT_RSS_TABLES - port_res.nic_pfs; 4382 4383 /* Each PF Pool's RSS Tables limit = 4384 * PF's Max VFs / Total_Max_VFs on Port * RSS Tables on Port 4385 */ 4386 adapter->pool_res.max_rss_tables = 4387 max_vfs * rss_tables_on_port / port_res.max_vfs; 4388 } 4389 4390 static int be_get_sriov_config(struct be_adapter *adapter) 4391 { 4392 struct be_resources res = {0}; 4393 int max_vfs, old_vfs; 4394 4395 be_cmd_get_profile_config(adapter, &res, NULL, ACTIVE_PROFILE_TYPE, 4396 RESOURCE_LIMITS, 0); 4397 4398 /* Some old versions of BE3 FW don't report max_vfs value */ 4399 if (BE3_chip(adapter) && !res.max_vfs) { 4400 max_vfs = pci_sriov_get_totalvfs(adapter->pdev); 4401 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0; 4402 } 4403 4404 adapter->pool_res = res; 4405 4406 /* If during previous unload of the driver, the VFs were not disabled, 4407 * then we cannot rely on the PF POOL limits for the TotalVFs value. 4408 * Instead use the TotalVFs value stored in the pci-dev struct. 4409 */ 4410 old_vfs = pci_num_vf(adapter->pdev); 4411 if (old_vfs) { 4412 dev_info(&adapter->pdev->dev, "%d VFs are already enabled\n", 4413 old_vfs); 4414 4415 adapter->pool_res.max_vfs = 4416 pci_sriov_get_totalvfs(adapter->pdev); 4417 adapter->num_vfs = old_vfs; 4418 } 4419 4420 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) { 4421 be_calculate_pf_pool_rss_tables(adapter); 4422 dev_info(&adapter->pdev->dev, 4423 "RSS can be enabled for all VFs if num_vfs <= %d\n", 4424 be_max_pf_pool_rss_tables(adapter)); 4425 } 4426 return 0; 4427 } 4428 4429 static void be_alloc_sriov_res(struct be_adapter *adapter) 4430 { 4431 int old_vfs = pci_num_vf(adapter->pdev); 4432 struct be_resources vft_res = {0}; 4433 int status; 4434 4435 be_get_sriov_config(adapter); 4436 4437 if (!old_vfs) 4438 pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter)); 4439 4440 /* When the HW is in SRIOV capable configuration, the PF-pool 4441 * resources are given to PF during driver load, if there are no 4442 * old VFs. This facility is not available in BE3 FW. 4443 * Also, this is done by FW in Lancer chip. 4444 */ 4445 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) { 4446 be_calculate_vf_res(adapter, 0, &vft_res); 4447 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0, 4448 &vft_res); 4449 if (status) 4450 dev_err(&adapter->pdev->dev, 4451 "Failed to optimize SRIOV resources\n"); 4452 } 4453 } 4454 4455 static int be_get_resources(struct be_adapter *adapter) 4456 { 4457 struct device *dev = &adapter->pdev->dev; 4458 struct be_resources res = {0}; 4459 int status; 4460 4461 /* For Lancer, SH etc read per-function resource limits from FW. 4462 * GET_FUNC_CONFIG returns per function guaranteed limits. 4463 * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits 4464 */ 4465 if (BEx_chip(adapter)) { 4466 BEx_get_resources(adapter, &res); 4467 } else { 4468 status = be_cmd_get_func_config(adapter, &res); 4469 if (status) 4470 return status; 4471 4472 /* If a deafault RXQ must be created, we'll use up one RSSQ*/ 4473 if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs && 4474 !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS)) 4475 res.max_rss_qs -= 1; 4476 } 4477 4478 /* If RoCE is supported stash away half the EQs for RoCE */ 4479 res.max_nic_evt_qs = be_roce_supported(adapter) ? 4480 res.max_evt_qs / 2 : res.max_evt_qs; 4481 adapter->res = res; 4482 4483 /* If FW supports RSS default queue, then skip creating non-RSS 4484 * queue for non-IP traffic. 4485 */ 4486 adapter->need_def_rxq = (be_if_cap_flags(adapter) & 4487 BE_IF_FLAGS_DEFQ_RSS) ? 0 : 1; 4488 4489 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n", 4490 be_max_txqs(adapter), be_max_rxqs(adapter), 4491 be_max_rss(adapter), be_max_nic_eqs(adapter), 4492 be_max_vfs(adapter)); 4493 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n", 4494 be_max_uc(adapter), be_max_mc(adapter), 4495 be_max_vlans(adapter)); 4496 4497 /* Ensure RX and TX queues are created in pairs at init time */ 4498 adapter->cfg_num_rx_irqs = 4499 min_t(u16, netif_get_num_default_rss_queues(), 4500 be_max_qp_irqs(adapter)); 4501 adapter->cfg_num_tx_irqs = adapter->cfg_num_rx_irqs; 4502 return 0; 4503 } 4504 4505 static int be_get_config(struct be_adapter *adapter) 4506 { 4507 int status, level; 4508 u16 profile_id; 4509 4510 status = be_cmd_get_cntl_attributes(adapter); 4511 if (status) 4512 return status; 4513 4514 status = be_cmd_query_fw_cfg(adapter); 4515 if (status) 4516 return status; 4517 4518 if (!lancer_chip(adapter) && be_physfn(adapter)) 4519 be_cmd_get_fat_dump_len(adapter, &adapter->fat_dump_len); 4520 4521 if (BEx_chip(adapter)) { 4522 level = be_cmd_get_fw_log_level(adapter); 4523 adapter->msg_enable = 4524 level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0; 4525 } 4526 4527 be_cmd_get_acpi_wol_cap(adapter); 4528 pci_enable_wake(adapter->pdev, PCI_D3hot, adapter->wol_en); 4529 pci_enable_wake(adapter->pdev, PCI_D3cold, adapter->wol_en); 4530 4531 be_cmd_query_port_name(adapter); 4532 4533 if (be_physfn(adapter)) { 4534 status = be_cmd_get_active_profile(adapter, &profile_id); 4535 if (!status) 4536 dev_info(&adapter->pdev->dev, 4537 "Using profile 0x%x\n", profile_id); 4538 } 4539 4540 return 0; 4541 } 4542 4543 static int be_mac_setup(struct be_adapter *adapter) 4544 { 4545 u8 mac[ETH_ALEN]; 4546 int status; 4547 4548 if (is_zero_ether_addr(adapter->netdev->dev_addr)) { 4549 status = be_cmd_get_perm_mac(adapter, mac); 4550 if (status) 4551 return status; 4552 4553 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN); 4554 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN); 4555 } 4556 4557 return 0; 4558 } 4559 4560 static void be_schedule_worker(struct be_adapter *adapter) 4561 { 4562 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000)); 4563 adapter->flags |= BE_FLAGS_WORKER_SCHEDULED; 4564 } 4565 4566 static void be_destroy_err_recovery_workq(void) 4567 { 4568 if (!be_err_recovery_workq) 4569 return; 4570 4571 flush_workqueue(be_err_recovery_workq); 4572 destroy_workqueue(be_err_recovery_workq); 4573 be_err_recovery_workq = NULL; 4574 } 4575 4576 static void be_schedule_err_detection(struct be_adapter *adapter, u32 delay) 4577 { 4578 struct be_error_recovery *err_rec = &adapter->error_recovery; 4579 4580 if (!be_err_recovery_workq) 4581 return; 4582 4583 queue_delayed_work(be_err_recovery_workq, &err_rec->err_detection_work, 4584 msecs_to_jiffies(delay)); 4585 adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED; 4586 } 4587 4588 static int be_setup_queues(struct be_adapter *adapter) 4589 { 4590 struct net_device *netdev = adapter->netdev; 4591 int status; 4592 4593 status = be_evt_queues_create(adapter); 4594 if (status) 4595 goto err; 4596 4597 status = be_tx_qs_create(adapter); 4598 if (status) 4599 goto err; 4600 4601 status = be_rx_cqs_create(adapter); 4602 if (status) 4603 goto err; 4604 4605 status = be_mcc_queues_create(adapter); 4606 if (status) 4607 goto err; 4608 4609 status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs); 4610 if (status) 4611 goto err; 4612 4613 status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs); 4614 if (status) 4615 goto err; 4616 4617 return 0; 4618 err: 4619 dev_err(&adapter->pdev->dev, "queue_setup failed\n"); 4620 return status; 4621 } 4622 4623 static int be_if_create(struct be_adapter *adapter) 4624 { 4625 u32 en_flags = BE_IF_FLAGS_RSS | BE_IF_FLAGS_DEFQ_RSS; 4626 u32 cap_flags = be_if_cap_flags(adapter); 4627 int status; 4628 4629 /* alloc required memory for other filtering fields */ 4630 adapter->pmac_id = kcalloc(be_max_uc(adapter), 4631 sizeof(*adapter->pmac_id), GFP_KERNEL); 4632 if (!adapter->pmac_id) 4633 return -ENOMEM; 4634 4635 adapter->mc_list = kcalloc(be_max_mc(adapter), 4636 sizeof(*adapter->mc_list), GFP_KERNEL); 4637 if (!adapter->mc_list) 4638 return -ENOMEM; 4639 4640 adapter->uc_list = kcalloc(be_max_uc(adapter), 4641 sizeof(*adapter->uc_list), GFP_KERNEL); 4642 if (!adapter->uc_list) 4643 return -ENOMEM; 4644 4645 if (adapter->cfg_num_rx_irqs == 1) 4646 cap_flags &= ~(BE_IF_FLAGS_DEFQ_RSS | BE_IF_FLAGS_RSS); 4647 4648 en_flags &= cap_flags; 4649 /* will enable all the needed filter flags in be_open() */ 4650 status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags, 4651 &adapter->if_handle, 0); 4652 4653 if (status) 4654 return status; 4655 4656 return 0; 4657 } 4658 4659 int be_update_queues(struct be_adapter *adapter) 4660 { 4661 struct net_device *netdev = adapter->netdev; 4662 int status; 4663 4664 if (netif_running(netdev)) 4665 be_close(netdev); 4666 4667 be_cancel_worker(adapter); 4668 4669 /* If any vectors have been shared with RoCE we cannot re-program 4670 * the MSIx table. 4671 */ 4672 if (!adapter->num_msix_roce_vec) 4673 be_msix_disable(adapter); 4674 4675 be_clear_queues(adapter); 4676 status = be_cmd_if_destroy(adapter, adapter->if_handle, 0); 4677 if (status) 4678 return status; 4679 4680 if (!msix_enabled(adapter)) { 4681 status = be_msix_enable(adapter); 4682 if (status) 4683 return status; 4684 } 4685 4686 status = be_if_create(adapter); 4687 if (status) 4688 return status; 4689 4690 status = be_setup_queues(adapter); 4691 if (status) 4692 return status; 4693 4694 be_schedule_worker(adapter); 4695 4696 if (netif_running(netdev)) 4697 status = be_open(netdev); 4698 4699 return status; 4700 } 4701 4702 static inline int fw_major_num(const char *fw_ver) 4703 { 4704 int fw_major = 0, i; 4705 4706 i = sscanf(fw_ver, "%d.", &fw_major); 4707 if (i != 1) 4708 return 0; 4709 4710 return fw_major; 4711 } 4712 4713 /* If it is error recovery, FLR the PF 4714 * Else if any VFs are already enabled don't FLR the PF 4715 */ 4716 static bool be_reset_required(struct be_adapter *adapter) 4717 { 4718 if (be_error_recovering(adapter)) 4719 return true; 4720 else 4721 return pci_num_vf(adapter->pdev) == 0; 4722 } 4723 4724 /* Wait for the FW to be ready and perform the required initialization */ 4725 static int be_func_init(struct be_adapter *adapter) 4726 { 4727 int status; 4728 4729 status = be_fw_wait_ready(adapter); 4730 if (status) 4731 return status; 4732 4733 /* FW is now ready; clear errors to allow cmds/doorbell */ 4734 be_clear_error(adapter, BE_CLEAR_ALL); 4735 4736 if (be_reset_required(adapter)) { 4737 status = be_cmd_reset_function(adapter); 4738 if (status) 4739 return status; 4740 4741 /* Wait for interrupts to quiesce after an FLR */ 4742 msleep(100); 4743 } 4744 4745 /* Tell FW we're ready to fire cmds */ 4746 status = be_cmd_fw_init(adapter); 4747 if (status) 4748 return status; 4749 4750 /* Allow interrupts for other ULPs running on NIC function */ 4751 be_intr_set(adapter, true); 4752 4753 return 0; 4754 } 4755 4756 static int be_setup(struct be_adapter *adapter) 4757 { 4758 struct device *dev = &adapter->pdev->dev; 4759 int status; 4760 4761 status = be_func_init(adapter); 4762 if (status) 4763 return status; 4764 4765 be_setup_init(adapter); 4766 4767 if (!lancer_chip(adapter)) 4768 be_cmd_req_native_mode(adapter); 4769 4770 /* invoke this cmd first to get pf_num and vf_num which are needed 4771 * for issuing profile related cmds 4772 */ 4773 if (!BEx_chip(adapter)) { 4774 status = be_cmd_get_func_config(adapter, NULL); 4775 if (status) 4776 return status; 4777 } 4778 4779 status = be_get_config(adapter); 4780 if (status) 4781 goto err; 4782 4783 if (!BE2_chip(adapter) && be_physfn(adapter)) 4784 be_alloc_sriov_res(adapter); 4785 4786 status = be_get_resources(adapter); 4787 if (status) 4788 goto err; 4789 4790 status = be_msix_enable(adapter); 4791 if (status) 4792 goto err; 4793 4794 /* will enable all the needed filter flags in be_open() */ 4795 status = be_if_create(adapter); 4796 if (status) 4797 goto err; 4798 4799 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 4800 rtnl_lock(); 4801 status = be_setup_queues(adapter); 4802 rtnl_unlock(); 4803 if (status) 4804 goto err; 4805 4806 be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0); 4807 4808 status = be_mac_setup(adapter); 4809 if (status) 4810 goto err; 4811 4812 be_cmd_get_fw_ver(adapter); 4813 dev_info(dev, "FW version is %s\n", adapter->fw_ver); 4814 4815 if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) { 4816 dev_err(dev, "Firmware on card is old(%s), IRQs may not work", 4817 adapter->fw_ver); 4818 dev_err(dev, "Please upgrade firmware to version >= 4.0\n"); 4819 } 4820 4821 status = be_cmd_set_flow_control(adapter, adapter->tx_fc, 4822 adapter->rx_fc); 4823 if (status) 4824 be_cmd_get_flow_control(adapter, &adapter->tx_fc, 4825 &adapter->rx_fc); 4826 4827 dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n", 4828 adapter->tx_fc, adapter->rx_fc); 4829 4830 if (be_physfn(adapter)) 4831 be_cmd_set_logical_link_config(adapter, 4832 IFLA_VF_LINK_STATE_AUTO, 0); 4833 4834 /* BE3 EVB echoes broadcast/multicast packets back to PF's vport 4835 * confusing a linux bridge or OVS that it might be connected to. 4836 * Set the EVB to PASSTHRU mode which effectively disables the EVB 4837 * when SRIOV is not enabled. 4838 */ 4839 if (BE3_chip(adapter)) 4840 be_cmd_set_hsw_config(adapter, 0, 0, adapter->if_handle, 4841 PORT_FWD_TYPE_PASSTHRU, 0); 4842 4843 if (adapter->num_vfs) 4844 be_vf_setup(adapter); 4845 4846 status = be_cmd_get_phy_info(adapter); 4847 if (!status && be_pause_supported(adapter)) 4848 adapter->phy.fc_autoneg = 1; 4849 4850 if (be_physfn(adapter) && !lancer_chip(adapter)) 4851 be_cmd_set_features(adapter); 4852 4853 be_schedule_worker(adapter); 4854 adapter->flags |= BE_FLAGS_SETUP_DONE; 4855 return 0; 4856 err: 4857 be_clear(adapter); 4858 return status; 4859 } 4860 4861 #ifdef CONFIG_NET_POLL_CONTROLLER 4862 static void be_netpoll(struct net_device *netdev) 4863 { 4864 struct be_adapter *adapter = netdev_priv(netdev); 4865 struct be_eq_obj *eqo; 4866 int i; 4867 4868 for_all_evt_queues(adapter, eqo, i) { 4869 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0); 4870 napi_schedule(&eqo->napi); 4871 } 4872 } 4873 #endif 4874 4875 int be_load_fw(struct be_adapter *adapter, u8 *fw_file) 4876 { 4877 const struct firmware *fw; 4878 int status; 4879 4880 if (!netif_running(adapter->netdev)) { 4881 dev_err(&adapter->pdev->dev, 4882 "Firmware load not allowed (interface is down)\n"); 4883 return -ENETDOWN; 4884 } 4885 4886 status = request_firmware(&fw, fw_file, &adapter->pdev->dev); 4887 if (status) 4888 goto fw_exit; 4889 4890 dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file); 4891 4892 if (lancer_chip(adapter)) 4893 status = lancer_fw_download(adapter, fw); 4894 else 4895 status = be_fw_download(adapter, fw); 4896 4897 if (!status) 4898 be_cmd_get_fw_ver(adapter); 4899 4900 fw_exit: 4901 release_firmware(fw); 4902 return status; 4903 } 4904 4905 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4906 u16 flags) 4907 { 4908 struct be_adapter *adapter = netdev_priv(dev); 4909 struct nlattr *attr, *br_spec; 4910 int rem; 4911 int status = 0; 4912 u16 mode = 0; 4913 4914 if (!sriov_enabled(adapter)) 4915 return -EOPNOTSUPP; 4916 4917 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4918 if (!br_spec) 4919 return -EINVAL; 4920 4921 nla_for_each_nested(attr, br_spec, rem) { 4922 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4923 continue; 4924 4925 if (nla_len(attr) < sizeof(mode)) 4926 return -EINVAL; 4927 4928 mode = nla_get_u16(attr); 4929 if (BE3_chip(adapter) && mode == BRIDGE_MODE_VEPA) 4930 return -EOPNOTSUPP; 4931 4932 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 4933 return -EINVAL; 4934 4935 status = be_cmd_set_hsw_config(adapter, 0, 0, 4936 adapter->if_handle, 4937 mode == BRIDGE_MODE_VEPA ? 4938 PORT_FWD_TYPE_VEPA : 4939 PORT_FWD_TYPE_VEB, 0); 4940 if (status) 4941 goto err; 4942 4943 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n", 4944 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 4945 4946 return status; 4947 } 4948 err: 4949 dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n", 4950 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 4951 4952 return status; 4953 } 4954 4955 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 4956 struct net_device *dev, u32 filter_mask, 4957 int nlflags) 4958 { 4959 struct be_adapter *adapter = netdev_priv(dev); 4960 int status = 0; 4961 u8 hsw_mode; 4962 4963 /* BE and Lancer chips support VEB mode only */ 4964 if (BEx_chip(adapter) || lancer_chip(adapter)) { 4965 /* VEB is disabled in non-SR-IOV profiles on BE3/Lancer */ 4966 if (!pci_sriov_get_totalvfs(adapter->pdev)) 4967 return 0; 4968 hsw_mode = PORT_FWD_TYPE_VEB; 4969 } else { 4970 status = be_cmd_get_hsw_config(adapter, NULL, 0, 4971 adapter->if_handle, &hsw_mode, 4972 NULL); 4973 if (status) 4974 return 0; 4975 4976 if (hsw_mode == PORT_FWD_TYPE_PASSTHRU) 4977 return 0; 4978 } 4979 4980 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 4981 hsw_mode == PORT_FWD_TYPE_VEPA ? 4982 BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB, 4983 0, 0, nlflags, filter_mask, NULL); 4984 } 4985 4986 static struct be_cmd_work *be_alloc_work(struct be_adapter *adapter, 4987 void (*func)(struct work_struct *)) 4988 { 4989 struct be_cmd_work *work; 4990 4991 work = kzalloc(sizeof(*work), GFP_ATOMIC); 4992 if (!work) { 4993 dev_err(&adapter->pdev->dev, 4994 "be_work memory allocation failed\n"); 4995 return NULL; 4996 } 4997 4998 INIT_WORK(&work->work, func); 4999 work->adapter = adapter; 5000 return work; 5001 } 5002 5003 /* VxLAN offload Notes: 5004 * 5005 * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't 5006 * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload 5007 * is expected to work across all types of IP tunnels once exported. Skyhawk 5008 * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN 5009 * offloads in hw_enc_features only when a VxLAN port is added. If other (non 5010 * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for 5011 * those other tunnels are unexported on the fly through ndo_features_check(). 5012 * 5013 * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack 5014 * adds more than one port, disable offloads and don't re-enable them again 5015 * until after all the tunnels are removed. 5016 */ 5017 static void be_work_add_vxlan_port(struct work_struct *work) 5018 { 5019 struct be_cmd_work *cmd_work = 5020 container_of(work, struct be_cmd_work, work); 5021 struct be_adapter *adapter = cmd_work->adapter; 5022 struct net_device *netdev = adapter->netdev; 5023 struct device *dev = &adapter->pdev->dev; 5024 __be16 port = cmd_work->info.vxlan_port; 5025 int status; 5026 5027 if (adapter->vxlan_port == port && adapter->vxlan_port_count) { 5028 adapter->vxlan_port_aliases++; 5029 goto done; 5030 } 5031 5032 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) { 5033 dev_info(dev, 5034 "Only one UDP port supported for VxLAN offloads\n"); 5035 dev_info(dev, "Disabling VxLAN offloads\n"); 5036 adapter->vxlan_port_count++; 5037 goto err; 5038 } 5039 5040 if (adapter->vxlan_port_count++ >= 1) 5041 goto done; 5042 5043 status = be_cmd_manage_iface(adapter, adapter->if_handle, 5044 OP_CONVERT_NORMAL_TO_TUNNEL); 5045 if (status) { 5046 dev_warn(dev, "Failed to convert normal interface to tunnel\n"); 5047 goto err; 5048 } 5049 5050 status = be_cmd_set_vxlan_port(adapter, port); 5051 if (status) { 5052 dev_warn(dev, "Failed to add VxLAN port\n"); 5053 goto err; 5054 } 5055 adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS; 5056 adapter->vxlan_port = port; 5057 5058 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 5059 NETIF_F_TSO | NETIF_F_TSO6 | 5060 NETIF_F_GSO_UDP_TUNNEL; 5061 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL; 5062 netdev->features |= NETIF_F_GSO_UDP_TUNNEL; 5063 5064 dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n", 5065 be16_to_cpu(port)); 5066 goto done; 5067 err: 5068 be_disable_vxlan_offloads(adapter); 5069 done: 5070 kfree(cmd_work); 5071 } 5072 5073 static void be_work_del_vxlan_port(struct work_struct *work) 5074 { 5075 struct be_cmd_work *cmd_work = 5076 container_of(work, struct be_cmd_work, work); 5077 struct be_adapter *adapter = cmd_work->adapter; 5078 __be16 port = cmd_work->info.vxlan_port; 5079 5080 if (adapter->vxlan_port != port) 5081 goto done; 5082 5083 if (adapter->vxlan_port_aliases) { 5084 adapter->vxlan_port_aliases--; 5085 goto out; 5086 } 5087 5088 be_disable_vxlan_offloads(adapter); 5089 5090 dev_info(&adapter->pdev->dev, 5091 "Disabled VxLAN offloads for UDP port %d\n", 5092 be16_to_cpu(port)); 5093 done: 5094 adapter->vxlan_port_count--; 5095 out: 5096 kfree(cmd_work); 5097 } 5098 5099 static void be_cfg_vxlan_port(struct net_device *netdev, 5100 struct udp_tunnel_info *ti, 5101 void (*func)(struct work_struct *)) 5102 { 5103 struct be_adapter *adapter = netdev_priv(netdev); 5104 struct be_cmd_work *cmd_work; 5105 5106 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 5107 return; 5108 5109 if (lancer_chip(adapter) || BEx_chip(adapter) || be_is_mc(adapter)) 5110 return; 5111 5112 cmd_work = be_alloc_work(adapter, func); 5113 if (cmd_work) { 5114 cmd_work->info.vxlan_port = ti->port; 5115 queue_work(be_wq, &cmd_work->work); 5116 } 5117 } 5118 5119 static void be_del_vxlan_port(struct net_device *netdev, 5120 struct udp_tunnel_info *ti) 5121 { 5122 be_cfg_vxlan_port(netdev, ti, be_work_del_vxlan_port); 5123 } 5124 5125 static void be_add_vxlan_port(struct net_device *netdev, 5126 struct udp_tunnel_info *ti) 5127 { 5128 be_cfg_vxlan_port(netdev, ti, be_work_add_vxlan_port); 5129 } 5130 5131 static netdev_features_t be_features_check(struct sk_buff *skb, 5132 struct net_device *dev, 5133 netdev_features_t features) 5134 { 5135 struct be_adapter *adapter = netdev_priv(dev); 5136 u8 l4_hdr = 0; 5137 5138 /* The code below restricts offload features for some tunneled packets. 5139 * Offload features for normal (non tunnel) packets are unchanged. 5140 */ 5141 if (!skb->encapsulation || 5142 !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)) 5143 return features; 5144 5145 /* It's an encapsulated packet and VxLAN offloads are enabled. We 5146 * should disable tunnel offload features if it's not a VxLAN packet, 5147 * as tunnel offloads have been enabled only for VxLAN. This is done to 5148 * allow other tunneled traffic like GRE work fine while VxLAN 5149 * offloads are configured in Skyhawk-R. 5150 */ 5151 switch (vlan_get_protocol(skb)) { 5152 case htons(ETH_P_IP): 5153 l4_hdr = ip_hdr(skb)->protocol; 5154 break; 5155 case htons(ETH_P_IPV6): 5156 l4_hdr = ipv6_hdr(skb)->nexthdr; 5157 break; 5158 default: 5159 return features; 5160 } 5161 5162 if (l4_hdr != IPPROTO_UDP || 5163 skb->inner_protocol_type != ENCAP_TYPE_ETHER || 5164 skb->inner_protocol != htons(ETH_P_TEB) || 5165 skb_inner_mac_header(skb) - skb_transport_header(skb) != 5166 sizeof(struct udphdr) + sizeof(struct vxlanhdr)) 5167 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 5168 5169 return features; 5170 } 5171 5172 static int be_get_phys_port_id(struct net_device *dev, 5173 struct netdev_phys_item_id *ppid) 5174 { 5175 int i, id_len = CNTL_SERIAL_NUM_WORDS * CNTL_SERIAL_NUM_WORD_SZ + 1; 5176 struct be_adapter *adapter = netdev_priv(dev); 5177 u8 *id; 5178 5179 if (MAX_PHYS_ITEM_ID_LEN < id_len) 5180 return -ENOSPC; 5181 5182 ppid->id[0] = adapter->hba_port_num + 1; 5183 id = &ppid->id[1]; 5184 for (i = CNTL_SERIAL_NUM_WORDS - 1; i >= 0; 5185 i--, id += CNTL_SERIAL_NUM_WORD_SZ) 5186 memcpy(id, &adapter->serial_num[i], CNTL_SERIAL_NUM_WORD_SZ); 5187 5188 ppid->id_len = id_len; 5189 5190 return 0; 5191 } 5192 5193 static void be_set_rx_mode(struct net_device *dev) 5194 { 5195 struct be_adapter *adapter = netdev_priv(dev); 5196 struct be_cmd_work *work; 5197 5198 work = be_alloc_work(adapter, be_work_set_rx_mode); 5199 if (work) 5200 queue_work(be_wq, &work->work); 5201 } 5202 5203 static const struct net_device_ops be_netdev_ops = { 5204 .ndo_open = be_open, 5205 .ndo_stop = be_close, 5206 .ndo_start_xmit = be_xmit, 5207 .ndo_set_rx_mode = be_set_rx_mode, 5208 .ndo_set_mac_address = be_mac_addr_set, 5209 .ndo_change_mtu = be_change_mtu, 5210 .ndo_get_stats64 = be_get_stats64, 5211 .ndo_validate_addr = eth_validate_addr, 5212 .ndo_vlan_rx_add_vid = be_vlan_add_vid, 5213 .ndo_vlan_rx_kill_vid = be_vlan_rem_vid, 5214 .ndo_set_vf_mac = be_set_vf_mac, 5215 .ndo_set_vf_vlan = be_set_vf_vlan, 5216 .ndo_set_vf_rate = be_set_vf_tx_rate, 5217 .ndo_get_vf_config = be_get_vf_config, 5218 .ndo_set_vf_link_state = be_set_vf_link_state, 5219 .ndo_set_vf_spoofchk = be_set_vf_spoofchk, 5220 #ifdef CONFIG_NET_POLL_CONTROLLER 5221 .ndo_poll_controller = be_netpoll, 5222 #endif 5223 .ndo_bridge_setlink = be_ndo_bridge_setlink, 5224 .ndo_bridge_getlink = be_ndo_bridge_getlink, 5225 #ifdef CONFIG_NET_RX_BUSY_POLL 5226 .ndo_busy_poll = be_busy_poll, 5227 #endif 5228 .ndo_udp_tunnel_add = be_add_vxlan_port, 5229 .ndo_udp_tunnel_del = be_del_vxlan_port, 5230 .ndo_features_check = be_features_check, 5231 .ndo_get_phys_port_id = be_get_phys_port_id, 5232 }; 5233 5234 static void be_netdev_init(struct net_device *netdev) 5235 { 5236 struct be_adapter *adapter = netdev_priv(netdev); 5237 5238 netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5239 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | 5240 NETIF_F_HW_VLAN_CTAG_TX; 5241 if ((be_if_cap_flags(adapter) & BE_IF_FLAGS_RSS)) 5242 netdev->hw_features |= NETIF_F_RXHASH; 5243 5244 netdev->features |= netdev->hw_features | 5245 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER; 5246 5247 netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5248 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 5249 5250 netdev->priv_flags |= IFF_UNICAST_FLT; 5251 5252 netdev->flags |= IFF_MULTICAST; 5253 5254 netif_set_gso_max_size(netdev, BE_MAX_GSO_SIZE - ETH_HLEN); 5255 5256 netdev->netdev_ops = &be_netdev_ops; 5257 5258 netdev->ethtool_ops = &be_ethtool_ops; 5259 } 5260 5261 static void be_cleanup(struct be_adapter *adapter) 5262 { 5263 struct net_device *netdev = adapter->netdev; 5264 5265 rtnl_lock(); 5266 netif_device_detach(netdev); 5267 if (netif_running(netdev)) 5268 be_close(netdev); 5269 rtnl_unlock(); 5270 5271 be_clear(adapter); 5272 } 5273 5274 static int be_resume(struct be_adapter *adapter) 5275 { 5276 struct net_device *netdev = adapter->netdev; 5277 int status; 5278 5279 status = be_setup(adapter); 5280 if (status) 5281 return status; 5282 5283 rtnl_lock(); 5284 if (netif_running(netdev)) 5285 status = be_open(netdev); 5286 rtnl_unlock(); 5287 5288 if (status) 5289 return status; 5290 5291 netif_device_attach(netdev); 5292 5293 return 0; 5294 } 5295 5296 static void be_soft_reset(struct be_adapter *adapter) 5297 { 5298 u32 val; 5299 5300 dev_info(&adapter->pdev->dev, "Initiating chip soft reset\n"); 5301 val = ioread32(adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET); 5302 val |= SLIPORT_SOFTRESET_SR_MASK; 5303 iowrite32(val, adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET); 5304 } 5305 5306 static bool be_err_is_recoverable(struct be_adapter *adapter) 5307 { 5308 struct be_error_recovery *err_rec = &adapter->error_recovery; 5309 unsigned long initial_idle_time = 5310 msecs_to_jiffies(ERR_RECOVERY_IDLE_TIME); 5311 unsigned long recovery_interval = 5312 msecs_to_jiffies(ERR_RECOVERY_INTERVAL); 5313 u16 ue_err_code; 5314 u32 val; 5315 5316 val = be_POST_stage_get(adapter); 5317 if ((val & POST_STAGE_RECOVERABLE_ERR) != POST_STAGE_RECOVERABLE_ERR) 5318 return false; 5319 ue_err_code = val & POST_ERR_RECOVERY_CODE_MASK; 5320 if (ue_err_code == 0) 5321 return false; 5322 5323 dev_err(&adapter->pdev->dev, "Recoverable HW error code: 0x%x\n", 5324 ue_err_code); 5325 5326 if (jiffies - err_rec->probe_time <= initial_idle_time) { 5327 dev_err(&adapter->pdev->dev, 5328 "Cannot recover within %lu sec from driver load\n", 5329 jiffies_to_msecs(initial_idle_time) / MSEC_PER_SEC); 5330 return false; 5331 } 5332 5333 if (err_rec->last_recovery_time && 5334 (jiffies - err_rec->last_recovery_time <= recovery_interval)) { 5335 dev_err(&adapter->pdev->dev, 5336 "Cannot recover within %lu sec from last recovery\n", 5337 jiffies_to_msecs(recovery_interval) / MSEC_PER_SEC); 5338 return false; 5339 } 5340 5341 if (ue_err_code == err_rec->last_err_code) { 5342 dev_err(&adapter->pdev->dev, 5343 "Cannot recover from a consecutive TPE error\n"); 5344 return false; 5345 } 5346 5347 err_rec->last_recovery_time = jiffies; 5348 err_rec->last_err_code = ue_err_code; 5349 return true; 5350 } 5351 5352 static int be_tpe_recover(struct be_adapter *adapter) 5353 { 5354 struct be_error_recovery *err_rec = &adapter->error_recovery; 5355 int status = -EAGAIN; 5356 u32 val; 5357 5358 switch (err_rec->recovery_state) { 5359 case ERR_RECOVERY_ST_NONE: 5360 err_rec->recovery_state = ERR_RECOVERY_ST_DETECT; 5361 err_rec->resched_delay = ERR_RECOVERY_UE_DETECT_DURATION; 5362 break; 5363 5364 case ERR_RECOVERY_ST_DETECT: 5365 val = be_POST_stage_get(adapter); 5366 if ((val & POST_STAGE_RECOVERABLE_ERR) != 5367 POST_STAGE_RECOVERABLE_ERR) { 5368 dev_err(&adapter->pdev->dev, 5369 "Unrecoverable HW error detected: 0x%x\n", val); 5370 status = -EINVAL; 5371 err_rec->resched_delay = 0; 5372 break; 5373 } 5374 5375 dev_err(&adapter->pdev->dev, "Recoverable HW error detected\n"); 5376 5377 /* Only PF0 initiates Chip Soft Reset. But PF0 must wait UE2SR 5378 * milliseconds before it checks for final error status in 5379 * SLIPORT_SEMAPHORE to determine if recovery criteria is met. 5380 * If it does, then PF0 initiates a Soft Reset. 5381 */ 5382 if (adapter->pf_num == 0) { 5383 err_rec->recovery_state = ERR_RECOVERY_ST_RESET; 5384 err_rec->resched_delay = err_rec->ue_to_reset_time - 5385 ERR_RECOVERY_UE_DETECT_DURATION; 5386 break; 5387 } 5388 5389 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL; 5390 err_rec->resched_delay = err_rec->ue_to_poll_time - 5391 ERR_RECOVERY_UE_DETECT_DURATION; 5392 break; 5393 5394 case ERR_RECOVERY_ST_RESET: 5395 if (!be_err_is_recoverable(adapter)) { 5396 dev_err(&adapter->pdev->dev, 5397 "Failed to meet recovery criteria\n"); 5398 status = -EIO; 5399 err_rec->resched_delay = 0; 5400 break; 5401 } 5402 be_soft_reset(adapter); 5403 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL; 5404 err_rec->resched_delay = err_rec->ue_to_poll_time - 5405 err_rec->ue_to_reset_time; 5406 break; 5407 5408 case ERR_RECOVERY_ST_PRE_POLL: 5409 err_rec->recovery_state = ERR_RECOVERY_ST_REINIT; 5410 err_rec->resched_delay = 0; 5411 status = 0; /* done */ 5412 break; 5413 5414 default: 5415 status = -EINVAL; 5416 err_rec->resched_delay = 0; 5417 break; 5418 } 5419 5420 return status; 5421 } 5422 5423 static int be_err_recover(struct be_adapter *adapter) 5424 { 5425 int status; 5426 5427 if (!lancer_chip(adapter)) { 5428 if (!adapter->error_recovery.recovery_supported || 5429 adapter->priv_flags & BE_DISABLE_TPE_RECOVERY) 5430 return -EIO; 5431 status = be_tpe_recover(adapter); 5432 if (status) 5433 goto err; 5434 } 5435 5436 /* Wait for adapter to reach quiescent state before 5437 * destroying queues 5438 */ 5439 status = be_fw_wait_ready(adapter); 5440 if (status) 5441 goto err; 5442 5443 adapter->flags |= BE_FLAGS_TRY_RECOVERY; 5444 5445 be_cleanup(adapter); 5446 5447 status = be_resume(adapter); 5448 if (status) 5449 goto err; 5450 5451 adapter->flags &= ~BE_FLAGS_TRY_RECOVERY; 5452 5453 err: 5454 return status; 5455 } 5456 5457 static void be_err_detection_task(struct work_struct *work) 5458 { 5459 struct be_error_recovery *err_rec = 5460 container_of(work, struct be_error_recovery, 5461 err_detection_work.work); 5462 struct be_adapter *adapter = 5463 container_of(err_rec, struct be_adapter, 5464 error_recovery); 5465 u32 resched_delay = ERR_RECOVERY_DETECTION_DELAY; 5466 struct device *dev = &adapter->pdev->dev; 5467 int recovery_status; 5468 5469 be_detect_error(adapter); 5470 if (!be_check_error(adapter, BE_ERROR_HW)) 5471 goto reschedule_task; 5472 5473 recovery_status = be_err_recover(adapter); 5474 if (!recovery_status) { 5475 err_rec->recovery_retries = 0; 5476 err_rec->recovery_state = ERR_RECOVERY_ST_NONE; 5477 dev_info(dev, "Adapter recovery successful\n"); 5478 goto reschedule_task; 5479 } else if (!lancer_chip(adapter) && err_rec->resched_delay) { 5480 /* BEx/SH recovery state machine */ 5481 if (adapter->pf_num == 0 && 5482 err_rec->recovery_state > ERR_RECOVERY_ST_DETECT) 5483 dev_err(&adapter->pdev->dev, 5484 "Adapter recovery in progress\n"); 5485 resched_delay = err_rec->resched_delay; 5486 goto reschedule_task; 5487 } else if (lancer_chip(adapter) && be_virtfn(adapter)) { 5488 /* For VFs, check if PF have allocated resources 5489 * every second. 5490 */ 5491 dev_err(dev, "Re-trying adapter recovery\n"); 5492 goto reschedule_task; 5493 } else if (lancer_chip(adapter) && err_rec->recovery_retries++ < 5494 ERR_RECOVERY_MAX_RETRY_COUNT) { 5495 /* In case of another error during recovery, it takes 30 sec 5496 * for adapter to come out of error. Retry error recovery after 5497 * this time interval. 5498 */ 5499 dev_err(&adapter->pdev->dev, "Re-trying adapter recovery\n"); 5500 resched_delay = ERR_RECOVERY_RETRY_DELAY; 5501 goto reschedule_task; 5502 } else { 5503 dev_err(dev, "Adapter recovery failed\n"); 5504 dev_err(dev, "Please reboot server to recover\n"); 5505 } 5506 5507 return; 5508 5509 reschedule_task: 5510 be_schedule_err_detection(adapter, resched_delay); 5511 } 5512 5513 static void be_log_sfp_info(struct be_adapter *adapter) 5514 { 5515 int status; 5516 5517 status = be_cmd_query_sfp_info(adapter); 5518 if (!status) { 5519 dev_err(&adapter->pdev->dev, 5520 "Port %c: %s Vendor: %s part no: %s", 5521 adapter->port_name, 5522 be_misconfig_evt_port_state[adapter->phy_state], 5523 adapter->phy.vendor_name, 5524 adapter->phy.vendor_pn); 5525 } 5526 adapter->flags &= ~BE_FLAGS_PHY_MISCONFIGURED; 5527 } 5528 5529 static void be_worker(struct work_struct *work) 5530 { 5531 struct be_adapter *adapter = 5532 container_of(work, struct be_adapter, work.work); 5533 struct be_rx_obj *rxo; 5534 int i; 5535 5536 if (be_physfn(adapter) && 5537 MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0) 5538 be_cmd_get_die_temperature(adapter); 5539 5540 /* when interrupts are not yet enabled, just reap any pending 5541 * mcc completions 5542 */ 5543 if (!netif_running(adapter->netdev)) { 5544 local_bh_disable(); 5545 be_process_mcc(adapter); 5546 local_bh_enable(); 5547 goto reschedule; 5548 } 5549 5550 if (!adapter->stats_cmd_sent) { 5551 if (lancer_chip(adapter)) 5552 lancer_cmd_get_pport_stats(adapter, 5553 &adapter->stats_cmd); 5554 else 5555 be_cmd_get_stats(adapter, &adapter->stats_cmd); 5556 } 5557 5558 for_all_rx_queues(adapter, rxo, i) { 5559 /* Replenish RX-queues starved due to memory 5560 * allocation failures. 5561 */ 5562 if (rxo->rx_post_starved) 5563 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST); 5564 } 5565 5566 /* EQ-delay update for Skyhawk is done while notifying EQ */ 5567 if (!skyhawk_chip(adapter)) 5568 be_eqd_update(adapter, false); 5569 5570 if (adapter->flags & BE_FLAGS_PHY_MISCONFIGURED) 5571 be_log_sfp_info(adapter); 5572 5573 reschedule: 5574 adapter->work_counter++; 5575 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000)); 5576 } 5577 5578 static void be_unmap_pci_bars(struct be_adapter *adapter) 5579 { 5580 if (adapter->csr) 5581 pci_iounmap(adapter->pdev, adapter->csr); 5582 if (adapter->db) 5583 pci_iounmap(adapter->pdev, adapter->db); 5584 if (adapter->pcicfg && adapter->pcicfg_mapped) 5585 pci_iounmap(adapter->pdev, adapter->pcicfg); 5586 } 5587 5588 static int db_bar(struct be_adapter *adapter) 5589 { 5590 if (lancer_chip(adapter) || be_virtfn(adapter)) 5591 return 0; 5592 else 5593 return 4; 5594 } 5595 5596 static int be_roce_map_pci_bars(struct be_adapter *adapter) 5597 { 5598 if (skyhawk_chip(adapter)) { 5599 adapter->roce_db.size = 4096; 5600 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev, 5601 db_bar(adapter)); 5602 adapter->roce_db.total_size = pci_resource_len(adapter->pdev, 5603 db_bar(adapter)); 5604 } 5605 return 0; 5606 } 5607 5608 static int be_map_pci_bars(struct be_adapter *adapter) 5609 { 5610 struct pci_dev *pdev = adapter->pdev; 5611 u8 __iomem *addr; 5612 u32 sli_intf; 5613 5614 pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf); 5615 adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >> 5616 SLI_INTF_FAMILY_SHIFT; 5617 adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0; 5618 5619 if (BEx_chip(adapter) && be_physfn(adapter)) { 5620 adapter->csr = pci_iomap(pdev, 2, 0); 5621 if (!adapter->csr) 5622 return -ENOMEM; 5623 } 5624 5625 addr = pci_iomap(pdev, db_bar(adapter), 0); 5626 if (!addr) 5627 goto pci_map_err; 5628 adapter->db = addr; 5629 5630 if (skyhawk_chip(adapter) || BEx_chip(adapter)) { 5631 if (be_physfn(adapter)) { 5632 /* PCICFG is the 2nd BAR in BE2 */ 5633 addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0); 5634 if (!addr) 5635 goto pci_map_err; 5636 adapter->pcicfg = addr; 5637 adapter->pcicfg_mapped = true; 5638 } else { 5639 adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET; 5640 adapter->pcicfg_mapped = false; 5641 } 5642 } 5643 5644 be_roce_map_pci_bars(adapter); 5645 return 0; 5646 5647 pci_map_err: 5648 dev_err(&pdev->dev, "Error in mapping PCI BARs\n"); 5649 be_unmap_pci_bars(adapter); 5650 return -ENOMEM; 5651 } 5652 5653 static void be_drv_cleanup(struct be_adapter *adapter) 5654 { 5655 struct be_dma_mem *mem = &adapter->mbox_mem_alloced; 5656 struct device *dev = &adapter->pdev->dev; 5657 5658 if (mem->va) 5659 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5660 5661 mem = &adapter->rx_filter; 5662 if (mem->va) 5663 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5664 5665 mem = &adapter->stats_cmd; 5666 if (mem->va) 5667 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5668 } 5669 5670 /* Allocate and initialize various fields in be_adapter struct */ 5671 static int be_drv_init(struct be_adapter *adapter) 5672 { 5673 struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced; 5674 struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem; 5675 struct be_dma_mem *rx_filter = &adapter->rx_filter; 5676 struct be_dma_mem *stats_cmd = &adapter->stats_cmd; 5677 struct device *dev = &adapter->pdev->dev; 5678 int status = 0; 5679 5680 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 5681 mbox_mem_alloc->va = dma_zalloc_coherent(dev, mbox_mem_alloc->size, 5682 &mbox_mem_alloc->dma, 5683 GFP_KERNEL); 5684 if (!mbox_mem_alloc->va) 5685 return -ENOMEM; 5686 5687 mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 5688 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 5689 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 5690 5691 rx_filter->size = sizeof(struct be_cmd_req_rx_filter); 5692 rx_filter->va = dma_zalloc_coherent(dev, rx_filter->size, 5693 &rx_filter->dma, GFP_KERNEL); 5694 if (!rx_filter->va) { 5695 status = -ENOMEM; 5696 goto free_mbox; 5697 } 5698 5699 if (lancer_chip(adapter)) 5700 stats_cmd->size = sizeof(struct lancer_cmd_req_pport_stats); 5701 else if (BE2_chip(adapter)) 5702 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v0); 5703 else if (BE3_chip(adapter)) 5704 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v1); 5705 else 5706 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v2); 5707 stats_cmd->va = dma_zalloc_coherent(dev, stats_cmd->size, 5708 &stats_cmd->dma, GFP_KERNEL); 5709 if (!stats_cmd->va) { 5710 status = -ENOMEM; 5711 goto free_rx_filter; 5712 } 5713 5714 mutex_init(&adapter->mbox_lock); 5715 mutex_init(&adapter->mcc_lock); 5716 mutex_init(&adapter->rx_filter_lock); 5717 spin_lock_init(&adapter->mcc_cq_lock); 5718 init_completion(&adapter->et_cmd_compl); 5719 5720 pci_save_state(adapter->pdev); 5721 5722 INIT_DELAYED_WORK(&adapter->work, be_worker); 5723 5724 adapter->error_recovery.recovery_state = ERR_RECOVERY_ST_NONE; 5725 adapter->error_recovery.resched_delay = 0; 5726 INIT_DELAYED_WORK(&adapter->error_recovery.err_detection_work, 5727 be_err_detection_task); 5728 5729 adapter->rx_fc = true; 5730 adapter->tx_fc = true; 5731 5732 /* Must be a power of 2 or else MODULO will BUG_ON */ 5733 adapter->be_get_temp_freq = 64; 5734 5735 return 0; 5736 5737 free_rx_filter: 5738 dma_free_coherent(dev, rx_filter->size, rx_filter->va, rx_filter->dma); 5739 free_mbox: 5740 dma_free_coherent(dev, mbox_mem_alloc->size, mbox_mem_alloc->va, 5741 mbox_mem_alloc->dma); 5742 return status; 5743 } 5744 5745 static void be_remove(struct pci_dev *pdev) 5746 { 5747 struct be_adapter *adapter = pci_get_drvdata(pdev); 5748 5749 if (!adapter) 5750 return; 5751 5752 be_roce_dev_remove(adapter); 5753 be_intr_set(adapter, false); 5754 5755 be_cancel_err_detection(adapter); 5756 5757 unregister_netdev(adapter->netdev); 5758 5759 be_clear(adapter); 5760 5761 if (!pci_vfs_assigned(adapter->pdev)) 5762 be_cmd_reset_function(adapter); 5763 5764 /* tell fw we're done with firing cmds */ 5765 be_cmd_fw_clean(adapter); 5766 5767 be_unmap_pci_bars(adapter); 5768 be_drv_cleanup(adapter); 5769 5770 pci_disable_pcie_error_reporting(pdev); 5771 5772 pci_release_regions(pdev); 5773 pci_disable_device(pdev); 5774 5775 free_netdev(adapter->netdev); 5776 } 5777 5778 static ssize_t be_hwmon_show_temp(struct device *dev, 5779 struct device_attribute *dev_attr, 5780 char *buf) 5781 { 5782 struct be_adapter *adapter = dev_get_drvdata(dev); 5783 5784 /* Unit: millidegree Celsius */ 5785 if (adapter->hwmon_info.be_on_die_temp == BE_INVALID_DIE_TEMP) 5786 return -EIO; 5787 else 5788 return sprintf(buf, "%u\n", 5789 adapter->hwmon_info.be_on_die_temp * 1000); 5790 } 5791 5792 static SENSOR_DEVICE_ATTR(temp1_input, S_IRUGO, 5793 be_hwmon_show_temp, NULL, 1); 5794 5795 static struct attribute *be_hwmon_attrs[] = { 5796 &sensor_dev_attr_temp1_input.dev_attr.attr, 5797 NULL 5798 }; 5799 5800 ATTRIBUTE_GROUPS(be_hwmon); 5801 5802 static char *mc_name(struct be_adapter *adapter) 5803 { 5804 char *str = ""; /* default */ 5805 5806 switch (adapter->mc_type) { 5807 case UMC: 5808 str = "UMC"; 5809 break; 5810 case FLEX10: 5811 str = "FLEX10"; 5812 break; 5813 case vNIC1: 5814 str = "vNIC-1"; 5815 break; 5816 case nPAR: 5817 str = "nPAR"; 5818 break; 5819 case UFP: 5820 str = "UFP"; 5821 break; 5822 case vNIC2: 5823 str = "vNIC-2"; 5824 break; 5825 default: 5826 str = ""; 5827 } 5828 5829 return str; 5830 } 5831 5832 static inline char *func_name(struct be_adapter *adapter) 5833 { 5834 return be_physfn(adapter) ? "PF" : "VF"; 5835 } 5836 5837 static inline char *nic_name(struct pci_dev *pdev) 5838 { 5839 switch (pdev->device) { 5840 case OC_DEVICE_ID1: 5841 return OC_NAME; 5842 case OC_DEVICE_ID2: 5843 return OC_NAME_BE; 5844 case OC_DEVICE_ID3: 5845 case OC_DEVICE_ID4: 5846 return OC_NAME_LANCER; 5847 case BE_DEVICE_ID2: 5848 return BE3_NAME; 5849 case OC_DEVICE_ID5: 5850 case OC_DEVICE_ID6: 5851 return OC_NAME_SH; 5852 default: 5853 return BE_NAME; 5854 } 5855 } 5856 5857 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id) 5858 { 5859 struct be_adapter *adapter; 5860 struct net_device *netdev; 5861 int status = 0; 5862 5863 dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER); 5864 5865 status = pci_enable_device(pdev); 5866 if (status) 5867 goto do_none; 5868 5869 status = pci_request_regions(pdev, DRV_NAME); 5870 if (status) 5871 goto disable_dev; 5872 pci_set_master(pdev); 5873 5874 netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS); 5875 if (!netdev) { 5876 status = -ENOMEM; 5877 goto rel_reg; 5878 } 5879 adapter = netdev_priv(netdev); 5880 adapter->pdev = pdev; 5881 pci_set_drvdata(pdev, adapter); 5882 adapter->netdev = netdev; 5883 SET_NETDEV_DEV(netdev, &pdev->dev); 5884 5885 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 5886 if (!status) { 5887 netdev->features |= NETIF_F_HIGHDMA; 5888 } else { 5889 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 5890 if (status) { 5891 dev_err(&pdev->dev, "Could not set PCI DMA Mask\n"); 5892 goto free_netdev; 5893 } 5894 } 5895 5896 status = pci_enable_pcie_error_reporting(pdev); 5897 if (!status) 5898 dev_info(&pdev->dev, "PCIe error reporting enabled\n"); 5899 5900 status = be_map_pci_bars(adapter); 5901 if (status) 5902 goto free_netdev; 5903 5904 status = be_drv_init(adapter); 5905 if (status) 5906 goto unmap_bars; 5907 5908 status = be_setup(adapter); 5909 if (status) 5910 goto drv_cleanup; 5911 5912 be_netdev_init(netdev); 5913 status = register_netdev(netdev); 5914 if (status != 0) 5915 goto unsetup; 5916 5917 be_roce_dev_add(adapter); 5918 5919 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 5920 adapter->error_recovery.probe_time = jiffies; 5921 5922 /* On Die temperature not supported for VF. */ 5923 if (be_physfn(adapter) && IS_ENABLED(CONFIG_BE2NET_HWMON)) { 5924 adapter->hwmon_info.hwmon_dev = 5925 devm_hwmon_device_register_with_groups(&pdev->dev, 5926 DRV_NAME, 5927 adapter, 5928 be_hwmon_groups); 5929 adapter->hwmon_info.be_on_die_temp = BE_INVALID_DIE_TEMP; 5930 } 5931 5932 dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev), 5933 func_name(adapter), mc_name(adapter), adapter->port_name); 5934 5935 return 0; 5936 5937 unsetup: 5938 be_clear(adapter); 5939 drv_cleanup: 5940 be_drv_cleanup(adapter); 5941 unmap_bars: 5942 be_unmap_pci_bars(adapter); 5943 free_netdev: 5944 free_netdev(netdev); 5945 rel_reg: 5946 pci_release_regions(pdev); 5947 disable_dev: 5948 pci_disable_device(pdev); 5949 do_none: 5950 dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev)); 5951 return status; 5952 } 5953 5954 static int be_suspend(struct pci_dev *pdev, pm_message_t state) 5955 { 5956 struct be_adapter *adapter = pci_get_drvdata(pdev); 5957 5958 be_intr_set(adapter, false); 5959 be_cancel_err_detection(adapter); 5960 5961 be_cleanup(adapter); 5962 5963 pci_save_state(pdev); 5964 pci_disable_device(pdev); 5965 pci_set_power_state(pdev, pci_choose_state(pdev, state)); 5966 return 0; 5967 } 5968 5969 static int be_pci_resume(struct pci_dev *pdev) 5970 { 5971 struct be_adapter *adapter = pci_get_drvdata(pdev); 5972 int status = 0; 5973 5974 status = pci_enable_device(pdev); 5975 if (status) 5976 return status; 5977 5978 pci_restore_state(pdev); 5979 5980 status = be_resume(adapter); 5981 if (status) 5982 return status; 5983 5984 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 5985 5986 return 0; 5987 } 5988 5989 /* 5990 * An FLR will stop BE from DMAing any data. 5991 */ 5992 static void be_shutdown(struct pci_dev *pdev) 5993 { 5994 struct be_adapter *adapter = pci_get_drvdata(pdev); 5995 5996 if (!adapter) 5997 return; 5998 5999 be_roce_dev_shutdown(adapter); 6000 cancel_delayed_work_sync(&adapter->work); 6001 be_cancel_err_detection(adapter); 6002 6003 netif_device_detach(adapter->netdev); 6004 6005 be_cmd_reset_function(adapter); 6006 6007 pci_disable_device(pdev); 6008 } 6009 6010 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev, 6011 pci_channel_state_t state) 6012 { 6013 struct be_adapter *adapter = pci_get_drvdata(pdev); 6014 6015 dev_err(&adapter->pdev->dev, "EEH error detected\n"); 6016 6017 be_roce_dev_remove(adapter); 6018 6019 if (!be_check_error(adapter, BE_ERROR_EEH)) { 6020 be_set_error(adapter, BE_ERROR_EEH); 6021 6022 be_cancel_err_detection(adapter); 6023 6024 be_cleanup(adapter); 6025 } 6026 6027 if (state == pci_channel_io_perm_failure) 6028 return PCI_ERS_RESULT_DISCONNECT; 6029 6030 pci_disable_device(pdev); 6031 6032 /* The error could cause the FW to trigger a flash debug dump. 6033 * Resetting the card while flash dump is in progress 6034 * can cause it not to recover; wait for it to finish. 6035 * Wait only for first function as it is needed only once per 6036 * adapter. 6037 */ 6038 if (pdev->devfn == 0) 6039 ssleep(30); 6040 6041 return PCI_ERS_RESULT_NEED_RESET; 6042 } 6043 6044 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev) 6045 { 6046 struct be_adapter *adapter = pci_get_drvdata(pdev); 6047 int status; 6048 6049 dev_info(&adapter->pdev->dev, "EEH reset\n"); 6050 6051 status = pci_enable_device(pdev); 6052 if (status) 6053 return PCI_ERS_RESULT_DISCONNECT; 6054 6055 pci_set_master(pdev); 6056 pci_restore_state(pdev); 6057 6058 /* Check if card is ok and fw is ready */ 6059 dev_info(&adapter->pdev->dev, 6060 "Waiting for FW to be ready after EEH reset\n"); 6061 status = be_fw_wait_ready(adapter); 6062 if (status) 6063 return PCI_ERS_RESULT_DISCONNECT; 6064 6065 pci_cleanup_aer_uncorrect_error_status(pdev); 6066 be_clear_error(adapter, BE_CLEAR_ALL); 6067 return PCI_ERS_RESULT_RECOVERED; 6068 } 6069 6070 static void be_eeh_resume(struct pci_dev *pdev) 6071 { 6072 int status = 0; 6073 struct be_adapter *adapter = pci_get_drvdata(pdev); 6074 6075 dev_info(&adapter->pdev->dev, "EEH resume\n"); 6076 6077 pci_save_state(pdev); 6078 6079 status = be_resume(adapter); 6080 if (status) 6081 goto err; 6082 6083 be_roce_dev_add(adapter); 6084 6085 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 6086 return; 6087 err: 6088 dev_err(&adapter->pdev->dev, "EEH resume failed\n"); 6089 } 6090 6091 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) 6092 { 6093 struct be_adapter *adapter = pci_get_drvdata(pdev); 6094 struct be_resources vft_res = {0}; 6095 int status; 6096 6097 if (!num_vfs) 6098 be_vf_clear(adapter); 6099 6100 adapter->num_vfs = num_vfs; 6101 6102 if (adapter->num_vfs == 0 && pci_vfs_assigned(pdev)) { 6103 dev_warn(&pdev->dev, 6104 "Cannot disable VFs while they are assigned\n"); 6105 return -EBUSY; 6106 } 6107 6108 /* When the HW is in SRIOV capable configuration, the PF-pool resources 6109 * are equally distributed across the max-number of VFs. The user may 6110 * request only a subset of the max-vfs to be enabled. 6111 * Based on num_vfs, redistribute the resources across num_vfs so that 6112 * each VF will have access to more number of resources. 6113 * This facility is not available in BE3 FW. 6114 * Also, this is done by FW in Lancer chip. 6115 */ 6116 if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) { 6117 be_calculate_vf_res(adapter, adapter->num_vfs, 6118 &vft_res); 6119 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 6120 adapter->num_vfs, &vft_res); 6121 if (status) 6122 dev_err(&pdev->dev, 6123 "Failed to optimize SR-IOV resources\n"); 6124 } 6125 6126 status = be_get_resources(adapter); 6127 if (status) 6128 return be_cmd_status(status); 6129 6130 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 6131 rtnl_lock(); 6132 status = be_update_queues(adapter); 6133 rtnl_unlock(); 6134 if (status) 6135 return be_cmd_status(status); 6136 6137 if (adapter->num_vfs) 6138 status = be_vf_setup(adapter); 6139 6140 if (!status) 6141 return adapter->num_vfs; 6142 6143 return 0; 6144 } 6145 6146 static const struct pci_error_handlers be_eeh_handlers = { 6147 .error_detected = be_eeh_err_detected, 6148 .slot_reset = be_eeh_reset, 6149 .resume = be_eeh_resume, 6150 }; 6151 6152 static struct pci_driver be_driver = { 6153 .name = DRV_NAME, 6154 .id_table = be_dev_ids, 6155 .probe = be_probe, 6156 .remove = be_remove, 6157 .suspend = be_suspend, 6158 .resume = be_pci_resume, 6159 .shutdown = be_shutdown, 6160 .sriov_configure = be_pci_sriov_configure, 6161 .err_handler = &be_eeh_handlers 6162 }; 6163 6164 static int __init be_init_module(void) 6165 { 6166 int status; 6167 6168 if (rx_frag_size != 8192 && rx_frag_size != 4096 && 6169 rx_frag_size != 2048) { 6170 printk(KERN_WARNING DRV_NAME 6171 " : Module param rx_frag_size must be 2048/4096/8192." 6172 " Using 2048\n"); 6173 rx_frag_size = 2048; 6174 } 6175 6176 if (num_vfs > 0) { 6177 pr_info(DRV_NAME " : Module param num_vfs is obsolete."); 6178 pr_info(DRV_NAME " : Use sysfs method to enable VFs\n"); 6179 } 6180 6181 be_wq = create_singlethread_workqueue("be_wq"); 6182 if (!be_wq) { 6183 pr_warn(DRV_NAME "workqueue creation failed\n"); 6184 return -1; 6185 } 6186 6187 be_err_recovery_workq = 6188 create_singlethread_workqueue("be_err_recover"); 6189 if (!be_err_recovery_workq) 6190 pr_warn(DRV_NAME "Could not create error recovery workqueue\n"); 6191 6192 status = pci_register_driver(&be_driver); 6193 if (status) { 6194 destroy_workqueue(be_wq); 6195 be_destroy_err_recovery_workq(); 6196 } 6197 return status; 6198 } 6199 module_init(be_init_module); 6200 6201 static void __exit be_exit_module(void) 6202 { 6203 pci_unregister_driver(&be_driver); 6204 6205 be_destroy_err_recovery_workq(); 6206 6207 if (be_wq) 6208 destroy_workqueue(be_wq); 6209 } 6210 module_exit(be_exit_module); 6211