1 /* 2 * Copyright (C) 2005 - 2011 Emulex 3 * All rights reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License version 2 7 * as published by the Free Software Foundation. The full GNU General 8 * Public License is included in this distribution in the file called COPYING. 9 * 10 * Contact Information: 11 * linux-drivers@emulex.com 12 * 13 * Emulex 14 * 3333 Susan Street 15 * Costa Mesa, CA 92626 16 */ 17 18 #include <linux/prefetch.h> 19 #include <linux/module.h> 20 #include "be.h" 21 #include "be_cmds.h" 22 #include <asm/div64.h> 23 24 MODULE_VERSION(DRV_VER); 25 MODULE_DEVICE_TABLE(pci, be_dev_ids); 26 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER); 27 MODULE_AUTHOR("ServerEngines Corporation"); 28 MODULE_LICENSE("GPL"); 29 30 static ushort rx_frag_size = 2048; 31 static unsigned int num_vfs; 32 module_param(rx_frag_size, ushort, S_IRUGO); 33 module_param(num_vfs, uint, S_IRUGO); 34 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data."); 35 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize"); 36 37 static DEFINE_PCI_DEVICE_TABLE(be_dev_ids) = { 38 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 39 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 40 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 41 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 42 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)}, 43 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)}, 44 { 0 } 45 }; 46 MODULE_DEVICE_TABLE(pci, be_dev_ids); 47 /* UE Status Low CSR */ 48 static const char * const ue_status_low_desc[] = { 49 "CEV", 50 "CTX", 51 "DBUF", 52 "ERX", 53 "Host", 54 "MPU", 55 "NDMA", 56 "PTC ", 57 "RDMA ", 58 "RXF ", 59 "RXIPS ", 60 "RXULP0 ", 61 "RXULP1 ", 62 "RXULP2 ", 63 "TIM ", 64 "TPOST ", 65 "TPRE ", 66 "TXIPS ", 67 "TXULP0 ", 68 "TXULP1 ", 69 "UC ", 70 "WDMA ", 71 "TXULP2 ", 72 "HOST1 ", 73 "P0_OB_LINK ", 74 "P1_OB_LINK ", 75 "HOST_GPIO ", 76 "MBOX ", 77 "AXGMAC0", 78 "AXGMAC1", 79 "JTAG", 80 "MPU_INTPEND" 81 }; 82 /* UE Status High CSR */ 83 static const char * const ue_status_hi_desc[] = { 84 "LPCMEMHOST", 85 "MGMT_MAC", 86 "PCS0ONLINE", 87 "MPU_IRAM", 88 "PCS1ONLINE", 89 "PCTL0", 90 "PCTL1", 91 "PMEM", 92 "RR", 93 "TXPB", 94 "RXPP", 95 "XAUI", 96 "TXP", 97 "ARM", 98 "IPC", 99 "HOST2", 100 "HOST3", 101 "HOST4", 102 "HOST5", 103 "HOST6", 104 "HOST7", 105 "HOST8", 106 "HOST9", 107 "NETC", 108 "Unknown", 109 "Unknown", 110 "Unknown", 111 "Unknown", 112 "Unknown", 113 "Unknown", 114 "Unknown", 115 "Unknown" 116 }; 117 118 /* Is BE in a multi-channel mode */ 119 static inline bool be_is_mc(struct be_adapter *adapter) { 120 return (adapter->function_mode & FLEX10_MODE || 121 adapter->function_mode & VNIC_MODE || 122 adapter->function_mode & UMC_ENABLED); 123 } 124 125 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q) 126 { 127 struct be_dma_mem *mem = &q->dma_mem; 128 if (mem->va) 129 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 130 mem->dma); 131 } 132 133 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q, 134 u16 len, u16 entry_size) 135 { 136 struct be_dma_mem *mem = &q->dma_mem; 137 138 memset(q, 0, sizeof(*q)); 139 q->len = len; 140 q->entry_size = entry_size; 141 mem->size = len * entry_size; 142 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma, 143 GFP_KERNEL); 144 if (!mem->va) 145 return -1; 146 memset(mem->va, 0, mem->size); 147 return 0; 148 } 149 150 static void be_intr_set(struct be_adapter *adapter, bool enable) 151 { 152 u32 reg, enabled; 153 154 if (adapter->eeh_err) 155 return; 156 157 pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, 158 ®); 159 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 160 161 if (!enabled && enable) 162 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 163 else if (enabled && !enable) 164 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 165 else 166 return; 167 168 pci_write_config_dword(adapter->pdev, 169 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg); 170 } 171 172 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted) 173 { 174 u32 val = 0; 175 val |= qid & DB_RQ_RING_ID_MASK; 176 val |= posted << DB_RQ_NUM_POSTED_SHIFT; 177 178 wmb(); 179 iowrite32(val, adapter->db + DB_RQ_OFFSET); 180 } 181 182 static void be_txq_notify(struct be_adapter *adapter, u16 qid, u16 posted) 183 { 184 u32 val = 0; 185 val |= qid & DB_TXULP_RING_ID_MASK; 186 val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT; 187 188 wmb(); 189 iowrite32(val, adapter->db + DB_TXULP1_OFFSET); 190 } 191 192 static void be_eq_notify(struct be_adapter *adapter, u16 qid, 193 bool arm, bool clear_int, u16 num_popped) 194 { 195 u32 val = 0; 196 val |= qid & DB_EQ_RING_ID_MASK; 197 val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << 198 DB_EQ_RING_ID_EXT_MASK_SHIFT); 199 200 if (adapter->eeh_err) 201 return; 202 203 if (arm) 204 val |= 1 << DB_EQ_REARM_SHIFT; 205 if (clear_int) 206 val |= 1 << DB_EQ_CLR_SHIFT; 207 val |= 1 << DB_EQ_EVNT_SHIFT; 208 val |= num_popped << DB_EQ_NUM_POPPED_SHIFT; 209 iowrite32(val, adapter->db + DB_EQ_OFFSET); 210 } 211 212 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped) 213 { 214 u32 val = 0; 215 val |= qid & DB_CQ_RING_ID_MASK; 216 val |= ((qid & DB_CQ_RING_ID_EXT_MASK) << 217 DB_CQ_RING_ID_EXT_MASK_SHIFT); 218 219 if (adapter->eeh_err) 220 return; 221 222 if (arm) 223 val |= 1 << DB_CQ_REARM_SHIFT; 224 val |= num_popped << DB_CQ_NUM_POPPED_SHIFT; 225 iowrite32(val, adapter->db + DB_CQ_OFFSET); 226 } 227 228 static int be_mac_addr_set(struct net_device *netdev, void *p) 229 { 230 struct be_adapter *adapter = netdev_priv(netdev); 231 struct sockaddr *addr = p; 232 int status = 0; 233 u8 current_mac[ETH_ALEN]; 234 u32 pmac_id = adapter->pmac_id; 235 236 if (!is_valid_ether_addr(addr->sa_data)) 237 return -EADDRNOTAVAIL; 238 239 status = be_cmd_mac_addr_query(adapter, current_mac, 240 MAC_ADDRESS_TYPE_NETWORK, false, adapter->if_handle); 241 if (status) 242 goto err; 243 244 if (memcmp(addr->sa_data, current_mac, ETH_ALEN)) { 245 status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data, 246 adapter->if_handle, &adapter->pmac_id, 0); 247 if (status) 248 goto err; 249 250 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0); 251 } 252 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 253 return 0; 254 err: 255 dev_err(&adapter->pdev->dev, "MAC %pM set Failed\n", addr->sa_data); 256 return status; 257 } 258 259 static void populate_be2_stats(struct be_adapter *adapter) 260 { 261 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 262 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 263 struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf; 264 struct be_port_rxf_stats_v0 *port_stats = 265 &rxf_stats->port[adapter->port_num]; 266 struct be_drv_stats *drvs = &adapter->drv_stats; 267 268 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 269 drvs->rx_pause_frames = port_stats->rx_pause_frames; 270 drvs->rx_crc_errors = port_stats->rx_crc_errors; 271 drvs->rx_control_frames = port_stats->rx_control_frames; 272 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 273 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 274 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 275 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 276 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 277 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 278 drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow; 279 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 280 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 281 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 282 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 283 drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow; 284 drvs->rx_dropped_header_too_small = 285 port_stats->rx_dropped_header_too_small; 286 drvs->rx_address_match_errors = port_stats->rx_address_match_errors; 287 drvs->rx_alignment_symbol_errors = 288 port_stats->rx_alignment_symbol_errors; 289 290 drvs->tx_pauseframes = port_stats->tx_pauseframes; 291 drvs->tx_controlframes = port_stats->tx_controlframes; 292 293 if (adapter->port_num) 294 drvs->jabber_events = rxf_stats->port1_jabber_events; 295 else 296 drvs->jabber_events = rxf_stats->port0_jabber_events; 297 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 298 drvs->rx_drops_no_txpb = rxf_stats->rx_drops_no_txpb; 299 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 300 drvs->rx_drops_invalid_ring = rxf_stats->rx_drops_invalid_ring; 301 drvs->forwarded_packets = rxf_stats->forwarded_packets; 302 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 303 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 304 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 305 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 306 } 307 308 static void populate_be3_stats(struct be_adapter *adapter) 309 { 310 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 311 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 312 struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf; 313 struct be_port_rxf_stats_v1 *port_stats = 314 &rxf_stats->port[adapter->port_num]; 315 struct be_drv_stats *drvs = &adapter->drv_stats; 316 317 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 318 drvs->rx_pause_frames = port_stats->rx_pause_frames; 319 drvs->rx_crc_errors = port_stats->rx_crc_errors; 320 drvs->rx_control_frames = port_stats->rx_control_frames; 321 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 322 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 323 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 324 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 325 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 326 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 327 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 328 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 329 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 330 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 331 drvs->rx_dropped_header_too_small = 332 port_stats->rx_dropped_header_too_small; 333 drvs->rx_input_fifo_overflow_drop = 334 port_stats->rx_input_fifo_overflow_drop; 335 drvs->rx_address_match_errors = port_stats->rx_address_match_errors; 336 drvs->rx_alignment_symbol_errors = 337 port_stats->rx_alignment_symbol_errors; 338 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 339 drvs->tx_pauseframes = port_stats->tx_pauseframes; 340 drvs->tx_controlframes = port_stats->tx_controlframes; 341 drvs->jabber_events = port_stats->jabber_events; 342 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 343 drvs->rx_drops_no_txpb = rxf_stats->rx_drops_no_txpb; 344 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 345 drvs->rx_drops_invalid_ring = rxf_stats->rx_drops_invalid_ring; 346 drvs->forwarded_packets = rxf_stats->forwarded_packets; 347 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 348 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 349 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 350 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 351 } 352 353 static void populate_lancer_stats(struct be_adapter *adapter) 354 { 355 356 struct be_drv_stats *drvs = &adapter->drv_stats; 357 struct lancer_pport_stats *pport_stats = 358 pport_stats_from_cmd(adapter); 359 360 be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats)); 361 drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo; 362 drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo; 363 drvs->rx_control_frames = pport_stats->rx_control_frames_lo; 364 drvs->rx_in_range_errors = pport_stats->rx_in_range_errors; 365 drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo; 366 drvs->rx_dropped_runt = pport_stats->rx_dropped_runt; 367 drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors; 368 drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors; 369 drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors; 370 drvs->rx_dropped_tcp_length = 371 pport_stats->rx_dropped_invalid_tcp_length; 372 drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small; 373 drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short; 374 drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors; 375 drvs->rx_dropped_header_too_small = 376 pport_stats->rx_dropped_header_too_small; 377 drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 378 drvs->rx_address_match_errors = pport_stats->rx_address_match_errors; 379 drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo; 380 drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 381 drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo; 382 drvs->tx_controlframes = pport_stats->tx_control_frames_lo; 383 drvs->jabber_events = pport_stats->rx_jabbers; 384 drvs->rx_drops_invalid_ring = pport_stats->rx_drops_invalid_queue; 385 drvs->forwarded_packets = pport_stats->num_forwards_lo; 386 drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo; 387 drvs->rx_drops_too_many_frags = 388 pport_stats->rx_drops_too_many_frags_lo; 389 } 390 391 static void accumulate_16bit_val(u32 *acc, u16 val) 392 { 393 #define lo(x) (x & 0xFFFF) 394 #define hi(x) (x & 0xFFFF0000) 395 bool wrapped = val < lo(*acc); 396 u32 newacc = hi(*acc) + val; 397 398 if (wrapped) 399 newacc += 65536; 400 ACCESS_ONCE(*acc) = newacc; 401 } 402 403 void be_parse_stats(struct be_adapter *adapter) 404 { 405 struct be_erx_stats_v1 *erx = be_erx_stats_from_cmd(adapter); 406 struct be_rx_obj *rxo; 407 int i; 408 409 if (adapter->generation == BE_GEN3) { 410 if (lancer_chip(adapter)) 411 populate_lancer_stats(adapter); 412 else 413 populate_be3_stats(adapter); 414 } else { 415 populate_be2_stats(adapter); 416 } 417 418 /* as erx_v1 is longer than v0, ok to use v1 defn for v0 access */ 419 for_all_rx_queues(adapter, rxo, i) { 420 /* below erx HW counter can actually wrap around after 421 * 65535. Driver accumulates a 32-bit value 422 */ 423 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags, 424 (u16)erx->rx_drops_no_fragments[rxo->q.id]); 425 } 426 } 427 428 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev, 429 struct rtnl_link_stats64 *stats) 430 { 431 struct be_adapter *adapter = netdev_priv(netdev); 432 struct be_drv_stats *drvs = &adapter->drv_stats; 433 struct be_rx_obj *rxo; 434 struct be_tx_obj *txo; 435 u64 pkts, bytes; 436 unsigned int start; 437 int i; 438 439 for_all_rx_queues(adapter, rxo, i) { 440 const struct be_rx_stats *rx_stats = rx_stats(rxo); 441 do { 442 start = u64_stats_fetch_begin_bh(&rx_stats->sync); 443 pkts = rx_stats(rxo)->rx_pkts; 444 bytes = rx_stats(rxo)->rx_bytes; 445 } while (u64_stats_fetch_retry_bh(&rx_stats->sync, start)); 446 stats->rx_packets += pkts; 447 stats->rx_bytes += bytes; 448 stats->multicast += rx_stats(rxo)->rx_mcast_pkts; 449 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs + 450 rx_stats(rxo)->rx_drops_no_frags; 451 } 452 453 for_all_tx_queues(adapter, txo, i) { 454 const struct be_tx_stats *tx_stats = tx_stats(txo); 455 do { 456 start = u64_stats_fetch_begin_bh(&tx_stats->sync); 457 pkts = tx_stats(txo)->tx_pkts; 458 bytes = tx_stats(txo)->tx_bytes; 459 } while (u64_stats_fetch_retry_bh(&tx_stats->sync, start)); 460 stats->tx_packets += pkts; 461 stats->tx_bytes += bytes; 462 } 463 464 /* bad pkts received */ 465 stats->rx_errors = drvs->rx_crc_errors + 466 drvs->rx_alignment_symbol_errors + 467 drvs->rx_in_range_errors + 468 drvs->rx_out_range_errors + 469 drvs->rx_frame_too_long + 470 drvs->rx_dropped_too_small + 471 drvs->rx_dropped_too_short + 472 drvs->rx_dropped_header_too_small + 473 drvs->rx_dropped_tcp_length + 474 drvs->rx_dropped_runt; 475 476 /* detailed rx errors */ 477 stats->rx_length_errors = drvs->rx_in_range_errors + 478 drvs->rx_out_range_errors + 479 drvs->rx_frame_too_long; 480 481 stats->rx_crc_errors = drvs->rx_crc_errors; 482 483 /* frame alignment errors */ 484 stats->rx_frame_errors = drvs->rx_alignment_symbol_errors; 485 486 /* receiver fifo overrun */ 487 /* drops_no_pbuf is no per i/f, it's per BE card */ 488 stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop + 489 drvs->rx_input_fifo_overflow_drop + 490 drvs->rx_drops_no_pbuf; 491 return stats; 492 } 493 494 void be_link_status_update(struct be_adapter *adapter, u32 link_status) 495 { 496 struct net_device *netdev = adapter->netdev; 497 498 /* when link status changes, link speed must be re-queried from card */ 499 adapter->link_speed = -1; 500 if ((link_status & LINK_STATUS_MASK) == LINK_UP) { 501 netif_carrier_on(netdev); 502 dev_info(&adapter->pdev->dev, "%s: Link up\n", netdev->name); 503 } else { 504 netif_carrier_off(netdev); 505 dev_info(&adapter->pdev->dev, "%s: Link down\n", netdev->name); 506 } 507 } 508 509 static void be_tx_stats_update(struct be_tx_obj *txo, 510 u32 wrb_cnt, u32 copied, u32 gso_segs, bool stopped) 511 { 512 struct be_tx_stats *stats = tx_stats(txo); 513 514 u64_stats_update_begin(&stats->sync); 515 stats->tx_reqs++; 516 stats->tx_wrbs += wrb_cnt; 517 stats->tx_bytes += copied; 518 stats->tx_pkts += (gso_segs ? gso_segs : 1); 519 if (stopped) 520 stats->tx_stops++; 521 u64_stats_update_end(&stats->sync); 522 } 523 524 /* Determine number of WRB entries needed to xmit data in an skb */ 525 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb, 526 bool *dummy) 527 { 528 int cnt = (skb->len > skb->data_len); 529 530 cnt += skb_shinfo(skb)->nr_frags; 531 532 /* to account for hdr wrb */ 533 cnt++; 534 if (lancer_chip(adapter) || !(cnt & 1)) { 535 *dummy = false; 536 } else { 537 /* add a dummy to make it an even num */ 538 cnt++; 539 *dummy = true; 540 } 541 BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT); 542 return cnt; 543 } 544 545 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len) 546 { 547 wrb->frag_pa_hi = upper_32_bits(addr); 548 wrb->frag_pa_lo = addr & 0xFFFFFFFF; 549 wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK; 550 } 551 552 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr, 553 struct sk_buff *skb, u32 wrb_cnt, u32 len) 554 { 555 u8 vlan_prio = 0; 556 u16 vlan_tag = 0; 557 558 memset(hdr, 0, sizeof(*hdr)); 559 560 AMAP_SET_BITS(struct amap_eth_hdr_wrb, crc, hdr, 1); 561 562 if (skb_is_gso(skb)) { 563 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso, hdr, 1); 564 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso_mss, 565 hdr, skb_shinfo(skb)->gso_size); 566 if (skb_is_gso_v6(skb) && !lancer_chip(adapter)) 567 AMAP_SET_BITS(struct amap_eth_hdr_wrb, lso6, hdr, 1); 568 if (lancer_chip(adapter) && adapter->sli_family == 569 LANCER_A0_SLI_FAMILY) { 570 AMAP_SET_BITS(struct amap_eth_hdr_wrb, ipcs, hdr, 1); 571 if (is_tcp_pkt(skb)) 572 AMAP_SET_BITS(struct amap_eth_hdr_wrb, 573 tcpcs, hdr, 1); 574 else if (is_udp_pkt(skb)) 575 AMAP_SET_BITS(struct amap_eth_hdr_wrb, 576 udpcs, hdr, 1); 577 } 578 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 579 if (is_tcp_pkt(skb)) 580 AMAP_SET_BITS(struct amap_eth_hdr_wrb, tcpcs, hdr, 1); 581 else if (is_udp_pkt(skb)) 582 AMAP_SET_BITS(struct amap_eth_hdr_wrb, udpcs, hdr, 1); 583 } 584 585 if (vlan_tx_tag_present(skb)) { 586 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan, hdr, 1); 587 vlan_tag = vlan_tx_tag_get(skb); 588 vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 589 /* If vlan priority provided by OS is NOT in available bmap */ 590 if (!(adapter->vlan_prio_bmap & (1 << vlan_prio))) 591 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) | 592 adapter->recommended_prio; 593 AMAP_SET_BITS(struct amap_eth_hdr_wrb, vlan_tag, hdr, vlan_tag); 594 } 595 596 AMAP_SET_BITS(struct amap_eth_hdr_wrb, event, hdr, 1); 597 AMAP_SET_BITS(struct amap_eth_hdr_wrb, complete, hdr, 1); 598 AMAP_SET_BITS(struct amap_eth_hdr_wrb, num_wrb, hdr, wrb_cnt); 599 AMAP_SET_BITS(struct amap_eth_hdr_wrb, len, hdr, len); 600 } 601 602 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb, 603 bool unmap_single) 604 { 605 dma_addr_t dma; 606 607 be_dws_le_to_cpu(wrb, sizeof(*wrb)); 608 609 dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo; 610 if (wrb->frag_len) { 611 if (unmap_single) 612 dma_unmap_single(dev, dma, wrb->frag_len, 613 DMA_TO_DEVICE); 614 else 615 dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE); 616 } 617 } 618 619 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq, 620 struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb) 621 { 622 dma_addr_t busaddr; 623 int i, copied = 0; 624 struct device *dev = &adapter->pdev->dev; 625 struct sk_buff *first_skb = skb; 626 struct be_eth_wrb *wrb; 627 struct be_eth_hdr_wrb *hdr; 628 bool map_single = false; 629 u16 map_head; 630 631 hdr = queue_head_node(txq); 632 queue_head_inc(txq); 633 map_head = txq->head; 634 635 if (skb->len > skb->data_len) { 636 int len = skb_headlen(skb); 637 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE); 638 if (dma_mapping_error(dev, busaddr)) 639 goto dma_err; 640 map_single = true; 641 wrb = queue_head_node(txq); 642 wrb_fill(wrb, busaddr, len); 643 be_dws_cpu_to_le(wrb, sizeof(*wrb)); 644 queue_head_inc(txq); 645 copied += len; 646 } 647 648 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 649 const struct skb_frag_struct *frag = 650 &skb_shinfo(skb)->frags[i]; 651 busaddr = skb_frag_dma_map(dev, frag, 0, 652 skb_frag_size(frag), DMA_TO_DEVICE); 653 if (dma_mapping_error(dev, busaddr)) 654 goto dma_err; 655 wrb = queue_head_node(txq); 656 wrb_fill(wrb, busaddr, skb_frag_size(frag)); 657 be_dws_cpu_to_le(wrb, sizeof(*wrb)); 658 queue_head_inc(txq); 659 copied += skb_frag_size(frag); 660 } 661 662 if (dummy_wrb) { 663 wrb = queue_head_node(txq); 664 wrb_fill(wrb, 0, 0); 665 be_dws_cpu_to_le(wrb, sizeof(*wrb)); 666 queue_head_inc(txq); 667 } 668 669 wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied); 670 be_dws_cpu_to_le(hdr, sizeof(*hdr)); 671 672 return copied; 673 dma_err: 674 txq->head = map_head; 675 while (copied) { 676 wrb = queue_head_node(txq); 677 unmap_tx_frag(dev, wrb, map_single); 678 map_single = false; 679 copied -= wrb->frag_len; 680 queue_head_inc(txq); 681 } 682 return 0; 683 } 684 685 static netdev_tx_t be_xmit(struct sk_buff *skb, 686 struct net_device *netdev) 687 { 688 struct be_adapter *adapter = netdev_priv(netdev); 689 struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)]; 690 struct be_queue_info *txq = &txo->q; 691 u32 wrb_cnt = 0, copied = 0; 692 u32 start = txq->head; 693 bool dummy_wrb, stopped = false; 694 695 wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb); 696 697 copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb); 698 if (copied) { 699 /* record the sent skb in the sent_skb table */ 700 BUG_ON(txo->sent_skb_list[start]); 701 txo->sent_skb_list[start] = skb; 702 703 /* Ensure txq has space for the next skb; Else stop the queue 704 * *BEFORE* ringing the tx doorbell, so that we serialze the 705 * tx compls of the current transmit which'll wake up the queue 706 */ 707 atomic_add(wrb_cnt, &txq->used); 708 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >= 709 txq->len) { 710 netif_stop_subqueue(netdev, skb_get_queue_mapping(skb)); 711 stopped = true; 712 } 713 714 be_txq_notify(adapter, txq->id, wrb_cnt); 715 716 be_tx_stats_update(txo, wrb_cnt, copied, 717 skb_shinfo(skb)->gso_segs, stopped); 718 } else { 719 txq->head = start; 720 dev_kfree_skb_any(skb); 721 } 722 return NETDEV_TX_OK; 723 } 724 725 static int be_change_mtu(struct net_device *netdev, int new_mtu) 726 { 727 struct be_adapter *adapter = netdev_priv(netdev); 728 if (new_mtu < BE_MIN_MTU || 729 new_mtu > (BE_MAX_JUMBO_FRAME_SIZE - 730 (ETH_HLEN + ETH_FCS_LEN))) { 731 dev_info(&adapter->pdev->dev, 732 "MTU must be between %d and %d bytes\n", 733 BE_MIN_MTU, 734 (BE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN))); 735 return -EINVAL; 736 } 737 dev_info(&adapter->pdev->dev, "MTU changed from %d to %d bytes\n", 738 netdev->mtu, new_mtu); 739 netdev->mtu = new_mtu; 740 return 0; 741 } 742 743 /* 744 * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE. 745 * If the user configures more, place BE in vlan promiscuous mode. 746 */ 747 static int be_vid_config(struct be_adapter *adapter, bool vf, u32 vf_num) 748 { 749 u16 vtag[BE_NUM_VLANS_SUPPORTED]; 750 u16 ntags = 0, i; 751 int status = 0; 752 u32 if_handle; 753 754 if (vf) { 755 if_handle = adapter->vf_cfg[vf_num].vf_if_handle; 756 vtag[0] = cpu_to_le16(adapter->vf_cfg[vf_num].vf_vlan_tag); 757 status = be_cmd_vlan_config(adapter, if_handle, vtag, 1, 1, 0); 758 } 759 760 /* No need to further configure vids if in promiscuous mode */ 761 if (adapter->promiscuous) 762 return 0; 763 764 if (adapter->vlans_added <= adapter->max_vlans) { 765 /* Construct VLAN Table to give to HW */ 766 for (i = 0; i < VLAN_N_VID; i++) { 767 if (adapter->vlan_tag[i]) { 768 vtag[ntags] = cpu_to_le16(i); 769 ntags++; 770 } 771 } 772 status = be_cmd_vlan_config(adapter, adapter->if_handle, 773 vtag, ntags, 1, 0); 774 } else { 775 status = be_cmd_vlan_config(adapter, adapter->if_handle, 776 NULL, 0, 1, 1); 777 } 778 779 return status; 780 } 781 782 static void be_vlan_add_vid(struct net_device *netdev, u16 vid) 783 { 784 struct be_adapter *adapter = netdev_priv(netdev); 785 786 adapter->vlans_added++; 787 if (!be_physfn(adapter)) 788 return; 789 790 adapter->vlan_tag[vid] = 1; 791 if (adapter->vlans_added <= (adapter->max_vlans + 1)) 792 be_vid_config(adapter, false, 0); 793 } 794 795 static void be_vlan_rem_vid(struct net_device *netdev, u16 vid) 796 { 797 struct be_adapter *adapter = netdev_priv(netdev); 798 799 adapter->vlans_added--; 800 801 if (!be_physfn(adapter)) 802 return; 803 804 adapter->vlan_tag[vid] = 0; 805 if (adapter->vlans_added <= adapter->max_vlans) 806 be_vid_config(adapter, false, 0); 807 } 808 809 static void be_set_rx_mode(struct net_device *netdev) 810 { 811 struct be_adapter *adapter = netdev_priv(netdev); 812 813 if (netdev->flags & IFF_PROMISC) { 814 be_cmd_rx_filter(adapter, IFF_PROMISC, ON); 815 adapter->promiscuous = true; 816 goto done; 817 } 818 819 /* BE was previously in promiscuous mode; disable it */ 820 if (adapter->promiscuous) { 821 adapter->promiscuous = false; 822 be_cmd_rx_filter(adapter, IFF_PROMISC, OFF); 823 824 if (adapter->vlans_added) 825 be_vid_config(adapter, false, 0); 826 } 827 828 /* Enable multicast promisc if num configured exceeds what we support */ 829 if (netdev->flags & IFF_ALLMULTI || 830 netdev_mc_count(netdev) > BE_MAX_MC) { 831 be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON); 832 goto done; 833 } 834 835 be_cmd_rx_filter(adapter, IFF_MULTICAST, ON); 836 done: 837 return; 838 } 839 840 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 841 { 842 struct be_adapter *adapter = netdev_priv(netdev); 843 int status; 844 845 if (!adapter->sriov_enabled) 846 return -EPERM; 847 848 if (!is_valid_ether_addr(mac) || (vf >= num_vfs)) 849 return -EINVAL; 850 851 if (adapter->vf_cfg[vf].vf_pmac_id != BE_INVALID_PMAC_ID) 852 status = be_cmd_pmac_del(adapter, 853 adapter->vf_cfg[vf].vf_if_handle, 854 adapter->vf_cfg[vf].vf_pmac_id, vf + 1); 855 856 status = be_cmd_pmac_add(adapter, mac, 857 adapter->vf_cfg[vf].vf_if_handle, 858 &adapter->vf_cfg[vf].vf_pmac_id, vf + 1); 859 860 if (status) 861 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed\n", 862 mac, vf); 863 else 864 memcpy(adapter->vf_cfg[vf].vf_mac_addr, mac, ETH_ALEN); 865 866 return status; 867 } 868 869 static int be_get_vf_config(struct net_device *netdev, int vf, 870 struct ifla_vf_info *vi) 871 { 872 struct be_adapter *adapter = netdev_priv(netdev); 873 874 if (!adapter->sriov_enabled) 875 return -EPERM; 876 877 if (vf >= num_vfs) 878 return -EINVAL; 879 880 vi->vf = vf; 881 vi->tx_rate = adapter->vf_cfg[vf].vf_tx_rate; 882 vi->vlan = adapter->vf_cfg[vf].vf_vlan_tag; 883 vi->qos = 0; 884 memcpy(&vi->mac, adapter->vf_cfg[vf].vf_mac_addr, ETH_ALEN); 885 886 return 0; 887 } 888 889 static int be_set_vf_vlan(struct net_device *netdev, 890 int vf, u16 vlan, u8 qos) 891 { 892 struct be_adapter *adapter = netdev_priv(netdev); 893 int status = 0; 894 895 if (!adapter->sriov_enabled) 896 return -EPERM; 897 898 if ((vf >= num_vfs) || (vlan > 4095)) 899 return -EINVAL; 900 901 if (vlan) { 902 adapter->vf_cfg[vf].vf_vlan_tag = vlan; 903 adapter->vlans_added++; 904 } else { 905 adapter->vf_cfg[vf].vf_vlan_tag = 0; 906 adapter->vlans_added--; 907 } 908 909 status = be_vid_config(adapter, true, vf); 910 911 if (status) 912 dev_info(&adapter->pdev->dev, 913 "VLAN %d config on VF %d failed\n", vlan, vf); 914 return status; 915 } 916 917 static int be_set_vf_tx_rate(struct net_device *netdev, 918 int vf, int rate) 919 { 920 struct be_adapter *adapter = netdev_priv(netdev); 921 int status = 0; 922 923 if (!adapter->sriov_enabled) 924 return -EPERM; 925 926 if ((vf >= num_vfs) || (rate < 0)) 927 return -EINVAL; 928 929 if (rate > 10000) 930 rate = 10000; 931 932 adapter->vf_cfg[vf].vf_tx_rate = rate; 933 status = be_cmd_set_qos(adapter, rate / 10, vf + 1); 934 935 if (status) 936 dev_info(&adapter->pdev->dev, 937 "tx rate %d on VF %d failed\n", rate, vf); 938 return status; 939 } 940 941 static void be_rx_eqd_update(struct be_adapter *adapter, struct be_rx_obj *rxo) 942 { 943 struct be_eq_obj *rx_eq = &rxo->rx_eq; 944 struct be_rx_stats *stats = rx_stats(rxo); 945 ulong now = jiffies; 946 ulong delta = now - stats->rx_jiffies; 947 u64 pkts; 948 unsigned int start, eqd; 949 950 if (!rx_eq->enable_aic) 951 return; 952 953 /* Wrapped around */ 954 if (time_before(now, stats->rx_jiffies)) { 955 stats->rx_jiffies = now; 956 return; 957 } 958 959 /* Update once a second */ 960 if (delta < HZ) 961 return; 962 963 do { 964 start = u64_stats_fetch_begin_bh(&stats->sync); 965 pkts = stats->rx_pkts; 966 } while (u64_stats_fetch_retry_bh(&stats->sync, start)); 967 968 stats->rx_pps = (unsigned long)(pkts - stats->rx_pkts_prev) / (delta / HZ); 969 stats->rx_pkts_prev = pkts; 970 stats->rx_jiffies = now; 971 eqd = stats->rx_pps / 110000; 972 eqd = eqd << 3; 973 if (eqd > rx_eq->max_eqd) 974 eqd = rx_eq->max_eqd; 975 if (eqd < rx_eq->min_eqd) 976 eqd = rx_eq->min_eqd; 977 if (eqd < 10) 978 eqd = 0; 979 if (eqd != rx_eq->cur_eqd) { 980 be_cmd_modify_eqd(adapter, rx_eq->q.id, eqd); 981 rx_eq->cur_eqd = eqd; 982 } 983 } 984 985 static void be_rx_stats_update(struct be_rx_obj *rxo, 986 struct be_rx_compl_info *rxcp) 987 { 988 struct be_rx_stats *stats = rx_stats(rxo); 989 990 u64_stats_update_begin(&stats->sync); 991 stats->rx_compl++; 992 stats->rx_bytes += rxcp->pkt_size; 993 stats->rx_pkts++; 994 if (rxcp->pkt_type == BE_MULTICAST_PACKET) 995 stats->rx_mcast_pkts++; 996 if (rxcp->err) 997 stats->rx_compl_err++; 998 u64_stats_update_end(&stats->sync); 999 } 1000 1001 static inline bool csum_passed(struct be_rx_compl_info *rxcp) 1002 { 1003 /* L4 checksum is not reliable for non TCP/UDP packets. 1004 * Also ignore ipcksm for ipv6 pkts */ 1005 return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum && 1006 (rxcp->ip_csum || rxcp->ipv6); 1007 } 1008 1009 static struct be_rx_page_info * 1010 get_rx_page_info(struct be_adapter *adapter, 1011 struct be_rx_obj *rxo, 1012 u16 frag_idx) 1013 { 1014 struct be_rx_page_info *rx_page_info; 1015 struct be_queue_info *rxq = &rxo->q; 1016 1017 rx_page_info = &rxo->page_info_tbl[frag_idx]; 1018 BUG_ON(!rx_page_info->page); 1019 1020 if (rx_page_info->last_page_user) { 1021 dma_unmap_page(&adapter->pdev->dev, 1022 dma_unmap_addr(rx_page_info, bus), 1023 adapter->big_page_size, DMA_FROM_DEVICE); 1024 rx_page_info->last_page_user = false; 1025 } 1026 1027 atomic_dec(&rxq->used); 1028 return rx_page_info; 1029 } 1030 1031 /* Throwaway the data in the Rx completion */ 1032 static void be_rx_compl_discard(struct be_adapter *adapter, 1033 struct be_rx_obj *rxo, 1034 struct be_rx_compl_info *rxcp) 1035 { 1036 struct be_queue_info *rxq = &rxo->q; 1037 struct be_rx_page_info *page_info; 1038 u16 i, num_rcvd = rxcp->num_rcvd; 1039 1040 for (i = 0; i < num_rcvd; i++) { 1041 page_info = get_rx_page_info(adapter, rxo, rxcp->rxq_idx); 1042 put_page(page_info->page); 1043 memset(page_info, 0, sizeof(*page_info)); 1044 index_inc(&rxcp->rxq_idx, rxq->len); 1045 } 1046 } 1047 1048 /* 1049 * skb_fill_rx_data forms a complete skb for an ether frame 1050 * indicated by rxcp. 1051 */ 1052 static void skb_fill_rx_data(struct be_adapter *adapter, struct be_rx_obj *rxo, 1053 struct sk_buff *skb, struct be_rx_compl_info *rxcp) 1054 { 1055 struct be_queue_info *rxq = &rxo->q; 1056 struct be_rx_page_info *page_info; 1057 u16 i, j; 1058 u16 hdr_len, curr_frag_len, remaining; 1059 u8 *start; 1060 1061 page_info = get_rx_page_info(adapter, rxo, rxcp->rxq_idx); 1062 start = page_address(page_info->page) + page_info->page_offset; 1063 prefetch(start); 1064 1065 /* Copy data in the first descriptor of this completion */ 1066 curr_frag_len = min(rxcp->pkt_size, rx_frag_size); 1067 1068 /* Copy the header portion into skb_data */ 1069 hdr_len = min(BE_HDR_LEN, curr_frag_len); 1070 memcpy(skb->data, start, hdr_len); 1071 skb->len = curr_frag_len; 1072 if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */ 1073 /* Complete packet has now been moved to data */ 1074 put_page(page_info->page); 1075 skb->data_len = 0; 1076 skb->tail += curr_frag_len; 1077 } else { 1078 skb_shinfo(skb)->nr_frags = 1; 1079 skb_frag_set_page(skb, 0, page_info->page); 1080 skb_shinfo(skb)->frags[0].page_offset = 1081 page_info->page_offset + hdr_len; 1082 skb_frag_size_set(&skb_shinfo(skb)->frags[0], curr_frag_len - hdr_len); 1083 skb->data_len = curr_frag_len - hdr_len; 1084 skb->truesize += rx_frag_size; 1085 skb->tail += hdr_len; 1086 } 1087 page_info->page = NULL; 1088 1089 if (rxcp->pkt_size <= rx_frag_size) { 1090 BUG_ON(rxcp->num_rcvd != 1); 1091 return; 1092 } 1093 1094 /* More frags present for this completion */ 1095 index_inc(&rxcp->rxq_idx, rxq->len); 1096 remaining = rxcp->pkt_size - curr_frag_len; 1097 for (i = 1, j = 0; i < rxcp->num_rcvd; i++) { 1098 page_info = get_rx_page_info(adapter, rxo, rxcp->rxq_idx); 1099 curr_frag_len = min(remaining, rx_frag_size); 1100 1101 /* Coalesce all frags from the same physical page in one slot */ 1102 if (page_info->page_offset == 0) { 1103 /* Fresh page */ 1104 j++; 1105 skb_frag_set_page(skb, j, page_info->page); 1106 skb_shinfo(skb)->frags[j].page_offset = 1107 page_info->page_offset; 1108 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 1109 skb_shinfo(skb)->nr_frags++; 1110 } else { 1111 put_page(page_info->page); 1112 } 1113 1114 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 1115 skb->len += curr_frag_len; 1116 skb->data_len += curr_frag_len; 1117 skb->truesize += rx_frag_size; 1118 remaining -= curr_frag_len; 1119 index_inc(&rxcp->rxq_idx, rxq->len); 1120 page_info->page = NULL; 1121 } 1122 BUG_ON(j > MAX_SKB_FRAGS); 1123 } 1124 1125 /* Process the RX completion indicated by rxcp when GRO is disabled */ 1126 static void be_rx_compl_process(struct be_adapter *adapter, 1127 struct be_rx_obj *rxo, 1128 struct be_rx_compl_info *rxcp) 1129 { 1130 struct net_device *netdev = adapter->netdev; 1131 struct sk_buff *skb; 1132 1133 skb = netdev_alloc_skb_ip_align(netdev, BE_HDR_LEN); 1134 if (unlikely(!skb)) { 1135 rx_stats(rxo)->rx_drops_no_skbs++; 1136 be_rx_compl_discard(adapter, rxo, rxcp); 1137 return; 1138 } 1139 1140 skb_fill_rx_data(adapter, rxo, skb, rxcp); 1141 1142 if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp))) 1143 skb->ip_summed = CHECKSUM_UNNECESSARY; 1144 else 1145 skb_checksum_none_assert(skb); 1146 1147 skb->protocol = eth_type_trans(skb, netdev); 1148 if (adapter->netdev->features & NETIF_F_RXHASH) 1149 skb->rxhash = rxcp->rss_hash; 1150 1151 1152 if (rxcp->vlanf) 1153 __vlan_hwaccel_put_tag(skb, rxcp->vlan_tag); 1154 1155 netif_receive_skb(skb); 1156 } 1157 1158 /* Process the RX completion indicated by rxcp when GRO is enabled */ 1159 static void be_rx_compl_process_gro(struct be_adapter *adapter, 1160 struct be_rx_obj *rxo, 1161 struct be_rx_compl_info *rxcp) 1162 { 1163 struct be_rx_page_info *page_info; 1164 struct sk_buff *skb = NULL; 1165 struct be_queue_info *rxq = &rxo->q; 1166 struct be_eq_obj *eq_obj = &rxo->rx_eq; 1167 u16 remaining, curr_frag_len; 1168 u16 i, j; 1169 1170 skb = napi_get_frags(&eq_obj->napi); 1171 if (!skb) { 1172 be_rx_compl_discard(adapter, rxo, rxcp); 1173 return; 1174 } 1175 1176 remaining = rxcp->pkt_size; 1177 for (i = 0, j = -1; i < rxcp->num_rcvd; i++) { 1178 page_info = get_rx_page_info(adapter, rxo, rxcp->rxq_idx); 1179 1180 curr_frag_len = min(remaining, rx_frag_size); 1181 1182 /* Coalesce all frags from the same physical page in one slot */ 1183 if (i == 0 || page_info->page_offset == 0) { 1184 /* First frag or Fresh page */ 1185 j++; 1186 skb_frag_set_page(skb, j, page_info->page); 1187 skb_shinfo(skb)->frags[j].page_offset = 1188 page_info->page_offset; 1189 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 1190 } else { 1191 put_page(page_info->page); 1192 } 1193 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 1194 skb->truesize += rx_frag_size; 1195 remaining -= curr_frag_len; 1196 index_inc(&rxcp->rxq_idx, rxq->len); 1197 memset(page_info, 0, sizeof(*page_info)); 1198 } 1199 BUG_ON(j > MAX_SKB_FRAGS); 1200 1201 skb_shinfo(skb)->nr_frags = j + 1; 1202 skb->len = rxcp->pkt_size; 1203 skb->data_len = rxcp->pkt_size; 1204 skb->ip_summed = CHECKSUM_UNNECESSARY; 1205 if (adapter->netdev->features & NETIF_F_RXHASH) 1206 skb->rxhash = rxcp->rss_hash; 1207 1208 if (rxcp->vlanf) 1209 __vlan_hwaccel_put_tag(skb, rxcp->vlan_tag); 1210 1211 napi_gro_frags(&eq_obj->napi); 1212 } 1213 1214 static void be_parse_rx_compl_v1(struct be_adapter *adapter, 1215 struct be_eth_rx_compl *compl, 1216 struct be_rx_compl_info *rxcp) 1217 { 1218 rxcp->pkt_size = 1219 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, pktsize, compl); 1220 rxcp->vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, vtp, compl); 1221 rxcp->err = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, err, compl); 1222 rxcp->tcpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, tcpf, compl); 1223 rxcp->udpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, udpf, compl); 1224 rxcp->ip_csum = 1225 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, ipcksm, compl); 1226 rxcp->l4_csum = 1227 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, l4_cksm, compl); 1228 rxcp->ipv6 = 1229 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, ip_version, compl); 1230 rxcp->rxq_idx = 1231 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, fragndx, compl); 1232 rxcp->num_rcvd = 1233 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, numfrags, compl); 1234 rxcp->pkt_type = 1235 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, cast_enc, compl); 1236 rxcp->rss_hash = 1237 AMAP_GET_BITS(struct amap_eth_rx_compl_v1, rsshash, rxcp); 1238 if (rxcp->vlanf) { 1239 rxcp->vtm = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, vtm, 1240 compl); 1241 rxcp->vlan_tag = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, vlan_tag, 1242 compl); 1243 } 1244 rxcp->port = AMAP_GET_BITS(struct amap_eth_rx_compl_v1, port, compl); 1245 } 1246 1247 static void be_parse_rx_compl_v0(struct be_adapter *adapter, 1248 struct be_eth_rx_compl *compl, 1249 struct be_rx_compl_info *rxcp) 1250 { 1251 rxcp->pkt_size = 1252 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, pktsize, compl); 1253 rxcp->vlanf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, vtp, compl); 1254 rxcp->err = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, err, compl); 1255 rxcp->tcpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, tcpf, compl); 1256 rxcp->udpf = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, udpf, compl); 1257 rxcp->ip_csum = 1258 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, ipcksm, compl); 1259 rxcp->l4_csum = 1260 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, l4_cksm, compl); 1261 rxcp->ipv6 = 1262 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, ip_version, compl); 1263 rxcp->rxq_idx = 1264 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, fragndx, compl); 1265 rxcp->num_rcvd = 1266 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, numfrags, compl); 1267 rxcp->pkt_type = 1268 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, cast_enc, compl); 1269 rxcp->rss_hash = 1270 AMAP_GET_BITS(struct amap_eth_rx_compl_v0, rsshash, rxcp); 1271 if (rxcp->vlanf) { 1272 rxcp->vtm = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, vtm, 1273 compl); 1274 rxcp->vlan_tag = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, vlan_tag, 1275 compl); 1276 } 1277 rxcp->port = AMAP_GET_BITS(struct amap_eth_rx_compl_v0, port, compl); 1278 } 1279 1280 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo) 1281 { 1282 struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq); 1283 struct be_rx_compl_info *rxcp = &rxo->rxcp; 1284 struct be_adapter *adapter = rxo->adapter; 1285 1286 /* For checking the valid bit it is Ok to use either definition as the 1287 * valid bit is at the same position in both v0 and v1 Rx compl */ 1288 if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0) 1289 return NULL; 1290 1291 rmb(); 1292 be_dws_le_to_cpu(compl, sizeof(*compl)); 1293 1294 if (adapter->be3_native) 1295 be_parse_rx_compl_v1(adapter, compl, rxcp); 1296 else 1297 be_parse_rx_compl_v0(adapter, compl, rxcp); 1298 1299 if (rxcp->vlanf) { 1300 /* vlanf could be wrongly set in some cards. 1301 * ignore if vtm is not set */ 1302 if ((adapter->function_mode & FLEX10_MODE) && !rxcp->vtm) 1303 rxcp->vlanf = 0; 1304 1305 if (!lancer_chip(adapter)) 1306 rxcp->vlan_tag = swab16(rxcp->vlan_tag); 1307 1308 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) && 1309 !adapter->vlan_tag[rxcp->vlan_tag]) 1310 rxcp->vlanf = 0; 1311 } 1312 1313 /* As the compl has been parsed, reset it; we wont touch it again */ 1314 compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0; 1315 1316 queue_tail_inc(&rxo->cq); 1317 return rxcp; 1318 } 1319 1320 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp) 1321 { 1322 u32 order = get_order(size); 1323 1324 if (order > 0) 1325 gfp |= __GFP_COMP; 1326 return alloc_pages(gfp, order); 1327 } 1328 1329 /* 1330 * Allocate a page, split it to fragments of size rx_frag_size and post as 1331 * receive buffers to BE 1332 */ 1333 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp) 1334 { 1335 struct be_adapter *adapter = rxo->adapter; 1336 struct be_rx_page_info *page_info_tbl = rxo->page_info_tbl; 1337 struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL; 1338 struct be_queue_info *rxq = &rxo->q; 1339 struct page *pagep = NULL; 1340 struct be_eth_rx_d *rxd; 1341 u64 page_dmaaddr = 0, frag_dmaaddr; 1342 u32 posted, page_offset = 0; 1343 1344 page_info = &rxo->page_info_tbl[rxq->head]; 1345 for (posted = 0; posted < MAX_RX_POST && !page_info->page; posted++) { 1346 if (!pagep) { 1347 pagep = be_alloc_pages(adapter->big_page_size, gfp); 1348 if (unlikely(!pagep)) { 1349 rx_stats(rxo)->rx_post_fail++; 1350 break; 1351 } 1352 page_dmaaddr = dma_map_page(&adapter->pdev->dev, pagep, 1353 0, adapter->big_page_size, 1354 DMA_FROM_DEVICE); 1355 page_info->page_offset = 0; 1356 } else { 1357 get_page(pagep); 1358 page_info->page_offset = page_offset + rx_frag_size; 1359 } 1360 page_offset = page_info->page_offset; 1361 page_info->page = pagep; 1362 dma_unmap_addr_set(page_info, bus, page_dmaaddr); 1363 frag_dmaaddr = page_dmaaddr + page_info->page_offset; 1364 1365 rxd = queue_head_node(rxq); 1366 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF); 1367 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr)); 1368 1369 /* Any space left in the current big page for another frag? */ 1370 if ((page_offset + rx_frag_size + rx_frag_size) > 1371 adapter->big_page_size) { 1372 pagep = NULL; 1373 page_info->last_page_user = true; 1374 } 1375 1376 prev_page_info = page_info; 1377 queue_head_inc(rxq); 1378 page_info = &page_info_tbl[rxq->head]; 1379 } 1380 if (pagep) 1381 prev_page_info->last_page_user = true; 1382 1383 if (posted) { 1384 atomic_add(posted, &rxq->used); 1385 be_rxq_notify(adapter, rxq->id, posted); 1386 } else if (atomic_read(&rxq->used) == 0) { 1387 /* Let be_worker replenish when memory is available */ 1388 rxo->rx_post_starved = true; 1389 } 1390 } 1391 1392 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq) 1393 { 1394 struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq); 1395 1396 if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0) 1397 return NULL; 1398 1399 rmb(); 1400 be_dws_le_to_cpu(txcp, sizeof(*txcp)); 1401 1402 txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0; 1403 1404 queue_tail_inc(tx_cq); 1405 return txcp; 1406 } 1407 1408 static u16 be_tx_compl_process(struct be_adapter *adapter, 1409 struct be_tx_obj *txo, u16 last_index) 1410 { 1411 struct be_queue_info *txq = &txo->q; 1412 struct be_eth_wrb *wrb; 1413 struct sk_buff **sent_skbs = txo->sent_skb_list; 1414 struct sk_buff *sent_skb; 1415 u16 cur_index, num_wrbs = 1; /* account for hdr wrb */ 1416 bool unmap_skb_hdr = true; 1417 1418 sent_skb = sent_skbs[txq->tail]; 1419 BUG_ON(!sent_skb); 1420 sent_skbs[txq->tail] = NULL; 1421 1422 /* skip header wrb */ 1423 queue_tail_inc(txq); 1424 1425 do { 1426 cur_index = txq->tail; 1427 wrb = queue_tail_node(txq); 1428 unmap_tx_frag(&adapter->pdev->dev, wrb, 1429 (unmap_skb_hdr && skb_headlen(sent_skb))); 1430 unmap_skb_hdr = false; 1431 1432 num_wrbs++; 1433 queue_tail_inc(txq); 1434 } while (cur_index != last_index); 1435 1436 kfree_skb(sent_skb); 1437 return num_wrbs; 1438 } 1439 1440 static inline struct be_eq_entry *event_get(struct be_eq_obj *eq_obj) 1441 { 1442 struct be_eq_entry *eqe = queue_tail_node(&eq_obj->q); 1443 1444 if (!eqe->evt) 1445 return NULL; 1446 1447 rmb(); 1448 eqe->evt = le32_to_cpu(eqe->evt); 1449 queue_tail_inc(&eq_obj->q); 1450 return eqe; 1451 } 1452 1453 static int event_handle(struct be_adapter *adapter, 1454 struct be_eq_obj *eq_obj, 1455 bool rearm) 1456 { 1457 struct be_eq_entry *eqe; 1458 u16 num = 0; 1459 1460 while ((eqe = event_get(eq_obj)) != NULL) { 1461 eqe->evt = 0; 1462 num++; 1463 } 1464 1465 /* Deal with any spurious interrupts that come 1466 * without events 1467 */ 1468 if (!num) 1469 rearm = true; 1470 1471 be_eq_notify(adapter, eq_obj->q.id, rearm, true, num); 1472 if (num) 1473 napi_schedule(&eq_obj->napi); 1474 1475 return num; 1476 } 1477 1478 /* Just read and notify events without processing them. 1479 * Used at the time of destroying event queues */ 1480 static void be_eq_clean(struct be_adapter *adapter, 1481 struct be_eq_obj *eq_obj) 1482 { 1483 struct be_eq_entry *eqe; 1484 u16 num = 0; 1485 1486 while ((eqe = event_get(eq_obj)) != NULL) { 1487 eqe->evt = 0; 1488 num++; 1489 } 1490 1491 if (num) 1492 be_eq_notify(adapter, eq_obj->q.id, false, true, num); 1493 } 1494 1495 static void be_rx_q_clean(struct be_adapter *adapter, struct be_rx_obj *rxo) 1496 { 1497 struct be_rx_page_info *page_info; 1498 struct be_queue_info *rxq = &rxo->q; 1499 struct be_queue_info *rx_cq = &rxo->cq; 1500 struct be_rx_compl_info *rxcp; 1501 u16 tail; 1502 1503 /* First cleanup pending rx completions */ 1504 while ((rxcp = be_rx_compl_get(rxo)) != NULL) { 1505 be_rx_compl_discard(adapter, rxo, rxcp); 1506 be_cq_notify(adapter, rx_cq->id, false, 1); 1507 } 1508 1509 /* Then free posted rx buffer that were not used */ 1510 tail = (rxq->head + rxq->len - atomic_read(&rxq->used)) % rxq->len; 1511 for (; atomic_read(&rxq->used) > 0; index_inc(&tail, rxq->len)) { 1512 page_info = get_rx_page_info(adapter, rxo, tail); 1513 put_page(page_info->page); 1514 memset(page_info, 0, sizeof(*page_info)); 1515 } 1516 BUG_ON(atomic_read(&rxq->used)); 1517 rxq->tail = rxq->head = 0; 1518 } 1519 1520 static void be_tx_compl_clean(struct be_adapter *adapter, 1521 struct be_tx_obj *txo) 1522 { 1523 struct be_queue_info *tx_cq = &txo->cq; 1524 struct be_queue_info *txq = &txo->q; 1525 struct be_eth_tx_compl *txcp; 1526 u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0; 1527 struct sk_buff **sent_skbs = txo->sent_skb_list; 1528 struct sk_buff *sent_skb; 1529 bool dummy_wrb; 1530 1531 /* Wait for a max of 200ms for all the tx-completions to arrive. */ 1532 do { 1533 while ((txcp = be_tx_compl_get(tx_cq))) { 1534 end_idx = AMAP_GET_BITS(struct amap_eth_tx_compl, 1535 wrb_index, txcp); 1536 num_wrbs += be_tx_compl_process(adapter, txo, end_idx); 1537 cmpl++; 1538 } 1539 if (cmpl) { 1540 be_cq_notify(adapter, tx_cq->id, false, cmpl); 1541 atomic_sub(num_wrbs, &txq->used); 1542 cmpl = 0; 1543 num_wrbs = 0; 1544 } 1545 1546 if (atomic_read(&txq->used) == 0 || ++timeo > 200) 1547 break; 1548 1549 mdelay(1); 1550 } while (true); 1551 1552 if (atomic_read(&txq->used)) 1553 dev_err(&adapter->pdev->dev, "%d pending tx-completions\n", 1554 atomic_read(&txq->used)); 1555 1556 /* free posted tx for which compls will never arrive */ 1557 while (atomic_read(&txq->used)) { 1558 sent_skb = sent_skbs[txq->tail]; 1559 end_idx = txq->tail; 1560 index_adv(&end_idx, 1561 wrb_cnt_for_skb(adapter, sent_skb, &dummy_wrb) - 1, 1562 txq->len); 1563 num_wrbs = be_tx_compl_process(adapter, txo, end_idx); 1564 atomic_sub(num_wrbs, &txq->used); 1565 } 1566 } 1567 1568 static void be_mcc_queues_destroy(struct be_adapter *adapter) 1569 { 1570 struct be_queue_info *q; 1571 1572 q = &adapter->mcc_obj.q; 1573 if (q->created) 1574 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ); 1575 be_queue_free(adapter, q); 1576 1577 q = &adapter->mcc_obj.cq; 1578 if (q->created) 1579 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 1580 be_queue_free(adapter, q); 1581 } 1582 1583 /* Must be called only after TX qs are created as MCC shares TX EQ */ 1584 static int be_mcc_queues_create(struct be_adapter *adapter) 1585 { 1586 struct be_queue_info *q, *cq; 1587 1588 /* Alloc MCC compl queue */ 1589 cq = &adapter->mcc_obj.cq; 1590 if (be_queue_alloc(adapter, cq, MCC_CQ_LEN, 1591 sizeof(struct be_mcc_compl))) 1592 goto err; 1593 1594 /* Ask BE to create MCC compl queue; share TX's eq */ 1595 if (be_cmd_cq_create(adapter, cq, &adapter->tx_eq.q, false, true, 0)) 1596 goto mcc_cq_free; 1597 1598 /* Alloc MCC queue */ 1599 q = &adapter->mcc_obj.q; 1600 if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 1601 goto mcc_cq_destroy; 1602 1603 /* Ask BE to create MCC queue */ 1604 if (be_cmd_mccq_create(adapter, q, cq)) 1605 goto mcc_q_free; 1606 1607 return 0; 1608 1609 mcc_q_free: 1610 be_queue_free(adapter, q); 1611 mcc_cq_destroy: 1612 be_cmd_q_destroy(adapter, cq, QTYPE_CQ); 1613 mcc_cq_free: 1614 be_queue_free(adapter, cq); 1615 err: 1616 return -1; 1617 } 1618 1619 static void be_tx_queues_destroy(struct be_adapter *adapter) 1620 { 1621 struct be_queue_info *q; 1622 struct be_tx_obj *txo; 1623 u8 i; 1624 1625 for_all_tx_queues(adapter, txo, i) { 1626 q = &txo->q; 1627 if (q->created) 1628 be_cmd_q_destroy(adapter, q, QTYPE_TXQ); 1629 be_queue_free(adapter, q); 1630 1631 q = &txo->cq; 1632 if (q->created) 1633 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 1634 be_queue_free(adapter, q); 1635 } 1636 1637 /* Clear any residual events */ 1638 be_eq_clean(adapter, &adapter->tx_eq); 1639 1640 q = &adapter->tx_eq.q; 1641 if (q->created) 1642 be_cmd_q_destroy(adapter, q, QTYPE_EQ); 1643 be_queue_free(adapter, q); 1644 } 1645 1646 static int be_num_txqs_want(struct be_adapter *adapter) 1647 { 1648 if ((num_vfs && adapter->sriov_enabled) || 1649 be_is_mc(adapter) || 1650 lancer_chip(adapter) || !be_physfn(adapter) || 1651 adapter->generation == BE_GEN2) 1652 return 1; 1653 else 1654 return MAX_TX_QS; 1655 } 1656 1657 /* One TX event queue is shared by all TX compl qs */ 1658 static int be_tx_queues_create(struct be_adapter *adapter) 1659 { 1660 struct be_queue_info *eq, *q, *cq; 1661 struct be_tx_obj *txo; 1662 u8 i; 1663 1664 adapter->num_tx_qs = be_num_txqs_want(adapter); 1665 if (adapter->num_tx_qs != MAX_TX_QS) 1666 netif_set_real_num_tx_queues(adapter->netdev, 1667 adapter->num_tx_qs); 1668 1669 adapter->tx_eq.max_eqd = 0; 1670 adapter->tx_eq.min_eqd = 0; 1671 adapter->tx_eq.cur_eqd = 96; 1672 adapter->tx_eq.enable_aic = false; 1673 1674 eq = &adapter->tx_eq.q; 1675 if (be_queue_alloc(adapter, eq, EVNT_Q_LEN, 1676 sizeof(struct be_eq_entry))) 1677 return -1; 1678 1679 if (be_cmd_eq_create(adapter, eq, adapter->tx_eq.cur_eqd)) 1680 goto err; 1681 adapter->tx_eq.eq_idx = adapter->eq_next_idx++; 1682 1683 for_all_tx_queues(adapter, txo, i) { 1684 cq = &txo->cq; 1685 if (be_queue_alloc(adapter, cq, TX_CQ_LEN, 1686 sizeof(struct be_eth_tx_compl))) 1687 goto err; 1688 1689 if (be_cmd_cq_create(adapter, cq, eq, false, false, 3)) 1690 goto err; 1691 1692 q = &txo->q; 1693 if (be_queue_alloc(adapter, q, TX_Q_LEN, 1694 sizeof(struct be_eth_wrb))) 1695 goto err; 1696 1697 if (be_cmd_txq_create(adapter, q, cq)) 1698 goto err; 1699 } 1700 return 0; 1701 1702 err: 1703 be_tx_queues_destroy(adapter); 1704 return -1; 1705 } 1706 1707 static void be_rx_queues_destroy(struct be_adapter *adapter) 1708 { 1709 struct be_queue_info *q; 1710 struct be_rx_obj *rxo; 1711 int i; 1712 1713 for_all_rx_queues(adapter, rxo, i) { 1714 be_queue_free(adapter, &rxo->q); 1715 1716 q = &rxo->cq; 1717 if (q->created) 1718 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 1719 be_queue_free(adapter, q); 1720 1721 q = &rxo->rx_eq.q; 1722 if (q->created) 1723 be_cmd_q_destroy(adapter, q, QTYPE_EQ); 1724 be_queue_free(adapter, q); 1725 } 1726 } 1727 1728 static u32 be_num_rxqs_want(struct be_adapter *adapter) 1729 { 1730 if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) && 1731 !adapter->sriov_enabled && be_physfn(adapter) && 1732 !be_is_mc(adapter)) { 1733 return 1 + MAX_RSS_QS; /* one default non-RSS queue */ 1734 } else { 1735 dev_warn(&adapter->pdev->dev, 1736 "No support for multiple RX queues\n"); 1737 return 1; 1738 } 1739 } 1740 1741 static int be_rx_queues_create(struct be_adapter *adapter) 1742 { 1743 struct be_queue_info *eq, *q, *cq; 1744 struct be_rx_obj *rxo; 1745 int rc, i; 1746 1747 adapter->num_rx_qs = min(be_num_rxqs_want(adapter), 1748 msix_enabled(adapter) ? 1749 adapter->num_msix_vec - 1 : 1); 1750 if (adapter->num_rx_qs != MAX_RX_QS) 1751 dev_warn(&adapter->pdev->dev, 1752 "Can create only %d RX queues", adapter->num_rx_qs); 1753 1754 adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE; 1755 for_all_rx_queues(adapter, rxo, i) { 1756 rxo->adapter = adapter; 1757 rxo->rx_eq.max_eqd = BE_MAX_EQD; 1758 rxo->rx_eq.enable_aic = true; 1759 1760 /* EQ */ 1761 eq = &rxo->rx_eq.q; 1762 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN, 1763 sizeof(struct be_eq_entry)); 1764 if (rc) 1765 goto err; 1766 1767 rc = be_cmd_eq_create(adapter, eq, rxo->rx_eq.cur_eqd); 1768 if (rc) 1769 goto err; 1770 1771 rxo->rx_eq.eq_idx = adapter->eq_next_idx++; 1772 1773 /* CQ */ 1774 cq = &rxo->cq; 1775 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN, 1776 sizeof(struct be_eth_rx_compl)); 1777 if (rc) 1778 goto err; 1779 1780 rc = be_cmd_cq_create(adapter, cq, eq, false, false, 3); 1781 if (rc) 1782 goto err; 1783 1784 /* Rx Q - will be created in be_open() */ 1785 q = &rxo->q; 1786 rc = be_queue_alloc(adapter, q, RX_Q_LEN, 1787 sizeof(struct be_eth_rx_d)); 1788 if (rc) 1789 goto err; 1790 1791 } 1792 1793 return 0; 1794 err: 1795 be_rx_queues_destroy(adapter); 1796 return -1; 1797 } 1798 1799 static bool event_peek(struct be_eq_obj *eq_obj) 1800 { 1801 struct be_eq_entry *eqe = queue_tail_node(&eq_obj->q); 1802 if (!eqe->evt) 1803 return false; 1804 else 1805 return true; 1806 } 1807 1808 static irqreturn_t be_intx(int irq, void *dev) 1809 { 1810 struct be_adapter *adapter = dev; 1811 struct be_rx_obj *rxo; 1812 int isr, i, tx = 0 , rx = 0; 1813 1814 if (lancer_chip(adapter)) { 1815 if (event_peek(&adapter->tx_eq)) 1816 tx = event_handle(adapter, &adapter->tx_eq, false); 1817 for_all_rx_queues(adapter, rxo, i) { 1818 if (event_peek(&rxo->rx_eq)) 1819 rx |= event_handle(adapter, &rxo->rx_eq, true); 1820 } 1821 1822 if (!(tx || rx)) 1823 return IRQ_NONE; 1824 1825 } else { 1826 isr = ioread32(adapter->csr + CEV_ISR0_OFFSET + 1827 (adapter->tx_eq.q.id / 8) * CEV_ISR_SIZE); 1828 if (!isr) 1829 return IRQ_NONE; 1830 1831 if ((1 << adapter->tx_eq.eq_idx & isr)) 1832 event_handle(adapter, &adapter->tx_eq, false); 1833 1834 for_all_rx_queues(adapter, rxo, i) { 1835 if ((1 << rxo->rx_eq.eq_idx & isr)) 1836 event_handle(adapter, &rxo->rx_eq, true); 1837 } 1838 } 1839 1840 return IRQ_HANDLED; 1841 } 1842 1843 static irqreturn_t be_msix_rx(int irq, void *dev) 1844 { 1845 struct be_rx_obj *rxo = dev; 1846 struct be_adapter *adapter = rxo->adapter; 1847 1848 event_handle(adapter, &rxo->rx_eq, true); 1849 1850 return IRQ_HANDLED; 1851 } 1852 1853 static irqreturn_t be_msix_tx_mcc(int irq, void *dev) 1854 { 1855 struct be_adapter *adapter = dev; 1856 1857 event_handle(adapter, &adapter->tx_eq, false); 1858 1859 return IRQ_HANDLED; 1860 } 1861 1862 static inline bool do_gro(struct be_rx_compl_info *rxcp) 1863 { 1864 return (rxcp->tcpf && !rxcp->err) ? true : false; 1865 } 1866 1867 static int be_poll_rx(struct napi_struct *napi, int budget) 1868 { 1869 struct be_eq_obj *rx_eq = container_of(napi, struct be_eq_obj, napi); 1870 struct be_rx_obj *rxo = container_of(rx_eq, struct be_rx_obj, rx_eq); 1871 struct be_adapter *adapter = rxo->adapter; 1872 struct be_queue_info *rx_cq = &rxo->cq; 1873 struct be_rx_compl_info *rxcp; 1874 u32 work_done; 1875 1876 rx_stats(rxo)->rx_polls++; 1877 for (work_done = 0; work_done < budget; work_done++) { 1878 rxcp = be_rx_compl_get(rxo); 1879 if (!rxcp) 1880 break; 1881 1882 /* Is it a flush compl that has no data */ 1883 if (unlikely(rxcp->num_rcvd == 0)) 1884 goto loop_continue; 1885 1886 /* Discard compl with partial DMA Lancer B0 */ 1887 if (unlikely(!rxcp->pkt_size)) { 1888 be_rx_compl_discard(adapter, rxo, rxcp); 1889 goto loop_continue; 1890 } 1891 1892 /* On BE drop pkts that arrive due to imperfect filtering in 1893 * promiscuous mode on some skews 1894 */ 1895 if (unlikely(rxcp->port != adapter->port_num && 1896 !lancer_chip(adapter))) { 1897 be_rx_compl_discard(adapter, rxo, rxcp); 1898 goto loop_continue; 1899 } 1900 1901 if (do_gro(rxcp)) 1902 be_rx_compl_process_gro(adapter, rxo, rxcp); 1903 else 1904 be_rx_compl_process(adapter, rxo, rxcp); 1905 loop_continue: 1906 be_rx_stats_update(rxo, rxcp); 1907 } 1908 1909 be_cq_notify(adapter, rx_cq->id, false, work_done); 1910 1911 /* Refill the queue */ 1912 if (work_done && atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM) 1913 be_post_rx_frags(rxo, GFP_ATOMIC); 1914 1915 /* All consumed */ 1916 if (work_done < budget) { 1917 napi_complete(napi); 1918 /* Arm CQ */ 1919 be_cq_notify(adapter, rx_cq->id, true, 0); 1920 } 1921 return work_done; 1922 } 1923 1924 /* As TX and MCC share the same EQ check for both TX and MCC completions. 1925 * For TX/MCC we don't honour budget; consume everything 1926 */ 1927 static int be_poll_tx_mcc(struct napi_struct *napi, int budget) 1928 { 1929 struct be_eq_obj *tx_eq = container_of(napi, struct be_eq_obj, napi); 1930 struct be_adapter *adapter = 1931 container_of(tx_eq, struct be_adapter, tx_eq); 1932 struct be_tx_obj *txo; 1933 struct be_eth_tx_compl *txcp; 1934 int tx_compl, mcc_compl, status = 0; 1935 u8 i; 1936 u16 num_wrbs; 1937 1938 for_all_tx_queues(adapter, txo, i) { 1939 tx_compl = 0; 1940 num_wrbs = 0; 1941 while ((txcp = be_tx_compl_get(&txo->cq))) { 1942 num_wrbs += be_tx_compl_process(adapter, txo, 1943 AMAP_GET_BITS(struct amap_eth_tx_compl, 1944 wrb_index, txcp)); 1945 tx_compl++; 1946 } 1947 if (tx_compl) { 1948 be_cq_notify(adapter, txo->cq.id, true, tx_compl); 1949 1950 atomic_sub(num_wrbs, &txo->q.used); 1951 1952 /* As Tx wrbs have been freed up, wake up netdev queue 1953 * if it was stopped due to lack of tx wrbs. */ 1954 if (__netif_subqueue_stopped(adapter->netdev, i) && 1955 atomic_read(&txo->q.used) < txo->q.len / 2) { 1956 netif_wake_subqueue(adapter->netdev, i); 1957 } 1958 1959 u64_stats_update_begin(&tx_stats(txo)->sync_compl); 1960 tx_stats(txo)->tx_compl += tx_compl; 1961 u64_stats_update_end(&tx_stats(txo)->sync_compl); 1962 } 1963 } 1964 1965 mcc_compl = be_process_mcc(adapter, &status); 1966 1967 if (mcc_compl) { 1968 struct be_mcc_obj *mcc_obj = &adapter->mcc_obj; 1969 be_cq_notify(adapter, mcc_obj->cq.id, true, mcc_compl); 1970 } 1971 1972 napi_complete(napi); 1973 1974 be_eq_notify(adapter, tx_eq->q.id, true, false, 0); 1975 adapter->drv_stats.tx_events++; 1976 return 1; 1977 } 1978 1979 void be_detect_dump_ue(struct be_adapter *adapter) 1980 { 1981 u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0; 1982 u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 1983 u32 i; 1984 1985 if (lancer_chip(adapter)) { 1986 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET); 1987 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 1988 sliport_err1 = ioread32(adapter->db + 1989 SLIPORT_ERROR1_OFFSET); 1990 sliport_err2 = ioread32(adapter->db + 1991 SLIPORT_ERROR2_OFFSET); 1992 } 1993 } else { 1994 pci_read_config_dword(adapter->pdev, 1995 PCICFG_UE_STATUS_LOW, &ue_lo); 1996 pci_read_config_dword(adapter->pdev, 1997 PCICFG_UE_STATUS_HIGH, &ue_hi); 1998 pci_read_config_dword(adapter->pdev, 1999 PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask); 2000 pci_read_config_dword(adapter->pdev, 2001 PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask); 2002 2003 ue_lo = (ue_lo & (~ue_lo_mask)); 2004 ue_hi = (ue_hi & (~ue_hi_mask)); 2005 } 2006 2007 if (ue_lo || ue_hi || 2008 sliport_status & SLIPORT_STATUS_ERR_MASK) { 2009 adapter->ue_detected = true; 2010 adapter->eeh_err = true; 2011 dev_err(&adapter->pdev->dev, "UE Detected!!\n"); 2012 } 2013 2014 if (ue_lo) { 2015 for (i = 0; ue_lo; ue_lo >>= 1, i++) { 2016 if (ue_lo & 1) 2017 dev_err(&adapter->pdev->dev, 2018 "UE: %s bit set\n", ue_status_low_desc[i]); 2019 } 2020 } 2021 if (ue_hi) { 2022 for (i = 0; ue_hi; ue_hi >>= 1, i++) { 2023 if (ue_hi & 1) 2024 dev_err(&adapter->pdev->dev, 2025 "UE: %s bit set\n", ue_status_hi_desc[i]); 2026 } 2027 } 2028 2029 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2030 dev_err(&adapter->pdev->dev, 2031 "sliport status 0x%x\n", sliport_status); 2032 dev_err(&adapter->pdev->dev, 2033 "sliport error1 0x%x\n", sliport_err1); 2034 dev_err(&adapter->pdev->dev, 2035 "sliport error2 0x%x\n", sliport_err2); 2036 } 2037 } 2038 2039 static void be_worker(struct work_struct *work) 2040 { 2041 struct be_adapter *adapter = 2042 container_of(work, struct be_adapter, work.work); 2043 struct be_rx_obj *rxo; 2044 int i; 2045 2046 if (!adapter->ue_detected) 2047 be_detect_dump_ue(adapter); 2048 2049 /* when interrupts are not yet enabled, just reap any pending 2050 * mcc completions */ 2051 if (!netif_running(adapter->netdev)) { 2052 int mcc_compl, status = 0; 2053 2054 mcc_compl = be_process_mcc(adapter, &status); 2055 2056 if (mcc_compl) { 2057 struct be_mcc_obj *mcc_obj = &adapter->mcc_obj; 2058 be_cq_notify(adapter, mcc_obj->cq.id, false, mcc_compl); 2059 } 2060 2061 goto reschedule; 2062 } 2063 2064 if (!adapter->stats_cmd_sent) { 2065 if (lancer_chip(adapter)) 2066 lancer_cmd_get_pport_stats(adapter, 2067 &adapter->stats_cmd); 2068 else 2069 be_cmd_get_stats(adapter, &adapter->stats_cmd); 2070 } 2071 2072 for_all_rx_queues(adapter, rxo, i) { 2073 be_rx_eqd_update(adapter, rxo); 2074 2075 if (rxo->rx_post_starved) { 2076 rxo->rx_post_starved = false; 2077 be_post_rx_frags(rxo, GFP_KERNEL); 2078 } 2079 } 2080 2081 reschedule: 2082 adapter->work_counter++; 2083 schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000)); 2084 } 2085 2086 static void be_msix_disable(struct be_adapter *adapter) 2087 { 2088 if (msix_enabled(adapter)) { 2089 pci_disable_msix(adapter->pdev); 2090 adapter->num_msix_vec = 0; 2091 } 2092 } 2093 2094 static void be_msix_enable(struct be_adapter *adapter) 2095 { 2096 #define BE_MIN_MSIX_VECTORS (1 + 1) /* Rx + Tx */ 2097 int i, status, num_vec; 2098 2099 num_vec = be_num_rxqs_want(adapter) + 1; 2100 2101 for (i = 0; i < num_vec; i++) 2102 adapter->msix_entries[i].entry = i; 2103 2104 status = pci_enable_msix(adapter->pdev, adapter->msix_entries, num_vec); 2105 if (status == 0) { 2106 goto done; 2107 } else if (status >= BE_MIN_MSIX_VECTORS) { 2108 num_vec = status; 2109 if (pci_enable_msix(adapter->pdev, adapter->msix_entries, 2110 num_vec) == 0) 2111 goto done; 2112 } 2113 return; 2114 done: 2115 adapter->num_msix_vec = num_vec; 2116 return; 2117 } 2118 2119 static int be_sriov_enable(struct be_adapter *adapter) 2120 { 2121 be_check_sriov_fn_type(adapter); 2122 #ifdef CONFIG_PCI_IOV 2123 if (be_physfn(adapter) && num_vfs) { 2124 int status, pos; 2125 u16 nvfs; 2126 2127 pos = pci_find_ext_capability(adapter->pdev, 2128 PCI_EXT_CAP_ID_SRIOV); 2129 pci_read_config_word(adapter->pdev, 2130 pos + PCI_SRIOV_TOTAL_VF, &nvfs); 2131 2132 if (num_vfs > nvfs) { 2133 dev_info(&adapter->pdev->dev, 2134 "Device supports %d VFs and not %d\n", 2135 nvfs, num_vfs); 2136 num_vfs = nvfs; 2137 } 2138 2139 status = pci_enable_sriov(adapter->pdev, num_vfs); 2140 adapter->sriov_enabled = status ? false : true; 2141 2142 if (adapter->sriov_enabled) { 2143 adapter->vf_cfg = kcalloc(num_vfs, 2144 sizeof(struct be_vf_cfg), 2145 GFP_KERNEL); 2146 if (!adapter->vf_cfg) 2147 return -ENOMEM; 2148 } 2149 } 2150 #endif 2151 return 0; 2152 } 2153 2154 static void be_sriov_disable(struct be_adapter *adapter) 2155 { 2156 #ifdef CONFIG_PCI_IOV 2157 if (adapter->sriov_enabled) { 2158 pci_disable_sriov(adapter->pdev); 2159 kfree(adapter->vf_cfg); 2160 adapter->sriov_enabled = false; 2161 } 2162 #endif 2163 } 2164 2165 static inline int be_msix_vec_get(struct be_adapter *adapter, 2166 struct be_eq_obj *eq_obj) 2167 { 2168 return adapter->msix_entries[eq_obj->eq_idx].vector; 2169 } 2170 2171 static int be_request_irq(struct be_adapter *adapter, 2172 struct be_eq_obj *eq_obj, 2173 void *handler, char *desc, void *context) 2174 { 2175 struct net_device *netdev = adapter->netdev; 2176 int vec; 2177 2178 sprintf(eq_obj->desc, "%s-%s", netdev->name, desc); 2179 vec = be_msix_vec_get(adapter, eq_obj); 2180 return request_irq(vec, handler, 0, eq_obj->desc, context); 2181 } 2182 2183 static void be_free_irq(struct be_adapter *adapter, struct be_eq_obj *eq_obj, 2184 void *context) 2185 { 2186 int vec = be_msix_vec_get(adapter, eq_obj); 2187 free_irq(vec, context); 2188 } 2189 2190 static int be_msix_register(struct be_adapter *adapter) 2191 { 2192 struct be_rx_obj *rxo; 2193 int status, i; 2194 char qname[10]; 2195 2196 status = be_request_irq(adapter, &adapter->tx_eq, be_msix_tx_mcc, "tx", 2197 adapter); 2198 if (status) 2199 goto err; 2200 2201 for_all_rx_queues(adapter, rxo, i) { 2202 sprintf(qname, "rxq%d", i); 2203 status = be_request_irq(adapter, &rxo->rx_eq, be_msix_rx, 2204 qname, rxo); 2205 if (status) 2206 goto err_msix; 2207 } 2208 2209 return 0; 2210 2211 err_msix: 2212 be_free_irq(adapter, &adapter->tx_eq, adapter); 2213 2214 for (i--, rxo = &adapter->rx_obj[i]; i >= 0; i--, rxo--) 2215 be_free_irq(adapter, &rxo->rx_eq, rxo); 2216 2217 err: 2218 dev_warn(&adapter->pdev->dev, 2219 "MSIX Request IRQ failed - err %d\n", status); 2220 be_msix_disable(adapter); 2221 return status; 2222 } 2223 2224 static int be_irq_register(struct be_adapter *adapter) 2225 { 2226 struct net_device *netdev = adapter->netdev; 2227 int status; 2228 2229 if (msix_enabled(adapter)) { 2230 status = be_msix_register(adapter); 2231 if (status == 0) 2232 goto done; 2233 /* INTx is not supported for VF */ 2234 if (!be_physfn(adapter)) 2235 return status; 2236 } 2237 2238 /* INTx */ 2239 netdev->irq = adapter->pdev->irq; 2240 status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name, 2241 adapter); 2242 if (status) { 2243 dev_err(&adapter->pdev->dev, 2244 "INTx request IRQ failed - err %d\n", status); 2245 return status; 2246 } 2247 done: 2248 adapter->isr_registered = true; 2249 return 0; 2250 } 2251 2252 static void be_irq_unregister(struct be_adapter *adapter) 2253 { 2254 struct net_device *netdev = adapter->netdev; 2255 struct be_rx_obj *rxo; 2256 int i; 2257 2258 if (!adapter->isr_registered) 2259 return; 2260 2261 /* INTx */ 2262 if (!msix_enabled(adapter)) { 2263 free_irq(netdev->irq, adapter); 2264 goto done; 2265 } 2266 2267 /* MSIx */ 2268 be_free_irq(adapter, &adapter->tx_eq, adapter); 2269 2270 for_all_rx_queues(adapter, rxo, i) 2271 be_free_irq(adapter, &rxo->rx_eq, rxo); 2272 2273 done: 2274 adapter->isr_registered = false; 2275 } 2276 2277 static void be_rx_queues_clear(struct be_adapter *adapter) 2278 { 2279 struct be_queue_info *q; 2280 struct be_rx_obj *rxo; 2281 int i; 2282 2283 for_all_rx_queues(adapter, rxo, i) { 2284 q = &rxo->q; 2285 if (q->created) { 2286 be_cmd_rxq_destroy(adapter, q); 2287 /* After the rxq is invalidated, wait for a grace time 2288 * of 1ms for all dma to end and the flush compl to 2289 * arrive 2290 */ 2291 mdelay(1); 2292 be_rx_q_clean(adapter, rxo); 2293 } 2294 2295 /* Clear any residual events */ 2296 q = &rxo->rx_eq.q; 2297 if (q->created) 2298 be_eq_clean(adapter, &rxo->rx_eq); 2299 } 2300 } 2301 2302 static int be_close(struct net_device *netdev) 2303 { 2304 struct be_adapter *adapter = netdev_priv(netdev); 2305 struct be_rx_obj *rxo; 2306 struct be_tx_obj *txo; 2307 struct be_eq_obj *tx_eq = &adapter->tx_eq; 2308 int vec, i; 2309 2310 be_async_mcc_disable(adapter); 2311 2312 if (!lancer_chip(adapter)) 2313 be_intr_set(adapter, false); 2314 2315 for_all_rx_queues(adapter, rxo, i) 2316 napi_disable(&rxo->rx_eq.napi); 2317 2318 napi_disable(&tx_eq->napi); 2319 2320 if (lancer_chip(adapter)) { 2321 be_cq_notify(adapter, adapter->mcc_obj.cq.id, false, 0); 2322 for_all_rx_queues(adapter, rxo, i) 2323 be_cq_notify(adapter, rxo->cq.id, false, 0); 2324 for_all_tx_queues(adapter, txo, i) 2325 be_cq_notify(adapter, txo->cq.id, false, 0); 2326 } 2327 2328 if (msix_enabled(adapter)) { 2329 vec = be_msix_vec_get(adapter, tx_eq); 2330 synchronize_irq(vec); 2331 2332 for_all_rx_queues(adapter, rxo, i) { 2333 vec = be_msix_vec_get(adapter, &rxo->rx_eq); 2334 synchronize_irq(vec); 2335 } 2336 } else { 2337 synchronize_irq(netdev->irq); 2338 } 2339 be_irq_unregister(adapter); 2340 2341 /* Wait for all pending tx completions to arrive so that 2342 * all tx skbs are freed. 2343 */ 2344 for_all_tx_queues(adapter, txo, i) 2345 be_tx_compl_clean(adapter, txo); 2346 2347 be_rx_queues_clear(adapter); 2348 return 0; 2349 } 2350 2351 static int be_rx_queues_setup(struct be_adapter *adapter) 2352 { 2353 struct be_rx_obj *rxo; 2354 int rc, i; 2355 u8 rsstable[MAX_RSS_QS]; 2356 2357 for_all_rx_queues(adapter, rxo, i) { 2358 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 2359 rx_frag_size, BE_MAX_JUMBO_FRAME_SIZE, 2360 adapter->if_handle, 2361 (i > 0) ? 1 : 0/* rss enable */, &rxo->rss_id); 2362 if (rc) 2363 return rc; 2364 } 2365 2366 if (be_multi_rxq(adapter)) { 2367 for_all_rss_queues(adapter, rxo, i) 2368 rsstable[i] = rxo->rss_id; 2369 2370 rc = be_cmd_rss_config(adapter, rsstable, 2371 adapter->num_rx_qs - 1); 2372 if (rc) 2373 return rc; 2374 } 2375 2376 /* First time posting */ 2377 for_all_rx_queues(adapter, rxo, i) { 2378 be_post_rx_frags(rxo, GFP_KERNEL); 2379 napi_enable(&rxo->rx_eq.napi); 2380 } 2381 return 0; 2382 } 2383 2384 static int be_open(struct net_device *netdev) 2385 { 2386 struct be_adapter *adapter = netdev_priv(netdev); 2387 struct be_eq_obj *tx_eq = &adapter->tx_eq; 2388 struct be_rx_obj *rxo; 2389 int status, i; 2390 2391 status = be_rx_queues_setup(adapter); 2392 if (status) 2393 goto err; 2394 2395 napi_enable(&tx_eq->napi); 2396 2397 be_irq_register(adapter); 2398 2399 if (!lancer_chip(adapter)) 2400 be_intr_set(adapter, true); 2401 2402 /* The evt queues are created in unarmed state; arm them */ 2403 for_all_rx_queues(adapter, rxo, i) { 2404 be_eq_notify(adapter, rxo->rx_eq.q.id, true, false, 0); 2405 be_cq_notify(adapter, rxo->cq.id, true, 0); 2406 } 2407 be_eq_notify(adapter, tx_eq->q.id, true, false, 0); 2408 2409 /* Now that interrupts are on we can process async mcc */ 2410 be_async_mcc_enable(adapter); 2411 2412 return 0; 2413 err: 2414 be_close(adapter->netdev); 2415 return -EIO; 2416 } 2417 2418 static int be_setup_wol(struct be_adapter *adapter, bool enable) 2419 { 2420 struct be_dma_mem cmd; 2421 int status = 0; 2422 u8 mac[ETH_ALEN]; 2423 2424 memset(mac, 0, ETH_ALEN); 2425 2426 cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config); 2427 cmd.va = dma_alloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma, 2428 GFP_KERNEL); 2429 if (cmd.va == NULL) 2430 return -1; 2431 memset(cmd.va, 0, cmd.size); 2432 2433 if (enable) { 2434 status = pci_write_config_dword(adapter->pdev, 2435 PCICFG_PM_CONTROL_OFFSET, PCICFG_PM_CONTROL_MASK); 2436 if (status) { 2437 dev_err(&adapter->pdev->dev, 2438 "Could not enable Wake-on-lan\n"); 2439 dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, 2440 cmd.dma); 2441 return status; 2442 } 2443 status = be_cmd_enable_magic_wol(adapter, 2444 adapter->netdev->dev_addr, &cmd); 2445 pci_enable_wake(adapter->pdev, PCI_D3hot, 1); 2446 pci_enable_wake(adapter->pdev, PCI_D3cold, 1); 2447 } else { 2448 status = be_cmd_enable_magic_wol(adapter, mac, &cmd); 2449 pci_enable_wake(adapter->pdev, PCI_D3hot, 0); 2450 pci_enable_wake(adapter->pdev, PCI_D3cold, 0); 2451 } 2452 2453 dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma); 2454 return status; 2455 } 2456 2457 /* 2458 * Generate a seed MAC address from the PF MAC Address using jhash. 2459 * MAC Address for VFs are assigned incrementally starting from the seed. 2460 * These addresses are programmed in the ASIC by the PF and the VF driver 2461 * queries for the MAC address during its probe. 2462 */ 2463 static inline int be_vf_eth_addr_config(struct be_adapter *adapter) 2464 { 2465 u32 vf; 2466 int status = 0; 2467 u8 mac[ETH_ALEN]; 2468 2469 be_vf_eth_addr_generate(adapter, mac); 2470 2471 for (vf = 0; vf < num_vfs; vf++) { 2472 status = be_cmd_pmac_add(adapter, mac, 2473 adapter->vf_cfg[vf].vf_if_handle, 2474 &adapter->vf_cfg[vf].vf_pmac_id, 2475 vf + 1); 2476 if (status) 2477 dev_err(&adapter->pdev->dev, 2478 "Mac address add failed for VF %d\n", vf); 2479 else 2480 memcpy(adapter->vf_cfg[vf].vf_mac_addr, mac, ETH_ALEN); 2481 2482 mac[5] += 1; 2483 } 2484 return status; 2485 } 2486 2487 static void be_vf_clear(struct be_adapter *adapter) 2488 { 2489 u32 vf; 2490 2491 for (vf = 0; vf < num_vfs; vf++) { 2492 if (adapter->vf_cfg[vf].vf_pmac_id != BE_INVALID_PMAC_ID) 2493 be_cmd_pmac_del(adapter, 2494 adapter->vf_cfg[vf].vf_if_handle, 2495 adapter->vf_cfg[vf].vf_pmac_id, vf + 1); 2496 } 2497 2498 for (vf = 0; vf < num_vfs; vf++) 2499 if (adapter->vf_cfg[vf].vf_if_handle) 2500 be_cmd_if_destroy(adapter, 2501 adapter->vf_cfg[vf].vf_if_handle, vf + 1); 2502 } 2503 2504 static int be_clear(struct be_adapter *adapter) 2505 { 2506 if (be_physfn(adapter) && adapter->sriov_enabled) 2507 be_vf_clear(adapter); 2508 2509 be_cmd_if_destroy(adapter, adapter->if_handle, 0); 2510 2511 be_mcc_queues_destroy(adapter); 2512 be_rx_queues_destroy(adapter); 2513 be_tx_queues_destroy(adapter); 2514 adapter->eq_next_idx = 0; 2515 2516 adapter->be3_native = false; 2517 adapter->promiscuous = false; 2518 2519 /* tell fw we're done with firing cmds */ 2520 be_cmd_fw_clean(adapter); 2521 return 0; 2522 } 2523 2524 static int be_vf_setup(struct be_adapter *adapter) 2525 { 2526 u32 cap_flags, en_flags, vf; 2527 u16 lnk_speed; 2528 int status; 2529 2530 cap_flags = en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST; 2531 for (vf = 0; vf < num_vfs; vf++) { 2532 status = be_cmd_if_create(adapter, cap_flags, en_flags, NULL, 2533 &adapter->vf_cfg[vf].vf_if_handle, 2534 NULL, vf+1); 2535 if (status) 2536 goto err; 2537 adapter->vf_cfg[vf].vf_pmac_id = BE_INVALID_PMAC_ID; 2538 } 2539 2540 if (!lancer_chip(adapter)) { 2541 status = be_vf_eth_addr_config(adapter); 2542 if (status) 2543 goto err; 2544 } 2545 2546 for (vf = 0; vf < num_vfs; vf++) { 2547 status = be_cmd_link_status_query(adapter, NULL, &lnk_speed, 2548 vf + 1); 2549 if (status) 2550 goto err; 2551 adapter->vf_cfg[vf].vf_tx_rate = lnk_speed * 10; 2552 } 2553 return 0; 2554 err: 2555 return status; 2556 } 2557 2558 static int be_setup(struct be_adapter *adapter) 2559 { 2560 struct net_device *netdev = adapter->netdev; 2561 u32 cap_flags, en_flags; 2562 u32 tx_fc, rx_fc; 2563 int status; 2564 u8 mac[ETH_ALEN]; 2565 2566 /* Allow all priorities by default. A GRP5 evt may modify this */ 2567 adapter->vlan_prio_bmap = 0xff; 2568 adapter->link_speed = -1; 2569 2570 be_cmd_req_native_mode(adapter); 2571 2572 status = be_tx_queues_create(adapter); 2573 if (status != 0) 2574 goto err; 2575 2576 status = be_rx_queues_create(adapter); 2577 if (status != 0) 2578 goto err; 2579 2580 status = be_mcc_queues_create(adapter); 2581 if (status != 0) 2582 goto err; 2583 2584 memset(mac, 0, ETH_ALEN); 2585 status = be_cmd_mac_addr_query(adapter, mac, MAC_ADDRESS_TYPE_NETWORK, 2586 true /*permanent */, 0); 2587 if (status) 2588 return status; 2589 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN); 2590 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN); 2591 2592 en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST | 2593 BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS; 2594 cap_flags = en_flags | BE_IF_FLAGS_MCAST_PROMISCUOUS | 2595 BE_IF_FLAGS_PROMISCUOUS; 2596 if (adapter->function_caps & BE_FUNCTION_CAPS_RSS) { 2597 cap_flags |= BE_IF_FLAGS_RSS; 2598 en_flags |= BE_IF_FLAGS_RSS; 2599 } 2600 status = be_cmd_if_create(adapter, cap_flags, en_flags, 2601 netdev->dev_addr, &adapter->if_handle, 2602 &adapter->pmac_id, 0); 2603 if (status != 0) 2604 goto err; 2605 2606 /* For BEx, the VF's permanent mac queried from card is incorrect. 2607 * Query the mac configued by the PF using if_handle 2608 */ 2609 if (!be_physfn(adapter) && !lancer_chip(adapter)) { 2610 status = be_cmd_mac_addr_query(adapter, mac, 2611 MAC_ADDRESS_TYPE_NETWORK, false, adapter->if_handle); 2612 if (!status) { 2613 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN); 2614 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN); 2615 } 2616 } 2617 2618 be_cmd_get_fw_ver(adapter, adapter->fw_ver, NULL); 2619 2620 status = be_vid_config(adapter, false, 0); 2621 if (status) 2622 goto err; 2623 2624 be_set_rx_mode(adapter->netdev); 2625 2626 status = be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc); 2627 if (status) 2628 goto err; 2629 if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc) { 2630 status = be_cmd_set_flow_control(adapter, adapter->tx_fc, 2631 adapter->rx_fc); 2632 if (status) 2633 goto err; 2634 } 2635 2636 pcie_set_readrq(adapter->pdev, 4096); 2637 2638 if (be_physfn(adapter) && adapter->sriov_enabled) { 2639 status = be_vf_setup(adapter); 2640 if (status) 2641 goto err; 2642 } 2643 2644 return 0; 2645 err: 2646 be_clear(adapter); 2647 return status; 2648 } 2649 2650 #define FW_FILE_HDR_SIGN "ServerEngines Corp. " 2651 static bool be_flash_redboot(struct be_adapter *adapter, 2652 const u8 *p, u32 img_start, int image_size, 2653 int hdr_size) 2654 { 2655 u32 crc_offset; 2656 u8 flashed_crc[4]; 2657 int status; 2658 2659 crc_offset = hdr_size + img_start + image_size - 4; 2660 2661 p += crc_offset; 2662 2663 status = be_cmd_get_flash_crc(adapter, flashed_crc, 2664 (image_size - 4)); 2665 if (status) { 2666 dev_err(&adapter->pdev->dev, 2667 "could not get crc from flash, not flashing redboot\n"); 2668 return false; 2669 } 2670 2671 /*update redboot only if crc does not match*/ 2672 if (!memcmp(flashed_crc, p, 4)) 2673 return false; 2674 else 2675 return true; 2676 } 2677 2678 static bool phy_flashing_required(struct be_adapter *adapter) 2679 { 2680 int status = 0; 2681 struct be_phy_info phy_info; 2682 2683 status = be_cmd_get_phy_info(adapter, &phy_info); 2684 if (status) 2685 return false; 2686 if ((phy_info.phy_type == TN_8022) && 2687 (phy_info.interface_type == PHY_TYPE_BASET_10GB)) { 2688 return true; 2689 } 2690 return false; 2691 } 2692 2693 static int be_flash_data(struct be_adapter *adapter, 2694 const struct firmware *fw, 2695 struct be_dma_mem *flash_cmd, int num_of_images) 2696 2697 { 2698 int status = 0, i, filehdr_size = 0; 2699 u32 total_bytes = 0, flash_op; 2700 int num_bytes; 2701 const u8 *p = fw->data; 2702 struct be_cmd_write_flashrom *req = flash_cmd->va; 2703 const struct flash_comp *pflashcomp; 2704 int num_comp; 2705 2706 static const struct flash_comp gen3_flash_types[10] = { 2707 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, IMG_TYPE_ISCSI_ACTIVE, 2708 FLASH_IMAGE_MAX_SIZE_g3}, 2709 { FLASH_REDBOOT_START_g3, IMG_TYPE_REDBOOT, 2710 FLASH_REDBOOT_IMAGE_MAX_SIZE_g3}, 2711 { FLASH_iSCSI_BIOS_START_g3, IMG_TYPE_BIOS, 2712 FLASH_BIOS_IMAGE_MAX_SIZE_g3}, 2713 { FLASH_PXE_BIOS_START_g3, IMG_TYPE_PXE_BIOS, 2714 FLASH_BIOS_IMAGE_MAX_SIZE_g3}, 2715 { FLASH_FCoE_BIOS_START_g3, IMG_TYPE_FCOE_BIOS, 2716 FLASH_BIOS_IMAGE_MAX_SIZE_g3}, 2717 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, IMG_TYPE_ISCSI_BACKUP, 2718 FLASH_IMAGE_MAX_SIZE_g3}, 2719 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, IMG_TYPE_FCOE_FW_ACTIVE, 2720 FLASH_IMAGE_MAX_SIZE_g3}, 2721 { FLASH_FCoE_BACKUP_IMAGE_START_g3, IMG_TYPE_FCOE_FW_BACKUP, 2722 FLASH_IMAGE_MAX_SIZE_g3}, 2723 { FLASH_NCSI_START_g3, IMG_TYPE_NCSI_FW, 2724 FLASH_NCSI_IMAGE_MAX_SIZE_g3}, 2725 { FLASH_PHY_FW_START_g3, IMG_TYPE_PHY_FW, 2726 FLASH_PHY_FW_IMAGE_MAX_SIZE_g3} 2727 }; 2728 static const struct flash_comp gen2_flash_types[8] = { 2729 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, IMG_TYPE_ISCSI_ACTIVE, 2730 FLASH_IMAGE_MAX_SIZE_g2}, 2731 { FLASH_REDBOOT_START_g2, IMG_TYPE_REDBOOT, 2732 FLASH_REDBOOT_IMAGE_MAX_SIZE_g2}, 2733 { FLASH_iSCSI_BIOS_START_g2, IMG_TYPE_BIOS, 2734 FLASH_BIOS_IMAGE_MAX_SIZE_g2}, 2735 { FLASH_PXE_BIOS_START_g2, IMG_TYPE_PXE_BIOS, 2736 FLASH_BIOS_IMAGE_MAX_SIZE_g2}, 2737 { FLASH_FCoE_BIOS_START_g2, IMG_TYPE_FCOE_BIOS, 2738 FLASH_BIOS_IMAGE_MAX_SIZE_g2}, 2739 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, IMG_TYPE_ISCSI_BACKUP, 2740 FLASH_IMAGE_MAX_SIZE_g2}, 2741 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, IMG_TYPE_FCOE_FW_ACTIVE, 2742 FLASH_IMAGE_MAX_SIZE_g2}, 2743 { FLASH_FCoE_BACKUP_IMAGE_START_g2, IMG_TYPE_FCOE_FW_BACKUP, 2744 FLASH_IMAGE_MAX_SIZE_g2} 2745 }; 2746 2747 if (adapter->generation == BE_GEN3) { 2748 pflashcomp = gen3_flash_types; 2749 filehdr_size = sizeof(struct flash_file_hdr_g3); 2750 num_comp = ARRAY_SIZE(gen3_flash_types); 2751 } else { 2752 pflashcomp = gen2_flash_types; 2753 filehdr_size = sizeof(struct flash_file_hdr_g2); 2754 num_comp = ARRAY_SIZE(gen2_flash_types); 2755 } 2756 for (i = 0; i < num_comp; i++) { 2757 if ((pflashcomp[i].optype == IMG_TYPE_NCSI_FW) && 2758 memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0) 2759 continue; 2760 if (pflashcomp[i].optype == IMG_TYPE_PHY_FW) { 2761 if (!phy_flashing_required(adapter)) 2762 continue; 2763 } 2764 if ((pflashcomp[i].optype == IMG_TYPE_REDBOOT) && 2765 (!be_flash_redboot(adapter, fw->data, 2766 pflashcomp[i].offset, pflashcomp[i].size, filehdr_size + 2767 (num_of_images * sizeof(struct image_hdr))))) 2768 continue; 2769 p = fw->data; 2770 p += filehdr_size + pflashcomp[i].offset 2771 + (num_of_images * sizeof(struct image_hdr)); 2772 if (p + pflashcomp[i].size > fw->data + fw->size) 2773 return -1; 2774 total_bytes = pflashcomp[i].size; 2775 while (total_bytes) { 2776 if (total_bytes > 32*1024) 2777 num_bytes = 32*1024; 2778 else 2779 num_bytes = total_bytes; 2780 total_bytes -= num_bytes; 2781 if (!total_bytes) { 2782 if (pflashcomp[i].optype == IMG_TYPE_PHY_FW) 2783 flash_op = FLASHROM_OPER_PHY_FLASH; 2784 else 2785 flash_op = FLASHROM_OPER_FLASH; 2786 } else { 2787 if (pflashcomp[i].optype == IMG_TYPE_PHY_FW) 2788 flash_op = FLASHROM_OPER_PHY_SAVE; 2789 else 2790 flash_op = FLASHROM_OPER_SAVE; 2791 } 2792 memcpy(req->params.data_buf, p, num_bytes); 2793 p += num_bytes; 2794 status = be_cmd_write_flashrom(adapter, flash_cmd, 2795 pflashcomp[i].optype, flash_op, num_bytes); 2796 if (status) { 2797 if ((status == ILLEGAL_IOCTL_REQ) && 2798 (pflashcomp[i].optype == 2799 IMG_TYPE_PHY_FW)) 2800 break; 2801 dev_err(&adapter->pdev->dev, 2802 "cmd to write to flash rom failed.\n"); 2803 return -1; 2804 } 2805 } 2806 } 2807 return 0; 2808 } 2809 2810 static int get_ufigen_type(struct flash_file_hdr_g2 *fhdr) 2811 { 2812 if (fhdr == NULL) 2813 return 0; 2814 if (fhdr->build[0] == '3') 2815 return BE_GEN3; 2816 else if (fhdr->build[0] == '2') 2817 return BE_GEN2; 2818 else 2819 return 0; 2820 } 2821 2822 static int lancer_fw_download(struct be_adapter *adapter, 2823 const struct firmware *fw) 2824 { 2825 #define LANCER_FW_DOWNLOAD_CHUNK (32 * 1024) 2826 #define LANCER_FW_DOWNLOAD_LOCATION "/prg" 2827 struct be_dma_mem flash_cmd; 2828 const u8 *data_ptr = NULL; 2829 u8 *dest_image_ptr = NULL; 2830 size_t image_size = 0; 2831 u32 chunk_size = 0; 2832 u32 data_written = 0; 2833 u32 offset = 0; 2834 int status = 0; 2835 u8 add_status = 0; 2836 2837 if (!IS_ALIGNED(fw->size, sizeof(u32))) { 2838 dev_err(&adapter->pdev->dev, 2839 "FW Image not properly aligned. " 2840 "Length must be 4 byte aligned.\n"); 2841 status = -EINVAL; 2842 goto lancer_fw_exit; 2843 } 2844 2845 flash_cmd.size = sizeof(struct lancer_cmd_req_write_object) 2846 + LANCER_FW_DOWNLOAD_CHUNK; 2847 flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size, 2848 &flash_cmd.dma, GFP_KERNEL); 2849 if (!flash_cmd.va) { 2850 status = -ENOMEM; 2851 dev_err(&adapter->pdev->dev, 2852 "Memory allocation failure while flashing\n"); 2853 goto lancer_fw_exit; 2854 } 2855 2856 dest_image_ptr = flash_cmd.va + 2857 sizeof(struct lancer_cmd_req_write_object); 2858 image_size = fw->size; 2859 data_ptr = fw->data; 2860 2861 while (image_size) { 2862 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK); 2863 2864 /* Copy the image chunk content. */ 2865 memcpy(dest_image_ptr, data_ptr, chunk_size); 2866 2867 status = lancer_cmd_write_object(adapter, &flash_cmd, 2868 chunk_size, offset, LANCER_FW_DOWNLOAD_LOCATION, 2869 &data_written, &add_status); 2870 2871 if (status) 2872 break; 2873 2874 offset += data_written; 2875 data_ptr += data_written; 2876 image_size -= data_written; 2877 } 2878 2879 if (!status) { 2880 /* Commit the FW written */ 2881 status = lancer_cmd_write_object(adapter, &flash_cmd, 2882 0, offset, LANCER_FW_DOWNLOAD_LOCATION, 2883 &data_written, &add_status); 2884 } 2885 2886 dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va, 2887 flash_cmd.dma); 2888 if (status) { 2889 dev_err(&adapter->pdev->dev, 2890 "Firmware load error. " 2891 "Status code: 0x%x Additional Status: 0x%x\n", 2892 status, add_status); 2893 goto lancer_fw_exit; 2894 } 2895 2896 dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n"); 2897 lancer_fw_exit: 2898 return status; 2899 } 2900 2901 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw) 2902 { 2903 struct flash_file_hdr_g2 *fhdr; 2904 struct flash_file_hdr_g3 *fhdr3; 2905 struct image_hdr *img_hdr_ptr = NULL; 2906 struct be_dma_mem flash_cmd; 2907 const u8 *p; 2908 int status = 0, i = 0, num_imgs = 0; 2909 2910 p = fw->data; 2911 fhdr = (struct flash_file_hdr_g2 *) p; 2912 2913 flash_cmd.size = sizeof(struct be_cmd_write_flashrom) + 32*1024; 2914 flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size, 2915 &flash_cmd.dma, GFP_KERNEL); 2916 if (!flash_cmd.va) { 2917 status = -ENOMEM; 2918 dev_err(&adapter->pdev->dev, 2919 "Memory allocation failure while flashing\n"); 2920 goto be_fw_exit; 2921 } 2922 2923 if ((adapter->generation == BE_GEN3) && 2924 (get_ufigen_type(fhdr) == BE_GEN3)) { 2925 fhdr3 = (struct flash_file_hdr_g3 *) fw->data; 2926 num_imgs = le32_to_cpu(fhdr3->num_imgs); 2927 for (i = 0; i < num_imgs; i++) { 2928 img_hdr_ptr = (struct image_hdr *) (fw->data + 2929 (sizeof(struct flash_file_hdr_g3) + 2930 i * sizeof(struct image_hdr))); 2931 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) 2932 status = be_flash_data(adapter, fw, &flash_cmd, 2933 num_imgs); 2934 } 2935 } else if ((adapter->generation == BE_GEN2) && 2936 (get_ufigen_type(fhdr) == BE_GEN2)) { 2937 status = be_flash_data(adapter, fw, &flash_cmd, 0); 2938 } else { 2939 dev_err(&adapter->pdev->dev, 2940 "UFI and Interface are not compatible for flashing\n"); 2941 status = -1; 2942 } 2943 2944 dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va, 2945 flash_cmd.dma); 2946 if (status) { 2947 dev_err(&adapter->pdev->dev, "Firmware load error\n"); 2948 goto be_fw_exit; 2949 } 2950 2951 dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n"); 2952 2953 be_fw_exit: 2954 return status; 2955 } 2956 2957 int be_load_fw(struct be_adapter *adapter, u8 *fw_file) 2958 { 2959 const struct firmware *fw; 2960 int status; 2961 2962 if (!netif_running(adapter->netdev)) { 2963 dev_err(&adapter->pdev->dev, 2964 "Firmware load not allowed (interface is down)\n"); 2965 return -1; 2966 } 2967 2968 status = request_firmware(&fw, fw_file, &adapter->pdev->dev); 2969 if (status) 2970 goto fw_exit; 2971 2972 dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file); 2973 2974 if (lancer_chip(adapter)) 2975 status = lancer_fw_download(adapter, fw); 2976 else 2977 status = be_fw_download(adapter, fw); 2978 2979 fw_exit: 2980 release_firmware(fw); 2981 return status; 2982 } 2983 2984 static struct net_device_ops be_netdev_ops = { 2985 .ndo_open = be_open, 2986 .ndo_stop = be_close, 2987 .ndo_start_xmit = be_xmit, 2988 .ndo_set_rx_mode = be_set_rx_mode, 2989 .ndo_set_mac_address = be_mac_addr_set, 2990 .ndo_change_mtu = be_change_mtu, 2991 .ndo_get_stats64 = be_get_stats64, 2992 .ndo_validate_addr = eth_validate_addr, 2993 .ndo_vlan_rx_add_vid = be_vlan_add_vid, 2994 .ndo_vlan_rx_kill_vid = be_vlan_rem_vid, 2995 .ndo_set_vf_mac = be_set_vf_mac, 2996 .ndo_set_vf_vlan = be_set_vf_vlan, 2997 .ndo_set_vf_tx_rate = be_set_vf_tx_rate, 2998 .ndo_get_vf_config = be_get_vf_config 2999 }; 3000 3001 static void be_netdev_init(struct net_device *netdev) 3002 { 3003 struct be_adapter *adapter = netdev_priv(netdev); 3004 struct be_rx_obj *rxo; 3005 int i; 3006 3007 netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 3008 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | 3009 NETIF_F_HW_VLAN_TX; 3010 if (be_multi_rxq(adapter)) 3011 netdev->hw_features |= NETIF_F_RXHASH; 3012 3013 netdev->features |= netdev->hw_features | 3014 NETIF_F_HW_VLAN_RX | NETIF_F_HW_VLAN_FILTER; 3015 3016 netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 3017 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 3018 3019 netdev->flags |= IFF_MULTICAST; 3020 3021 netif_set_gso_max_size(netdev, 65535); 3022 3023 BE_SET_NETDEV_OPS(netdev, &be_netdev_ops); 3024 3025 SET_ETHTOOL_OPS(netdev, &be_ethtool_ops); 3026 3027 for_all_rx_queues(adapter, rxo, i) 3028 netif_napi_add(netdev, &rxo->rx_eq.napi, be_poll_rx, 3029 BE_NAPI_WEIGHT); 3030 3031 netif_napi_add(netdev, &adapter->tx_eq.napi, be_poll_tx_mcc, 3032 BE_NAPI_WEIGHT); 3033 } 3034 3035 static void be_unmap_pci_bars(struct be_adapter *adapter) 3036 { 3037 if (adapter->csr) 3038 iounmap(adapter->csr); 3039 if (adapter->db) 3040 iounmap(adapter->db); 3041 } 3042 3043 static int be_map_pci_bars(struct be_adapter *adapter) 3044 { 3045 u8 __iomem *addr; 3046 int db_reg; 3047 3048 if (lancer_chip(adapter)) { 3049 addr = ioremap_nocache(pci_resource_start(adapter->pdev, 0), 3050 pci_resource_len(adapter->pdev, 0)); 3051 if (addr == NULL) 3052 return -ENOMEM; 3053 adapter->db = addr; 3054 return 0; 3055 } 3056 3057 if (be_physfn(adapter)) { 3058 addr = ioremap_nocache(pci_resource_start(adapter->pdev, 2), 3059 pci_resource_len(adapter->pdev, 2)); 3060 if (addr == NULL) 3061 return -ENOMEM; 3062 adapter->csr = addr; 3063 } 3064 3065 if (adapter->generation == BE_GEN2) { 3066 db_reg = 4; 3067 } else { 3068 if (be_physfn(adapter)) 3069 db_reg = 4; 3070 else 3071 db_reg = 0; 3072 } 3073 addr = ioremap_nocache(pci_resource_start(adapter->pdev, db_reg), 3074 pci_resource_len(adapter->pdev, db_reg)); 3075 if (addr == NULL) 3076 goto pci_map_err; 3077 adapter->db = addr; 3078 3079 return 0; 3080 pci_map_err: 3081 be_unmap_pci_bars(adapter); 3082 return -ENOMEM; 3083 } 3084 3085 3086 static void be_ctrl_cleanup(struct be_adapter *adapter) 3087 { 3088 struct be_dma_mem *mem = &adapter->mbox_mem_alloced; 3089 3090 be_unmap_pci_bars(adapter); 3091 3092 if (mem->va) 3093 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 3094 mem->dma); 3095 3096 mem = &adapter->rx_filter; 3097 if (mem->va) 3098 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 3099 mem->dma); 3100 } 3101 3102 static int be_ctrl_init(struct be_adapter *adapter) 3103 { 3104 struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced; 3105 struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem; 3106 struct be_dma_mem *rx_filter = &adapter->rx_filter; 3107 int status; 3108 3109 status = be_map_pci_bars(adapter); 3110 if (status) 3111 goto done; 3112 3113 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 3114 mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev, 3115 mbox_mem_alloc->size, 3116 &mbox_mem_alloc->dma, 3117 GFP_KERNEL); 3118 if (!mbox_mem_alloc->va) { 3119 status = -ENOMEM; 3120 goto unmap_pci_bars; 3121 } 3122 mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 3123 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 3124 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 3125 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox)); 3126 3127 rx_filter->size = sizeof(struct be_cmd_req_rx_filter); 3128 rx_filter->va = dma_alloc_coherent(&adapter->pdev->dev, rx_filter->size, 3129 &rx_filter->dma, GFP_KERNEL); 3130 if (rx_filter->va == NULL) { 3131 status = -ENOMEM; 3132 goto free_mbox; 3133 } 3134 memset(rx_filter->va, 0, rx_filter->size); 3135 3136 mutex_init(&adapter->mbox_lock); 3137 spin_lock_init(&adapter->mcc_lock); 3138 spin_lock_init(&adapter->mcc_cq_lock); 3139 3140 init_completion(&adapter->flash_compl); 3141 pci_save_state(adapter->pdev); 3142 return 0; 3143 3144 free_mbox: 3145 dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size, 3146 mbox_mem_alloc->va, mbox_mem_alloc->dma); 3147 3148 unmap_pci_bars: 3149 be_unmap_pci_bars(adapter); 3150 3151 done: 3152 return status; 3153 } 3154 3155 static void be_stats_cleanup(struct be_adapter *adapter) 3156 { 3157 struct be_dma_mem *cmd = &adapter->stats_cmd; 3158 3159 if (cmd->va) 3160 dma_free_coherent(&adapter->pdev->dev, cmd->size, 3161 cmd->va, cmd->dma); 3162 } 3163 3164 static int be_stats_init(struct be_adapter *adapter) 3165 { 3166 struct be_dma_mem *cmd = &adapter->stats_cmd; 3167 3168 if (adapter->generation == BE_GEN2) { 3169 cmd->size = sizeof(struct be_cmd_req_get_stats_v0); 3170 } else { 3171 if (lancer_chip(adapter)) 3172 cmd->size = sizeof(struct lancer_cmd_req_pport_stats); 3173 else 3174 cmd->size = sizeof(struct be_cmd_req_get_stats_v1); 3175 } 3176 cmd->va = dma_alloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma, 3177 GFP_KERNEL); 3178 if (cmd->va == NULL) 3179 return -1; 3180 memset(cmd->va, 0, cmd->size); 3181 return 0; 3182 } 3183 3184 static void __devexit be_remove(struct pci_dev *pdev) 3185 { 3186 struct be_adapter *adapter = pci_get_drvdata(pdev); 3187 3188 if (!adapter) 3189 return; 3190 3191 cancel_delayed_work_sync(&adapter->work); 3192 3193 unregister_netdev(adapter->netdev); 3194 3195 be_clear(adapter); 3196 3197 be_stats_cleanup(adapter); 3198 3199 be_ctrl_cleanup(adapter); 3200 3201 be_sriov_disable(adapter); 3202 3203 be_msix_disable(adapter); 3204 3205 pci_set_drvdata(pdev, NULL); 3206 pci_release_regions(pdev); 3207 pci_disable_device(pdev); 3208 3209 free_netdev(adapter->netdev); 3210 } 3211 3212 static int be_get_config(struct be_adapter *adapter) 3213 { 3214 int status; 3215 3216 status = be_cmd_query_fw_cfg(adapter, &adapter->port_num, 3217 &adapter->function_mode, &adapter->function_caps); 3218 if (status) 3219 return status; 3220 3221 if (adapter->function_mode & FLEX10_MODE) 3222 adapter->max_vlans = BE_NUM_VLANS_SUPPORTED/4; 3223 else 3224 adapter->max_vlans = BE_NUM_VLANS_SUPPORTED; 3225 3226 status = be_cmd_get_cntl_attributes(adapter); 3227 if (status) 3228 return status; 3229 3230 return 0; 3231 } 3232 3233 static int be_dev_family_check(struct be_adapter *adapter) 3234 { 3235 struct pci_dev *pdev = adapter->pdev; 3236 u32 sli_intf = 0, if_type; 3237 3238 switch (pdev->device) { 3239 case BE_DEVICE_ID1: 3240 case OC_DEVICE_ID1: 3241 adapter->generation = BE_GEN2; 3242 break; 3243 case BE_DEVICE_ID2: 3244 case OC_DEVICE_ID2: 3245 adapter->generation = BE_GEN3; 3246 break; 3247 case OC_DEVICE_ID3: 3248 case OC_DEVICE_ID4: 3249 pci_read_config_dword(pdev, SLI_INTF_REG_OFFSET, &sli_intf); 3250 if_type = (sli_intf & SLI_INTF_IF_TYPE_MASK) >> 3251 SLI_INTF_IF_TYPE_SHIFT; 3252 3253 if (((sli_intf & SLI_INTF_VALID_MASK) != SLI_INTF_VALID) || 3254 if_type != 0x02) { 3255 dev_err(&pdev->dev, "SLI_INTF reg val is not valid\n"); 3256 return -EINVAL; 3257 } 3258 adapter->sli_family = ((sli_intf & SLI_INTF_FAMILY_MASK) >> 3259 SLI_INTF_FAMILY_SHIFT); 3260 adapter->generation = BE_GEN3; 3261 break; 3262 default: 3263 adapter->generation = 0; 3264 } 3265 return 0; 3266 } 3267 3268 static int lancer_wait_ready(struct be_adapter *adapter) 3269 { 3270 #define SLIPORT_READY_TIMEOUT 500 3271 u32 sliport_status; 3272 int status = 0, i; 3273 3274 for (i = 0; i < SLIPORT_READY_TIMEOUT; i++) { 3275 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET); 3276 if (sliport_status & SLIPORT_STATUS_RDY_MASK) 3277 break; 3278 3279 msleep(20); 3280 } 3281 3282 if (i == SLIPORT_READY_TIMEOUT) 3283 status = -1; 3284 3285 return status; 3286 } 3287 3288 static int lancer_test_and_set_rdy_state(struct be_adapter *adapter) 3289 { 3290 int status; 3291 u32 sliport_status, err, reset_needed; 3292 status = lancer_wait_ready(adapter); 3293 if (!status) { 3294 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET); 3295 err = sliport_status & SLIPORT_STATUS_ERR_MASK; 3296 reset_needed = sliport_status & SLIPORT_STATUS_RN_MASK; 3297 if (err && reset_needed) { 3298 iowrite32(SLI_PORT_CONTROL_IP_MASK, 3299 adapter->db + SLIPORT_CONTROL_OFFSET); 3300 3301 /* check adapter has corrected the error */ 3302 status = lancer_wait_ready(adapter); 3303 sliport_status = ioread32(adapter->db + 3304 SLIPORT_STATUS_OFFSET); 3305 sliport_status &= (SLIPORT_STATUS_ERR_MASK | 3306 SLIPORT_STATUS_RN_MASK); 3307 if (status || sliport_status) 3308 status = -1; 3309 } else if (err || reset_needed) { 3310 status = -1; 3311 } 3312 } 3313 return status; 3314 } 3315 3316 static int __devinit be_probe(struct pci_dev *pdev, 3317 const struct pci_device_id *pdev_id) 3318 { 3319 int status = 0; 3320 struct be_adapter *adapter; 3321 struct net_device *netdev; 3322 3323 status = pci_enable_device(pdev); 3324 if (status) 3325 goto do_none; 3326 3327 status = pci_request_regions(pdev, DRV_NAME); 3328 if (status) 3329 goto disable_dev; 3330 pci_set_master(pdev); 3331 3332 netdev = alloc_etherdev_mq(sizeof(struct be_adapter), MAX_TX_QS); 3333 if (netdev == NULL) { 3334 status = -ENOMEM; 3335 goto rel_reg; 3336 } 3337 adapter = netdev_priv(netdev); 3338 adapter->pdev = pdev; 3339 pci_set_drvdata(pdev, adapter); 3340 3341 status = be_dev_family_check(adapter); 3342 if (status) 3343 goto free_netdev; 3344 3345 adapter->netdev = netdev; 3346 SET_NETDEV_DEV(netdev, &pdev->dev); 3347 3348 status = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); 3349 if (!status) { 3350 netdev->features |= NETIF_F_HIGHDMA; 3351 } else { 3352 status = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 3353 if (status) { 3354 dev_err(&pdev->dev, "Could not set PCI DMA Mask\n"); 3355 goto free_netdev; 3356 } 3357 } 3358 3359 status = be_sriov_enable(adapter); 3360 if (status) 3361 goto free_netdev; 3362 3363 status = be_ctrl_init(adapter); 3364 if (status) 3365 goto disable_sriov; 3366 3367 if (lancer_chip(adapter)) { 3368 status = lancer_test_and_set_rdy_state(adapter); 3369 if (status) { 3370 dev_err(&pdev->dev, "Adapter in non recoverable error\n"); 3371 goto ctrl_clean; 3372 } 3373 } 3374 3375 /* sync up with fw's ready state */ 3376 if (be_physfn(adapter)) { 3377 status = be_cmd_POST(adapter); 3378 if (status) 3379 goto ctrl_clean; 3380 } 3381 3382 /* tell fw we're ready to fire cmds */ 3383 status = be_cmd_fw_init(adapter); 3384 if (status) 3385 goto ctrl_clean; 3386 3387 status = be_cmd_reset_function(adapter); 3388 if (status) 3389 goto ctrl_clean; 3390 3391 status = be_stats_init(adapter); 3392 if (status) 3393 goto ctrl_clean; 3394 3395 status = be_get_config(adapter); 3396 if (status) 3397 goto stats_clean; 3398 3399 /* The INTR bit may be set in the card when probed by a kdump kernel 3400 * after a crash. 3401 */ 3402 if (!lancer_chip(adapter)) 3403 be_intr_set(adapter, false); 3404 3405 be_msix_enable(adapter); 3406 3407 INIT_DELAYED_WORK(&adapter->work, be_worker); 3408 adapter->rx_fc = adapter->tx_fc = true; 3409 3410 status = be_setup(adapter); 3411 if (status) 3412 goto msix_disable; 3413 3414 be_netdev_init(netdev); 3415 status = register_netdev(netdev); 3416 if (status != 0) 3417 goto unsetup; 3418 3419 dev_info(&pdev->dev, "%s port %d\n", nic_name(pdev), adapter->port_num); 3420 3421 schedule_delayed_work(&adapter->work, msecs_to_jiffies(100)); 3422 return 0; 3423 3424 unsetup: 3425 be_clear(adapter); 3426 msix_disable: 3427 be_msix_disable(adapter); 3428 stats_clean: 3429 be_stats_cleanup(adapter); 3430 ctrl_clean: 3431 be_ctrl_cleanup(adapter); 3432 disable_sriov: 3433 be_sriov_disable(adapter); 3434 free_netdev: 3435 free_netdev(netdev); 3436 pci_set_drvdata(pdev, NULL); 3437 rel_reg: 3438 pci_release_regions(pdev); 3439 disable_dev: 3440 pci_disable_device(pdev); 3441 do_none: 3442 dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev)); 3443 return status; 3444 } 3445 3446 static int be_suspend(struct pci_dev *pdev, pm_message_t state) 3447 { 3448 struct be_adapter *adapter = pci_get_drvdata(pdev); 3449 struct net_device *netdev = adapter->netdev; 3450 3451 cancel_delayed_work_sync(&adapter->work); 3452 if (adapter->wol) 3453 be_setup_wol(adapter, true); 3454 3455 netif_device_detach(netdev); 3456 if (netif_running(netdev)) { 3457 rtnl_lock(); 3458 be_close(netdev); 3459 rtnl_unlock(); 3460 } 3461 be_clear(adapter); 3462 3463 be_msix_disable(adapter); 3464 pci_save_state(pdev); 3465 pci_disable_device(pdev); 3466 pci_set_power_state(pdev, pci_choose_state(pdev, state)); 3467 return 0; 3468 } 3469 3470 static int be_resume(struct pci_dev *pdev) 3471 { 3472 int status = 0; 3473 struct be_adapter *adapter = pci_get_drvdata(pdev); 3474 struct net_device *netdev = adapter->netdev; 3475 3476 netif_device_detach(netdev); 3477 3478 status = pci_enable_device(pdev); 3479 if (status) 3480 return status; 3481 3482 pci_set_power_state(pdev, 0); 3483 pci_restore_state(pdev); 3484 3485 be_msix_enable(adapter); 3486 /* tell fw we're ready to fire cmds */ 3487 status = be_cmd_fw_init(adapter); 3488 if (status) 3489 return status; 3490 3491 be_setup(adapter); 3492 if (netif_running(netdev)) { 3493 rtnl_lock(); 3494 be_open(netdev); 3495 rtnl_unlock(); 3496 } 3497 netif_device_attach(netdev); 3498 3499 if (adapter->wol) 3500 be_setup_wol(adapter, false); 3501 3502 schedule_delayed_work(&adapter->work, msecs_to_jiffies(100)); 3503 return 0; 3504 } 3505 3506 /* 3507 * An FLR will stop BE from DMAing any data. 3508 */ 3509 static void be_shutdown(struct pci_dev *pdev) 3510 { 3511 struct be_adapter *adapter = pci_get_drvdata(pdev); 3512 3513 if (!adapter) 3514 return; 3515 3516 cancel_delayed_work_sync(&adapter->work); 3517 3518 netif_device_detach(adapter->netdev); 3519 3520 if (adapter->wol) 3521 be_setup_wol(adapter, true); 3522 3523 be_cmd_reset_function(adapter); 3524 3525 pci_disable_device(pdev); 3526 } 3527 3528 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev, 3529 pci_channel_state_t state) 3530 { 3531 struct be_adapter *adapter = pci_get_drvdata(pdev); 3532 struct net_device *netdev = adapter->netdev; 3533 3534 dev_err(&adapter->pdev->dev, "EEH error detected\n"); 3535 3536 adapter->eeh_err = true; 3537 3538 netif_device_detach(netdev); 3539 3540 if (netif_running(netdev)) { 3541 rtnl_lock(); 3542 be_close(netdev); 3543 rtnl_unlock(); 3544 } 3545 be_clear(adapter); 3546 3547 if (state == pci_channel_io_perm_failure) 3548 return PCI_ERS_RESULT_DISCONNECT; 3549 3550 pci_disable_device(pdev); 3551 3552 return PCI_ERS_RESULT_NEED_RESET; 3553 } 3554 3555 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev) 3556 { 3557 struct be_adapter *adapter = pci_get_drvdata(pdev); 3558 int status; 3559 3560 dev_info(&adapter->pdev->dev, "EEH reset\n"); 3561 adapter->eeh_err = false; 3562 3563 status = pci_enable_device(pdev); 3564 if (status) 3565 return PCI_ERS_RESULT_DISCONNECT; 3566 3567 pci_set_master(pdev); 3568 pci_set_power_state(pdev, 0); 3569 pci_restore_state(pdev); 3570 3571 /* Check if card is ok and fw is ready */ 3572 status = be_cmd_POST(adapter); 3573 if (status) 3574 return PCI_ERS_RESULT_DISCONNECT; 3575 3576 return PCI_ERS_RESULT_RECOVERED; 3577 } 3578 3579 static void be_eeh_resume(struct pci_dev *pdev) 3580 { 3581 int status = 0; 3582 struct be_adapter *adapter = pci_get_drvdata(pdev); 3583 struct net_device *netdev = adapter->netdev; 3584 3585 dev_info(&adapter->pdev->dev, "EEH resume\n"); 3586 3587 pci_save_state(pdev); 3588 3589 /* tell fw we're ready to fire cmds */ 3590 status = be_cmd_fw_init(adapter); 3591 if (status) 3592 goto err; 3593 3594 status = be_setup(adapter); 3595 if (status) 3596 goto err; 3597 3598 if (netif_running(netdev)) { 3599 status = be_open(netdev); 3600 if (status) 3601 goto err; 3602 } 3603 netif_device_attach(netdev); 3604 return; 3605 err: 3606 dev_err(&adapter->pdev->dev, "EEH resume failed\n"); 3607 } 3608 3609 static struct pci_error_handlers be_eeh_handlers = { 3610 .error_detected = be_eeh_err_detected, 3611 .slot_reset = be_eeh_reset, 3612 .resume = be_eeh_resume, 3613 }; 3614 3615 static struct pci_driver be_driver = { 3616 .name = DRV_NAME, 3617 .id_table = be_dev_ids, 3618 .probe = be_probe, 3619 .remove = be_remove, 3620 .suspend = be_suspend, 3621 .resume = be_resume, 3622 .shutdown = be_shutdown, 3623 .err_handler = &be_eeh_handlers 3624 }; 3625 3626 static int __init be_init_module(void) 3627 { 3628 if (rx_frag_size != 8192 && rx_frag_size != 4096 && 3629 rx_frag_size != 2048) { 3630 printk(KERN_WARNING DRV_NAME 3631 " : Module param rx_frag_size must be 2048/4096/8192." 3632 " Using 2048\n"); 3633 rx_frag_size = 2048; 3634 } 3635 3636 return pci_register_driver(&be_driver); 3637 } 3638 module_init(be_init_module); 3639 3640 static void __exit be_exit_module(void) 3641 { 3642 pci_unregister_driver(&be_driver); 3643 } 3644 module_exit(be_exit_module); 3645