1 /******************************************************************************* 2 3 Intel 82599 Virtual Function driver 4 Copyright(c) 1999 - 2015 Intel Corporation. 5 6 This program is free software; you can redistribute it and/or modify it 7 under the terms and conditions of the GNU General Public License, 8 version 2, as published by the Free Software Foundation. 9 10 This program is distributed in the hope it will be useful, but WITHOUT 11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 more details. 14 15 You should have received a copy of the GNU General Public License along with 16 this program; if not, see <http://www.gnu.org/licenses/>. 17 18 The full GNU General Public License is included in this distribution in 19 the file called "COPYING". 20 21 Contact Information: 22 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 25 *******************************************************************************/ 26 27 /****************************************************************************** 28 Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code 29 ******************************************************************************/ 30 31 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 32 33 #include <linux/types.h> 34 #include <linux/bitops.h> 35 #include <linux/module.h> 36 #include <linux/pci.h> 37 #include <linux/netdevice.h> 38 #include <linux/vmalloc.h> 39 #include <linux/string.h> 40 #include <linux/in.h> 41 #include <linux/ip.h> 42 #include <linux/tcp.h> 43 #include <linux/sctp.h> 44 #include <linux/ipv6.h> 45 #include <linux/slab.h> 46 #include <net/checksum.h> 47 #include <net/ip6_checksum.h> 48 #include <linux/ethtool.h> 49 #include <linux/if.h> 50 #include <linux/if_vlan.h> 51 #include <linux/prefetch.h> 52 53 #include "ixgbevf.h" 54 55 const char ixgbevf_driver_name[] = "ixgbevf"; 56 static const char ixgbevf_driver_string[] = 57 "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver"; 58 59 #define DRV_VERSION "2.12.1-k" 60 const char ixgbevf_driver_version[] = DRV_VERSION; 61 static char ixgbevf_copyright[] = 62 "Copyright (c) 2009 - 2015 Intel Corporation."; 63 64 static const struct ixgbevf_info *ixgbevf_info_tbl[] = { 65 [board_82599_vf] = &ixgbevf_82599_vf_info, 66 [board_82599_vf_hv] = &ixgbevf_82599_vf_hv_info, 67 [board_X540_vf] = &ixgbevf_X540_vf_info, 68 [board_X540_vf_hv] = &ixgbevf_X540_vf_hv_info, 69 [board_X550_vf] = &ixgbevf_X550_vf_info, 70 [board_X550_vf_hv] = &ixgbevf_X550_vf_hv_info, 71 [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info, 72 [board_X550EM_x_vf_hv] = &ixgbevf_X550EM_x_vf_hv_info, 73 }; 74 75 /* ixgbevf_pci_tbl - PCI Device ID Table 76 * 77 * Wildcard entries (PCI_ANY_ID) should come last 78 * Last entry must be all 0s 79 * 80 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 81 * Class, Class Mask, private data (not used) } 82 */ 83 static const struct pci_device_id ixgbevf_pci_tbl[] = { 84 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf }, 85 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF_HV), board_82599_vf_hv }, 86 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf }, 87 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF_HV), board_X540_vf_hv }, 88 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf }, 89 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF_HV), board_X550_vf_hv }, 90 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf }, 91 {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV), board_X550EM_x_vf_hv}, 92 /* required last entry */ 93 {0, } 94 }; 95 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl); 96 97 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 98 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver"); 99 MODULE_LICENSE("GPL"); 100 MODULE_VERSION(DRV_VERSION); 101 102 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK) 103 static int debug = -1; 104 module_param(debug, int, 0); 105 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 106 107 static struct workqueue_struct *ixgbevf_wq; 108 109 static void ixgbevf_service_event_schedule(struct ixgbevf_adapter *adapter) 110 { 111 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 112 !test_bit(__IXGBEVF_REMOVING, &adapter->state) && 113 !test_and_set_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)) 114 queue_work(ixgbevf_wq, &adapter->service_task); 115 } 116 117 static void ixgbevf_service_event_complete(struct ixgbevf_adapter *adapter) 118 { 119 BUG_ON(!test_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state)); 120 121 /* flush memory to make sure state is correct before next watchdog */ 122 smp_mb__before_atomic(); 123 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 124 } 125 126 /* forward decls */ 127 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter); 128 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector); 129 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter); 130 131 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw) 132 { 133 struct ixgbevf_adapter *adapter = hw->back; 134 135 if (!hw->hw_addr) 136 return; 137 hw->hw_addr = NULL; 138 dev_err(&adapter->pdev->dev, "Adapter removed\n"); 139 if (test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 140 ixgbevf_service_event_schedule(adapter); 141 } 142 143 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg) 144 { 145 u32 value; 146 147 /* The following check not only optimizes a bit by not 148 * performing a read on the status register when the 149 * register just read was a status register read that 150 * returned IXGBE_FAILED_READ_REG. It also blocks any 151 * potential recursion. 152 */ 153 if (reg == IXGBE_VFSTATUS) { 154 ixgbevf_remove_adapter(hw); 155 return; 156 } 157 value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS); 158 if (value == IXGBE_FAILED_READ_REG) 159 ixgbevf_remove_adapter(hw); 160 } 161 162 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg) 163 { 164 u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr); 165 u32 value; 166 167 if (IXGBE_REMOVED(reg_addr)) 168 return IXGBE_FAILED_READ_REG; 169 value = readl(reg_addr + reg); 170 if (unlikely(value == IXGBE_FAILED_READ_REG)) 171 ixgbevf_check_remove(hw, reg); 172 return value; 173 } 174 175 /** 176 * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors 177 * @adapter: pointer to adapter struct 178 * @direction: 0 for Rx, 1 for Tx, -1 for other causes 179 * @queue: queue to map the corresponding interrupt to 180 * @msix_vector: the vector to map to the corresponding queue 181 **/ 182 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction, 183 u8 queue, u8 msix_vector) 184 { 185 u32 ivar, index; 186 struct ixgbe_hw *hw = &adapter->hw; 187 188 if (direction == -1) { 189 /* other causes */ 190 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 191 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC); 192 ivar &= ~0xFF; 193 ivar |= msix_vector; 194 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar); 195 } else { 196 /* Tx or Rx causes */ 197 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 198 index = ((16 * (queue & 1)) + (8 * direction)); 199 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1)); 200 ivar &= ~(0xFF << index); 201 ivar |= (msix_vector << index); 202 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar); 203 } 204 } 205 206 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring, 207 struct ixgbevf_tx_buffer *tx_buffer) 208 { 209 if (tx_buffer->skb) { 210 dev_kfree_skb_any(tx_buffer->skb); 211 if (dma_unmap_len(tx_buffer, len)) 212 dma_unmap_single(tx_ring->dev, 213 dma_unmap_addr(tx_buffer, dma), 214 dma_unmap_len(tx_buffer, len), 215 DMA_TO_DEVICE); 216 } else if (dma_unmap_len(tx_buffer, len)) { 217 dma_unmap_page(tx_ring->dev, 218 dma_unmap_addr(tx_buffer, dma), 219 dma_unmap_len(tx_buffer, len), 220 DMA_TO_DEVICE); 221 } 222 tx_buffer->next_to_watch = NULL; 223 tx_buffer->skb = NULL; 224 dma_unmap_len_set(tx_buffer, len, 0); 225 /* tx_buffer must be completely set up in the transmit path */ 226 } 227 228 static u64 ixgbevf_get_tx_completed(struct ixgbevf_ring *ring) 229 { 230 return ring->stats.packets; 231 } 232 233 static u32 ixgbevf_get_tx_pending(struct ixgbevf_ring *ring) 234 { 235 struct ixgbevf_adapter *adapter = netdev_priv(ring->netdev); 236 struct ixgbe_hw *hw = &adapter->hw; 237 238 u32 head = IXGBE_READ_REG(hw, IXGBE_VFTDH(ring->reg_idx)); 239 u32 tail = IXGBE_READ_REG(hw, IXGBE_VFTDT(ring->reg_idx)); 240 241 if (head != tail) 242 return (head < tail) ? 243 tail - head : (tail + ring->count - head); 244 245 return 0; 246 } 247 248 static inline bool ixgbevf_check_tx_hang(struct ixgbevf_ring *tx_ring) 249 { 250 u32 tx_done = ixgbevf_get_tx_completed(tx_ring); 251 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; 252 u32 tx_pending = ixgbevf_get_tx_pending(tx_ring); 253 254 clear_check_for_tx_hang(tx_ring); 255 256 /* Check for a hung queue, but be thorough. This verifies 257 * that a transmit has been completed since the previous 258 * check AND there is at least one packet pending. The 259 * ARMED bit is set to indicate a potential hang. 260 */ 261 if ((tx_done_old == tx_done) && tx_pending) { 262 /* make sure it is true for two checks in a row */ 263 return test_and_set_bit(__IXGBEVF_HANG_CHECK_ARMED, 264 &tx_ring->state); 265 } 266 /* reset the countdown */ 267 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &tx_ring->state); 268 269 /* update completed stats and continue */ 270 tx_ring->tx_stats.tx_done_old = tx_done; 271 272 return false; 273 } 274 275 static void ixgbevf_tx_timeout_reset(struct ixgbevf_adapter *adapter) 276 { 277 /* Do the reset outside of interrupt context */ 278 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 279 set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state); 280 ixgbevf_service_event_schedule(adapter); 281 } 282 } 283 284 /** 285 * ixgbevf_tx_timeout - Respond to a Tx Hang 286 * @netdev: network interface device structure 287 **/ 288 static void ixgbevf_tx_timeout(struct net_device *netdev) 289 { 290 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 291 292 ixgbevf_tx_timeout_reset(adapter); 293 } 294 295 /** 296 * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes 297 * @q_vector: board private structure 298 * @tx_ring: tx ring to clean 299 * @napi_budget: Used to determine if we are in netpoll 300 **/ 301 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, 302 struct ixgbevf_ring *tx_ring, int napi_budget) 303 { 304 struct ixgbevf_adapter *adapter = q_vector->adapter; 305 struct ixgbevf_tx_buffer *tx_buffer; 306 union ixgbe_adv_tx_desc *tx_desc; 307 unsigned int total_bytes = 0, total_packets = 0; 308 unsigned int budget = tx_ring->count / 2; 309 unsigned int i = tx_ring->next_to_clean; 310 311 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 312 return true; 313 314 tx_buffer = &tx_ring->tx_buffer_info[i]; 315 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 316 i -= tx_ring->count; 317 318 do { 319 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch; 320 321 /* if next_to_watch is not set then there is no work pending */ 322 if (!eop_desc) 323 break; 324 325 /* prevent any other reads prior to eop_desc */ 326 read_barrier_depends(); 327 328 /* if DD is not set pending work has not been completed */ 329 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) 330 break; 331 332 /* clear next_to_watch to prevent false hangs */ 333 tx_buffer->next_to_watch = NULL; 334 335 /* update the statistics for this packet */ 336 total_bytes += tx_buffer->bytecount; 337 total_packets += tx_buffer->gso_segs; 338 339 /* free the skb */ 340 napi_consume_skb(tx_buffer->skb, napi_budget); 341 342 /* unmap skb header data */ 343 dma_unmap_single(tx_ring->dev, 344 dma_unmap_addr(tx_buffer, dma), 345 dma_unmap_len(tx_buffer, len), 346 DMA_TO_DEVICE); 347 348 /* clear tx_buffer data */ 349 tx_buffer->skb = NULL; 350 dma_unmap_len_set(tx_buffer, len, 0); 351 352 /* unmap remaining buffers */ 353 while (tx_desc != eop_desc) { 354 tx_buffer++; 355 tx_desc++; 356 i++; 357 if (unlikely(!i)) { 358 i -= tx_ring->count; 359 tx_buffer = tx_ring->tx_buffer_info; 360 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 361 } 362 363 /* unmap any remaining paged data */ 364 if (dma_unmap_len(tx_buffer, len)) { 365 dma_unmap_page(tx_ring->dev, 366 dma_unmap_addr(tx_buffer, dma), 367 dma_unmap_len(tx_buffer, len), 368 DMA_TO_DEVICE); 369 dma_unmap_len_set(tx_buffer, len, 0); 370 } 371 } 372 373 /* move us one more past the eop_desc for start of next pkt */ 374 tx_buffer++; 375 tx_desc++; 376 i++; 377 if (unlikely(!i)) { 378 i -= tx_ring->count; 379 tx_buffer = tx_ring->tx_buffer_info; 380 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 381 } 382 383 /* issue prefetch for next Tx descriptor */ 384 prefetch(tx_desc); 385 386 /* update budget accounting */ 387 budget--; 388 } while (likely(budget)); 389 390 i += tx_ring->count; 391 tx_ring->next_to_clean = i; 392 u64_stats_update_begin(&tx_ring->syncp); 393 tx_ring->stats.bytes += total_bytes; 394 tx_ring->stats.packets += total_packets; 395 u64_stats_update_end(&tx_ring->syncp); 396 q_vector->tx.total_bytes += total_bytes; 397 q_vector->tx.total_packets += total_packets; 398 399 if (check_for_tx_hang(tx_ring) && ixgbevf_check_tx_hang(tx_ring)) { 400 struct ixgbe_hw *hw = &adapter->hw; 401 union ixgbe_adv_tx_desc *eop_desc; 402 403 eop_desc = tx_ring->tx_buffer_info[i].next_to_watch; 404 405 pr_err("Detected Tx Unit Hang\n" 406 " Tx Queue <%d>\n" 407 " TDH, TDT <%x>, <%x>\n" 408 " next_to_use <%x>\n" 409 " next_to_clean <%x>\n" 410 "tx_buffer_info[next_to_clean]\n" 411 " next_to_watch <%p>\n" 412 " eop_desc->wb.status <%x>\n" 413 " time_stamp <%lx>\n" 414 " jiffies <%lx>\n", 415 tx_ring->queue_index, 416 IXGBE_READ_REG(hw, IXGBE_VFTDH(tx_ring->reg_idx)), 417 IXGBE_READ_REG(hw, IXGBE_VFTDT(tx_ring->reg_idx)), 418 tx_ring->next_to_use, i, 419 eop_desc, (eop_desc ? eop_desc->wb.status : 0), 420 tx_ring->tx_buffer_info[i].time_stamp, jiffies); 421 422 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 423 424 /* schedule immediate reset if we believe we hung */ 425 ixgbevf_tx_timeout_reset(adapter); 426 427 return true; 428 } 429 430 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) 431 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 432 (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) { 433 /* Make sure that anybody stopping the queue after this 434 * sees the new next_to_clean. 435 */ 436 smp_mb(); 437 438 if (__netif_subqueue_stopped(tx_ring->netdev, 439 tx_ring->queue_index) && 440 !test_bit(__IXGBEVF_DOWN, &adapter->state)) { 441 netif_wake_subqueue(tx_ring->netdev, 442 tx_ring->queue_index); 443 ++tx_ring->tx_stats.restart_queue; 444 } 445 } 446 447 return !!budget; 448 } 449 450 /** 451 * ixgbevf_rx_skb - Helper function to determine proper Rx method 452 * @q_vector: structure containing interrupt and ring information 453 * @skb: packet to send up 454 **/ 455 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector, 456 struct sk_buff *skb) 457 { 458 #ifdef CONFIG_NET_RX_BUSY_POLL 459 skb_mark_napi_id(skb, &q_vector->napi); 460 461 if (ixgbevf_qv_busy_polling(q_vector)) { 462 netif_receive_skb(skb); 463 /* exit early if we busy polled */ 464 return; 465 } 466 #endif /* CONFIG_NET_RX_BUSY_POLL */ 467 468 napi_gro_receive(&q_vector->napi, skb); 469 } 470 471 #define IXGBE_RSS_L4_TYPES_MASK \ 472 ((1ul << IXGBE_RXDADV_RSSTYPE_IPV4_TCP) | \ 473 (1ul << IXGBE_RXDADV_RSSTYPE_IPV4_UDP) | \ 474 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_TCP) | \ 475 (1ul << IXGBE_RXDADV_RSSTYPE_IPV6_UDP)) 476 477 static inline void ixgbevf_rx_hash(struct ixgbevf_ring *ring, 478 union ixgbe_adv_rx_desc *rx_desc, 479 struct sk_buff *skb) 480 { 481 u16 rss_type; 482 483 if (!(ring->netdev->features & NETIF_F_RXHASH)) 484 return; 485 486 rss_type = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.pkt_info) & 487 IXGBE_RXDADV_RSSTYPE_MASK; 488 489 if (!rss_type) 490 return; 491 492 skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), 493 (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? 494 PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); 495 } 496 497 /** 498 * ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum 499 * @ring: structure containig ring specific data 500 * @rx_desc: current Rx descriptor being processed 501 * @skb: skb currently being received and modified 502 **/ 503 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring, 504 union ixgbe_adv_rx_desc *rx_desc, 505 struct sk_buff *skb) 506 { 507 skb_checksum_none_assert(skb); 508 509 /* Rx csum disabled */ 510 if (!(ring->netdev->features & NETIF_F_RXCSUM)) 511 return; 512 513 /* if IP and error */ 514 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && 515 ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { 516 ring->rx_stats.csum_err++; 517 return; 518 } 519 520 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS)) 521 return; 522 523 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) { 524 ring->rx_stats.csum_err++; 525 return; 526 } 527 528 /* It must be a TCP or UDP packet with a valid checksum */ 529 skb->ip_summed = CHECKSUM_UNNECESSARY; 530 } 531 532 /** 533 * ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor 534 * @rx_ring: rx descriptor ring packet is being transacted on 535 * @rx_desc: pointer to the EOP Rx descriptor 536 * @skb: pointer to current skb being populated 537 * 538 * This function checks the ring, descriptor, and packet information in 539 * order to populate the checksum, VLAN, protocol, and other fields within 540 * the skb. 541 **/ 542 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring, 543 union ixgbe_adv_rx_desc *rx_desc, 544 struct sk_buff *skb) 545 { 546 ixgbevf_rx_hash(rx_ring, rx_desc, skb); 547 ixgbevf_rx_checksum(rx_ring, rx_desc, skb); 548 549 if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { 550 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); 551 unsigned long *active_vlans = netdev_priv(rx_ring->netdev); 552 553 if (test_bit(vid & VLAN_VID_MASK, active_vlans)) 554 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); 555 } 556 557 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 558 } 559 560 /** 561 * ixgbevf_is_non_eop - process handling of non-EOP buffers 562 * @rx_ring: Rx ring being processed 563 * @rx_desc: Rx descriptor for current buffer 564 * @skb: current socket buffer containing buffer in progress 565 * 566 * This function updates next to clean. If the buffer is an EOP buffer 567 * this function exits returning false, otherwise it will place the 568 * sk_buff in the next buffer to be chained and return true indicating 569 * that this is in fact a non-EOP buffer. 570 **/ 571 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring, 572 union ixgbe_adv_rx_desc *rx_desc) 573 { 574 u32 ntc = rx_ring->next_to_clean + 1; 575 576 /* fetch, update, and store next to clean */ 577 ntc = (ntc < rx_ring->count) ? ntc : 0; 578 rx_ring->next_to_clean = ntc; 579 580 prefetch(IXGBEVF_RX_DESC(rx_ring, ntc)); 581 582 if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) 583 return false; 584 585 return true; 586 } 587 588 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring, 589 struct ixgbevf_rx_buffer *bi) 590 { 591 struct page *page = bi->page; 592 dma_addr_t dma = bi->dma; 593 594 /* since we are recycling buffers we should seldom need to alloc */ 595 if (likely(page)) 596 return true; 597 598 /* alloc new page for storage */ 599 page = dev_alloc_page(); 600 if (unlikely(!page)) { 601 rx_ring->rx_stats.alloc_rx_page_failed++; 602 return false; 603 } 604 605 /* map page for use */ 606 dma = dma_map_page(rx_ring->dev, page, 0, 607 PAGE_SIZE, DMA_FROM_DEVICE); 608 609 /* if mapping failed free memory back to system since 610 * there isn't much point in holding memory we can't use 611 */ 612 if (dma_mapping_error(rx_ring->dev, dma)) { 613 __free_page(page); 614 615 rx_ring->rx_stats.alloc_rx_buff_failed++; 616 return false; 617 } 618 619 bi->dma = dma; 620 bi->page = page; 621 bi->page_offset = 0; 622 623 return true; 624 } 625 626 /** 627 * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split 628 * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on 629 * @cleaned_count: number of buffers to replace 630 **/ 631 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring, 632 u16 cleaned_count) 633 { 634 union ixgbe_adv_rx_desc *rx_desc; 635 struct ixgbevf_rx_buffer *bi; 636 unsigned int i = rx_ring->next_to_use; 637 638 /* nothing to do or no valid netdev defined */ 639 if (!cleaned_count || !rx_ring->netdev) 640 return; 641 642 rx_desc = IXGBEVF_RX_DESC(rx_ring, i); 643 bi = &rx_ring->rx_buffer_info[i]; 644 i -= rx_ring->count; 645 646 do { 647 if (!ixgbevf_alloc_mapped_page(rx_ring, bi)) 648 break; 649 650 /* Refresh the desc even if pkt_addr didn't change 651 * because each write-back erases this info. 652 */ 653 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 654 655 rx_desc++; 656 bi++; 657 i++; 658 if (unlikely(!i)) { 659 rx_desc = IXGBEVF_RX_DESC(rx_ring, 0); 660 bi = rx_ring->rx_buffer_info; 661 i -= rx_ring->count; 662 } 663 664 /* clear the hdr_addr for the next_to_use descriptor */ 665 rx_desc->read.hdr_addr = 0; 666 667 cleaned_count--; 668 } while (cleaned_count); 669 670 i += rx_ring->count; 671 672 if (rx_ring->next_to_use != i) { 673 /* record the next descriptor to use */ 674 rx_ring->next_to_use = i; 675 676 /* update next to alloc since we have filled the ring */ 677 rx_ring->next_to_alloc = i; 678 679 /* Force memory writes to complete before letting h/w 680 * know there are new descriptors to fetch. (Only 681 * applicable for weak-ordered memory model archs, 682 * such as IA-64). 683 */ 684 wmb(); 685 ixgbevf_write_tail(rx_ring, i); 686 } 687 } 688 689 /** 690 * ixgbevf_cleanup_headers - Correct corrupted or empty headers 691 * @rx_ring: rx descriptor ring packet is being transacted on 692 * @rx_desc: pointer to the EOP Rx descriptor 693 * @skb: pointer to current skb being fixed 694 * 695 * Check for corrupted packet headers caused by senders on the local L2 696 * embedded NIC switch not setting up their Tx Descriptors right. These 697 * should be very rare. 698 * 699 * Also address the case where we are pulling data in on pages only 700 * and as such no data is present in the skb header. 701 * 702 * In addition if skb is not at least 60 bytes we need to pad it so that 703 * it is large enough to qualify as a valid Ethernet frame. 704 * 705 * Returns true if an error was encountered and skb was freed. 706 **/ 707 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring, 708 union ixgbe_adv_rx_desc *rx_desc, 709 struct sk_buff *skb) 710 { 711 /* verify that the packet does not have any known errors */ 712 if (unlikely(ixgbevf_test_staterr(rx_desc, 713 IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) { 714 struct net_device *netdev = rx_ring->netdev; 715 716 if (!(netdev->features & NETIF_F_RXALL)) { 717 dev_kfree_skb_any(skb); 718 return true; 719 } 720 } 721 722 /* if eth_skb_pad returns an error the skb was freed */ 723 if (eth_skb_pad(skb)) 724 return true; 725 726 return false; 727 } 728 729 /** 730 * ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring 731 * @rx_ring: rx descriptor ring to store buffers on 732 * @old_buff: donor buffer to have page reused 733 * 734 * Synchronizes page for reuse by the adapter 735 **/ 736 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring, 737 struct ixgbevf_rx_buffer *old_buff) 738 { 739 struct ixgbevf_rx_buffer *new_buff; 740 u16 nta = rx_ring->next_to_alloc; 741 742 new_buff = &rx_ring->rx_buffer_info[nta]; 743 744 /* update, and store next to alloc */ 745 nta++; 746 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 747 748 /* transfer page from old buffer to new buffer */ 749 new_buff->page = old_buff->page; 750 new_buff->dma = old_buff->dma; 751 new_buff->page_offset = old_buff->page_offset; 752 753 /* sync the buffer for use by the device */ 754 dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma, 755 new_buff->page_offset, 756 IXGBEVF_RX_BUFSZ, 757 DMA_FROM_DEVICE); 758 } 759 760 static inline bool ixgbevf_page_is_reserved(struct page *page) 761 { 762 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page); 763 } 764 765 /** 766 * ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff 767 * @rx_ring: rx descriptor ring to transact packets on 768 * @rx_buffer: buffer containing page to add 769 * @rx_desc: descriptor containing length of buffer written by hardware 770 * @skb: sk_buff to place the data into 771 * 772 * This function will add the data contained in rx_buffer->page to the skb. 773 * This is done either through a direct copy if the data in the buffer is 774 * less than the skb header size, otherwise it will just attach the page as 775 * a frag to the skb. 776 * 777 * The function will then update the page offset if necessary and return 778 * true if the buffer can be reused by the adapter. 779 **/ 780 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring, 781 struct ixgbevf_rx_buffer *rx_buffer, 782 union ixgbe_adv_rx_desc *rx_desc, 783 struct sk_buff *skb) 784 { 785 struct page *page = rx_buffer->page; 786 unsigned char *va = page_address(page) + rx_buffer->page_offset; 787 unsigned int size = le16_to_cpu(rx_desc->wb.upper.length); 788 #if (PAGE_SIZE < 8192) 789 unsigned int truesize = IXGBEVF_RX_BUFSZ; 790 #else 791 unsigned int truesize = ALIGN(size, L1_CACHE_BYTES); 792 #endif 793 unsigned int pull_len; 794 795 if (unlikely(skb_is_nonlinear(skb))) 796 goto add_tail_frag; 797 798 if (likely(size <= IXGBEVF_RX_HDR_SIZE)) { 799 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long))); 800 801 /* page is not reserved, we can reuse buffer as is */ 802 if (likely(!ixgbevf_page_is_reserved(page))) 803 return true; 804 805 /* this page cannot be reused so discard it */ 806 put_page(page); 807 return false; 808 } 809 810 /* we need the header to contain the greater of either ETH_HLEN or 811 * 60 bytes if the skb->len is less than 60 for skb_pad. 812 */ 813 pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE); 814 815 /* align pull length to size of long to optimize memcpy performance */ 816 memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long))); 817 818 /* update all of the pointers */ 819 va += pull_len; 820 size -= pull_len; 821 822 add_tail_frag: 823 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, 824 (unsigned long)va & ~PAGE_MASK, size, truesize); 825 826 /* avoid re-using remote pages */ 827 if (unlikely(ixgbevf_page_is_reserved(page))) 828 return false; 829 830 #if (PAGE_SIZE < 8192) 831 /* if we are only owner of page we can reuse it */ 832 if (unlikely(page_count(page) != 1)) 833 return false; 834 835 /* flip page offset to other buffer */ 836 rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ; 837 838 #else 839 /* move offset up to the next cache line */ 840 rx_buffer->page_offset += truesize; 841 842 if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ)) 843 return false; 844 845 #endif 846 /* Even if we own the page, we are not allowed to use atomic_set() 847 * This would break get_page_unless_zero() users. 848 */ 849 page_ref_inc(page); 850 851 return true; 852 } 853 854 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring, 855 union ixgbe_adv_rx_desc *rx_desc, 856 struct sk_buff *skb) 857 { 858 struct ixgbevf_rx_buffer *rx_buffer; 859 struct page *page; 860 861 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; 862 page = rx_buffer->page; 863 prefetchw(page); 864 865 if (likely(!skb)) { 866 void *page_addr = page_address(page) + 867 rx_buffer->page_offset; 868 869 /* prefetch first cache line of first page */ 870 prefetch(page_addr); 871 #if L1_CACHE_BYTES < 128 872 prefetch(page_addr + L1_CACHE_BYTES); 873 #endif 874 875 /* allocate a skb to store the frags */ 876 skb = netdev_alloc_skb_ip_align(rx_ring->netdev, 877 IXGBEVF_RX_HDR_SIZE); 878 if (unlikely(!skb)) { 879 rx_ring->rx_stats.alloc_rx_buff_failed++; 880 return NULL; 881 } 882 883 /* we will be copying header into skb->data in 884 * pskb_may_pull so it is in our interest to prefetch 885 * it now to avoid a possible cache miss 886 */ 887 prefetchw(skb->data); 888 } 889 890 /* we are reusing so sync this buffer for CPU use */ 891 dma_sync_single_range_for_cpu(rx_ring->dev, 892 rx_buffer->dma, 893 rx_buffer->page_offset, 894 IXGBEVF_RX_BUFSZ, 895 DMA_FROM_DEVICE); 896 897 /* pull page into skb */ 898 if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) { 899 /* hand second half of page back to the ring */ 900 ixgbevf_reuse_rx_page(rx_ring, rx_buffer); 901 } else { 902 /* we are not reusing the buffer so unmap it */ 903 dma_unmap_page(rx_ring->dev, rx_buffer->dma, 904 PAGE_SIZE, DMA_FROM_DEVICE); 905 } 906 907 /* clear contents of buffer_info */ 908 rx_buffer->dma = 0; 909 rx_buffer->page = NULL; 910 911 return skb; 912 } 913 914 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, 915 u32 qmask) 916 { 917 struct ixgbe_hw *hw = &adapter->hw; 918 919 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask); 920 } 921 922 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector, 923 struct ixgbevf_ring *rx_ring, 924 int budget) 925 { 926 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 927 u16 cleaned_count = ixgbevf_desc_unused(rx_ring); 928 struct sk_buff *skb = rx_ring->skb; 929 930 while (likely(total_rx_packets < budget)) { 931 union ixgbe_adv_rx_desc *rx_desc; 932 933 /* return some buffers to hardware, one at a time is too slow */ 934 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) { 935 ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count); 936 cleaned_count = 0; 937 } 938 939 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean); 940 941 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD)) 942 break; 943 944 /* This memory barrier is needed to keep us from reading 945 * any other fields out of the rx_desc until we know the 946 * RXD_STAT_DD bit is set 947 */ 948 rmb(); 949 950 /* retrieve a buffer from the ring */ 951 skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb); 952 953 /* exit if we failed to retrieve a buffer */ 954 if (!skb) 955 break; 956 957 cleaned_count++; 958 959 /* fetch next buffer in frame if non-eop */ 960 if (ixgbevf_is_non_eop(rx_ring, rx_desc)) 961 continue; 962 963 /* verify the packet layout is correct */ 964 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) { 965 skb = NULL; 966 continue; 967 } 968 969 /* probably a little skewed due to removing CRC */ 970 total_rx_bytes += skb->len; 971 972 /* Workaround hardware that can't do proper VEPA multicast 973 * source pruning. 974 */ 975 if ((skb->pkt_type == PACKET_BROADCAST || 976 skb->pkt_type == PACKET_MULTICAST) && 977 ether_addr_equal(rx_ring->netdev->dev_addr, 978 eth_hdr(skb)->h_source)) { 979 dev_kfree_skb_irq(skb); 980 continue; 981 } 982 983 /* populate checksum, VLAN, and protocol */ 984 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb); 985 986 ixgbevf_rx_skb(q_vector, skb); 987 988 /* reset skb pointer */ 989 skb = NULL; 990 991 /* update budget accounting */ 992 total_rx_packets++; 993 } 994 995 /* place incomplete frames back on ring for completion */ 996 rx_ring->skb = skb; 997 998 u64_stats_update_begin(&rx_ring->syncp); 999 rx_ring->stats.packets += total_rx_packets; 1000 rx_ring->stats.bytes += total_rx_bytes; 1001 u64_stats_update_end(&rx_ring->syncp); 1002 q_vector->rx.total_packets += total_rx_packets; 1003 q_vector->rx.total_bytes += total_rx_bytes; 1004 1005 return total_rx_packets; 1006 } 1007 1008 /** 1009 * ixgbevf_poll - NAPI polling calback 1010 * @napi: napi struct with our devices info in it 1011 * @budget: amount of work driver is allowed to do this pass, in packets 1012 * 1013 * This function will clean more than one or more rings associated with a 1014 * q_vector. 1015 **/ 1016 static int ixgbevf_poll(struct napi_struct *napi, int budget) 1017 { 1018 struct ixgbevf_q_vector *q_vector = 1019 container_of(napi, struct ixgbevf_q_vector, napi); 1020 struct ixgbevf_adapter *adapter = q_vector->adapter; 1021 struct ixgbevf_ring *ring; 1022 int per_ring_budget, work_done = 0; 1023 bool clean_complete = true; 1024 1025 ixgbevf_for_each_ring(ring, q_vector->tx) { 1026 if (!ixgbevf_clean_tx_irq(q_vector, ring, budget)) 1027 clean_complete = false; 1028 } 1029 1030 if (budget <= 0) 1031 return budget; 1032 #ifdef CONFIG_NET_RX_BUSY_POLL 1033 if (!ixgbevf_qv_lock_napi(q_vector)) 1034 return budget; 1035 #endif 1036 1037 /* attempt to distribute budget to each queue fairly, but don't allow 1038 * the budget to go below 1 because we'll exit polling 1039 */ 1040 if (q_vector->rx.count > 1) 1041 per_ring_budget = max(budget/q_vector->rx.count, 1); 1042 else 1043 per_ring_budget = budget; 1044 1045 ixgbevf_for_each_ring(ring, q_vector->rx) { 1046 int cleaned = ixgbevf_clean_rx_irq(q_vector, ring, 1047 per_ring_budget); 1048 work_done += cleaned; 1049 if (cleaned >= per_ring_budget) 1050 clean_complete = false; 1051 } 1052 1053 #ifdef CONFIG_NET_RX_BUSY_POLL 1054 ixgbevf_qv_unlock_napi(q_vector); 1055 #endif 1056 1057 /* If all work not completed, return budget and keep polling */ 1058 if (!clean_complete) 1059 return budget; 1060 /* all work done, exit the polling mode */ 1061 napi_complete_done(napi, work_done); 1062 if (adapter->rx_itr_setting == 1) 1063 ixgbevf_set_itr(q_vector); 1064 if (!test_bit(__IXGBEVF_DOWN, &adapter->state) && 1065 !test_bit(__IXGBEVF_REMOVING, &adapter->state)) 1066 ixgbevf_irq_enable_queues(adapter, 1067 BIT(q_vector->v_idx)); 1068 1069 return 0; 1070 } 1071 1072 /** 1073 * ixgbevf_write_eitr - write VTEITR register in hardware specific way 1074 * @q_vector: structure containing interrupt and ring information 1075 **/ 1076 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector) 1077 { 1078 struct ixgbevf_adapter *adapter = q_vector->adapter; 1079 struct ixgbe_hw *hw = &adapter->hw; 1080 int v_idx = q_vector->v_idx; 1081 u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR; 1082 1083 /* set the WDIS bit to not clear the timer bits and cause an 1084 * immediate assertion of the interrupt 1085 */ 1086 itr_reg |= IXGBE_EITR_CNT_WDIS; 1087 1088 IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg); 1089 } 1090 1091 #ifdef CONFIG_NET_RX_BUSY_POLL 1092 /* must be called with local_bh_disable()d */ 1093 static int ixgbevf_busy_poll_recv(struct napi_struct *napi) 1094 { 1095 struct ixgbevf_q_vector *q_vector = 1096 container_of(napi, struct ixgbevf_q_vector, napi); 1097 struct ixgbevf_adapter *adapter = q_vector->adapter; 1098 struct ixgbevf_ring *ring; 1099 int found = 0; 1100 1101 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 1102 return LL_FLUSH_FAILED; 1103 1104 if (!ixgbevf_qv_lock_poll(q_vector)) 1105 return LL_FLUSH_BUSY; 1106 1107 ixgbevf_for_each_ring(ring, q_vector->rx) { 1108 found = ixgbevf_clean_rx_irq(q_vector, ring, 4); 1109 #ifdef BP_EXTENDED_STATS 1110 if (found) 1111 ring->stats.cleaned += found; 1112 else 1113 ring->stats.misses++; 1114 #endif 1115 if (found) 1116 break; 1117 } 1118 1119 ixgbevf_qv_unlock_poll(q_vector); 1120 1121 return found; 1122 } 1123 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1124 1125 /** 1126 * ixgbevf_configure_msix - Configure MSI-X hardware 1127 * @adapter: board private structure 1128 * 1129 * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X 1130 * interrupts. 1131 **/ 1132 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter) 1133 { 1134 struct ixgbevf_q_vector *q_vector; 1135 int q_vectors, v_idx; 1136 1137 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1138 adapter->eims_enable_mask = 0; 1139 1140 /* Populate the IVAR table and set the ITR values to the 1141 * corresponding register. 1142 */ 1143 for (v_idx = 0; v_idx < q_vectors; v_idx++) { 1144 struct ixgbevf_ring *ring; 1145 1146 q_vector = adapter->q_vector[v_idx]; 1147 1148 ixgbevf_for_each_ring(ring, q_vector->rx) 1149 ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx); 1150 1151 ixgbevf_for_each_ring(ring, q_vector->tx) 1152 ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx); 1153 1154 if (q_vector->tx.ring && !q_vector->rx.ring) { 1155 /* Tx only vector */ 1156 if (adapter->tx_itr_setting == 1) 1157 q_vector->itr = IXGBE_12K_ITR; 1158 else 1159 q_vector->itr = adapter->tx_itr_setting; 1160 } else { 1161 /* Rx or Rx/Tx vector */ 1162 if (adapter->rx_itr_setting == 1) 1163 q_vector->itr = IXGBE_20K_ITR; 1164 else 1165 q_vector->itr = adapter->rx_itr_setting; 1166 } 1167 1168 /* add q_vector eims value to global eims_enable_mask */ 1169 adapter->eims_enable_mask |= BIT(v_idx); 1170 1171 ixgbevf_write_eitr(q_vector); 1172 } 1173 1174 ixgbevf_set_ivar(adapter, -1, 1, v_idx); 1175 /* setup eims_other and add value to global eims_enable_mask */ 1176 adapter->eims_other = BIT(v_idx); 1177 adapter->eims_enable_mask |= adapter->eims_other; 1178 } 1179 1180 enum latency_range { 1181 lowest_latency = 0, 1182 low_latency = 1, 1183 bulk_latency = 2, 1184 latency_invalid = 255 1185 }; 1186 1187 /** 1188 * ixgbevf_update_itr - update the dynamic ITR value based on statistics 1189 * @q_vector: structure containing interrupt and ring information 1190 * @ring_container: structure containing ring performance data 1191 * 1192 * Stores a new ITR value based on packets and byte 1193 * counts during the last interrupt. The advantage of per interrupt 1194 * computation is faster updates and more accurate ITR for the current 1195 * traffic pattern. Constants in this function were computed 1196 * based on theoretical maximum wire speed and thresholds were set based 1197 * on testing data as well as attempting to minimize response time 1198 * while increasing bulk throughput. 1199 **/ 1200 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector, 1201 struct ixgbevf_ring_container *ring_container) 1202 { 1203 int bytes = ring_container->total_bytes; 1204 int packets = ring_container->total_packets; 1205 u32 timepassed_us; 1206 u64 bytes_perint; 1207 u8 itr_setting = ring_container->itr; 1208 1209 if (packets == 0) 1210 return; 1211 1212 /* simple throttle rate management 1213 * 0-20MB/s lowest (100000 ints/s) 1214 * 20-100MB/s low (20000 ints/s) 1215 * 100-1249MB/s bulk (12000 ints/s) 1216 */ 1217 /* what was last interrupt timeslice? */ 1218 timepassed_us = q_vector->itr >> 2; 1219 bytes_perint = bytes / timepassed_us; /* bytes/usec */ 1220 1221 switch (itr_setting) { 1222 case lowest_latency: 1223 if (bytes_perint > 10) 1224 itr_setting = low_latency; 1225 break; 1226 case low_latency: 1227 if (bytes_perint > 20) 1228 itr_setting = bulk_latency; 1229 else if (bytes_perint <= 10) 1230 itr_setting = lowest_latency; 1231 break; 1232 case bulk_latency: 1233 if (bytes_perint <= 20) 1234 itr_setting = low_latency; 1235 break; 1236 } 1237 1238 /* clear work counters since we have the values we need */ 1239 ring_container->total_bytes = 0; 1240 ring_container->total_packets = 0; 1241 1242 /* write updated itr to ring container */ 1243 ring_container->itr = itr_setting; 1244 } 1245 1246 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector) 1247 { 1248 u32 new_itr = q_vector->itr; 1249 u8 current_itr; 1250 1251 ixgbevf_update_itr(q_vector, &q_vector->tx); 1252 ixgbevf_update_itr(q_vector, &q_vector->rx); 1253 1254 current_itr = max(q_vector->rx.itr, q_vector->tx.itr); 1255 1256 switch (current_itr) { 1257 /* counts and packets in update_itr are dependent on these numbers */ 1258 case lowest_latency: 1259 new_itr = IXGBE_100K_ITR; 1260 break; 1261 case low_latency: 1262 new_itr = IXGBE_20K_ITR; 1263 break; 1264 case bulk_latency: 1265 new_itr = IXGBE_12K_ITR; 1266 break; 1267 default: 1268 break; 1269 } 1270 1271 if (new_itr != q_vector->itr) { 1272 /* do an exponential smoothing */ 1273 new_itr = (10 * new_itr * q_vector->itr) / 1274 ((9 * new_itr) + q_vector->itr); 1275 1276 /* save the algorithm value here */ 1277 q_vector->itr = new_itr; 1278 1279 ixgbevf_write_eitr(q_vector); 1280 } 1281 } 1282 1283 static irqreturn_t ixgbevf_msix_other(int irq, void *data) 1284 { 1285 struct ixgbevf_adapter *adapter = data; 1286 struct ixgbe_hw *hw = &adapter->hw; 1287 1288 hw->mac.get_link_status = 1; 1289 1290 ixgbevf_service_event_schedule(adapter); 1291 1292 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other); 1293 1294 return IRQ_HANDLED; 1295 } 1296 1297 /** 1298 * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues) 1299 * @irq: unused 1300 * @data: pointer to our q_vector struct for this interrupt vector 1301 **/ 1302 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data) 1303 { 1304 struct ixgbevf_q_vector *q_vector = data; 1305 1306 /* EIAM disabled interrupts (on this vector) for us */ 1307 if (q_vector->rx.ring || q_vector->tx.ring) 1308 napi_schedule_irqoff(&q_vector->napi); 1309 1310 return IRQ_HANDLED; 1311 } 1312 1313 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx, 1314 int r_idx) 1315 { 1316 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1317 1318 a->rx_ring[r_idx]->next = q_vector->rx.ring; 1319 q_vector->rx.ring = a->rx_ring[r_idx]; 1320 q_vector->rx.count++; 1321 } 1322 1323 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx, 1324 int t_idx) 1325 { 1326 struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx]; 1327 1328 a->tx_ring[t_idx]->next = q_vector->tx.ring; 1329 q_vector->tx.ring = a->tx_ring[t_idx]; 1330 q_vector->tx.count++; 1331 } 1332 1333 /** 1334 * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors 1335 * @adapter: board private structure to initialize 1336 * 1337 * This function maps descriptor rings to the queue-specific vectors 1338 * we were allotted through the MSI-X enabling code. Ideally, we'd have 1339 * one vector per ring/queue, but on a constrained vector budget, we 1340 * group the rings as "efficiently" as possible. You would add new 1341 * mapping configurations in here. 1342 **/ 1343 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter) 1344 { 1345 int q_vectors; 1346 int v_start = 0; 1347 int rxr_idx = 0, txr_idx = 0; 1348 int rxr_remaining = adapter->num_rx_queues; 1349 int txr_remaining = adapter->num_tx_queues; 1350 int i, j; 1351 int rqpv, tqpv; 1352 1353 q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1354 1355 /* The ideal configuration... 1356 * We have enough vectors to map one per queue. 1357 */ 1358 if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) { 1359 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) 1360 map_vector_to_rxq(adapter, v_start, rxr_idx); 1361 1362 for (; txr_idx < txr_remaining; v_start++, txr_idx++) 1363 map_vector_to_txq(adapter, v_start, txr_idx); 1364 return 0; 1365 } 1366 1367 /* If we don't have enough vectors for a 1-to-1 1368 * mapping, we'll have to group them so there are 1369 * multiple queues per vector. 1370 */ 1371 /* Re-adjusting *qpv takes care of the remainder. */ 1372 for (i = v_start; i < q_vectors; i++) { 1373 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i); 1374 for (j = 0; j < rqpv; j++) { 1375 map_vector_to_rxq(adapter, i, rxr_idx); 1376 rxr_idx++; 1377 rxr_remaining--; 1378 } 1379 } 1380 for (i = v_start; i < q_vectors; i++) { 1381 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i); 1382 for (j = 0; j < tqpv; j++) { 1383 map_vector_to_txq(adapter, i, txr_idx); 1384 txr_idx++; 1385 txr_remaining--; 1386 } 1387 } 1388 1389 return 0; 1390 } 1391 1392 /** 1393 * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts 1394 * @adapter: board private structure 1395 * 1396 * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests 1397 * interrupts from the kernel. 1398 **/ 1399 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter) 1400 { 1401 struct net_device *netdev = adapter->netdev; 1402 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1403 int vector, err; 1404 int ri = 0, ti = 0; 1405 1406 for (vector = 0; vector < q_vectors; vector++) { 1407 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector]; 1408 struct msix_entry *entry = &adapter->msix_entries[vector]; 1409 1410 if (q_vector->tx.ring && q_vector->rx.ring) { 1411 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1412 "%s-%s-%d", netdev->name, "TxRx", ri++); 1413 ti++; 1414 } else if (q_vector->rx.ring) { 1415 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1416 "%s-%s-%d", netdev->name, "rx", ri++); 1417 } else if (q_vector->tx.ring) { 1418 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1419 "%s-%s-%d", netdev->name, "tx", ti++); 1420 } else { 1421 /* skip this unused q_vector */ 1422 continue; 1423 } 1424 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0, 1425 q_vector->name, q_vector); 1426 if (err) { 1427 hw_dbg(&adapter->hw, 1428 "request_irq failed for MSIX interrupt Error: %d\n", 1429 err); 1430 goto free_queue_irqs; 1431 } 1432 } 1433 1434 err = request_irq(adapter->msix_entries[vector].vector, 1435 &ixgbevf_msix_other, 0, netdev->name, adapter); 1436 if (err) { 1437 hw_dbg(&adapter->hw, "request_irq for msix_other failed: %d\n", 1438 err); 1439 goto free_queue_irqs; 1440 } 1441 1442 return 0; 1443 1444 free_queue_irqs: 1445 while (vector) { 1446 vector--; 1447 free_irq(adapter->msix_entries[vector].vector, 1448 adapter->q_vector[vector]); 1449 } 1450 /* This failure is non-recoverable - it indicates the system is 1451 * out of MSIX vector resources and the VF driver cannot run 1452 * without them. Set the number of msix vectors to zero 1453 * indicating that not enough can be allocated. The error 1454 * will be returned to the user indicating device open failed. 1455 * Any further attempts to force the driver to open will also 1456 * fail. The only way to recover is to unload the driver and 1457 * reload it again. If the system has recovered some MSIX 1458 * vectors then it may succeed. 1459 */ 1460 adapter->num_msix_vectors = 0; 1461 return err; 1462 } 1463 1464 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter) 1465 { 1466 int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1467 1468 for (i = 0; i < q_vectors; i++) { 1469 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i]; 1470 1471 q_vector->rx.ring = NULL; 1472 q_vector->tx.ring = NULL; 1473 q_vector->rx.count = 0; 1474 q_vector->tx.count = 0; 1475 } 1476 } 1477 1478 /** 1479 * ixgbevf_request_irq - initialize interrupts 1480 * @adapter: board private structure 1481 * 1482 * Attempts to configure interrupts using the best available 1483 * capabilities of the hardware and kernel. 1484 **/ 1485 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter) 1486 { 1487 int err = ixgbevf_request_msix_irqs(adapter); 1488 1489 if (err) 1490 hw_dbg(&adapter->hw, "request_irq failed, Error %d\n", err); 1491 1492 return err; 1493 } 1494 1495 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter) 1496 { 1497 int i, q_vectors; 1498 1499 q_vectors = adapter->num_msix_vectors; 1500 i = q_vectors - 1; 1501 1502 free_irq(adapter->msix_entries[i].vector, adapter); 1503 i--; 1504 1505 for (; i >= 0; i--) { 1506 /* free only the irqs that were actually requested */ 1507 if (!adapter->q_vector[i]->rx.ring && 1508 !adapter->q_vector[i]->tx.ring) 1509 continue; 1510 1511 free_irq(adapter->msix_entries[i].vector, 1512 adapter->q_vector[i]); 1513 } 1514 1515 ixgbevf_reset_q_vectors(adapter); 1516 } 1517 1518 /** 1519 * ixgbevf_irq_disable - Mask off interrupt generation on the NIC 1520 * @adapter: board private structure 1521 **/ 1522 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter) 1523 { 1524 struct ixgbe_hw *hw = &adapter->hw; 1525 int i; 1526 1527 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0); 1528 IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0); 1529 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0); 1530 1531 IXGBE_WRITE_FLUSH(hw); 1532 1533 for (i = 0; i < adapter->num_msix_vectors; i++) 1534 synchronize_irq(adapter->msix_entries[i].vector); 1535 } 1536 1537 /** 1538 * ixgbevf_irq_enable - Enable default interrupt generation settings 1539 * @adapter: board private structure 1540 **/ 1541 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter) 1542 { 1543 struct ixgbe_hw *hw = &adapter->hw; 1544 1545 IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask); 1546 IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask); 1547 IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask); 1548 } 1549 1550 /** 1551 * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset 1552 * @adapter: board private structure 1553 * @ring: structure containing ring specific data 1554 * 1555 * Configure the Tx descriptor ring after a reset. 1556 **/ 1557 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, 1558 struct ixgbevf_ring *ring) 1559 { 1560 struct ixgbe_hw *hw = &adapter->hw; 1561 u64 tdba = ring->dma; 1562 int wait_loop = 10; 1563 u32 txdctl = IXGBE_TXDCTL_ENABLE; 1564 u8 reg_idx = ring->reg_idx; 1565 1566 /* disable queue to avoid issues while updating state */ 1567 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH); 1568 IXGBE_WRITE_FLUSH(hw); 1569 1570 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 1571 IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32); 1572 IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx), 1573 ring->count * sizeof(union ixgbe_adv_tx_desc)); 1574 1575 /* disable head writeback */ 1576 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0); 1577 IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0); 1578 1579 /* enable relaxed ordering */ 1580 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx), 1581 (IXGBE_DCA_TXCTRL_DESC_RRO_EN | 1582 IXGBE_DCA_TXCTRL_DATA_RRO_EN)); 1583 1584 /* reset head and tail pointers */ 1585 IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0); 1586 IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0); 1587 ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx); 1588 1589 /* reset ntu and ntc to place SW in sync with hardwdare */ 1590 ring->next_to_clean = 0; 1591 ring->next_to_use = 0; 1592 1593 /* In order to avoid issues WTHRESH + PTHRESH should always be equal 1594 * to or less than the number of on chip descriptors, which is 1595 * currently 40. 1596 */ 1597 txdctl |= (8 << 16); /* WTHRESH = 8 */ 1598 1599 /* Setting PTHRESH to 32 both improves performance */ 1600 txdctl |= (1u << 8) | /* HTHRESH = 1 */ 1601 32; /* PTHRESH = 32 */ 1602 1603 clear_bit(__IXGBEVF_HANG_CHECK_ARMED, &ring->state); 1604 1605 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl); 1606 1607 /* poll to verify queue is enabled */ 1608 do { 1609 usleep_range(1000, 2000); 1610 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx)); 1611 } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE)); 1612 if (!wait_loop) 1613 pr_err("Could not enable Tx Queue %d\n", reg_idx); 1614 } 1615 1616 /** 1617 * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset 1618 * @adapter: board private structure 1619 * 1620 * Configure the Tx unit of the MAC after a reset. 1621 **/ 1622 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter) 1623 { 1624 u32 i; 1625 1626 /* Setup the HW Tx Head and Tail descriptor pointers */ 1627 for (i = 0; i < adapter->num_tx_queues; i++) 1628 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]); 1629 } 1630 1631 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 1632 1633 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index) 1634 { 1635 struct ixgbe_hw *hw = &adapter->hw; 1636 u32 srrctl; 1637 1638 srrctl = IXGBE_SRRCTL_DROP_EN; 1639 1640 srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; 1641 srrctl |= IXGBEVF_RX_BUFSZ >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; 1642 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1643 1644 IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl); 1645 } 1646 1647 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter) 1648 { 1649 struct ixgbe_hw *hw = &adapter->hw; 1650 1651 /* PSRTYPE must be initialized in 82599 */ 1652 u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1653 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR | 1654 IXGBE_PSRTYPE_L2HDR; 1655 1656 if (adapter->num_rx_queues > 1) 1657 psrtype |= BIT(29); 1658 1659 IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype); 1660 } 1661 1662 #define IXGBEVF_MAX_RX_DESC_POLL 10 1663 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter, 1664 struct ixgbevf_ring *ring) 1665 { 1666 struct ixgbe_hw *hw = &adapter->hw; 1667 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1668 u32 rxdctl; 1669 u8 reg_idx = ring->reg_idx; 1670 1671 if (IXGBE_REMOVED(hw->hw_addr)) 1672 return; 1673 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1674 rxdctl &= ~IXGBE_RXDCTL_ENABLE; 1675 1676 /* write value back with RXDCTL.ENABLE bit cleared */ 1677 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1678 1679 /* the hardware may take up to 100us to really disable the Rx queue */ 1680 do { 1681 udelay(10); 1682 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1683 } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE)); 1684 1685 if (!wait_loop) 1686 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n", 1687 reg_idx); 1688 } 1689 1690 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter, 1691 struct ixgbevf_ring *ring) 1692 { 1693 struct ixgbe_hw *hw = &adapter->hw; 1694 int wait_loop = IXGBEVF_MAX_RX_DESC_POLL; 1695 u32 rxdctl; 1696 u8 reg_idx = ring->reg_idx; 1697 1698 if (IXGBE_REMOVED(hw->hw_addr)) 1699 return; 1700 do { 1701 usleep_range(1000, 2000); 1702 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1703 } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE)); 1704 1705 if (!wait_loop) 1706 pr_err("RXDCTL.ENABLE queue %d not set while polling\n", 1707 reg_idx); 1708 } 1709 1710 static void ixgbevf_setup_vfmrqc(struct ixgbevf_adapter *adapter) 1711 { 1712 struct ixgbe_hw *hw = &adapter->hw; 1713 u32 vfmrqc = 0, vfreta = 0; 1714 u16 rss_i = adapter->num_rx_queues; 1715 u8 i, j; 1716 1717 /* Fill out hash function seeds */ 1718 netdev_rss_key_fill(adapter->rss_key, sizeof(adapter->rss_key)); 1719 for (i = 0; i < IXGBEVF_VFRSSRK_REGS; i++) 1720 IXGBE_WRITE_REG(hw, IXGBE_VFRSSRK(i), adapter->rss_key[i]); 1721 1722 for (i = 0, j = 0; i < IXGBEVF_X550_VFRETA_SIZE; i++, j++) { 1723 if (j == rss_i) 1724 j = 0; 1725 1726 adapter->rss_indir_tbl[i] = j; 1727 1728 vfreta |= j << (i & 0x3) * 8; 1729 if ((i & 3) == 3) { 1730 IXGBE_WRITE_REG(hw, IXGBE_VFRETA(i >> 2), vfreta); 1731 vfreta = 0; 1732 } 1733 } 1734 1735 /* Perform hash on these packet types */ 1736 vfmrqc |= IXGBE_VFMRQC_RSS_FIELD_IPV4 | 1737 IXGBE_VFMRQC_RSS_FIELD_IPV4_TCP | 1738 IXGBE_VFMRQC_RSS_FIELD_IPV6 | 1739 IXGBE_VFMRQC_RSS_FIELD_IPV6_TCP; 1740 1741 vfmrqc |= IXGBE_VFMRQC_RSSEN; 1742 1743 IXGBE_WRITE_REG(hw, IXGBE_VFMRQC, vfmrqc); 1744 } 1745 1746 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter, 1747 struct ixgbevf_ring *ring) 1748 { 1749 struct ixgbe_hw *hw = &adapter->hw; 1750 u64 rdba = ring->dma; 1751 u32 rxdctl; 1752 u8 reg_idx = ring->reg_idx; 1753 1754 /* disable queue to avoid issues while updating state */ 1755 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx)); 1756 ixgbevf_disable_rx_queue(adapter, ring); 1757 1758 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 1759 IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32); 1760 IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx), 1761 ring->count * sizeof(union ixgbe_adv_rx_desc)); 1762 1763 #ifndef CONFIG_SPARC 1764 /* enable relaxed ordering */ 1765 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1766 IXGBE_DCA_RXCTRL_DESC_RRO_EN); 1767 #else 1768 IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx), 1769 IXGBE_DCA_RXCTRL_DESC_RRO_EN | 1770 IXGBE_DCA_RXCTRL_DATA_WRO_EN); 1771 #endif 1772 1773 /* reset head and tail pointers */ 1774 IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0); 1775 IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0); 1776 ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx); 1777 1778 /* reset ntu and ntc to place SW in sync with hardwdare */ 1779 ring->next_to_clean = 0; 1780 ring->next_to_use = 0; 1781 ring->next_to_alloc = 0; 1782 1783 ixgbevf_configure_srrctl(adapter, reg_idx); 1784 1785 /* allow any size packet since we can handle overflow */ 1786 rxdctl &= ~IXGBE_RXDCTL_RLPML_EN; 1787 1788 rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME; 1789 IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl); 1790 1791 ixgbevf_rx_desc_queue_enable(adapter, ring); 1792 ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring)); 1793 } 1794 1795 /** 1796 * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset 1797 * @adapter: board private structure 1798 * 1799 * Configure the Rx unit of the MAC after a reset. 1800 **/ 1801 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter) 1802 { 1803 int i; 1804 struct ixgbe_hw *hw = &adapter->hw; 1805 struct net_device *netdev = adapter->netdev; 1806 1807 ixgbevf_setup_psrtype(adapter); 1808 if (hw->mac.type >= ixgbe_mac_X550_vf) 1809 ixgbevf_setup_vfmrqc(adapter); 1810 1811 /* notify the PF of our intent to use this size of frame */ 1812 hw->mac.ops.set_rlpml(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN); 1813 1814 /* Setup the HW Rx Head and Tail Descriptor Pointers and 1815 * the Base and Length of the Rx Descriptor Ring 1816 */ 1817 for (i = 0; i < adapter->num_rx_queues; i++) 1818 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]); 1819 } 1820 1821 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev, 1822 __be16 proto, u16 vid) 1823 { 1824 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1825 struct ixgbe_hw *hw = &adapter->hw; 1826 int err; 1827 1828 spin_lock_bh(&adapter->mbx_lock); 1829 1830 /* add VID to filter table */ 1831 err = hw->mac.ops.set_vfta(hw, vid, 0, true); 1832 1833 spin_unlock_bh(&adapter->mbx_lock); 1834 1835 /* translate error return types so error makes sense */ 1836 if (err == IXGBE_ERR_MBX) 1837 return -EIO; 1838 1839 if (err == IXGBE_ERR_INVALID_ARGUMENT) 1840 return -EACCES; 1841 1842 set_bit(vid, adapter->active_vlans); 1843 1844 return err; 1845 } 1846 1847 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev, 1848 __be16 proto, u16 vid) 1849 { 1850 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1851 struct ixgbe_hw *hw = &adapter->hw; 1852 int err; 1853 1854 spin_lock_bh(&adapter->mbx_lock); 1855 1856 /* remove VID from filter table */ 1857 err = hw->mac.ops.set_vfta(hw, vid, 0, false); 1858 1859 spin_unlock_bh(&adapter->mbx_lock); 1860 1861 clear_bit(vid, adapter->active_vlans); 1862 1863 return err; 1864 } 1865 1866 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter) 1867 { 1868 u16 vid; 1869 1870 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1871 ixgbevf_vlan_rx_add_vid(adapter->netdev, 1872 htons(ETH_P_8021Q), vid); 1873 } 1874 1875 static int ixgbevf_write_uc_addr_list(struct net_device *netdev) 1876 { 1877 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1878 struct ixgbe_hw *hw = &adapter->hw; 1879 int count = 0; 1880 1881 if ((netdev_uc_count(netdev)) > 10) { 1882 pr_err("Too many unicast filters - No Space\n"); 1883 return -ENOSPC; 1884 } 1885 1886 if (!netdev_uc_empty(netdev)) { 1887 struct netdev_hw_addr *ha; 1888 1889 netdev_for_each_uc_addr(ha, netdev) { 1890 hw->mac.ops.set_uc_addr(hw, ++count, ha->addr); 1891 udelay(200); 1892 } 1893 } else { 1894 /* If the list is empty then send message to PF driver to 1895 * clear all MAC VLANs on this VF. 1896 */ 1897 hw->mac.ops.set_uc_addr(hw, 0, NULL); 1898 } 1899 1900 return count; 1901 } 1902 1903 /** 1904 * ixgbevf_set_rx_mode - Multicast and unicast set 1905 * @netdev: network interface device structure 1906 * 1907 * The set_rx_method entry point is called whenever the multicast address 1908 * list, unicast address list or the network interface flags are updated. 1909 * This routine is responsible for configuring the hardware for proper 1910 * multicast mode and configuring requested unicast filters. 1911 **/ 1912 static void ixgbevf_set_rx_mode(struct net_device *netdev) 1913 { 1914 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 1915 struct ixgbe_hw *hw = &adapter->hw; 1916 unsigned int flags = netdev->flags; 1917 int xcast_mode; 1918 1919 xcast_mode = (flags & IFF_ALLMULTI) ? IXGBEVF_XCAST_MODE_ALLMULTI : 1920 (flags & (IFF_BROADCAST | IFF_MULTICAST)) ? 1921 IXGBEVF_XCAST_MODE_MULTI : IXGBEVF_XCAST_MODE_NONE; 1922 1923 spin_lock_bh(&adapter->mbx_lock); 1924 1925 hw->mac.ops.update_xcast_mode(hw, xcast_mode); 1926 1927 /* reprogram multicast list */ 1928 hw->mac.ops.update_mc_addr_list(hw, netdev); 1929 1930 ixgbevf_write_uc_addr_list(netdev); 1931 1932 spin_unlock_bh(&adapter->mbx_lock); 1933 } 1934 1935 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter) 1936 { 1937 int q_idx; 1938 struct ixgbevf_q_vector *q_vector; 1939 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1940 1941 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1942 q_vector = adapter->q_vector[q_idx]; 1943 #ifdef CONFIG_NET_RX_BUSY_POLL 1944 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]); 1945 #endif 1946 napi_enable(&q_vector->napi); 1947 } 1948 } 1949 1950 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter) 1951 { 1952 int q_idx; 1953 struct ixgbevf_q_vector *q_vector; 1954 int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 1955 1956 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1957 q_vector = adapter->q_vector[q_idx]; 1958 napi_disable(&q_vector->napi); 1959 #ifdef CONFIG_NET_RX_BUSY_POLL 1960 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) { 1961 pr_info("QV %d locked\n", q_idx); 1962 usleep_range(1000, 20000); 1963 } 1964 #endif /* CONFIG_NET_RX_BUSY_POLL */ 1965 } 1966 } 1967 1968 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter) 1969 { 1970 struct ixgbe_hw *hw = &adapter->hw; 1971 unsigned int def_q = 0; 1972 unsigned int num_tcs = 0; 1973 unsigned int num_rx_queues = adapter->num_rx_queues; 1974 unsigned int num_tx_queues = adapter->num_tx_queues; 1975 int err; 1976 1977 spin_lock_bh(&adapter->mbx_lock); 1978 1979 /* fetch queue configuration from the PF */ 1980 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 1981 1982 spin_unlock_bh(&adapter->mbx_lock); 1983 1984 if (err) 1985 return err; 1986 1987 if (num_tcs > 1) { 1988 /* we need only one Tx queue */ 1989 num_tx_queues = 1; 1990 1991 /* update default Tx ring register index */ 1992 adapter->tx_ring[0]->reg_idx = def_q; 1993 1994 /* we need as many queues as traffic classes */ 1995 num_rx_queues = num_tcs; 1996 } 1997 1998 /* if we have a bad config abort request queue reset */ 1999 if ((adapter->num_rx_queues != num_rx_queues) || 2000 (adapter->num_tx_queues != num_tx_queues)) { 2001 /* force mailbox timeout to prevent further messages */ 2002 hw->mbx.timeout = 0; 2003 2004 /* wait for watchdog to come around and bail us out */ 2005 set_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, &adapter->state); 2006 } 2007 2008 return 0; 2009 } 2010 2011 static void ixgbevf_configure(struct ixgbevf_adapter *adapter) 2012 { 2013 ixgbevf_configure_dcb(adapter); 2014 2015 ixgbevf_set_rx_mode(adapter->netdev); 2016 2017 ixgbevf_restore_vlan(adapter); 2018 2019 ixgbevf_configure_tx(adapter); 2020 ixgbevf_configure_rx(adapter); 2021 } 2022 2023 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter) 2024 { 2025 /* Only save pre-reset stats if there are some */ 2026 if (adapter->stats.vfgprc || adapter->stats.vfgptc) { 2027 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc - 2028 adapter->stats.base_vfgprc; 2029 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc - 2030 adapter->stats.base_vfgptc; 2031 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc - 2032 adapter->stats.base_vfgorc; 2033 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc - 2034 adapter->stats.base_vfgotc; 2035 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc - 2036 adapter->stats.base_vfmprc; 2037 } 2038 } 2039 2040 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter) 2041 { 2042 struct ixgbe_hw *hw = &adapter->hw; 2043 2044 adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC); 2045 adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB); 2046 adapter->stats.last_vfgorc |= 2047 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32); 2048 adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC); 2049 adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB); 2050 adapter->stats.last_vfgotc |= 2051 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32); 2052 adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC); 2053 2054 adapter->stats.base_vfgprc = adapter->stats.last_vfgprc; 2055 adapter->stats.base_vfgorc = adapter->stats.last_vfgorc; 2056 adapter->stats.base_vfgptc = adapter->stats.last_vfgptc; 2057 adapter->stats.base_vfgotc = adapter->stats.last_vfgotc; 2058 adapter->stats.base_vfmprc = adapter->stats.last_vfmprc; 2059 } 2060 2061 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) 2062 { 2063 struct ixgbe_hw *hw = &adapter->hw; 2064 int api[] = { ixgbe_mbox_api_12, 2065 ixgbe_mbox_api_11, 2066 ixgbe_mbox_api_10, 2067 ixgbe_mbox_api_unknown }; 2068 int err, idx = 0; 2069 2070 spin_lock_bh(&adapter->mbx_lock); 2071 2072 while (api[idx] != ixgbe_mbox_api_unknown) { 2073 err = hw->mac.ops.negotiate_api_version(hw, api[idx]); 2074 if (!err) 2075 break; 2076 idx++; 2077 } 2078 2079 spin_unlock_bh(&adapter->mbx_lock); 2080 } 2081 2082 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter) 2083 { 2084 struct net_device *netdev = adapter->netdev; 2085 struct ixgbe_hw *hw = &adapter->hw; 2086 2087 ixgbevf_configure_msix(adapter); 2088 2089 spin_lock_bh(&adapter->mbx_lock); 2090 2091 if (is_valid_ether_addr(hw->mac.addr)) 2092 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0); 2093 else 2094 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0); 2095 2096 spin_unlock_bh(&adapter->mbx_lock); 2097 2098 smp_mb__before_atomic(); 2099 clear_bit(__IXGBEVF_DOWN, &adapter->state); 2100 ixgbevf_napi_enable_all(adapter); 2101 2102 /* clear any pending interrupts, may auto mask */ 2103 IXGBE_READ_REG(hw, IXGBE_VTEICR); 2104 ixgbevf_irq_enable(adapter); 2105 2106 /* enable transmits */ 2107 netif_tx_start_all_queues(netdev); 2108 2109 ixgbevf_save_reset_stats(adapter); 2110 ixgbevf_init_last_counter_stats(adapter); 2111 2112 hw->mac.get_link_status = 1; 2113 mod_timer(&adapter->service_timer, jiffies); 2114 } 2115 2116 void ixgbevf_up(struct ixgbevf_adapter *adapter) 2117 { 2118 ixgbevf_configure(adapter); 2119 2120 ixgbevf_up_complete(adapter); 2121 } 2122 2123 /** 2124 * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue 2125 * @rx_ring: ring to free buffers from 2126 **/ 2127 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) 2128 { 2129 struct device *dev = rx_ring->dev; 2130 unsigned long size; 2131 unsigned int i; 2132 2133 /* Free Rx ring sk_buff */ 2134 if (rx_ring->skb) { 2135 dev_kfree_skb(rx_ring->skb); 2136 rx_ring->skb = NULL; 2137 } 2138 2139 /* ring already cleared, nothing to do */ 2140 if (!rx_ring->rx_buffer_info) 2141 return; 2142 2143 /* Free all the Rx ring pages */ 2144 for (i = 0; i < rx_ring->count; i++) { 2145 struct ixgbevf_rx_buffer *rx_buffer; 2146 2147 rx_buffer = &rx_ring->rx_buffer_info[i]; 2148 if (rx_buffer->dma) 2149 dma_unmap_page(dev, rx_buffer->dma, 2150 PAGE_SIZE, DMA_FROM_DEVICE); 2151 rx_buffer->dma = 0; 2152 if (rx_buffer->page) 2153 __free_page(rx_buffer->page); 2154 rx_buffer->page = NULL; 2155 } 2156 2157 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 2158 memset(rx_ring->rx_buffer_info, 0, size); 2159 2160 /* Zero out the descriptor ring */ 2161 memset(rx_ring->desc, 0, rx_ring->size); 2162 } 2163 2164 /** 2165 * ixgbevf_clean_tx_ring - Free Tx Buffers 2166 * @tx_ring: ring to be cleaned 2167 **/ 2168 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) 2169 { 2170 struct ixgbevf_tx_buffer *tx_buffer_info; 2171 unsigned long size; 2172 unsigned int i; 2173 2174 if (!tx_ring->tx_buffer_info) 2175 return; 2176 2177 /* Free all the Tx ring sk_buffs */ 2178 for (i = 0; i < tx_ring->count; i++) { 2179 tx_buffer_info = &tx_ring->tx_buffer_info[i]; 2180 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info); 2181 } 2182 2183 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2184 memset(tx_ring->tx_buffer_info, 0, size); 2185 2186 memset(tx_ring->desc, 0, tx_ring->size); 2187 } 2188 2189 /** 2190 * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues 2191 * @adapter: board private structure 2192 **/ 2193 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter) 2194 { 2195 int i; 2196 2197 for (i = 0; i < adapter->num_rx_queues; i++) 2198 ixgbevf_clean_rx_ring(adapter->rx_ring[i]); 2199 } 2200 2201 /** 2202 * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues 2203 * @adapter: board private structure 2204 **/ 2205 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter) 2206 { 2207 int i; 2208 2209 for (i = 0; i < adapter->num_tx_queues; i++) 2210 ixgbevf_clean_tx_ring(adapter->tx_ring[i]); 2211 } 2212 2213 void ixgbevf_down(struct ixgbevf_adapter *adapter) 2214 { 2215 struct net_device *netdev = adapter->netdev; 2216 struct ixgbe_hw *hw = &adapter->hw; 2217 int i; 2218 2219 /* signal that we are down to the interrupt handler */ 2220 if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state)) 2221 return; /* do nothing if already down */ 2222 2223 /* disable all enabled Rx queues */ 2224 for (i = 0; i < adapter->num_rx_queues; i++) 2225 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]); 2226 2227 usleep_range(10000, 20000); 2228 2229 netif_tx_stop_all_queues(netdev); 2230 2231 /* call carrier off first to avoid false dev_watchdog timeouts */ 2232 netif_carrier_off(netdev); 2233 netif_tx_disable(netdev); 2234 2235 ixgbevf_irq_disable(adapter); 2236 2237 ixgbevf_napi_disable_all(adapter); 2238 2239 del_timer_sync(&adapter->service_timer); 2240 2241 /* disable transmits in the hardware now that interrupts are off */ 2242 for (i = 0; i < adapter->num_tx_queues; i++) { 2243 u8 reg_idx = adapter->tx_ring[i]->reg_idx; 2244 2245 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), 2246 IXGBE_TXDCTL_SWFLSH); 2247 } 2248 2249 if (!pci_channel_offline(adapter->pdev)) 2250 ixgbevf_reset(adapter); 2251 2252 ixgbevf_clean_all_tx_rings(adapter); 2253 ixgbevf_clean_all_rx_rings(adapter); 2254 } 2255 2256 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter) 2257 { 2258 WARN_ON(in_interrupt()); 2259 2260 while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state)) 2261 msleep(1); 2262 2263 ixgbevf_down(adapter); 2264 ixgbevf_up(adapter); 2265 2266 clear_bit(__IXGBEVF_RESETTING, &adapter->state); 2267 } 2268 2269 void ixgbevf_reset(struct ixgbevf_adapter *adapter) 2270 { 2271 struct ixgbe_hw *hw = &adapter->hw; 2272 struct net_device *netdev = adapter->netdev; 2273 2274 if (hw->mac.ops.reset_hw(hw)) { 2275 hw_dbg(hw, "PF still resetting\n"); 2276 } else { 2277 hw->mac.ops.init_hw(hw); 2278 ixgbevf_negotiate_api(adapter); 2279 } 2280 2281 if (is_valid_ether_addr(adapter->hw.mac.addr)) { 2282 ether_addr_copy(netdev->dev_addr, adapter->hw.mac.addr); 2283 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); 2284 } 2285 2286 adapter->last_reset = jiffies; 2287 } 2288 2289 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter, 2290 int vectors) 2291 { 2292 int vector_threshold; 2293 2294 /* We'll want at least 2 (vector_threshold): 2295 * 1) TxQ[0] + RxQ[0] handler 2296 * 2) Other (Link Status Change, etc.) 2297 */ 2298 vector_threshold = MIN_MSIX_COUNT; 2299 2300 /* The more we get, the more we will assign to Tx/Rx Cleanup 2301 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 2302 * Right now, we simply care about how many we'll get; we'll 2303 * set them up later while requesting irq's. 2304 */ 2305 vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2306 vector_threshold, vectors); 2307 2308 if (vectors < 0) { 2309 dev_err(&adapter->pdev->dev, 2310 "Unable to allocate MSI-X interrupts\n"); 2311 kfree(adapter->msix_entries); 2312 adapter->msix_entries = NULL; 2313 return vectors; 2314 } 2315 2316 /* Adjust for only the vectors we'll use, which is minimum 2317 * of max_msix_q_vectors + NON_Q_VECTORS, or the number of 2318 * vectors we were allocated. 2319 */ 2320 adapter->num_msix_vectors = vectors; 2321 2322 return 0; 2323 } 2324 2325 /** 2326 * ixgbevf_set_num_queues - Allocate queues for device, feature dependent 2327 * @adapter: board private structure to initialize 2328 * 2329 * This is the top level queue allocation routine. The order here is very 2330 * important, starting with the "most" number of features turned on at once, 2331 * and ending with the smallest set of features. This way large combinations 2332 * can be allocated if they're turned on, and smaller combinations are the 2333 * fallthrough conditions. 2334 * 2335 **/ 2336 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) 2337 { 2338 struct ixgbe_hw *hw = &adapter->hw; 2339 unsigned int def_q = 0; 2340 unsigned int num_tcs = 0; 2341 int err; 2342 2343 /* Start with base case */ 2344 adapter->num_rx_queues = 1; 2345 adapter->num_tx_queues = 1; 2346 2347 spin_lock_bh(&adapter->mbx_lock); 2348 2349 /* fetch queue configuration from the PF */ 2350 err = ixgbevf_get_queues(hw, &num_tcs, &def_q); 2351 2352 spin_unlock_bh(&adapter->mbx_lock); 2353 2354 if (err) 2355 return; 2356 2357 /* we need as many queues as traffic classes */ 2358 if (num_tcs > 1) { 2359 adapter->num_rx_queues = num_tcs; 2360 } else { 2361 u16 rss = min_t(u16, num_online_cpus(), IXGBEVF_MAX_RSS_QUEUES); 2362 2363 switch (hw->api_version) { 2364 case ixgbe_mbox_api_11: 2365 case ixgbe_mbox_api_12: 2366 adapter->num_rx_queues = rss; 2367 adapter->num_tx_queues = rss; 2368 default: 2369 break; 2370 } 2371 } 2372 } 2373 2374 /** 2375 * ixgbevf_alloc_queues - Allocate memory for all rings 2376 * @adapter: board private structure to initialize 2377 * 2378 * We allocate one ring per queue at run-time since we don't know the 2379 * number of queues at compile-time. The polling_netdev array is 2380 * intended for Multiqueue, but should work fine with a single queue. 2381 **/ 2382 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter) 2383 { 2384 struct ixgbevf_ring *ring; 2385 int rx = 0, tx = 0; 2386 2387 for (; tx < adapter->num_tx_queues; tx++) { 2388 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2389 if (!ring) 2390 goto err_allocation; 2391 2392 ring->dev = &adapter->pdev->dev; 2393 ring->netdev = adapter->netdev; 2394 ring->count = adapter->tx_ring_count; 2395 ring->queue_index = tx; 2396 ring->reg_idx = tx; 2397 2398 adapter->tx_ring[tx] = ring; 2399 } 2400 2401 for (; rx < adapter->num_rx_queues; rx++) { 2402 ring = kzalloc(sizeof(*ring), GFP_KERNEL); 2403 if (!ring) 2404 goto err_allocation; 2405 2406 ring->dev = &adapter->pdev->dev; 2407 ring->netdev = adapter->netdev; 2408 2409 ring->count = adapter->rx_ring_count; 2410 ring->queue_index = rx; 2411 ring->reg_idx = rx; 2412 2413 adapter->rx_ring[rx] = ring; 2414 } 2415 2416 return 0; 2417 2418 err_allocation: 2419 while (tx) { 2420 kfree(adapter->tx_ring[--tx]); 2421 adapter->tx_ring[tx] = NULL; 2422 } 2423 2424 while (rx) { 2425 kfree(adapter->rx_ring[--rx]); 2426 adapter->rx_ring[rx] = NULL; 2427 } 2428 return -ENOMEM; 2429 } 2430 2431 /** 2432 * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported 2433 * @adapter: board private structure to initialize 2434 * 2435 * Attempt to configure the interrupts using the best available 2436 * capabilities of the hardware and the kernel. 2437 **/ 2438 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter) 2439 { 2440 struct net_device *netdev = adapter->netdev; 2441 int err; 2442 int vector, v_budget; 2443 2444 /* It's easy to be greedy for MSI-X vectors, but it really 2445 * doesn't do us much good if we have a lot more vectors 2446 * than CPU's. So let's be conservative and only ask for 2447 * (roughly) the same number of vectors as there are CPU's. 2448 * The default is to use pairs of vectors. 2449 */ 2450 v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues); 2451 v_budget = min_t(int, v_budget, num_online_cpus()); 2452 v_budget += NON_Q_VECTORS; 2453 2454 /* A failure in MSI-X entry allocation isn't fatal, but it does 2455 * mean we disable MSI-X capabilities of the adapter. 2456 */ 2457 adapter->msix_entries = kcalloc(v_budget, 2458 sizeof(struct msix_entry), GFP_KERNEL); 2459 if (!adapter->msix_entries) 2460 return -ENOMEM; 2461 2462 for (vector = 0; vector < v_budget; vector++) 2463 adapter->msix_entries[vector].entry = vector; 2464 2465 err = ixgbevf_acquire_msix_vectors(adapter, v_budget); 2466 if (err) 2467 return err; 2468 2469 err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues); 2470 if (err) 2471 return err; 2472 2473 return netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues); 2474 } 2475 2476 /** 2477 * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors 2478 * @adapter: board private structure to initialize 2479 * 2480 * We allocate one q_vector per queue interrupt. If allocation fails we 2481 * return -ENOMEM. 2482 **/ 2483 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter) 2484 { 2485 int q_idx, num_q_vectors; 2486 struct ixgbevf_q_vector *q_vector; 2487 2488 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2489 2490 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2491 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL); 2492 if (!q_vector) 2493 goto err_out; 2494 q_vector->adapter = adapter; 2495 q_vector->v_idx = q_idx; 2496 netif_napi_add(adapter->netdev, &q_vector->napi, 2497 ixgbevf_poll, 64); 2498 adapter->q_vector[q_idx] = q_vector; 2499 } 2500 2501 return 0; 2502 2503 err_out: 2504 while (q_idx) { 2505 q_idx--; 2506 q_vector = adapter->q_vector[q_idx]; 2507 #ifdef CONFIG_NET_RX_BUSY_POLL 2508 napi_hash_del(&q_vector->napi); 2509 #endif 2510 netif_napi_del(&q_vector->napi); 2511 kfree(q_vector); 2512 adapter->q_vector[q_idx] = NULL; 2513 } 2514 return -ENOMEM; 2515 } 2516 2517 /** 2518 * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors 2519 * @adapter: board private structure to initialize 2520 * 2521 * This function frees the memory allocated to the q_vectors. In addition if 2522 * NAPI is enabled it will delete any references to the NAPI struct prior 2523 * to freeing the q_vector. 2524 **/ 2525 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter) 2526 { 2527 int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; 2528 2529 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 2530 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx]; 2531 2532 adapter->q_vector[q_idx] = NULL; 2533 #ifdef CONFIG_NET_RX_BUSY_POLL 2534 napi_hash_del(&q_vector->napi); 2535 #endif 2536 netif_napi_del(&q_vector->napi); 2537 kfree(q_vector); 2538 } 2539 } 2540 2541 /** 2542 * ixgbevf_reset_interrupt_capability - Reset MSIX setup 2543 * @adapter: board private structure 2544 * 2545 **/ 2546 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter) 2547 { 2548 pci_disable_msix(adapter->pdev); 2549 kfree(adapter->msix_entries); 2550 adapter->msix_entries = NULL; 2551 } 2552 2553 /** 2554 * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init 2555 * @adapter: board private structure to initialize 2556 * 2557 **/ 2558 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter) 2559 { 2560 int err; 2561 2562 /* Number of supported queues */ 2563 ixgbevf_set_num_queues(adapter); 2564 2565 err = ixgbevf_set_interrupt_capability(adapter); 2566 if (err) { 2567 hw_dbg(&adapter->hw, 2568 "Unable to setup interrupt capabilities\n"); 2569 goto err_set_interrupt; 2570 } 2571 2572 err = ixgbevf_alloc_q_vectors(adapter); 2573 if (err) { 2574 hw_dbg(&adapter->hw, "Unable to allocate memory for queue vectors\n"); 2575 goto err_alloc_q_vectors; 2576 } 2577 2578 err = ixgbevf_alloc_queues(adapter); 2579 if (err) { 2580 pr_err("Unable to allocate memory for queues\n"); 2581 goto err_alloc_queues; 2582 } 2583 2584 hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, Tx Queue count = %u\n", 2585 (adapter->num_rx_queues > 1) ? "Enabled" : 2586 "Disabled", adapter->num_rx_queues, adapter->num_tx_queues); 2587 2588 set_bit(__IXGBEVF_DOWN, &adapter->state); 2589 2590 return 0; 2591 err_alloc_queues: 2592 ixgbevf_free_q_vectors(adapter); 2593 err_alloc_q_vectors: 2594 ixgbevf_reset_interrupt_capability(adapter); 2595 err_set_interrupt: 2596 return err; 2597 } 2598 2599 /** 2600 * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings 2601 * @adapter: board private structure to clear interrupt scheme on 2602 * 2603 * We go through and clear interrupt specific resources and reset the structure 2604 * to pre-load conditions 2605 **/ 2606 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter) 2607 { 2608 int i; 2609 2610 for (i = 0; i < adapter->num_tx_queues; i++) { 2611 kfree(adapter->tx_ring[i]); 2612 adapter->tx_ring[i] = NULL; 2613 } 2614 for (i = 0; i < adapter->num_rx_queues; i++) { 2615 kfree(adapter->rx_ring[i]); 2616 adapter->rx_ring[i] = NULL; 2617 } 2618 2619 adapter->num_tx_queues = 0; 2620 adapter->num_rx_queues = 0; 2621 2622 ixgbevf_free_q_vectors(adapter); 2623 ixgbevf_reset_interrupt_capability(adapter); 2624 } 2625 2626 /** 2627 * ixgbevf_sw_init - Initialize general software structures 2628 * @adapter: board private structure to initialize 2629 * 2630 * ixgbevf_sw_init initializes the Adapter private data structure. 2631 * Fields are initialized based on PCI device information and 2632 * OS network device settings (MTU size). 2633 **/ 2634 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter) 2635 { 2636 struct ixgbe_hw *hw = &adapter->hw; 2637 struct pci_dev *pdev = adapter->pdev; 2638 struct net_device *netdev = adapter->netdev; 2639 int err; 2640 2641 /* PCI config space info */ 2642 hw->vendor_id = pdev->vendor; 2643 hw->device_id = pdev->device; 2644 hw->revision_id = pdev->revision; 2645 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2646 hw->subsystem_device_id = pdev->subsystem_device; 2647 2648 hw->mbx.ops.init_params(hw); 2649 2650 /* assume legacy case in which PF would only give VF 2 queues */ 2651 hw->mac.max_tx_queues = 2; 2652 hw->mac.max_rx_queues = 2; 2653 2654 /* lock to protect mailbox accesses */ 2655 spin_lock_init(&adapter->mbx_lock); 2656 2657 err = hw->mac.ops.reset_hw(hw); 2658 if (err) { 2659 dev_info(&pdev->dev, 2660 "PF still in reset state. Is the PF interface up?\n"); 2661 } else { 2662 err = hw->mac.ops.init_hw(hw); 2663 if (err) { 2664 pr_err("init_shared_code failed: %d\n", err); 2665 goto out; 2666 } 2667 ixgbevf_negotiate_api(adapter); 2668 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr); 2669 if (err) 2670 dev_info(&pdev->dev, "Error reading MAC address\n"); 2671 else if (is_zero_ether_addr(adapter->hw.mac.addr)) 2672 dev_info(&pdev->dev, 2673 "MAC address not assigned by administrator.\n"); 2674 ether_addr_copy(netdev->dev_addr, hw->mac.addr); 2675 } 2676 2677 if (!is_valid_ether_addr(netdev->dev_addr)) { 2678 dev_info(&pdev->dev, "Assigning random MAC address\n"); 2679 eth_hw_addr_random(netdev); 2680 ether_addr_copy(hw->mac.addr, netdev->dev_addr); 2681 ether_addr_copy(hw->mac.perm_addr, netdev->dev_addr); 2682 } 2683 2684 /* Enable dynamic interrupt throttling rates */ 2685 adapter->rx_itr_setting = 1; 2686 adapter->tx_itr_setting = 1; 2687 2688 /* set default ring sizes */ 2689 adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD; 2690 adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD; 2691 2692 set_bit(__IXGBEVF_DOWN, &adapter->state); 2693 return 0; 2694 2695 out: 2696 return err; 2697 } 2698 2699 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter) \ 2700 { \ 2701 u32 current_counter = IXGBE_READ_REG(hw, reg); \ 2702 if (current_counter < last_counter) \ 2703 counter += 0x100000000LL; \ 2704 last_counter = current_counter; \ 2705 counter &= 0xFFFFFFFF00000000LL; \ 2706 counter |= current_counter; \ 2707 } 2708 2709 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \ 2710 { \ 2711 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb); \ 2712 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb); \ 2713 u64 current_counter = (current_counter_msb << 32) | \ 2714 current_counter_lsb; \ 2715 if (current_counter < last_counter) \ 2716 counter += 0x1000000000LL; \ 2717 last_counter = current_counter; \ 2718 counter &= 0xFFFFFFF000000000LL; \ 2719 counter |= current_counter; \ 2720 } 2721 /** 2722 * ixgbevf_update_stats - Update the board statistics counters. 2723 * @adapter: board private structure 2724 **/ 2725 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter) 2726 { 2727 struct ixgbe_hw *hw = &adapter->hw; 2728 int i; 2729 2730 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2731 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2732 return; 2733 2734 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc, 2735 adapter->stats.vfgprc); 2736 UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc, 2737 adapter->stats.vfgptc); 2738 UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB, 2739 adapter->stats.last_vfgorc, 2740 adapter->stats.vfgorc); 2741 UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB, 2742 adapter->stats.last_vfgotc, 2743 adapter->stats.vfgotc); 2744 UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc, 2745 adapter->stats.vfmprc); 2746 2747 for (i = 0; i < adapter->num_rx_queues; i++) { 2748 adapter->hw_csum_rx_error += 2749 adapter->rx_ring[i]->hw_csum_rx_error; 2750 adapter->rx_ring[i]->hw_csum_rx_error = 0; 2751 } 2752 } 2753 2754 /** 2755 * ixgbevf_service_timer - Timer Call-back 2756 * @data: pointer to adapter cast into an unsigned long 2757 **/ 2758 static void ixgbevf_service_timer(unsigned long data) 2759 { 2760 struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data; 2761 2762 /* Reset the timer */ 2763 mod_timer(&adapter->service_timer, (HZ * 2) + jiffies); 2764 2765 ixgbevf_service_event_schedule(adapter); 2766 } 2767 2768 static void ixgbevf_reset_subtask(struct ixgbevf_adapter *adapter) 2769 { 2770 if (!test_and_clear_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state)) 2771 return; 2772 2773 /* If we're already down or resetting, just bail */ 2774 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2775 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2776 return; 2777 2778 adapter->tx_timeout_count++; 2779 2780 ixgbevf_reinit_locked(adapter); 2781 } 2782 2783 /** 2784 * ixgbevf_check_hang_subtask - check for hung queues and dropped interrupts 2785 * @adapter: pointer to the device adapter structure 2786 * 2787 * This function serves two purposes. First it strobes the interrupt lines 2788 * in order to make certain interrupts are occurring. Secondly it sets the 2789 * bits needed to check for TX hangs. As a result we should immediately 2790 * determine if a hang has occurred. 2791 **/ 2792 static void ixgbevf_check_hang_subtask(struct ixgbevf_adapter *adapter) 2793 { 2794 struct ixgbe_hw *hw = &adapter->hw; 2795 u32 eics = 0; 2796 int i; 2797 2798 /* If we're down or resetting, just bail */ 2799 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2800 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2801 return; 2802 2803 /* Force detection of hung controller */ 2804 if (netif_carrier_ok(adapter->netdev)) { 2805 for (i = 0; i < adapter->num_tx_queues; i++) 2806 set_check_for_tx_hang(adapter->tx_ring[i]); 2807 } 2808 2809 /* get one bit for every active Tx/Rx interrupt vector */ 2810 for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) { 2811 struct ixgbevf_q_vector *qv = adapter->q_vector[i]; 2812 2813 if (qv->rx.ring || qv->tx.ring) 2814 eics |= BIT(i); 2815 } 2816 2817 /* Cause software interrupt to ensure rings are cleaned */ 2818 IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics); 2819 } 2820 2821 /** 2822 * ixgbevf_watchdog_update_link - update the link status 2823 * @adapter: pointer to the device adapter structure 2824 **/ 2825 static void ixgbevf_watchdog_update_link(struct ixgbevf_adapter *adapter) 2826 { 2827 struct ixgbe_hw *hw = &adapter->hw; 2828 u32 link_speed = adapter->link_speed; 2829 bool link_up = adapter->link_up; 2830 s32 err; 2831 2832 spin_lock_bh(&adapter->mbx_lock); 2833 2834 err = hw->mac.ops.check_link(hw, &link_speed, &link_up, false); 2835 2836 spin_unlock_bh(&adapter->mbx_lock); 2837 2838 /* if check for link returns error we will need to reset */ 2839 if (err && time_after(jiffies, adapter->last_reset + (10 * HZ))) { 2840 set_bit(__IXGBEVF_RESET_REQUESTED, &adapter->state); 2841 link_up = false; 2842 } 2843 2844 adapter->link_up = link_up; 2845 adapter->link_speed = link_speed; 2846 } 2847 2848 /** 2849 * ixgbevf_watchdog_link_is_up - update netif_carrier status and 2850 * print link up message 2851 * @adapter: pointer to the device adapter structure 2852 **/ 2853 static void ixgbevf_watchdog_link_is_up(struct ixgbevf_adapter *adapter) 2854 { 2855 struct net_device *netdev = adapter->netdev; 2856 2857 /* only continue if link was previously down */ 2858 if (netif_carrier_ok(netdev)) 2859 return; 2860 2861 dev_info(&adapter->pdev->dev, "NIC Link is Up %s\n", 2862 (adapter->link_speed == IXGBE_LINK_SPEED_10GB_FULL) ? 2863 "10 Gbps" : 2864 (adapter->link_speed == IXGBE_LINK_SPEED_1GB_FULL) ? 2865 "1 Gbps" : 2866 (adapter->link_speed == IXGBE_LINK_SPEED_100_FULL) ? 2867 "100 Mbps" : 2868 "unknown speed"); 2869 2870 netif_carrier_on(netdev); 2871 } 2872 2873 /** 2874 * ixgbevf_watchdog_link_is_down - update netif_carrier status and 2875 * print link down message 2876 * @adapter: pointer to the adapter structure 2877 **/ 2878 static void ixgbevf_watchdog_link_is_down(struct ixgbevf_adapter *adapter) 2879 { 2880 struct net_device *netdev = adapter->netdev; 2881 2882 adapter->link_speed = 0; 2883 2884 /* only continue if link was up previously */ 2885 if (!netif_carrier_ok(netdev)) 2886 return; 2887 2888 dev_info(&adapter->pdev->dev, "NIC Link is Down\n"); 2889 2890 netif_carrier_off(netdev); 2891 } 2892 2893 /** 2894 * ixgbevf_watchdog_subtask - worker thread to bring link up 2895 * @work: pointer to work_struct containing our data 2896 **/ 2897 static void ixgbevf_watchdog_subtask(struct ixgbevf_adapter *adapter) 2898 { 2899 /* if interface is down do nothing */ 2900 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 2901 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 2902 return; 2903 2904 ixgbevf_watchdog_update_link(adapter); 2905 2906 if (adapter->link_up) 2907 ixgbevf_watchdog_link_is_up(adapter); 2908 else 2909 ixgbevf_watchdog_link_is_down(adapter); 2910 2911 ixgbevf_update_stats(adapter); 2912 } 2913 2914 /** 2915 * ixgbevf_service_task - manages and runs subtasks 2916 * @work: pointer to work_struct containing our data 2917 **/ 2918 static void ixgbevf_service_task(struct work_struct *work) 2919 { 2920 struct ixgbevf_adapter *adapter = container_of(work, 2921 struct ixgbevf_adapter, 2922 service_task); 2923 struct ixgbe_hw *hw = &adapter->hw; 2924 2925 if (IXGBE_REMOVED(hw->hw_addr)) { 2926 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) { 2927 rtnl_lock(); 2928 ixgbevf_down(adapter); 2929 rtnl_unlock(); 2930 } 2931 return; 2932 } 2933 2934 ixgbevf_queue_reset_subtask(adapter); 2935 ixgbevf_reset_subtask(adapter); 2936 ixgbevf_watchdog_subtask(adapter); 2937 ixgbevf_check_hang_subtask(adapter); 2938 2939 ixgbevf_service_event_complete(adapter); 2940 } 2941 2942 /** 2943 * ixgbevf_free_tx_resources - Free Tx Resources per Queue 2944 * @tx_ring: Tx descriptor ring for a specific queue 2945 * 2946 * Free all transmit software resources 2947 **/ 2948 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring) 2949 { 2950 ixgbevf_clean_tx_ring(tx_ring); 2951 2952 vfree(tx_ring->tx_buffer_info); 2953 tx_ring->tx_buffer_info = NULL; 2954 2955 /* if not set, then don't free */ 2956 if (!tx_ring->desc) 2957 return; 2958 2959 dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc, 2960 tx_ring->dma); 2961 2962 tx_ring->desc = NULL; 2963 } 2964 2965 /** 2966 * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues 2967 * @adapter: board private structure 2968 * 2969 * Free all transmit software resources 2970 **/ 2971 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter) 2972 { 2973 int i; 2974 2975 for (i = 0; i < adapter->num_tx_queues; i++) 2976 if (adapter->tx_ring[i]->desc) 2977 ixgbevf_free_tx_resources(adapter->tx_ring[i]); 2978 } 2979 2980 /** 2981 * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors) 2982 * @tx_ring: Tx descriptor ring (for a specific queue) to setup 2983 * 2984 * Return 0 on success, negative on failure 2985 **/ 2986 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) 2987 { 2988 int size; 2989 2990 size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count; 2991 tx_ring->tx_buffer_info = vzalloc(size); 2992 if (!tx_ring->tx_buffer_info) 2993 goto err; 2994 2995 /* round up to nearest 4K */ 2996 tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc); 2997 tx_ring->size = ALIGN(tx_ring->size, 4096); 2998 2999 tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size, 3000 &tx_ring->dma, GFP_KERNEL); 3001 if (!tx_ring->desc) 3002 goto err; 3003 3004 return 0; 3005 3006 err: 3007 vfree(tx_ring->tx_buffer_info); 3008 tx_ring->tx_buffer_info = NULL; 3009 hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit descriptor ring\n"); 3010 return -ENOMEM; 3011 } 3012 3013 /** 3014 * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources 3015 * @adapter: board private structure 3016 * 3017 * If this function returns with an error, then it's possible one or 3018 * more of the rings is populated (while the rest are not). It is the 3019 * callers duty to clean those orphaned rings. 3020 * 3021 * Return 0 on success, negative on failure 3022 **/ 3023 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter) 3024 { 3025 int i, err = 0; 3026 3027 for (i = 0; i < adapter->num_tx_queues; i++) { 3028 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]); 3029 if (!err) 3030 continue; 3031 hw_dbg(&adapter->hw, "Allocation for Tx Queue %u failed\n", i); 3032 break; 3033 } 3034 3035 return err; 3036 } 3037 3038 /** 3039 * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors) 3040 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 3041 * 3042 * Returns 0 on success, negative on failure 3043 **/ 3044 int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring) 3045 { 3046 int size; 3047 3048 size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count; 3049 rx_ring->rx_buffer_info = vzalloc(size); 3050 if (!rx_ring->rx_buffer_info) 3051 goto err; 3052 3053 /* Round up to nearest 4K */ 3054 rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); 3055 rx_ring->size = ALIGN(rx_ring->size, 4096); 3056 3057 rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size, 3058 &rx_ring->dma, GFP_KERNEL); 3059 3060 if (!rx_ring->desc) 3061 goto err; 3062 3063 return 0; 3064 err: 3065 vfree(rx_ring->rx_buffer_info); 3066 rx_ring->rx_buffer_info = NULL; 3067 dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n"); 3068 return -ENOMEM; 3069 } 3070 3071 /** 3072 * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources 3073 * @adapter: board private structure 3074 * 3075 * If this function returns with an error, then it's possible one or 3076 * more of the rings is populated (while the rest are not). It is the 3077 * callers duty to clean those orphaned rings. 3078 * 3079 * Return 0 on success, negative on failure 3080 **/ 3081 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter) 3082 { 3083 int i, err = 0; 3084 3085 for (i = 0; i < adapter->num_rx_queues; i++) { 3086 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]); 3087 if (!err) 3088 continue; 3089 hw_dbg(&adapter->hw, "Allocation for Rx Queue %u failed\n", i); 3090 break; 3091 } 3092 return err; 3093 } 3094 3095 /** 3096 * ixgbevf_free_rx_resources - Free Rx Resources 3097 * @rx_ring: ring to clean the resources from 3098 * 3099 * Free all receive software resources 3100 **/ 3101 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring) 3102 { 3103 ixgbevf_clean_rx_ring(rx_ring); 3104 3105 vfree(rx_ring->rx_buffer_info); 3106 rx_ring->rx_buffer_info = NULL; 3107 3108 dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc, 3109 rx_ring->dma); 3110 3111 rx_ring->desc = NULL; 3112 } 3113 3114 /** 3115 * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues 3116 * @adapter: board private structure 3117 * 3118 * Free all receive software resources 3119 **/ 3120 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter) 3121 { 3122 int i; 3123 3124 for (i = 0; i < adapter->num_rx_queues; i++) 3125 if (adapter->rx_ring[i]->desc) 3126 ixgbevf_free_rx_resources(adapter->rx_ring[i]); 3127 } 3128 3129 /** 3130 * ixgbevf_open - Called when a network interface is made active 3131 * @netdev: network interface device structure 3132 * 3133 * Returns 0 on success, negative value on failure 3134 * 3135 * The open entry point is called when a network interface is made 3136 * active by the system (IFF_UP). At this point all resources needed 3137 * for transmit and receive operations are allocated, the interrupt 3138 * handler is registered with the OS, the watchdog timer is started, 3139 * and the stack is notified that the interface is ready. 3140 **/ 3141 int ixgbevf_open(struct net_device *netdev) 3142 { 3143 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3144 struct ixgbe_hw *hw = &adapter->hw; 3145 int err; 3146 3147 /* A previous failure to open the device because of a lack of 3148 * available MSIX vector resources may have reset the number 3149 * of msix vectors variable to zero. The only way to recover 3150 * is to unload/reload the driver and hope that the system has 3151 * been able to recover some MSIX vector resources. 3152 */ 3153 if (!adapter->num_msix_vectors) 3154 return -ENOMEM; 3155 3156 if (hw->adapter_stopped) { 3157 ixgbevf_reset(adapter); 3158 /* if adapter is still stopped then PF isn't up and 3159 * the VF can't start. 3160 */ 3161 if (hw->adapter_stopped) { 3162 err = IXGBE_ERR_MBX; 3163 pr_err("Unable to start - perhaps the PF Driver isn't up yet\n"); 3164 goto err_setup_reset; 3165 } 3166 } 3167 3168 /* disallow open during test */ 3169 if (test_bit(__IXGBEVF_TESTING, &adapter->state)) 3170 return -EBUSY; 3171 3172 netif_carrier_off(netdev); 3173 3174 /* allocate transmit descriptors */ 3175 err = ixgbevf_setup_all_tx_resources(adapter); 3176 if (err) 3177 goto err_setup_tx; 3178 3179 /* allocate receive descriptors */ 3180 err = ixgbevf_setup_all_rx_resources(adapter); 3181 if (err) 3182 goto err_setup_rx; 3183 3184 ixgbevf_configure(adapter); 3185 3186 /* Map the Tx/Rx rings to the vectors we were allotted. 3187 * if request_irq will be called in this function map_rings 3188 * must be called *before* up_complete 3189 */ 3190 ixgbevf_map_rings_to_vectors(adapter); 3191 3192 err = ixgbevf_request_irq(adapter); 3193 if (err) 3194 goto err_req_irq; 3195 3196 ixgbevf_up_complete(adapter); 3197 3198 return 0; 3199 3200 err_req_irq: 3201 ixgbevf_down(adapter); 3202 err_setup_rx: 3203 ixgbevf_free_all_rx_resources(adapter); 3204 err_setup_tx: 3205 ixgbevf_free_all_tx_resources(adapter); 3206 ixgbevf_reset(adapter); 3207 3208 err_setup_reset: 3209 3210 return err; 3211 } 3212 3213 /** 3214 * ixgbevf_close - Disables a network interface 3215 * @netdev: network interface device structure 3216 * 3217 * Returns 0, this is not allowed to fail 3218 * 3219 * The close entry point is called when an interface is de-activated 3220 * by the OS. The hardware is still under the drivers control, but 3221 * needs to be disabled. A global MAC reset is issued to stop the 3222 * hardware, and all transmit and receive resources are freed. 3223 **/ 3224 int ixgbevf_close(struct net_device *netdev) 3225 { 3226 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3227 3228 ixgbevf_down(adapter); 3229 ixgbevf_free_irq(adapter); 3230 3231 ixgbevf_free_all_tx_resources(adapter); 3232 ixgbevf_free_all_rx_resources(adapter); 3233 3234 return 0; 3235 } 3236 3237 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter) 3238 { 3239 struct net_device *dev = adapter->netdev; 3240 3241 if (!test_and_clear_bit(__IXGBEVF_QUEUE_RESET_REQUESTED, 3242 &adapter->state)) 3243 return; 3244 3245 /* if interface is down do nothing */ 3246 if (test_bit(__IXGBEVF_DOWN, &adapter->state) || 3247 test_bit(__IXGBEVF_RESETTING, &adapter->state)) 3248 return; 3249 3250 /* Hardware has to reinitialize queues and interrupts to 3251 * match packet buffer alignment. Unfortunately, the 3252 * hardware is not flexible enough to do this dynamically. 3253 */ 3254 if (netif_running(dev)) 3255 ixgbevf_close(dev); 3256 3257 ixgbevf_clear_interrupt_scheme(adapter); 3258 ixgbevf_init_interrupt_scheme(adapter); 3259 3260 if (netif_running(dev)) 3261 ixgbevf_open(dev); 3262 } 3263 3264 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, 3265 u32 vlan_macip_lens, u32 type_tucmd, 3266 u32 mss_l4len_idx) 3267 { 3268 struct ixgbe_adv_tx_context_desc *context_desc; 3269 u16 i = tx_ring->next_to_use; 3270 3271 context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i); 3272 3273 i++; 3274 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3275 3276 /* set bits to identify this as an advanced context descriptor */ 3277 type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT; 3278 3279 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens); 3280 context_desc->seqnum_seed = 0; 3281 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd); 3282 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx); 3283 } 3284 3285 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, 3286 struct ixgbevf_tx_buffer *first, 3287 u8 *hdr_len) 3288 { 3289 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; 3290 struct sk_buff *skb = first->skb; 3291 union { 3292 struct iphdr *v4; 3293 struct ipv6hdr *v6; 3294 unsigned char *hdr; 3295 } ip; 3296 union { 3297 struct tcphdr *tcp; 3298 unsigned char *hdr; 3299 } l4; 3300 u32 paylen, l4_offset; 3301 int err; 3302 3303 if (skb->ip_summed != CHECKSUM_PARTIAL) 3304 return 0; 3305 3306 if (!skb_is_gso(skb)) 3307 return 0; 3308 3309 err = skb_cow_head(skb, 0); 3310 if (err < 0) 3311 return err; 3312 3313 ip.hdr = skb_network_header(skb); 3314 l4.hdr = skb_checksum_start(skb); 3315 3316 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ 3317 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3318 3319 /* initialize outer IP header fields */ 3320 if (ip.v4->version == 4) { 3321 /* IP header will have to cancel out any data that 3322 * is not a part of the outer IP header 3323 */ 3324 ip.v4->check = csum_fold(csum_add(lco_csum(skb), 3325 csum_unfold(l4.tcp->check))); 3326 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4; 3327 3328 ip.v4->tot_len = 0; 3329 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3330 IXGBE_TX_FLAGS_CSUM | 3331 IXGBE_TX_FLAGS_IPV4; 3332 } else { 3333 ip.v6->payload_len = 0; 3334 first->tx_flags |= IXGBE_TX_FLAGS_TSO | 3335 IXGBE_TX_FLAGS_CSUM; 3336 } 3337 3338 /* determine offset of inner transport header */ 3339 l4_offset = l4.hdr - skb->data; 3340 3341 /* compute length of segmentation header */ 3342 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 3343 3344 /* remove payload length from inner checksum */ 3345 paylen = skb->len - l4_offset; 3346 csum_replace_by_diff(&l4.tcp->check, htonl(paylen)); 3347 3348 /* update gso size and bytecount with header size */ 3349 first->gso_segs = skb_shinfo(skb)->gso_segs; 3350 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3351 3352 /* mss_l4len_id: use 1 as index for TSO */ 3353 mss_l4len_idx = (*hdr_len - l4_offset) << IXGBE_ADVTXD_L4LEN_SHIFT; 3354 mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT; 3355 mss_l4len_idx |= (1u << IXGBE_ADVTXD_IDX_SHIFT); 3356 3357 /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */ 3358 vlan_macip_lens = l4.hdr - ip.hdr; 3359 vlan_macip_lens |= (ip.hdr - skb->data) << IXGBE_ADVTXD_MACLEN_SHIFT; 3360 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3361 3362 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, 3363 type_tucmd, mss_l4len_idx); 3364 3365 return 1; 3366 } 3367 3368 static inline bool ixgbevf_ipv6_csum_is_sctp(struct sk_buff *skb) 3369 { 3370 unsigned int offset = 0; 3371 3372 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL); 3373 3374 return offset == skb_checksum_start_offset(skb); 3375 } 3376 3377 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, 3378 struct ixgbevf_tx_buffer *first) 3379 { 3380 struct sk_buff *skb = first->skb; 3381 u32 vlan_macip_lens = 0; 3382 u32 type_tucmd = 0; 3383 3384 if (skb->ip_summed != CHECKSUM_PARTIAL) 3385 goto no_csum; 3386 3387 switch (skb->csum_offset) { 3388 case offsetof(struct tcphdr, check): 3389 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP; 3390 /* fall through */ 3391 case offsetof(struct udphdr, check): 3392 break; 3393 case offsetof(struct sctphdr, checksum): 3394 /* validate that this is actually an SCTP request */ 3395 if (((first->protocol == htons(ETH_P_IP)) && 3396 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) || 3397 ((first->protocol == htons(ETH_P_IPV6)) && 3398 ixgbevf_ipv6_csum_is_sctp(skb))) { 3399 type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_SCTP; 3400 break; 3401 } 3402 /* fall through */ 3403 default: 3404 skb_checksum_help(skb); 3405 goto no_csum; 3406 } 3407 /* update TX checksum flag */ 3408 first->tx_flags |= IXGBE_TX_FLAGS_CSUM; 3409 vlan_macip_lens = skb_checksum_start_offset(skb) - 3410 skb_network_offset(skb); 3411 no_csum: 3412 /* vlan_macip_lens: MACLEN, VLAN tag */ 3413 vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT; 3414 vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK; 3415 3416 ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0); 3417 } 3418 3419 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags) 3420 { 3421 /* set type for advanced descriptor with frame checksum insertion */ 3422 __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA | 3423 IXGBE_ADVTXD_DCMD_IFCS | 3424 IXGBE_ADVTXD_DCMD_DEXT); 3425 3426 /* set HW VLAN bit if VLAN is present */ 3427 if (tx_flags & IXGBE_TX_FLAGS_VLAN) 3428 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE); 3429 3430 /* set segmentation enable bits for TSO/FSO */ 3431 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3432 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE); 3433 3434 return cmd_type; 3435 } 3436 3437 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, 3438 u32 tx_flags, unsigned int paylen) 3439 { 3440 __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT); 3441 3442 /* enable L4 checksum for TSO and TX checksum offload */ 3443 if (tx_flags & IXGBE_TX_FLAGS_CSUM) 3444 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM); 3445 3446 /* enble IPv4 checksum for TSO */ 3447 if (tx_flags & IXGBE_TX_FLAGS_IPV4) 3448 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM); 3449 3450 /* use index 1 context for TSO/FSO/FCOE */ 3451 if (tx_flags & IXGBE_TX_FLAGS_TSO) 3452 olinfo_status |= cpu_to_le32(1u << IXGBE_ADVTXD_IDX_SHIFT); 3453 3454 /* Check Context must be set if Tx switch is enabled, which it 3455 * always is for case where virtual functions are running 3456 */ 3457 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC); 3458 3459 tx_desc->read.olinfo_status = olinfo_status; 3460 } 3461 3462 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, 3463 struct ixgbevf_tx_buffer *first, 3464 const u8 hdr_len) 3465 { 3466 dma_addr_t dma; 3467 struct sk_buff *skb = first->skb; 3468 struct ixgbevf_tx_buffer *tx_buffer; 3469 union ixgbe_adv_tx_desc *tx_desc; 3470 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0]; 3471 unsigned int data_len = skb->data_len; 3472 unsigned int size = skb_headlen(skb); 3473 unsigned int paylen = skb->len - hdr_len; 3474 u32 tx_flags = first->tx_flags; 3475 __le32 cmd_type; 3476 u16 i = tx_ring->next_to_use; 3477 3478 tx_desc = IXGBEVF_TX_DESC(tx_ring, i); 3479 3480 ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen); 3481 cmd_type = ixgbevf_tx_cmd_type(tx_flags); 3482 3483 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3484 if (dma_mapping_error(tx_ring->dev, dma)) 3485 goto dma_error; 3486 3487 /* record length, and DMA address */ 3488 dma_unmap_len_set(first, len, size); 3489 dma_unmap_addr_set(first, dma, dma); 3490 3491 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3492 3493 for (;;) { 3494 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) { 3495 tx_desc->read.cmd_type_len = 3496 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD); 3497 3498 i++; 3499 tx_desc++; 3500 if (i == tx_ring->count) { 3501 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3502 i = 0; 3503 } 3504 3505 dma += IXGBE_MAX_DATA_PER_TXD; 3506 size -= IXGBE_MAX_DATA_PER_TXD; 3507 3508 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3509 tx_desc->read.olinfo_status = 0; 3510 } 3511 3512 if (likely(!data_len)) 3513 break; 3514 3515 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size); 3516 3517 i++; 3518 tx_desc++; 3519 if (i == tx_ring->count) { 3520 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0); 3521 i = 0; 3522 } 3523 3524 size = skb_frag_size(frag); 3525 data_len -= size; 3526 3527 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3528 DMA_TO_DEVICE); 3529 if (dma_mapping_error(tx_ring->dev, dma)) 3530 goto dma_error; 3531 3532 tx_buffer = &tx_ring->tx_buffer_info[i]; 3533 dma_unmap_len_set(tx_buffer, len, size); 3534 dma_unmap_addr_set(tx_buffer, dma, dma); 3535 3536 tx_desc->read.buffer_addr = cpu_to_le64(dma); 3537 tx_desc->read.olinfo_status = 0; 3538 3539 frag++; 3540 } 3541 3542 /* write last descriptor with RS and EOP bits */ 3543 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD); 3544 tx_desc->read.cmd_type_len = cmd_type; 3545 3546 /* set the timestamp */ 3547 first->time_stamp = jiffies; 3548 3549 /* Force memory writes to complete before letting h/w know there 3550 * are new descriptors to fetch. (Only applicable for weak-ordered 3551 * memory model archs, such as IA-64). 3552 * 3553 * We also need this memory barrier (wmb) to make certain all of the 3554 * status bits have been updated before next_to_watch is written. 3555 */ 3556 wmb(); 3557 3558 /* set next_to_watch value indicating a packet is present */ 3559 first->next_to_watch = tx_desc; 3560 3561 i++; 3562 if (i == tx_ring->count) 3563 i = 0; 3564 3565 tx_ring->next_to_use = i; 3566 3567 /* notify HW of packet */ 3568 ixgbevf_write_tail(tx_ring, i); 3569 3570 return; 3571 dma_error: 3572 dev_err(tx_ring->dev, "TX DMA map failed\n"); 3573 3574 /* clear dma mappings for failed tx_buffer_info map */ 3575 for (;;) { 3576 tx_buffer = &tx_ring->tx_buffer_info[i]; 3577 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer); 3578 if (tx_buffer == first) 3579 break; 3580 if (i == 0) 3581 i = tx_ring->count; 3582 i--; 3583 } 3584 3585 tx_ring->next_to_use = i; 3586 } 3587 3588 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3589 { 3590 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 3591 /* Herbert's original patch had: 3592 * smp_mb__after_netif_stop_queue(); 3593 * but since that doesn't exist yet, just open code it. 3594 */ 3595 smp_mb(); 3596 3597 /* We need to check again in a case another CPU has just 3598 * made room available. 3599 */ 3600 if (likely(ixgbevf_desc_unused(tx_ring) < size)) 3601 return -EBUSY; 3602 3603 /* A reprieve! - use start_queue because it doesn't call schedule */ 3604 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 3605 ++tx_ring->tx_stats.restart_queue; 3606 3607 return 0; 3608 } 3609 3610 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) 3611 { 3612 if (likely(ixgbevf_desc_unused(tx_ring) >= size)) 3613 return 0; 3614 return __ixgbevf_maybe_stop_tx(tx_ring, size); 3615 } 3616 3617 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3618 { 3619 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3620 struct ixgbevf_tx_buffer *first; 3621 struct ixgbevf_ring *tx_ring; 3622 int tso; 3623 u32 tx_flags = 0; 3624 u16 count = TXD_USE_COUNT(skb_headlen(skb)); 3625 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3626 unsigned short f; 3627 #endif 3628 u8 hdr_len = 0; 3629 u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL); 3630 3631 if (!dst_mac || is_link_local_ether_addr(dst_mac)) { 3632 dev_kfree_skb_any(skb); 3633 return NETDEV_TX_OK; 3634 } 3635 3636 tx_ring = adapter->tx_ring[skb->queue_mapping]; 3637 3638 /* need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD, 3639 * + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD, 3640 * + 2 desc gap to keep tail from touching head, 3641 * + 1 desc for context descriptor, 3642 * otherwise try next time 3643 */ 3644 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD 3645 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) 3646 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size); 3647 #else 3648 count += skb_shinfo(skb)->nr_frags; 3649 #endif 3650 if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) { 3651 tx_ring->tx_stats.tx_busy++; 3652 return NETDEV_TX_BUSY; 3653 } 3654 3655 /* record the location of the first descriptor for this packet */ 3656 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use]; 3657 first->skb = skb; 3658 first->bytecount = skb->len; 3659 first->gso_segs = 1; 3660 3661 if (skb_vlan_tag_present(skb)) { 3662 tx_flags |= skb_vlan_tag_get(skb); 3663 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT; 3664 tx_flags |= IXGBE_TX_FLAGS_VLAN; 3665 } 3666 3667 /* record initial flags and protocol */ 3668 first->tx_flags = tx_flags; 3669 first->protocol = vlan_get_protocol(skb); 3670 3671 tso = ixgbevf_tso(tx_ring, first, &hdr_len); 3672 if (tso < 0) 3673 goto out_drop; 3674 else if (!tso) 3675 ixgbevf_tx_csum(tx_ring, first); 3676 3677 ixgbevf_tx_map(tx_ring, first, hdr_len); 3678 3679 ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED); 3680 3681 return NETDEV_TX_OK; 3682 3683 out_drop: 3684 dev_kfree_skb_any(first->skb); 3685 first->skb = NULL; 3686 3687 return NETDEV_TX_OK; 3688 } 3689 3690 /** 3691 * ixgbevf_set_mac - Change the Ethernet Address of the NIC 3692 * @netdev: network interface device structure 3693 * @p: pointer to an address structure 3694 * 3695 * Returns 0 on success, negative on failure 3696 **/ 3697 static int ixgbevf_set_mac(struct net_device *netdev, void *p) 3698 { 3699 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3700 struct ixgbe_hw *hw = &adapter->hw; 3701 struct sockaddr *addr = p; 3702 int err; 3703 3704 if (!is_valid_ether_addr(addr->sa_data)) 3705 return -EADDRNOTAVAIL; 3706 3707 spin_lock_bh(&adapter->mbx_lock); 3708 3709 err = hw->mac.ops.set_rar(hw, 0, addr->sa_data, 0); 3710 3711 spin_unlock_bh(&adapter->mbx_lock); 3712 3713 if (err) 3714 return -EPERM; 3715 3716 ether_addr_copy(hw->mac.addr, addr->sa_data); 3717 ether_addr_copy(netdev->dev_addr, addr->sa_data); 3718 3719 return 0; 3720 } 3721 3722 /** 3723 * ixgbevf_change_mtu - Change the Maximum Transfer Unit 3724 * @netdev: network interface device structure 3725 * @new_mtu: new value for maximum frame size 3726 * 3727 * Returns 0 on success, negative on failure 3728 **/ 3729 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu) 3730 { 3731 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3732 struct ixgbe_hw *hw = &adapter->hw; 3733 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN; 3734 int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE; 3735 3736 switch (adapter->hw.api_version) { 3737 case ixgbe_mbox_api_11: 3738 case ixgbe_mbox_api_12: 3739 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3740 break; 3741 default: 3742 if (adapter->hw.mac.type != ixgbe_mac_82599_vf) 3743 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE; 3744 break; 3745 } 3746 3747 /* MTU < 68 is an error and causes problems on some kernels */ 3748 if ((new_mtu < 68) || (max_frame > max_possible_frame)) 3749 return -EINVAL; 3750 3751 hw_dbg(hw, "changing MTU from %d to %d\n", 3752 netdev->mtu, new_mtu); 3753 /* must set new MTU before calling down or up */ 3754 netdev->mtu = new_mtu; 3755 3756 /* notify the PF of our intent to use this size of frame */ 3757 hw->mac.ops.set_rlpml(hw, max_frame); 3758 3759 return 0; 3760 } 3761 3762 #ifdef CONFIG_NET_POLL_CONTROLLER 3763 /* Polling 'interrupt' - used by things like netconsole to send skbs 3764 * without having to re-enable interrupts. It's not called while 3765 * the interrupt routine is executing. 3766 */ 3767 static void ixgbevf_netpoll(struct net_device *netdev) 3768 { 3769 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3770 int i; 3771 3772 /* if interface is down do nothing */ 3773 if (test_bit(__IXGBEVF_DOWN, &adapter->state)) 3774 return; 3775 for (i = 0; i < adapter->num_rx_queues; i++) 3776 ixgbevf_msix_clean_rings(0, adapter->q_vector[i]); 3777 } 3778 #endif /* CONFIG_NET_POLL_CONTROLLER */ 3779 3780 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state) 3781 { 3782 struct net_device *netdev = pci_get_drvdata(pdev); 3783 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3784 #ifdef CONFIG_PM 3785 int retval = 0; 3786 #endif 3787 3788 netif_device_detach(netdev); 3789 3790 if (netif_running(netdev)) { 3791 rtnl_lock(); 3792 ixgbevf_down(adapter); 3793 ixgbevf_free_irq(adapter); 3794 ixgbevf_free_all_tx_resources(adapter); 3795 ixgbevf_free_all_rx_resources(adapter); 3796 rtnl_unlock(); 3797 } 3798 3799 ixgbevf_clear_interrupt_scheme(adapter); 3800 3801 #ifdef CONFIG_PM 3802 retval = pci_save_state(pdev); 3803 if (retval) 3804 return retval; 3805 3806 #endif 3807 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 3808 pci_disable_device(pdev); 3809 3810 return 0; 3811 } 3812 3813 #ifdef CONFIG_PM 3814 static int ixgbevf_resume(struct pci_dev *pdev) 3815 { 3816 struct net_device *netdev = pci_get_drvdata(pdev); 3817 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3818 u32 err; 3819 3820 pci_restore_state(pdev); 3821 /* pci_restore_state clears dev->state_saved so call 3822 * pci_save_state to restore it. 3823 */ 3824 pci_save_state(pdev); 3825 3826 err = pci_enable_device_mem(pdev); 3827 if (err) { 3828 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 3829 return err; 3830 } 3831 smp_mb__before_atomic(); 3832 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 3833 pci_set_master(pdev); 3834 3835 ixgbevf_reset(adapter); 3836 3837 rtnl_lock(); 3838 err = ixgbevf_init_interrupt_scheme(adapter); 3839 rtnl_unlock(); 3840 if (err) { 3841 dev_err(&pdev->dev, "Cannot initialize interrupts\n"); 3842 return err; 3843 } 3844 3845 if (netif_running(netdev)) { 3846 err = ixgbevf_open(netdev); 3847 if (err) 3848 return err; 3849 } 3850 3851 netif_device_attach(netdev); 3852 3853 return err; 3854 } 3855 3856 #endif /* CONFIG_PM */ 3857 static void ixgbevf_shutdown(struct pci_dev *pdev) 3858 { 3859 ixgbevf_suspend(pdev, PMSG_SUSPEND); 3860 } 3861 3862 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev, 3863 struct rtnl_link_stats64 *stats) 3864 { 3865 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 3866 unsigned int start; 3867 u64 bytes, packets; 3868 const struct ixgbevf_ring *ring; 3869 int i; 3870 3871 ixgbevf_update_stats(adapter); 3872 3873 stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc; 3874 3875 for (i = 0; i < adapter->num_rx_queues; i++) { 3876 ring = adapter->rx_ring[i]; 3877 do { 3878 start = u64_stats_fetch_begin_irq(&ring->syncp); 3879 bytes = ring->stats.bytes; 3880 packets = ring->stats.packets; 3881 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3882 stats->rx_bytes += bytes; 3883 stats->rx_packets += packets; 3884 } 3885 3886 for (i = 0; i < adapter->num_tx_queues; i++) { 3887 ring = adapter->tx_ring[i]; 3888 do { 3889 start = u64_stats_fetch_begin_irq(&ring->syncp); 3890 bytes = ring->stats.bytes; 3891 packets = ring->stats.packets; 3892 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 3893 stats->tx_bytes += bytes; 3894 stats->tx_packets += packets; 3895 } 3896 3897 return stats; 3898 } 3899 3900 #define IXGBEVF_MAX_MAC_HDR_LEN 127 3901 #define IXGBEVF_MAX_NETWORK_HDR_LEN 511 3902 3903 static netdev_features_t 3904 ixgbevf_features_check(struct sk_buff *skb, struct net_device *dev, 3905 netdev_features_t features) 3906 { 3907 unsigned int network_hdr_len, mac_hdr_len; 3908 3909 /* Make certain the headers can be described by a context descriptor */ 3910 mac_hdr_len = skb_network_header(skb) - skb->data; 3911 if (unlikely(mac_hdr_len > IXGBEVF_MAX_MAC_HDR_LEN)) 3912 return features & ~(NETIF_F_HW_CSUM | 3913 NETIF_F_SCTP_CRC | 3914 NETIF_F_HW_VLAN_CTAG_TX | 3915 NETIF_F_TSO | 3916 NETIF_F_TSO6); 3917 3918 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); 3919 if (unlikely(network_hdr_len > IXGBEVF_MAX_NETWORK_HDR_LEN)) 3920 return features & ~(NETIF_F_HW_CSUM | 3921 NETIF_F_SCTP_CRC | 3922 NETIF_F_TSO | 3923 NETIF_F_TSO6); 3924 3925 /* We can only support IPV4 TSO in tunnels if we can mangle the 3926 * inner IP ID field, so strip TSO if MANGLEID is not supported. 3927 */ 3928 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) 3929 features &= ~NETIF_F_TSO; 3930 3931 return features; 3932 } 3933 3934 static const struct net_device_ops ixgbevf_netdev_ops = { 3935 .ndo_open = ixgbevf_open, 3936 .ndo_stop = ixgbevf_close, 3937 .ndo_start_xmit = ixgbevf_xmit_frame, 3938 .ndo_set_rx_mode = ixgbevf_set_rx_mode, 3939 .ndo_get_stats64 = ixgbevf_get_stats, 3940 .ndo_validate_addr = eth_validate_addr, 3941 .ndo_set_mac_address = ixgbevf_set_mac, 3942 .ndo_change_mtu = ixgbevf_change_mtu, 3943 .ndo_tx_timeout = ixgbevf_tx_timeout, 3944 .ndo_vlan_rx_add_vid = ixgbevf_vlan_rx_add_vid, 3945 .ndo_vlan_rx_kill_vid = ixgbevf_vlan_rx_kill_vid, 3946 #ifdef CONFIG_NET_RX_BUSY_POLL 3947 .ndo_busy_poll = ixgbevf_busy_poll_recv, 3948 #endif 3949 #ifdef CONFIG_NET_POLL_CONTROLLER 3950 .ndo_poll_controller = ixgbevf_netpoll, 3951 #endif 3952 .ndo_features_check = ixgbevf_features_check, 3953 }; 3954 3955 static void ixgbevf_assign_netdev_ops(struct net_device *dev) 3956 { 3957 dev->netdev_ops = &ixgbevf_netdev_ops; 3958 ixgbevf_set_ethtool_ops(dev); 3959 dev->watchdog_timeo = 5 * HZ; 3960 } 3961 3962 /** 3963 * ixgbevf_probe - Device Initialization Routine 3964 * @pdev: PCI device information struct 3965 * @ent: entry in ixgbevf_pci_tbl 3966 * 3967 * Returns 0 on success, negative on failure 3968 * 3969 * ixgbevf_probe initializes an adapter identified by a pci_dev structure. 3970 * The OS initialization, configuring of the adapter private structure, 3971 * and a hardware reset occur. 3972 **/ 3973 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 3974 { 3975 struct net_device *netdev; 3976 struct ixgbevf_adapter *adapter = NULL; 3977 struct ixgbe_hw *hw = NULL; 3978 const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data]; 3979 int err, pci_using_dac; 3980 bool disable_dev = false; 3981 3982 err = pci_enable_device(pdev); 3983 if (err) 3984 return err; 3985 3986 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { 3987 pci_using_dac = 1; 3988 } else { 3989 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 3990 if (err) { 3991 dev_err(&pdev->dev, "No usable DMA configuration, aborting\n"); 3992 goto err_dma; 3993 } 3994 pci_using_dac = 0; 3995 } 3996 3997 err = pci_request_regions(pdev, ixgbevf_driver_name); 3998 if (err) { 3999 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err); 4000 goto err_pci_reg; 4001 } 4002 4003 pci_set_master(pdev); 4004 4005 netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter), 4006 MAX_TX_QUEUES); 4007 if (!netdev) { 4008 err = -ENOMEM; 4009 goto err_alloc_etherdev; 4010 } 4011 4012 SET_NETDEV_DEV(netdev, &pdev->dev); 4013 4014 adapter = netdev_priv(netdev); 4015 4016 adapter->netdev = netdev; 4017 adapter->pdev = pdev; 4018 hw = &adapter->hw; 4019 hw->back = adapter; 4020 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); 4021 4022 /* call save state here in standalone driver because it relies on 4023 * adapter struct to exist, and needs to call netdev_priv 4024 */ 4025 pci_save_state(pdev); 4026 4027 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 4028 pci_resource_len(pdev, 0)); 4029 adapter->io_addr = hw->hw_addr; 4030 if (!hw->hw_addr) { 4031 err = -EIO; 4032 goto err_ioremap; 4033 } 4034 4035 ixgbevf_assign_netdev_ops(netdev); 4036 4037 /* Setup HW API */ 4038 memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); 4039 hw->mac.type = ii->mac; 4040 4041 memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, 4042 sizeof(struct ixgbe_mbx_operations)); 4043 4044 /* setup the private structure */ 4045 err = ixgbevf_sw_init(adapter); 4046 if (err) 4047 goto err_sw_init; 4048 4049 /* The HW MAC address was set and/or determined in sw_init */ 4050 if (!is_valid_ether_addr(netdev->dev_addr)) { 4051 pr_err("invalid MAC address\n"); 4052 err = -EIO; 4053 goto err_sw_init; 4054 } 4055 4056 netdev->hw_features = NETIF_F_SG | 4057 NETIF_F_TSO | 4058 NETIF_F_TSO6 | 4059 NETIF_F_RXCSUM | 4060 NETIF_F_HW_CSUM | 4061 NETIF_F_SCTP_CRC; 4062 4063 #define IXGBEVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 4064 NETIF_F_GSO_GRE_CSUM | \ 4065 NETIF_F_GSO_IPXIP4 | \ 4066 NETIF_F_GSO_IPXIP6 | \ 4067 NETIF_F_GSO_UDP_TUNNEL | \ 4068 NETIF_F_GSO_UDP_TUNNEL_CSUM) 4069 4070 netdev->gso_partial_features = IXGBEVF_GSO_PARTIAL_FEATURES; 4071 netdev->hw_features |= NETIF_F_GSO_PARTIAL | 4072 IXGBEVF_GSO_PARTIAL_FEATURES; 4073 4074 netdev->features = netdev->hw_features; 4075 4076 if (pci_using_dac) 4077 netdev->features |= NETIF_F_HIGHDMA; 4078 4079 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID; 4080 netdev->mpls_features |= NETIF_F_HW_CSUM; 4081 netdev->hw_enc_features |= netdev->vlan_features; 4082 4083 /* set this bit last since it cannot be part of vlan_features */ 4084 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | 4085 NETIF_F_HW_VLAN_CTAG_RX | 4086 NETIF_F_HW_VLAN_CTAG_TX; 4087 4088 netdev->priv_flags |= IFF_UNICAST_FLT; 4089 4090 if (IXGBE_REMOVED(hw->hw_addr)) { 4091 err = -EIO; 4092 goto err_sw_init; 4093 } 4094 4095 setup_timer(&adapter->service_timer, &ixgbevf_service_timer, 4096 (unsigned long)adapter); 4097 4098 INIT_WORK(&adapter->service_task, ixgbevf_service_task); 4099 set_bit(__IXGBEVF_SERVICE_INITED, &adapter->state); 4100 clear_bit(__IXGBEVF_SERVICE_SCHED, &adapter->state); 4101 4102 err = ixgbevf_init_interrupt_scheme(adapter); 4103 if (err) 4104 goto err_sw_init; 4105 4106 strcpy(netdev->name, "eth%d"); 4107 4108 err = register_netdev(netdev); 4109 if (err) 4110 goto err_register; 4111 4112 pci_set_drvdata(pdev, netdev); 4113 netif_carrier_off(netdev); 4114 4115 ixgbevf_init_last_counter_stats(adapter); 4116 4117 /* print the VF info */ 4118 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 4119 dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type); 4120 4121 switch (hw->mac.type) { 4122 case ixgbe_mac_X550_vf: 4123 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n"); 4124 break; 4125 case ixgbe_mac_X540_vf: 4126 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n"); 4127 break; 4128 case ixgbe_mac_82599_vf: 4129 default: 4130 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n"); 4131 break; 4132 } 4133 4134 return 0; 4135 4136 err_register: 4137 ixgbevf_clear_interrupt_scheme(adapter); 4138 err_sw_init: 4139 ixgbevf_reset_interrupt_capability(adapter); 4140 iounmap(adapter->io_addr); 4141 err_ioremap: 4142 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4143 free_netdev(netdev); 4144 err_alloc_etherdev: 4145 pci_release_regions(pdev); 4146 err_pci_reg: 4147 err_dma: 4148 if (!adapter || disable_dev) 4149 pci_disable_device(pdev); 4150 return err; 4151 } 4152 4153 /** 4154 * ixgbevf_remove - Device Removal Routine 4155 * @pdev: PCI device information struct 4156 * 4157 * ixgbevf_remove is called by the PCI subsystem to alert the driver 4158 * that it should release a PCI device. The could be caused by a 4159 * Hot-Plug event, or because the driver is going to be removed from 4160 * memory. 4161 **/ 4162 static void ixgbevf_remove(struct pci_dev *pdev) 4163 { 4164 struct net_device *netdev = pci_get_drvdata(pdev); 4165 struct ixgbevf_adapter *adapter; 4166 bool disable_dev; 4167 4168 if (!netdev) 4169 return; 4170 4171 adapter = netdev_priv(netdev); 4172 4173 set_bit(__IXGBEVF_REMOVING, &adapter->state); 4174 cancel_work_sync(&adapter->service_task); 4175 4176 if (netdev->reg_state == NETREG_REGISTERED) 4177 unregister_netdev(netdev); 4178 4179 ixgbevf_clear_interrupt_scheme(adapter); 4180 ixgbevf_reset_interrupt_capability(adapter); 4181 4182 iounmap(adapter->io_addr); 4183 pci_release_regions(pdev); 4184 4185 hw_dbg(&adapter->hw, "Remove complete\n"); 4186 4187 disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state); 4188 free_netdev(netdev); 4189 4190 if (disable_dev) 4191 pci_disable_device(pdev); 4192 } 4193 4194 /** 4195 * ixgbevf_io_error_detected - called when PCI error is detected 4196 * @pdev: Pointer to PCI device 4197 * @state: The current pci connection state 4198 * 4199 * This function is called after a PCI bus error affecting 4200 * this device has been detected. 4201 **/ 4202 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev, 4203 pci_channel_state_t state) 4204 { 4205 struct net_device *netdev = pci_get_drvdata(pdev); 4206 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4207 4208 if (!test_bit(__IXGBEVF_SERVICE_INITED, &adapter->state)) 4209 return PCI_ERS_RESULT_DISCONNECT; 4210 4211 rtnl_lock(); 4212 netif_device_detach(netdev); 4213 4214 if (state == pci_channel_io_perm_failure) { 4215 rtnl_unlock(); 4216 return PCI_ERS_RESULT_DISCONNECT; 4217 } 4218 4219 if (netif_running(netdev)) 4220 ixgbevf_down(adapter); 4221 4222 if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state)) 4223 pci_disable_device(pdev); 4224 rtnl_unlock(); 4225 4226 /* Request a slot slot reset. */ 4227 return PCI_ERS_RESULT_NEED_RESET; 4228 } 4229 4230 /** 4231 * ixgbevf_io_slot_reset - called after the pci bus has been reset. 4232 * @pdev: Pointer to PCI device 4233 * 4234 * Restart the card from scratch, as if from a cold-boot. Implementation 4235 * resembles the first-half of the ixgbevf_resume routine. 4236 **/ 4237 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev) 4238 { 4239 struct net_device *netdev = pci_get_drvdata(pdev); 4240 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4241 4242 if (pci_enable_device_mem(pdev)) { 4243 dev_err(&pdev->dev, 4244 "Cannot re-enable PCI device after reset.\n"); 4245 return PCI_ERS_RESULT_DISCONNECT; 4246 } 4247 4248 smp_mb__before_atomic(); 4249 clear_bit(__IXGBEVF_DISABLED, &adapter->state); 4250 pci_set_master(pdev); 4251 4252 ixgbevf_reset(adapter); 4253 4254 return PCI_ERS_RESULT_RECOVERED; 4255 } 4256 4257 /** 4258 * ixgbevf_io_resume - called when traffic can start flowing again. 4259 * @pdev: Pointer to PCI device 4260 * 4261 * This callback is called when the error recovery driver tells us that 4262 * its OK to resume normal operation. Implementation resembles the 4263 * second-half of the ixgbevf_resume routine. 4264 **/ 4265 static void ixgbevf_io_resume(struct pci_dev *pdev) 4266 { 4267 struct net_device *netdev = pci_get_drvdata(pdev); 4268 struct ixgbevf_adapter *adapter = netdev_priv(netdev); 4269 4270 if (netif_running(netdev)) 4271 ixgbevf_up(adapter); 4272 4273 netif_device_attach(netdev); 4274 } 4275 4276 /* PCI Error Recovery (ERS) */ 4277 static const struct pci_error_handlers ixgbevf_err_handler = { 4278 .error_detected = ixgbevf_io_error_detected, 4279 .slot_reset = ixgbevf_io_slot_reset, 4280 .resume = ixgbevf_io_resume, 4281 }; 4282 4283 static struct pci_driver ixgbevf_driver = { 4284 .name = ixgbevf_driver_name, 4285 .id_table = ixgbevf_pci_tbl, 4286 .probe = ixgbevf_probe, 4287 .remove = ixgbevf_remove, 4288 #ifdef CONFIG_PM 4289 /* Power Management Hooks */ 4290 .suspend = ixgbevf_suspend, 4291 .resume = ixgbevf_resume, 4292 #endif 4293 .shutdown = ixgbevf_shutdown, 4294 .err_handler = &ixgbevf_err_handler 4295 }; 4296 4297 /** 4298 * ixgbevf_init_module - Driver Registration Routine 4299 * 4300 * ixgbevf_init_module is the first routine called when the driver is 4301 * loaded. All it does is register with the PCI subsystem. 4302 **/ 4303 static int __init ixgbevf_init_module(void) 4304 { 4305 pr_info("%s - version %s\n", ixgbevf_driver_string, 4306 ixgbevf_driver_version); 4307 4308 pr_info("%s\n", ixgbevf_copyright); 4309 ixgbevf_wq = create_singlethread_workqueue(ixgbevf_driver_name); 4310 if (!ixgbevf_wq) { 4311 pr_err("%s: Failed to create workqueue\n", ixgbevf_driver_name); 4312 return -ENOMEM; 4313 } 4314 4315 return pci_register_driver(&ixgbevf_driver); 4316 } 4317 4318 module_init(ixgbevf_init_module); 4319 4320 /** 4321 * ixgbevf_exit_module - Driver Exit Cleanup Routine 4322 * 4323 * ixgbevf_exit_module is called just before the driver is removed 4324 * from memory. 4325 **/ 4326 static void __exit ixgbevf_exit_module(void) 4327 { 4328 pci_unregister_driver(&ixgbevf_driver); 4329 if (ixgbevf_wq) { 4330 destroy_workqueue(ixgbevf_wq); 4331 ixgbevf_wq = NULL; 4332 } 4333 } 4334 4335 #ifdef DEBUG 4336 /** 4337 * ixgbevf_get_hw_dev_name - return device name string 4338 * used by hardware layer to print debugging information 4339 **/ 4340 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw) 4341 { 4342 struct ixgbevf_adapter *adapter = hw->back; 4343 4344 return adapter->netdev->name; 4345 } 4346 4347 #endif 4348 module_exit(ixgbevf_exit_module); 4349 4350 /* ixgbevf_main.c */ 4351