1 /* 2 * Linux driver for VMware's vmxnet3 ethernet NIC. 3 * 4 * Copyright (C) 2008-2009, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * The full GNU General Public License is included in this distribution in 21 * the file called "COPYING". 22 * 23 * Maintained by: Shreyas Bhatewara <pv-drivers@vmware.com> 24 * 25 */ 26 27 #include <linux/module.h> 28 #include <net/ip6_checksum.h> 29 30 #include "vmxnet3_int.h" 31 32 char vmxnet3_driver_name[] = "vmxnet3"; 33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver" 34 35 /* 36 * PCI Device ID Table 37 * Last entry must be all 0s 38 */ 39 static DEFINE_PCI_DEVICE_TABLE(vmxnet3_pciid_table) = { 40 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)}, 41 {0} 42 }; 43 44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table); 45 46 static atomic_t devices_found; 47 48 #define VMXNET3_MAX_DEVICES 10 49 static int enable_mq = 1; 50 static int irq_share_mode; 51 52 static void 53 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac); 54 55 /* 56 * Enable/Disable the given intr 57 */ 58 static void 59 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 60 { 61 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0); 62 } 63 64 65 static void 66 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 67 { 68 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1); 69 } 70 71 72 /* 73 * Enable/Disable all intrs used by the device 74 */ 75 static void 76 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) 77 { 78 int i; 79 80 for (i = 0; i < adapter->intr.num_intrs; i++) 81 vmxnet3_enable_intr(adapter, i); 82 adapter->shared->devRead.intrConf.intrCtrl &= 83 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 84 } 85 86 87 static void 88 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) 89 { 90 int i; 91 92 adapter->shared->devRead.intrConf.intrCtrl |= 93 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 94 for (i = 0; i < adapter->intr.num_intrs; i++) 95 vmxnet3_disable_intr(adapter, i); 96 } 97 98 99 static void 100 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events) 101 { 102 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events); 103 } 104 105 106 static bool 107 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 108 { 109 return tq->stopped; 110 } 111 112 113 static void 114 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 115 { 116 tq->stopped = false; 117 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue); 118 } 119 120 121 static void 122 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 123 { 124 tq->stopped = false; 125 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 126 } 127 128 129 static void 130 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 131 { 132 tq->stopped = true; 133 tq->num_stop++; 134 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 135 } 136 137 138 /* 139 * Check the link state. This may start or stop the tx queue. 140 */ 141 static void 142 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 143 { 144 u32 ret; 145 int i; 146 unsigned long flags; 147 148 spin_lock_irqsave(&adapter->cmd_lock, flags); 149 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 150 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 151 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 152 153 adapter->link_speed = ret >> 16; 154 if (ret & 1) { /* Link is up. */ 155 printk(KERN_INFO "%s: NIC Link is Up %d Mbps\n", 156 adapter->netdev->name, adapter->link_speed); 157 netif_carrier_on(adapter->netdev); 158 159 if (affectTxQueue) { 160 for (i = 0; i < adapter->num_tx_queues; i++) 161 vmxnet3_tq_start(&adapter->tx_queue[i], 162 adapter); 163 } 164 } else { 165 printk(KERN_INFO "%s: NIC Link is Down\n", 166 adapter->netdev->name); 167 netif_carrier_off(adapter->netdev); 168 169 if (affectTxQueue) { 170 for (i = 0; i < adapter->num_tx_queues; i++) 171 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter); 172 } 173 } 174 } 175 176 static void 177 vmxnet3_process_events(struct vmxnet3_adapter *adapter) 178 { 179 int i; 180 unsigned long flags; 181 u32 events = le32_to_cpu(adapter->shared->ecr); 182 if (!events) 183 return; 184 185 vmxnet3_ack_events(adapter, events); 186 187 /* Check if link state has changed */ 188 if (events & VMXNET3_ECR_LINK) 189 vmxnet3_check_link(adapter, true); 190 191 /* Check if there is an error on xmit/recv queues */ 192 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 193 spin_lock_irqsave(&adapter->cmd_lock, flags); 194 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 195 VMXNET3_CMD_GET_QUEUE_STATUS); 196 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 197 198 for (i = 0; i < adapter->num_tx_queues; i++) 199 if (adapter->tqd_start[i].status.stopped) 200 dev_err(&adapter->netdev->dev, 201 "%s: tq[%d] error 0x%x\n", 202 adapter->netdev->name, i, le32_to_cpu( 203 adapter->tqd_start[i].status.error)); 204 for (i = 0; i < adapter->num_rx_queues; i++) 205 if (adapter->rqd_start[i].status.stopped) 206 dev_err(&adapter->netdev->dev, 207 "%s: rq[%d] error 0x%x\n", 208 adapter->netdev->name, i, 209 adapter->rqd_start[i].status.error); 210 211 schedule_work(&adapter->work); 212 } 213 } 214 215 #ifdef __BIG_ENDIAN_BITFIELD 216 /* 217 * The device expects the bitfields in shared structures to be written in 218 * little endian. When CPU is big endian, the following routines are used to 219 * correctly read and write into ABI. 220 * The general technique used here is : double word bitfields are defined in 221 * opposite order for big endian architecture. Then before reading them in 222 * driver the complete double word is translated using le32_to_cpu. Similarly 223 * After the driver writes into bitfields, cpu_to_le32 is used to translate the 224 * double words into required format. 225 * In order to avoid touching bits in shared structure more than once, temporary 226 * descriptors are used. These are passed as srcDesc to following functions. 227 */ 228 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc, 229 struct Vmxnet3_RxDesc *dstDesc) 230 { 231 u32 *src = (u32 *)srcDesc + 2; 232 u32 *dst = (u32 *)dstDesc + 2; 233 dstDesc->addr = le64_to_cpu(srcDesc->addr); 234 *dst = le32_to_cpu(*src); 235 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1); 236 } 237 238 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc, 239 struct Vmxnet3_TxDesc *dstDesc) 240 { 241 int i; 242 u32 *src = (u32 *)(srcDesc + 1); 243 u32 *dst = (u32 *)(dstDesc + 1); 244 245 /* Working backwards so that the gen bit is set at the end. */ 246 for (i = 2; i > 0; i--) { 247 src--; 248 dst--; 249 *dst = cpu_to_le32(*src); 250 } 251 } 252 253 254 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc, 255 struct Vmxnet3_RxCompDesc *dstDesc) 256 { 257 int i = 0; 258 u32 *src = (u32 *)srcDesc; 259 u32 *dst = (u32 *)dstDesc; 260 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) { 261 *dst = le32_to_cpu(*src); 262 src++; 263 dst++; 264 } 265 } 266 267 268 /* Used to read bitfield values from double words. */ 269 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size) 270 { 271 u32 temp = le32_to_cpu(*bitfield); 272 u32 mask = ((1 << size) - 1) << pos; 273 temp &= mask; 274 temp >>= pos; 275 return temp; 276 } 277 278 279 280 #endif /* __BIG_ENDIAN_BITFIELD */ 281 282 #ifdef __BIG_ENDIAN_BITFIELD 283 284 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \ 285 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \ 286 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE) 287 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \ 288 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \ 289 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE) 290 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \ 291 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \ 292 VMXNET3_TCD_GEN_SIZE) 293 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \ 294 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE) 295 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \ 296 (dstrcd) = (tmp); \ 297 vmxnet3_RxCompToCPU((rcd), (tmp)); \ 298 } while (0) 299 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \ 300 (dstrxd) = (tmp); \ 301 vmxnet3_RxDescToCPU((rxd), (tmp)); \ 302 } while (0) 303 304 #else 305 306 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen) 307 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop) 308 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen) 309 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx) 310 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd) 311 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd) 312 313 #endif /* __BIG_ENDIAN_BITFIELD */ 314 315 316 static void 317 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi, 318 struct pci_dev *pdev) 319 { 320 if (tbi->map_type == VMXNET3_MAP_SINGLE) 321 pci_unmap_single(pdev, tbi->dma_addr, tbi->len, 322 PCI_DMA_TODEVICE); 323 else if (tbi->map_type == VMXNET3_MAP_PAGE) 324 pci_unmap_page(pdev, tbi->dma_addr, tbi->len, 325 PCI_DMA_TODEVICE); 326 else 327 BUG_ON(tbi->map_type != VMXNET3_MAP_NONE); 328 329 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */ 330 } 331 332 333 static int 334 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq, 335 struct pci_dev *pdev, struct vmxnet3_adapter *adapter) 336 { 337 struct sk_buff *skb; 338 int entries = 0; 339 340 /* no out of order completion */ 341 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp); 342 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1); 343 344 skb = tq->buf_info[eop_idx].skb; 345 BUG_ON(skb == NULL); 346 tq->buf_info[eop_idx].skb = NULL; 347 348 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size); 349 350 while (tq->tx_ring.next2comp != eop_idx) { 351 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp, 352 pdev); 353 354 /* update next2comp w/o tx_lock. Since we are marking more, 355 * instead of less, tx ring entries avail, the worst case is 356 * that the tx routine incorrectly re-queues a pkt due to 357 * insufficient tx ring entries. 358 */ 359 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 360 entries++; 361 } 362 363 dev_kfree_skb_any(skb); 364 return entries; 365 } 366 367 368 static int 369 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq, 370 struct vmxnet3_adapter *adapter) 371 { 372 int completed = 0; 373 union Vmxnet3_GenericDesc *gdesc; 374 375 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 376 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) { 377 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX( 378 &gdesc->tcd), tq, adapter->pdev, 379 adapter); 380 381 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring); 382 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 383 } 384 385 if (completed) { 386 spin_lock(&tq->tx_lock); 387 if (unlikely(vmxnet3_tq_stopped(tq, adapter) && 388 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) > 389 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) && 390 netif_carrier_ok(adapter->netdev))) { 391 vmxnet3_tq_wake(tq, adapter); 392 } 393 spin_unlock(&tq->tx_lock); 394 } 395 return completed; 396 } 397 398 399 static void 400 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq, 401 struct vmxnet3_adapter *adapter) 402 { 403 int i; 404 405 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) { 406 struct vmxnet3_tx_buf_info *tbi; 407 408 tbi = tq->buf_info + tq->tx_ring.next2comp; 409 410 vmxnet3_unmap_tx_buf(tbi, adapter->pdev); 411 if (tbi->skb) { 412 dev_kfree_skb_any(tbi->skb); 413 tbi->skb = NULL; 414 } 415 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 416 } 417 418 /* sanity check, verify all buffers are indeed unmapped and freed */ 419 for (i = 0; i < tq->tx_ring.size; i++) { 420 BUG_ON(tq->buf_info[i].skb != NULL || 421 tq->buf_info[i].map_type != VMXNET3_MAP_NONE); 422 } 423 424 tq->tx_ring.gen = VMXNET3_INIT_GEN; 425 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 426 427 tq->comp_ring.gen = VMXNET3_INIT_GEN; 428 tq->comp_ring.next2proc = 0; 429 } 430 431 432 static void 433 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 434 struct vmxnet3_adapter *adapter) 435 { 436 if (tq->tx_ring.base) { 437 pci_free_consistent(adapter->pdev, tq->tx_ring.size * 438 sizeof(struct Vmxnet3_TxDesc), 439 tq->tx_ring.base, tq->tx_ring.basePA); 440 tq->tx_ring.base = NULL; 441 } 442 if (tq->data_ring.base) { 443 pci_free_consistent(adapter->pdev, tq->data_ring.size * 444 sizeof(struct Vmxnet3_TxDataDesc), 445 tq->data_ring.base, tq->data_ring.basePA); 446 tq->data_ring.base = NULL; 447 } 448 if (tq->comp_ring.base) { 449 pci_free_consistent(adapter->pdev, tq->comp_ring.size * 450 sizeof(struct Vmxnet3_TxCompDesc), 451 tq->comp_ring.base, tq->comp_ring.basePA); 452 tq->comp_ring.base = NULL; 453 } 454 kfree(tq->buf_info); 455 tq->buf_info = NULL; 456 } 457 458 459 /* Destroy all tx queues */ 460 void 461 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter) 462 { 463 int i; 464 465 for (i = 0; i < adapter->num_tx_queues; i++) 466 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter); 467 } 468 469 470 static void 471 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 472 struct vmxnet3_adapter *adapter) 473 { 474 int i; 475 476 /* reset the tx ring contents to 0 and reset the tx ring states */ 477 memset(tq->tx_ring.base, 0, tq->tx_ring.size * 478 sizeof(struct Vmxnet3_TxDesc)); 479 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 480 tq->tx_ring.gen = VMXNET3_INIT_GEN; 481 482 memset(tq->data_ring.base, 0, tq->data_ring.size * 483 sizeof(struct Vmxnet3_TxDataDesc)); 484 485 /* reset the tx comp ring contents to 0 and reset comp ring states */ 486 memset(tq->comp_ring.base, 0, tq->comp_ring.size * 487 sizeof(struct Vmxnet3_TxCompDesc)); 488 tq->comp_ring.next2proc = 0; 489 tq->comp_ring.gen = VMXNET3_INIT_GEN; 490 491 /* reset the bookkeeping data */ 492 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size); 493 for (i = 0; i < tq->tx_ring.size; i++) 494 tq->buf_info[i].map_type = VMXNET3_MAP_NONE; 495 496 /* stats are not reset */ 497 } 498 499 500 static int 501 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq, 502 struct vmxnet3_adapter *adapter) 503 { 504 BUG_ON(tq->tx_ring.base || tq->data_ring.base || 505 tq->comp_ring.base || tq->buf_info); 506 507 tq->tx_ring.base = pci_alloc_consistent(adapter->pdev, tq->tx_ring.size 508 * sizeof(struct Vmxnet3_TxDesc), 509 &tq->tx_ring.basePA); 510 if (!tq->tx_ring.base) { 511 printk(KERN_ERR "%s: failed to allocate tx ring\n", 512 adapter->netdev->name); 513 goto err; 514 } 515 516 tq->data_ring.base = pci_alloc_consistent(adapter->pdev, 517 tq->data_ring.size * 518 sizeof(struct Vmxnet3_TxDataDesc), 519 &tq->data_ring.basePA); 520 if (!tq->data_ring.base) { 521 printk(KERN_ERR "%s: failed to allocate data ring\n", 522 adapter->netdev->name); 523 goto err; 524 } 525 526 tq->comp_ring.base = pci_alloc_consistent(adapter->pdev, 527 tq->comp_ring.size * 528 sizeof(struct Vmxnet3_TxCompDesc), 529 &tq->comp_ring.basePA); 530 if (!tq->comp_ring.base) { 531 printk(KERN_ERR "%s: failed to allocate tx comp ring\n", 532 adapter->netdev->name); 533 goto err; 534 } 535 536 tq->buf_info = kcalloc(tq->tx_ring.size, sizeof(tq->buf_info[0]), 537 GFP_KERNEL); 538 if (!tq->buf_info) 539 goto err; 540 541 return 0; 542 543 err: 544 vmxnet3_tq_destroy(tq, adapter); 545 return -ENOMEM; 546 } 547 548 static void 549 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter) 550 { 551 int i; 552 553 for (i = 0; i < adapter->num_tx_queues; i++) 554 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter); 555 } 556 557 /* 558 * starting from ring->next2fill, allocate rx buffers for the given ring 559 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers 560 * are allocated or allocation fails 561 */ 562 563 static int 564 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx, 565 int num_to_alloc, struct vmxnet3_adapter *adapter) 566 { 567 int num_allocated = 0; 568 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx]; 569 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx]; 570 u32 val; 571 572 while (num_allocated <= num_to_alloc) { 573 struct vmxnet3_rx_buf_info *rbi; 574 union Vmxnet3_GenericDesc *gd; 575 576 rbi = rbi_base + ring->next2fill; 577 gd = ring->base + ring->next2fill; 578 579 if (rbi->buf_type == VMXNET3_RX_BUF_SKB) { 580 if (rbi->skb == NULL) { 581 rbi->skb = dev_alloc_skb(rbi->len + 582 NET_IP_ALIGN); 583 if (unlikely(rbi->skb == NULL)) { 584 rq->stats.rx_buf_alloc_failure++; 585 break; 586 } 587 rbi->skb->dev = adapter->netdev; 588 589 skb_reserve(rbi->skb, NET_IP_ALIGN); 590 rbi->dma_addr = pci_map_single(adapter->pdev, 591 rbi->skb->data, rbi->len, 592 PCI_DMA_FROMDEVICE); 593 } else { 594 /* rx buffer skipped by the device */ 595 } 596 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 597 } else { 598 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE || 599 rbi->len != PAGE_SIZE); 600 601 if (rbi->page == NULL) { 602 rbi->page = alloc_page(GFP_ATOMIC); 603 if (unlikely(rbi->page == NULL)) { 604 rq->stats.rx_buf_alloc_failure++; 605 break; 606 } 607 rbi->dma_addr = pci_map_page(adapter->pdev, 608 rbi->page, 0, PAGE_SIZE, 609 PCI_DMA_FROMDEVICE); 610 } else { 611 /* rx buffers skipped by the device */ 612 } 613 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT; 614 } 615 616 BUG_ON(rbi->dma_addr == 0); 617 gd->rxd.addr = cpu_to_le64(rbi->dma_addr); 618 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT) 619 | val | rbi->len); 620 621 /* Fill the last buffer but dont mark it ready, or else the 622 * device will think that the queue is full */ 623 if (num_allocated == num_to_alloc) 624 break; 625 626 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT); 627 num_allocated++; 628 vmxnet3_cmd_ring_adv_next2fill(ring); 629 } 630 rq->uncommitted[ring_idx] += num_allocated; 631 632 dev_dbg(&adapter->netdev->dev, 633 "alloc_rx_buf: %d allocated, next2fill %u, next2comp " 634 "%u, uncommitted %u\n", num_allocated, ring->next2fill, 635 ring->next2comp, rq->uncommitted[ring_idx]); 636 637 /* so that the device can distinguish a full ring and an empty ring */ 638 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp); 639 640 return num_allocated; 641 } 642 643 644 static void 645 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd, 646 struct vmxnet3_rx_buf_info *rbi) 647 { 648 struct skb_frag_struct *frag = skb_shinfo(skb)->frags + 649 skb_shinfo(skb)->nr_frags; 650 651 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); 652 653 __skb_frag_set_page(frag, rbi->page); 654 frag->page_offset = 0; 655 skb_frag_size_set(frag, rcd->len); 656 skb->data_len += rcd->len; 657 skb->truesize += PAGE_SIZE; 658 skb_shinfo(skb)->nr_frags++; 659 } 660 661 662 static void 663 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx, 664 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev, 665 struct vmxnet3_adapter *adapter) 666 { 667 u32 dw2, len; 668 unsigned long buf_offset; 669 int i; 670 union Vmxnet3_GenericDesc *gdesc; 671 struct vmxnet3_tx_buf_info *tbi = NULL; 672 673 BUG_ON(ctx->copy_size > skb_headlen(skb)); 674 675 /* use the previous gen bit for the SOP desc */ 676 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 677 678 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; 679 gdesc = ctx->sop_txd; /* both loops below can be skipped */ 680 681 /* no need to map the buffer if headers are copied */ 682 if (ctx->copy_size) { 683 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA + 684 tq->tx_ring.next2fill * 685 sizeof(struct Vmxnet3_TxDataDesc)); 686 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size); 687 ctx->sop_txd->dword[3] = 0; 688 689 tbi = tq->buf_info + tq->tx_ring.next2fill; 690 tbi->map_type = VMXNET3_MAP_NONE; 691 692 dev_dbg(&adapter->netdev->dev, 693 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 694 tq->tx_ring.next2fill, 695 le64_to_cpu(ctx->sop_txd->txd.addr), 696 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]); 697 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 698 699 /* use the right gen for non-SOP desc */ 700 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 701 } 702 703 /* linear part can use multiple tx desc if it's big */ 704 len = skb_headlen(skb) - ctx->copy_size; 705 buf_offset = ctx->copy_size; 706 while (len) { 707 u32 buf_size; 708 709 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 710 buf_size = len; 711 dw2 |= len; 712 } else { 713 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 714 /* spec says that for TxDesc.len, 0 == 2^14 */ 715 } 716 717 tbi = tq->buf_info + tq->tx_ring.next2fill; 718 tbi->map_type = VMXNET3_MAP_SINGLE; 719 tbi->dma_addr = pci_map_single(adapter->pdev, 720 skb->data + buf_offset, buf_size, 721 PCI_DMA_TODEVICE); 722 723 tbi->len = buf_size; 724 725 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 726 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 727 728 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 729 gdesc->dword[2] = cpu_to_le32(dw2); 730 gdesc->dword[3] = 0; 731 732 dev_dbg(&adapter->netdev->dev, 733 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 734 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 735 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 736 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 737 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 738 739 len -= buf_size; 740 buf_offset += buf_size; 741 } 742 743 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 744 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; 745 u32 buf_size; 746 747 buf_offset = 0; 748 len = skb_frag_size(frag); 749 while (len) { 750 tbi = tq->buf_info + tq->tx_ring.next2fill; 751 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 752 buf_size = len; 753 dw2 |= len; 754 } else { 755 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 756 /* spec says that for TxDesc.len, 0 == 2^14 */ 757 } 758 tbi->map_type = VMXNET3_MAP_PAGE; 759 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag, 760 buf_offset, buf_size, 761 DMA_TO_DEVICE); 762 763 tbi->len = buf_size; 764 765 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 766 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 767 768 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 769 gdesc->dword[2] = cpu_to_le32(dw2); 770 gdesc->dword[3] = 0; 771 772 dev_dbg(&adapter->netdev->dev, 773 "txd[%u]: 0x%llu %u %u\n", 774 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 775 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 776 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 777 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 778 779 len -= buf_size; 780 buf_offset += buf_size; 781 } 782 } 783 784 ctx->eop_txd = gdesc; 785 786 /* set the last buf_info for the pkt */ 787 tbi->skb = skb; 788 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base; 789 } 790 791 792 /* Init all tx queues */ 793 static void 794 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter) 795 { 796 int i; 797 798 for (i = 0; i < adapter->num_tx_queues; i++) 799 vmxnet3_tq_init(&adapter->tx_queue[i], adapter); 800 } 801 802 803 /* 804 * parse and copy relevant protocol headers: 805 * For a tso pkt, relevant headers are L2/3/4 including options 806 * For a pkt requesting csum offloading, they are L2/3 and may include L4 807 * if it's a TCP/UDP pkt 808 * 809 * Returns: 810 * -1: error happens during parsing 811 * 0: protocol headers parsed, but too big to be copied 812 * 1: protocol headers parsed and copied 813 * 814 * Other effects: 815 * 1. related *ctx fields are updated. 816 * 2. ctx->copy_size is # of bytes copied 817 * 3. the portion copied is guaranteed to be in the linear part 818 * 819 */ 820 static int 821 vmxnet3_parse_and_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 822 struct vmxnet3_tx_ctx *ctx, 823 struct vmxnet3_adapter *adapter) 824 { 825 struct Vmxnet3_TxDataDesc *tdd; 826 827 if (ctx->mss) { /* TSO */ 828 ctx->eth_ip_hdr_size = skb_transport_offset(skb); 829 ctx->l4_hdr_size = tcp_hdrlen(skb); 830 ctx->copy_size = ctx->eth_ip_hdr_size + ctx->l4_hdr_size; 831 } else { 832 if (skb->ip_summed == CHECKSUM_PARTIAL) { 833 ctx->eth_ip_hdr_size = skb_checksum_start_offset(skb); 834 835 if (ctx->ipv4) { 836 const struct iphdr *iph = ip_hdr(skb); 837 838 if (iph->protocol == IPPROTO_TCP) 839 ctx->l4_hdr_size = tcp_hdrlen(skb); 840 else if (iph->protocol == IPPROTO_UDP) 841 ctx->l4_hdr_size = sizeof(struct udphdr); 842 else 843 ctx->l4_hdr_size = 0; 844 } else { 845 /* for simplicity, don't copy L4 headers */ 846 ctx->l4_hdr_size = 0; 847 } 848 ctx->copy_size = min(ctx->eth_ip_hdr_size + 849 ctx->l4_hdr_size, skb->len); 850 } else { 851 ctx->eth_ip_hdr_size = 0; 852 ctx->l4_hdr_size = 0; 853 /* copy as much as allowed */ 854 ctx->copy_size = min((unsigned int)VMXNET3_HDR_COPY_SIZE 855 , skb_headlen(skb)); 856 } 857 858 /* make sure headers are accessible directly */ 859 if (unlikely(!pskb_may_pull(skb, ctx->copy_size))) 860 goto err; 861 } 862 863 if (unlikely(ctx->copy_size > VMXNET3_HDR_COPY_SIZE)) { 864 tq->stats.oversized_hdr++; 865 ctx->copy_size = 0; 866 return 0; 867 } 868 869 tdd = tq->data_ring.base + tq->tx_ring.next2fill; 870 871 memcpy(tdd->data, skb->data, ctx->copy_size); 872 dev_dbg(&adapter->netdev->dev, 873 "copy %u bytes to dataRing[%u]\n", 874 ctx->copy_size, tq->tx_ring.next2fill); 875 return 1; 876 877 err: 878 return -1; 879 } 880 881 882 static void 883 vmxnet3_prepare_tso(struct sk_buff *skb, 884 struct vmxnet3_tx_ctx *ctx) 885 { 886 struct tcphdr *tcph = tcp_hdr(skb); 887 888 if (ctx->ipv4) { 889 struct iphdr *iph = ip_hdr(skb); 890 891 iph->check = 0; 892 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 893 IPPROTO_TCP, 0); 894 } else { 895 struct ipv6hdr *iph = ipv6_hdr(skb); 896 897 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0, 898 IPPROTO_TCP, 0); 899 } 900 } 901 902 static int txd_estimate(const struct sk_buff *skb) 903 { 904 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 905 int i; 906 907 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 908 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; 909 910 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag)); 911 } 912 return count; 913 } 914 915 /* 916 * Transmits a pkt thru a given tq 917 * Returns: 918 * NETDEV_TX_OK: descriptors are setup successfully 919 * NETDEV_TX_OK: error occurred, the pkt is dropped 920 * NETDEV_TX_BUSY: tx ring is full, queue is stopped 921 * 922 * Side-effects: 923 * 1. tx ring may be changed 924 * 2. tq stats may be updated accordingly 925 * 3. shared->txNumDeferred may be updated 926 */ 927 928 static int 929 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 930 struct vmxnet3_adapter *adapter, struct net_device *netdev) 931 { 932 int ret; 933 u32 count; 934 unsigned long flags; 935 struct vmxnet3_tx_ctx ctx; 936 union Vmxnet3_GenericDesc *gdesc; 937 #ifdef __BIG_ENDIAN_BITFIELD 938 /* Use temporary descriptor to avoid touching bits multiple times */ 939 union Vmxnet3_GenericDesc tempTxDesc; 940 #endif 941 942 count = txd_estimate(skb); 943 944 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP)); 945 946 ctx.mss = skb_shinfo(skb)->gso_size; 947 if (ctx.mss) { 948 if (skb_header_cloned(skb)) { 949 if (unlikely(pskb_expand_head(skb, 0, 0, 950 GFP_ATOMIC) != 0)) { 951 tq->stats.drop_tso++; 952 goto drop_pkt; 953 } 954 tq->stats.copy_skb_header++; 955 } 956 vmxnet3_prepare_tso(skb, &ctx); 957 } else { 958 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) { 959 960 /* non-tso pkts must not use more than 961 * VMXNET3_MAX_TXD_PER_PKT entries 962 */ 963 if (skb_linearize(skb) != 0) { 964 tq->stats.drop_too_many_frags++; 965 goto drop_pkt; 966 } 967 tq->stats.linearized++; 968 969 /* recalculate the # of descriptors to use */ 970 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 971 } 972 } 973 974 spin_lock_irqsave(&tq->tx_lock, flags); 975 976 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) { 977 tq->stats.tx_ring_full++; 978 dev_dbg(&adapter->netdev->dev, 979 "tx queue stopped on %s, next2comp %u" 980 " next2fill %u\n", adapter->netdev->name, 981 tq->tx_ring.next2comp, tq->tx_ring.next2fill); 982 983 vmxnet3_tq_stop(tq, adapter); 984 spin_unlock_irqrestore(&tq->tx_lock, flags); 985 return NETDEV_TX_BUSY; 986 } 987 988 989 ret = vmxnet3_parse_and_copy_hdr(skb, tq, &ctx, adapter); 990 if (ret >= 0) { 991 BUG_ON(ret <= 0 && ctx.copy_size != 0); 992 /* hdrs parsed, check against other limits */ 993 if (ctx.mss) { 994 if (unlikely(ctx.eth_ip_hdr_size + ctx.l4_hdr_size > 995 VMXNET3_MAX_TX_BUF_SIZE)) { 996 goto hdr_too_big; 997 } 998 } else { 999 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1000 if (unlikely(ctx.eth_ip_hdr_size + 1001 skb->csum_offset > 1002 VMXNET3_MAX_CSUM_OFFSET)) { 1003 goto hdr_too_big; 1004 } 1005 } 1006 } 1007 } else { 1008 tq->stats.drop_hdr_inspect_err++; 1009 goto unlock_drop_pkt; 1010 } 1011 1012 /* fill tx descs related to addr & len */ 1013 vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter); 1014 1015 /* setup the EOP desc */ 1016 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); 1017 1018 /* setup the SOP desc */ 1019 #ifdef __BIG_ENDIAN_BITFIELD 1020 gdesc = &tempTxDesc; 1021 gdesc->dword[2] = ctx.sop_txd->dword[2]; 1022 gdesc->dword[3] = ctx.sop_txd->dword[3]; 1023 #else 1024 gdesc = ctx.sop_txd; 1025 #endif 1026 if (ctx.mss) { 1027 gdesc->txd.hlen = ctx.eth_ip_hdr_size + ctx.l4_hdr_size; 1028 gdesc->txd.om = VMXNET3_OM_TSO; 1029 gdesc->txd.msscof = ctx.mss; 1030 le32_add_cpu(&tq->shared->txNumDeferred, (skb->len - 1031 gdesc->txd.hlen + ctx.mss - 1) / ctx.mss); 1032 } else { 1033 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1034 gdesc->txd.hlen = ctx.eth_ip_hdr_size; 1035 gdesc->txd.om = VMXNET3_OM_CSUM; 1036 gdesc->txd.msscof = ctx.eth_ip_hdr_size + 1037 skb->csum_offset; 1038 } else { 1039 gdesc->txd.om = 0; 1040 gdesc->txd.msscof = 0; 1041 } 1042 le32_add_cpu(&tq->shared->txNumDeferred, 1); 1043 } 1044 1045 if (vlan_tx_tag_present(skb)) { 1046 gdesc->txd.ti = 1; 1047 gdesc->txd.tci = vlan_tx_tag_get(skb); 1048 } 1049 1050 /* finally flips the GEN bit of the SOP desc. */ 1051 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ 1052 VMXNET3_TXD_GEN); 1053 #ifdef __BIG_ENDIAN_BITFIELD 1054 /* Finished updating in bitfields of Tx Desc, so write them in original 1055 * place. 1056 */ 1057 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc, 1058 (struct Vmxnet3_TxDesc *)ctx.sop_txd); 1059 gdesc = ctx.sop_txd; 1060 #endif 1061 dev_dbg(&adapter->netdev->dev, 1062 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n", 1063 (u32)(ctx.sop_txd - 1064 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr), 1065 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3])); 1066 1067 spin_unlock_irqrestore(&tq->tx_lock, flags); 1068 1069 if (le32_to_cpu(tq->shared->txNumDeferred) >= 1070 le32_to_cpu(tq->shared->txThreshold)) { 1071 tq->shared->txNumDeferred = 0; 1072 VMXNET3_WRITE_BAR0_REG(adapter, 1073 VMXNET3_REG_TXPROD + tq->qid * 8, 1074 tq->tx_ring.next2fill); 1075 } 1076 1077 return NETDEV_TX_OK; 1078 1079 hdr_too_big: 1080 tq->stats.drop_oversized_hdr++; 1081 unlock_drop_pkt: 1082 spin_unlock_irqrestore(&tq->tx_lock, flags); 1083 drop_pkt: 1084 tq->stats.drop_total++; 1085 dev_kfree_skb(skb); 1086 return NETDEV_TX_OK; 1087 } 1088 1089 1090 static netdev_tx_t 1091 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1092 { 1093 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1094 1095 BUG_ON(skb->queue_mapping > adapter->num_tx_queues); 1096 return vmxnet3_tq_xmit(skb, 1097 &adapter->tx_queue[skb->queue_mapping], 1098 adapter, netdev); 1099 } 1100 1101 1102 static void 1103 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, 1104 struct sk_buff *skb, 1105 union Vmxnet3_GenericDesc *gdesc) 1106 { 1107 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) { 1108 /* typical case: TCP/UDP over IP and both csums are correct */ 1109 if ((le32_to_cpu(gdesc->dword[3]) & VMXNET3_RCD_CSUM_OK) == 1110 VMXNET3_RCD_CSUM_OK) { 1111 skb->ip_summed = CHECKSUM_UNNECESSARY; 1112 BUG_ON(!(gdesc->rcd.tcp || gdesc->rcd.udp)); 1113 BUG_ON(!(gdesc->rcd.v4 || gdesc->rcd.v6)); 1114 BUG_ON(gdesc->rcd.frg); 1115 } else { 1116 if (gdesc->rcd.csum) { 1117 skb->csum = htons(gdesc->rcd.csum); 1118 skb->ip_summed = CHECKSUM_PARTIAL; 1119 } else { 1120 skb_checksum_none_assert(skb); 1121 } 1122 } 1123 } else { 1124 skb_checksum_none_assert(skb); 1125 } 1126 } 1127 1128 1129 static void 1130 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd, 1131 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter) 1132 { 1133 rq->stats.drop_err++; 1134 if (!rcd->fcs) 1135 rq->stats.drop_fcs++; 1136 1137 rq->stats.drop_total++; 1138 1139 /* 1140 * We do not unmap and chain the rx buffer to the skb. 1141 * We basically pretend this buffer is not used and will be recycled 1142 * by vmxnet3_rq_alloc_rx_buf() 1143 */ 1144 1145 /* 1146 * ctx->skb may be NULL if this is the first and the only one 1147 * desc for the pkt 1148 */ 1149 if (ctx->skb) 1150 dev_kfree_skb_irq(ctx->skb); 1151 1152 ctx->skb = NULL; 1153 } 1154 1155 1156 static int 1157 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, 1158 struct vmxnet3_adapter *adapter, int quota) 1159 { 1160 static const u32 rxprod_reg[2] = { 1161 VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2 1162 }; 1163 u32 num_rxd = 0; 1164 bool skip_page_frags = false; 1165 struct Vmxnet3_RxCompDesc *rcd; 1166 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; 1167 #ifdef __BIG_ENDIAN_BITFIELD 1168 struct Vmxnet3_RxDesc rxCmdDesc; 1169 struct Vmxnet3_RxCompDesc rxComp; 1170 #endif 1171 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, 1172 &rxComp); 1173 while (rcd->gen == rq->comp_ring.gen) { 1174 struct vmxnet3_rx_buf_info *rbi; 1175 struct sk_buff *skb, *new_skb = NULL; 1176 struct page *new_page = NULL; 1177 int num_to_alloc; 1178 struct Vmxnet3_RxDesc *rxd; 1179 u32 idx, ring_idx; 1180 struct vmxnet3_cmd_ring *ring = NULL; 1181 if (num_rxd >= quota) { 1182 /* we may stop even before we see the EOP desc of 1183 * the current pkt 1184 */ 1185 break; 1186 } 1187 num_rxd++; 1188 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2); 1189 idx = rcd->rxdIdx; 1190 ring_idx = rcd->rqID < adapter->num_rx_queues ? 0 : 1; 1191 ring = rq->rx_ring + ring_idx; 1192 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1193 &rxCmdDesc); 1194 rbi = rq->buf_info[ring_idx] + idx; 1195 1196 BUG_ON(rxd->addr != rbi->dma_addr || 1197 rxd->len != rbi->len); 1198 1199 if (unlikely(rcd->eop && rcd->err)) { 1200 vmxnet3_rx_error(rq, rcd, ctx, adapter); 1201 goto rcd_done; 1202 } 1203 1204 if (rcd->sop) { /* first buf of the pkt */ 1205 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD || 1206 rcd->rqID != rq->qid); 1207 1208 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB); 1209 BUG_ON(ctx->skb != NULL || rbi->skb == NULL); 1210 1211 if (unlikely(rcd->len == 0)) { 1212 /* Pretend the rx buffer is skipped. */ 1213 BUG_ON(!(rcd->sop && rcd->eop)); 1214 dev_dbg(&adapter->netdev->dev, 1215 "rxRing[%u][%u] 0 length\n", 1216 ring_idx, idx); 1217 goto rcd_done; 1218 } 1219 1220 skip_page_frags = false; 1221 ctx->skb = rbi->skb; 1222 new_skb = dev_alloc_skb(rbi->len + NET_IP_ALIGN); 1223 if (new_skb == NULL) { 1224 /* Skb allocation failed, do not handover this 1225 * skb to stack. Reuse it. Drop the existing pkt 1226 */ 1227 rq->stats.rx_buf_alloc_failure++; 1228 ctx->skb = NULL; 1229 rq->stats.drop_total++; 1230 skip_page_frags = true; 1231 goto rcd_done; 1232 } 1233 1234 pci_unmap_single(adapter->pdev, rbi->dma_addr, rbi->len, 1235 PCI_DMA_FROMDEVICE); 1236 1237 skb_put(ctx->skb, rcd->len); 1238 1239 /* Immediate refill */ 1240 new_skb->dev = adapter->netdev; 1241 skb_reserve(new_skb, NET_IP_ALIGN); 1242 rbi->skb = new_skb; 1243 rbi->dma_addr = pci_map_single(adapter->pdev, 1244 rbi->skb->data, rbi->len, 1245 PCI_DMA_FROMDEVICE); 1246 rxd->addr = cpu_to_le64(rbi->dma_addr); 1247 rxd->len = rbi->len; 1248 1249 } else { 1250 BUG_ON(ctx->skb == NULL && !skip_page_frags); 1251 1252 /* non SOP buffer must be type 1 in most cases */ 1253 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE); 1254 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY); 1255 1256 /* If an sop buffer was dropped, skip all 1257 * following non-sop fragments. They will be reused. 1258 */ 1259 if (skip_page_frags) 1260 goto rcd_done; 1261 1262 new_page = alloc_page(GFP_ATOMIC); 1263 if (unlikely(new_page == NULL)) { 1264 /* Replacement page frag could not be allocated. 1265 * Reuse this page. Drop the pkt and free the 1266 * skb which contained this page as a frag. Skip 1267 * processing all the following non-sop frags. 1268 */ 1269 rq->stats.rx_buf_alloc_failure++; 1270 dev_kfree_skb(ctx->skb); 1271 ctx->skb = NULL; 1272 skip_page_frags = true; 1273 goto rcd_done; 1274 } 1275 1276 if (rcd->len) { 1277 pci_unmap_page(adapter->pdev, 1278 rbi->dma_addr, rbi->len, 1279 PCI_DMA_FROMDEVICE); 1280 1281 vmxnet3_append_frag(ctx->skb, rcd, rbi); 1282 } 1283 1284 /* Immediate refill */ 1285 rbi->page = new_page; 1286 rbi->dma_addr = pci_map_page(adapter->pdev, rbi->page, 1287 0, PAGE_SIZE, 1288 PCI_DMA_FROMDEVICE); 1289 rxd->addr = cpu_to_le64(rbi->dma_addr); 1290 rxd->len = rbi->len; 1291 } 1292 1293 1294 skb = ctx->skb; 1295 if (rcd->eop) { 1296 skb->len += skb->data_len; 1297 1298 vmxnet3_rx_csum(adapter, skb, 1299 (union Vmxnet3_GenericDesc *)rcd); 1300 skb->protocol = eth_type_trans(skb, adapter->netdev); 1301 1302 if (unlikely(rcd->ts)) 1303 __vlan_hwaccel_put_tag(skb, rcd->tci); 1304 1305 if (adapter->netdev->features & NETIF_F_LRO) 1306 netif_receive_skb(skb); 1307 else 1308 napi_gro_receive(&rq->napi, skb); 1309 1310 ctx->skb = NULL; 1311 } 1312 1313 rcd_done: 1314 /* device may have skipped some rx descs */ 1315 ring->next2comp = idx; 1316 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring); 1317 ring = rq->rx_ring + ring_idx; 1318 while (num_to_alloc) { 1319 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd, 1320 &rxCmdDesc); 1321 BUG_ON(!rxd->addr); 1322 1323 /* Recv desc is ready to be used by the device */ 1324 rxd->gen = ring->gen; 1325 vmxnet3_cmd_ring_adv_next2fill(ring); 1326 num_to_alloc--; 1327 } 1328 1329 /* if needed, update the register */ 1330 if (unlikely(rq->shared->updateRxProd)) { 1331 VMXNET3_WRITE_BAR0_REG(adapter, 1332 rxprod_reg[ring_idx] + rq->qid * 8, 1333 ring->next2fill); 1334 rq->uncommitted[ring_idx] = 0; 1335 } 1336 1337 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring); 1338 vmxnet3_getRxComp(rcd, 1339 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp); 1340 } 1341 1342 return num_rxd; 1343 } 1344 1345 1346 static void 1347 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, 1348 struct vmxnet3_adapter *adapter) 1349 { 1350 u32 i, ring_idx; 1351 struct Vmxnet3_RxDesc *rxd; 1352 1353 for (ring_idx = 0; ring_idx < 2; ring_idx++) { 1354 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) { 1355 #ifdef __BIG_ENDIAN_BITFIELD 1356 struct Vmxnet3_RxDesc rxDesc; 1357 #endif 1358 vmxnet3_getRxDesc(rxd, 1359 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc); 1360 1361 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 1362 rq->buf_info[ring_idx][i].skb) { 1363 pci_unmap_single(adapter->pdev, rxd->addr, 1364 rxd->len, PCI_DMA_FROMDEVICE); 1365 dev_kfree_skb(rq->buf_info[ring_idx][i].skb); 1366 rq->buf_info[ring_idx][i].skb = NULL; 1367 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY && 1368 rq->buf_info[ring_idx][i].page) { 1369 pci_unmap_page(adapter->pdev, rxd->addr, 1370 rxd->len, PCI_DMA_FROMDEVICE); 1371 put_page(rq->buf_info[ring_idx][i].page); 1372 rq->buf_info[ring_idx][i].page = NULL; 1373 } 1374 } 1375 1376 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN; 1377 rq->rx_ring[ring_idx].next2fill = 1378 rq->rx_ring[ring_idx].next2comp = 0; 1379 rq->uncommitted[ring_idx] = 0; 1380 } 1381 1382 rq->comp_ring.gen = VMXNET3_INIT_GEN; 1383 rq->comp_ring.next2proc = 0; 1384 } 1385 1386 1387 static void 1388 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter) 1389 { 1390 int i; 1391 1392 for (i = 0; i < adapter->num_rx_queues; i++) 1393 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter); 1394 } 1395 1396 1397 void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 1398 struct vmxnet3_adapter *adapter) 1399 { 1400 int i; 1401 int j; 1402 1403 /* all rx buffers must have already been freed */ 1404 for (i = 0; i < 2; i++) { 1405 if (rq->buf_info[i]) { 1406 for (j = 0; j < rq->rx_ring[i].size; j++) 1407 BUG_ON(rq->buf_info[i][j].page != NULL); 1408 } 1409 } 1410 1411 1412 kfree(rq->buf_info[0]); 1413 1414 for (i = 0; i < 2; i++) { 1415 if (rq->rx_ring[i].base) { 1416 pci_free_consistent(adapter->pdev, rq->rx_ring[i].size 1417 * sizeof(struct Vmxnet3_RxDesc), 1418 rq->rx_ring[i].base, 1419 rq->rx_ring[i].basePA); 1420 rq->rx_ring[i].base = NULL; 1421 } 1422 rq->buf_info[i] = NULL; 1423 } 1424 1425 if (rq->comp_ring.base) { 1426 pci_free_consistent(adapter->pdev, rq->comp_ring.size * 1427 sizeof(struct Vmxnet3_RxCompDesc), 1428 rq->comp_ring.base, rq->comp_ring.basePA); 1429 rq->comp_ring.base = NULL; 1430 } 1431 } 1432 1433 1434 static int 1435 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq, 1436 struct vmxnet3_adapter *adapter) 1437 { 1438 int i; 1439 1440 /* initialize buf_info */ 1441 for (i = 0; i < rq->rx_ring[0].size; i++) { 1442 1443 /* 1st buf for a pkt is skbuff */ 1444 if (i % adapter->rx_buf_per_pkt == 0) { 1445 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB; 1446 rq->buf_info[0][i].len = adapter->skb_buf_size; 1447 } else { /* subsequent bufs for a pkt is frag */ 1448 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE; 1449 rq->buf_info[0][i].len = PAGE_SIZE; 1450 } 1451 } 1452 for (i = 0; i < rq->rx_ring[1].size; i++) { 1453 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE; 1454 rq->buf_info[1][i].len = PAGE_SIZE; 1455 } 1456 1457 /* reset internal state and allocate buffers for both rings */ 1458 for (i = 0; i < 2; i++) { 1459 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0; 1460 rq->uncommitted[i] = 0; 1461 1462 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size * 1463 sizeof(struct Vmxnet3_RxDesc)); 1464 rq->rx_ring[i].gen = VMXNET3_INIT_GEN; 1465 } 1466 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1, 1467 adapter) == 0) { 1468 /* at least has 1 rx buffer for the 1st ring */ 1469 return -ENOMEM; 1470 } 1471 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter); 1472 1473 /* reset the comp ring */ 1474 rq->comp_ring.next2proc = 0; 1475 memset(rq->comp_ring.base, 0, rq->comp_ring.size * 1476 sizeof(struct Vmxnet3_RxCompDesc)); 1477 rq->comp_ring.gen = VMXNET3_INIT_GEN; 1478 1479 /* reset rxctx */ 1480 rq->rx_ctx.skb = NULL; 1481 1482 /* stats are not reset */ 1483 return 0; 1484 } 1485 1486 1487 static int 1488 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter) 1489 { 1490 int i, err = 0; 1491 1492 for (i = 0; i < adapter->num_rx_queues; i++) { 1493 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter); 1494 if (unlikely(err)) { 1495 dev_err(&adapter->netdev->dev, "%s: failed to " 1496 "initialize rx queue%i\n", 1497 adapter->netdev->name, i); 1498 break; 1499 } 1500 } 1501 return err; 1502 1503 } 1504 1505 1506 static int 1507 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 1508 { 1509 int i; 1510 size_t sz; 1511 struct vmxnet3_rx_buf_info *bi; 1512 1513 for (i = 0; i < 2; i++) { 1514 1515 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc); 1516 rq->rx_ring[i].base = pci_alloc_consistent(adapter->pdev, sz, 1517 &rq->rx_ring[i].basePA); 1518 if (!rq->rx_ring[i].base) { 1519 printk(KERN_ERR "%s: failed to allocate rx ring %d\n", 1520 adapter->netdev->name, i); 1521 goto err; 1522 } 1523 } 1524 1525 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc); 1526 rq->comp_ring.base = pci_alloc_consistent(adapter->pdev, sz, 1527 &rq->comp_ring.basePA); 1528 if (!rq->comp_ring.base) { 1529 printk(KERN_ERR "%s: failed to allocate rx comp ring\n", 1530 adapter->netdev->name); 1531 goto err; 1532 } 1533 1534 sz = sizeof(struct vmxnet3_rx_buf_info) * (rq->rx_ring[0].size + 1535 rq->rx_ring[1].size); 1536 bi = kzalloc(sz, GFP_KERNEL); 1537 if (!bi) 1538 goto err; 1539 1540 rq->buf_info[0] = bi; 1541 rq->buf_info[1] = bi + rq->rx_ring[0].size; 1542 1543 return 0; 1544 1545 err: 1546 vmxnet3_rq_destroy(rq, adapter); 1547 return -ENOMEM; 1548 } 1549 1550 1551 static int 1552 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter) 1553 { 1554 int i, err = 0; 1555 1556 for (i = 0; i < adapter->num_rx_queues; i++) { 1557 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter); 1558 if (unlikely(err)) { 1559 dev_err(&adapter->netdev->dev, 1560 "%s: failed to create rx queue%i\n", 1561 adapter->netdev->name, i); 1562 goto err_out; 1563 } 1564 } 1565 return err; 1566 err_out: 1567 vmxnet3_rq_destroy_all(adapter); 1568 return err; 1569 1570 } 1571 1572 /* Multiple queue aware polling function for tx and rx */ 1573 1574 static int 1575 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 1576 { 1577 int rcd_done = 0, i; 1578 if (unlikely(adapter->shared->ecr)) 1579 vmxnet3_process_events(adapter); 1580 for (i = 0; i < adapter->num_tx_queues; i++) 1581 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter); 1582 1583 for (i = 0; i < adapter->num_rx_queues; i++) 1584 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i], 1585 adapter, budget); 1586 return rcd_done; 1587 } 1588 1589 1590 static int 1591 vmxnet3_poll(struct napi_struct *napi, int budget) 1592 { 1593 struct vmxnet3_rx_queue *rx_queue = container_of(napi, 1594 struct vmxnet3_rx_queue, napi); 1595 int rxd_done; 1596 1597 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget); 1598 1599 if (rxd_done < budget) { 1600 napi_complete(napi); 1601 vmxnet3_enable_all_intrs(rx_queue->adapter); 1602 } 1603 return rxd_done; 1604 } 1605 1606 /* 1607 * NAPI polling function for MSI-X mode with multiple Rx queues 1608 * Returns the # of the NAPI credit consumed (# of rx descriptors processed) 1609 */ 1610 1611 static int 1612 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget) 1613 { 1614 struct vmxnet3_rx_queue *rq = container_of(napi, 1615 struct vmxnet3_rx_queue, napi); 1616 struct vmxnet3_adapter *adapter = rq->adapter; 1617 int rxd_done; 1618 1619 /* When sharing interrupt with corresponding tx queue, process 1620 * tx completions in that queue as well 1621 */ 1622 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { 1623 struct vmxnet3_tx_queue *tq = 1624 &adapter->tx_queue[rq - adapter->rx_queue]; 1625 vmxnet3_tq_tx_complete(tq, adapter); 1626 } 1627 1628 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget); 1629 1630 if (rxd_done < budget) { 1631 napi_complete(napi); 1632 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx); 1633 } 1634 return rxd_done; 1635 } 1636 1637 1638 #ifdef CONFIG_PCI_MSI 1639 1640 /* 1641 * Handle completion interrupts on tx queues 1642 * Returns whether or not the intr is handled 1643 */ 1644 1645 static irqreturn_t 1646 vmxnet3_msix_tx(int irq, void *data) 1647 { 1648 struct vmxnet3_tx_queue *tq = data; 1649 struct vmxnet3_adapter *adapter = tq->adapter; 1650 1651 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 1652 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx); 1653 1654 /* Handle the case where only one irq is allocate for all tx queues */ 1655 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 1656 int i; 1657 for (i = 0; i < adapter->num_tx_queues; i++) { 1658 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i]; 1659 vmxnet3_tq_tx_complete(txq, adapter); 1660 } 1661 } else { 1662 vmxnet3_tq_tx_complete(tq, adapter); 1663 } 1664 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx); 1665 1666 return IRQ_HANDLED; 1667 } 1668 1669 1670 /* 1671 * Handle completion interrupts on rx queues. Returns whether or not the 1672 * intr is handled 1673 */ 1674 1675 static irqreturn_t 1676 vmxnet3_msix_rx(int irq, void *data) 1677 { 1678 struct vmxnet3_rx_queue *rq = data; 1679 struct vmxnet3_adapter *adapter = rq->adapter; 1680 1681 /* disable intr if needed */ 1682 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 1683 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx); 1684 napi_schedule(&rq->napi); 1685 1686 return IRQ_HANDLED; 1687 } 1688 1689 /* 1690 *---------------------------------------------------------------------------- 1691 * 1692 * vmxnet3_msix_event -- 1693 * 1694 * vmxnet3 msix event intr handler 1695 * 1696 * Result: 1697 * whether or not the intr is handled 1698 * 1699 *---------------------------------------------------------------------------- 1700 */ 1701 1702 static irqreturn_t 1703 vmxnet3_msix_event(int irq, void *data) 1704 { 1705 struct net_device *dev = data; 1706 struct vmxnet3_adapter *adapter = netdev_priv(dev); 1707 1708 /* disable intr if needed */ 1709 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 1710 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx); 1711 1712 if (adapter->shared->ecr) 1713 vmxnet3_process_events(adapter); 1714 1715 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx); 1716 1717 return IRQ_HANDLED; 1718 } 1719 1720 #endif /* CONFIG_PCI_MSI */ 1721 1722 1723 /* Interrupt handler for vmxnet3 */ 1724 static irqreturn_t 1725 vmxnet3_intr(int irq, void *dev_id) 1726 { 1727 struct net_device *dev = dev_id; 1728 struct vmxnet3_adapter *adapter = netdev_priv(dev); 1729 1730 if (adapter->intr.type == VMXNET3_IT_INTX) { 1731 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 1732 if (unlikely(icr == 0)) 1733 /* not ours */ 1734 return IRQ_NONE; 1735 } 1736 1737 1738 /* disable intr if needed */ 1739 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 1740 vmxnet3_disable_all_intrs(adapter); 1741 1742 napi_schedule(&adapter->rx_queue[0].napi); 1743 1744 return IRQ_HANDLED; 1745 } 1746 1747 #ifdef CONFIG_NET_POLL_CONTROLLER 1748 1749 /* netpoll callback. */ 1750 static void 1751 vmxnet3_netpoll(struct net_device *netdev) 1752 { 1753 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1754 1755 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 1756 vmxnet3_disable_all_intrs(adapter); 1757 1758 vmxnet3_do_poll(adapter, adapter->rx_queue[0].rx_ring[0].size); 1759 vmxnet3_enable_all_intrs(adapter); 1760 1761 } 1762 #endif /* CONFIG_NET_POLL_CONTROLLER */ 1763 1764 static int 1765 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 1766 { 1767 struct vmxnet3_intr *intr = &adapter->intr; 1768 int err = 0, i; 1769 int vector = 0; 1770 1771 #ifdef CONFIG_PCI_MSI 1772 if (adapter->intr.type == VMXNET3_IT_MSIX) { 1773 for (i = 0; i < adapter->num_tx_queues; i++) { 1774 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 1775 sprintf(adapter->tx_queue[i].name, "%s-tx-%d", 1776 adapter->netdev->name, vector); 1777 err = request_irq( 1778 intr->msix_entries[vector].vector, 1779 vmxnet3_msix_tx, 0, 1780 adapter->tx_queue[i].name, 1781 &adapter->tx_queue[i]); 1782 } else { 1783 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d", 1784 adapter->netdev->name, vector); 1785 } 1786 if (err) { 1787 dev_err(&adapter->netdev->dev, 1788 "Failed to request irq for MSIX, %s, " 1789 "error %d\n", 1790 adapter->tx_queue[i].name, err); 1791 return err; 1792 } 1793 1794 /* Handle the case where only 1 MSIx was allocated for 1795 * all tx queues */ 1796 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 1797 for (; i < adapter->num_tx_queues; i++) 1798 adapter->tx_queue[i].comp_ring.intr_idx 1799 = vector; 1800 vector++; 1801 break; 1802 } else { 1803 adapter->tx_queue[i].comp_ring.intr_idx 1804 = vector++; 1805 } 1806 } 1807 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) 1808 vector = 0; 1809 1810 for (i = 0; i < adapter->num_rx_queues; i++) { 1811 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) 1812 sprintf(adapter->rx_queue[i].name, "%s-rx-%d", 1813 adapter->netdev->name, vector); 1814 else 1815 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d", 1816 adapter->netdev->name, vector); 1817 err = request_irq(intr->msix_entries[vector].vector, 1818 vmxnet3_msix_rx, 0, 1819 adapter->rx_queue[i].name, 1820 &(adapter->rx_queue[i])); 1821 if (err) { 1822 printk(KERN_ERR "Failed to request irq for MSIX" 1823 ", %s, error %d\n", 1824 adapter->rx_queue[i].name, err); 1825 return err; 1826 } 1827 1828 adapter->rx_queue[i].comp_ring.intr_idx = vector++; 1829 } 1830 1831 sprintf(intr->event_msi_vector_name, "%s-event-%d", 1832 adapter->netdev->name, vector); 1833 err = request_irq(intr->msix_entries[vector].vector, 1834 vmxnet3_msix_event, 0, 1835 intr->event_msi_vector_name, adapter->netdev); 1836 intr->event_intr_idx = vector; 1837 1838 } else if (intr->type == VMXNET3_IT_MSI) { 1839 adapter->num_rx_queues = 1; 1840 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 1841 adapter->netdev->name, adapter->netdev); 1842 } else { 1843 #endif 1844 adapter->num_rx_queues = 1; 1845 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 1846 IRQF_SHARED, adapter->netdev->name, 1847 adapter->netdev); 1848 #ifdef CONFIG_PCI_MSI 1849 } 1850 #endif 1851 intr->num_intrs = vector + 1; 1852 if (err) { 1853 printk(KERN_ERR "Failed to request irq %s (intr type:%d), error" 1854 ":%d\n", adapter->netdev->name, intr->type, err); 1855 } else { 1856 /* Number of rx queues will not change after this */ 1857 for (i = 0; i < adapter->num_rx_queues; i++) { 1858 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 1859 rq->qid = i; 1860 rq->qid2 = i + adapter->num_rx_queues; 1861 } 1862 1863 1864 1865 /* init our intr settings */ 1866 for (i = 0; i < intr->num_intrs; i++) 1867 intr->mod_levels[i] = UPT1_IML_ADAPTIVE; 1868 if (adapter->intr.type != VMXNET3_IT_MSIX) { 1869 adapter->intr.event_intr_idx = 0; 1870 for (i = 0; i < adapter->num_tx_queues; i++) 1871 adapter->tx_queue[i].comp_ring.intr_idx = 0; 1872 adapter->rx_queue[0].comp_ring.intr_idx = 0; 1873 } 1874 1875 printk(KERN_INFO "%s: intr type %u, mode %u, %u vectors " 1876 "allocated\n", adapter->netdev->name, intr->type, 1877 intr->mask_mode, intr->num_intrs); 1878 } 1879 1880 return err; 1881 } 1882 1883 1884 static void 1885 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 1886 { 1887 struct vmxnet3_intr *intr = &adapter->intr; 1888 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0); 1889 1890 switch (intr->type) { 1891 #ifdef CONFIG_PCI_MSI 1892 case VMXNET3_IT_MSIX: 1893 { 1894 int i, vector = 0; 1895 1896 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 1897 for (i = 0; i < adapter->num_tx_queues; i++) { 1898 free_irq(intr->msix_entries[vector++].vector, 1899 &(adapter->tx_queue[i])); 1900 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) 1901 break; 1902 } 1903 } 1904 1905 for (i = 0; i < adapter->num_rx_queues; i++) { 1906 free_irq(intr->msix_entries[vector++].vector, 1907 &(adapter->rx_queue[i])); 1908 } 1909 1910 free_irq(intr->msix_entries[vector].vector, 1911 adapter->netdev); 1912 BUG_ON(vector >= intr->num_intrs); 1913 break; 1914 } 1915 #endif 1916 case VMXNET3_IT_MSI: 1917 free_irq(adapter->pdev->irq, adapter->netdev); 1918 break; 1919 case VMXNET3_IT_INTX: 1920 free_irq(adapter->pdev->irq, adapter->netdev); 1921 break; 1922 default: 1923 BUG(); 1924 } 1925 } 1926 1927 1928 static void 1929 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter) 1930 { 1931 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 1932 u16 vid; 1933 1934 /* allow untagged pkts */ 1935 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0); 1936 1937 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 1938 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 1939 } 1940 1941 1942 static int 1943 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, u16 vid) 1944 { 1945 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1946 1947 if (!(netdev->flags & IFF_PROMISC)) { 1948 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 1949 unsigned long flags; 1950 1951 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 1952 spin_lock_irqsave(&adapter->cmd_lock, flags); 1953 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 1954 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1955 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 1956 } 1957 1958 set_bit(vid, adapter->active_vlans); 1959 1960 return 0; 1961 } 1962 1963 1964 static int 1965 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, u16 vid) 1966 { 1967 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1968 1969 if (!(netdev->flags & IFF_PROMISC)) { 1970 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 1971 unsigned long flags; 1972 1973 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid); 1974 spin_lock_irqsave(&adapter->cmd_lock, flags); 1975 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 1976 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 1977 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 1978 } 1979 1980 clear_bit(vid, adapter->active_vlans); 1981 1982 return 0; 1983 } 1984 1985 1986 static u8 * 1987 vmxnet3_copy_mc(struct net_device *netdev) 1988 { 1989 u8 *buf = NULL; 1990 u32 sz = netdev_mc_count(netdev) * ETH_ALEN; 1991 1992 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */ 1993 if (sz <= 0xffff) { 1994 /* We may be called with BH disabled */ 1995 buf = kmalloc(sz, GFP_ATOMIC); 1996 if (buf) { 1997 struct netdev_hw_addr *ha; 1998 int i = 0; 1999 2000 netdev_for_each_mc_addr(ha, netdev) 2001 memcpy(buf + i++ * ETH_ALEN, ha->addr, 2002 ETH_ALEN); 2003 } 2004 } 2005 return buf; 2006 } 2007 2008 2009 static void 2010 vmxnet3_set_mc(struct net_device *netdev) 2011 { 2012 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2013 unsigned long flags; 2014 struct Vmxnet3_RxFilterConf *rxConf = 2015 &adapter->shared->devRead.rxFilterConf; 2016 u8 *new_table = NULL; 2017 u32 new_mode = VMXNET3_RXM_UCAST; 2018 2019 if (netdev->flags & IFF_PROMISC) { 2020 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2021 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable)); 2022 2023 new_mode |= VMXNET3_RXM_PROMISC; 2024 } else { 2025 vmxnet3_restore_vlan(adapter); 2026 } 2027 2028 if (netdev->flags & IFF_BROADCAST) 2029 new_mode |= VMXNET3_RXM_BCAST; 2030 2031 if (netdev->flags & IFF_ALLMULTI) 2032 new_mode |= VMXNET3_RXM_ALL_MULTI; 2033 else 2034 if (!netdev_mc_empty(netdev)) { 2035 new_table = vmxnet3_copy_mc(netdev); 2036 if (new_table) { 2037 new_mode |= VMXNET3_RXM_MCAST; 2038 rxConf->mfTableLen = cpu_to_le16( 2039 netdev_mc_count(netdev) * ETH_ALEN); 2040 rxConf->mfTablePA = cpu_to_le64(virt_to_phys( 2041 new_table)); 2042 } else { 2043 printk(KERN_INFO "%s: failed to copy mcast list" 2044 ", setting ALL_MULTI\n", netdev->name); 2045 new_mode |= VMXNET3_RXM_ALL_MULTI; 2046 } 2047 } 2048 2049 2050 if (!(new_mode & VMXNET3_RXM_MCAST)) { 2051 rxConf->mfTableLen = 0; 2052 rxConf->mfTablePA = 0; 2053 } 2054 2055 spin_lock_irqsave(&adapter->cmd_lock, flags); 2056 if (new_mode != rxConf->rxMode) { 2057 rxConf->rxMode = cpu_to_le32(new_mode); 2058 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2059 VMXNET3_CMD_UPDATE_RX_MODE); 2060 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2061 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2062 } 2063 2064 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2065 VMXNET3_CMD_UPDATE_MAC_FILTERS); 2066 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2067 2068 kfree(new_table); 2069 } 2070 2071 void 2072 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter) 2073 { 2074 int i; 2075 2076 for (i = 0; i < adapter->num_rx_queues; i++) 2077 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter); 2078 } 2079 2080 2081 /* 2082 * Set up driver_shared based on settings in adapter. 2083 */ 2084 2085 static void 2086 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter) 2087 { 2088 struct Vmxnet3_DriverShared *shared = adapter->shared; 2089 struct Vmxnet3_DSDevRead *devRead = &shared->devRead; 2090 struct Vmxnet3_TxQueueConf *tqc; 2091 struct Vmxnet3_RxQueueConf *rqc; 2092 int i; 2093 2094 memset(shared, 0, sizeof(*shared)); 2095 2096 /* driver settings */ 2097 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC); 2098 devRead->misc.driverInfo.version = cpu_to_le32( 2099 VMXNET3_DRIVER_VERSION_NUM); 2100 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ? 2101 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64); 2102 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 2103 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32( 2104 *((u32 *)&devRead->misc.driverInfo.gos)); 2105 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1); 2106 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1); 2107 2108 devRead->misc.ddPA = cpu_to_le64(virt_to_phys(adapter)); 2109 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter)); 2110 2111 /* set up feature flags */ 2112 if (adapter->netdev->features & NETIF_F_RXCSUM) 2113 devRead->misc.uptFeatures |= UPT1_F_RXCSUM; 2114 2115 if (adapter->netdev->features & NETIF_F_LRO) { 2116 devRead->misc.uptFeatures |= UPT1_F_LRO; 2117 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS); 2118 } 2119 if (adapter->netdev->features & NETIF_F_HW_VLAN_RX) 2120 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 2121 2122 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2123 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2124 devRead->misc.queueDescLen = cpu_to_le32( 2125 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 2126 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc)); 2127 2128 /* tx queue settings */ 2129 devRead->misc.numTxQueues = adapter->num_tx_queues; 2130 for (i = 0; i < adapter->num_tx_queues; i++) { 2131 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 2132 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL); 2133 tqc = &adapter->tqd_start[i].conf; 2134 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA); 2135 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA); 2136 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA); 2137 tqc->ddPA = cpu_to_le64(virt_to_phys(tq->buf_info)); 2138 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size); 2139 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size); 2140 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size); 2141 tqc->ddLen = cpu_to_le32( 2142 sizeof(struct vmxnet3_tx_buf_info) * 2143 tqc->txRingSize); 2144 tqc->intrIdx = tq->comp_ring.intr_idx; 2145 } 2146 2147 /* rx queue settings */ 2148 devRead->misc.numRxQueues = adapter->num_rx_queues; 2149 for (i = 0; i < adapter->num_rx_queues; i++) { 2150 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2151 rqc = &adapter->rqd_start[i].conf; 2152 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA); 2153 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA); 2154 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA); 2155 rqc->ddPA = cpu_to_le64(virt_to_phys( 2156 rq->buf_info)); 2157 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size); 2158 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size); 2159 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size); 2160 rqc->ddLen = cpu_to_le32( 2161 sizeof(struct vmxnet3_rx_buf_info) * 2162 (rqc->rxRingSize[0] + 2163 rqc->rxRingSize[1])); 2164 rqc->intrIdx = rq->comp_ring.intr_idx; 2165 } 2166 2167 #ifdef VMXNET3_RSS 2168 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf)); 2169 2170 if (adapter->rss) { 2171 struct UPT1_RSSConf *rssConf = adapter->rss_conf; 2172 devRead->misc.uptFeatures |= UPT1_F_RSS; 2173 devRead->misc.numRxQueues = adapter->num_rx_queues; 2174 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 | 2175 UPT1_RSS_HASH_TYPE_IPV4 | 2176 UPT1_RSS_HASH_TYPE_TCP_IPV6 | 2177 UPT1_RSS_HASH_TYPE_IPV6; 2178 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ; 2179 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE; 2180 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE; 2181 get_random_bytes(&rssConf->hashKey[0], rssConf->hashKeySize); 2182 for (i = 0; i < rssConf->indTableSize; i++) 2183 rssConf->indTable[i] = ethtool_rxfh_indir_default( 2184 i, adapter->num_rx_queues); 2185 2186 devRead->rssConfDesc.confVer = 1; 2187 devRead->rssConfDesc.confLen = sizeof(*rssConf); 2188 devRead->rssConfDesc.confPA = virt_to_phys(rssConf); 2189 } 2190 2191 #endif /* VMXNET3_RSS */ 2192 2193 /* intr settings */ 2194 devRead->intrConf.autoMask = adapter->intr.mask_mode == 2195 VMXNET3_IMM_AUTO; 2196 devRead->intrConf.numIntrs = adapter->intr.num_intrs; 2197 for (i = 0; i < adapter->intr.num_intrs; i++) 2198 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i]; 2199 2200 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx; 2201 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2202 2203 /* rx filter settings */ 2204 devRead->rxFilterConf.rxMode = 0; 2205 vmxnet3_restore_vlan(adapter); 2206 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr); 2207 2208 /* the rest are already zeroed */ 2209 } 2210 2211 2212 int 2213 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 2214 { 2215 int err, i; 2216 u32 ret; 2217 unsigned long flags; 2218 2219 dev_dbg(&adapter->netdev->dev, "%s: skb_buf_size %d, rx_buf_per_pkt %d," 2220 " ring sizes %u %u %u\n", adapter->netdev->name, 2221 adapter->skb_buf_size, adapter->rx_buf_per_pkt, 2222 adapter->tx_queue[0].tx_ring.size, 2223 adapter->rx_queue[0].rx_ring[0].size, 2224 adapter->rx_queue[0].rx_ring[1].size); 2225 2226 vmxnet3_tq_init_all(adapter); 2227 err = vmxnet3_rq_init_all(adapter); 2228 if (err) { 2229 printk(KERN_ERR "Failed to init rx queue for %s: error %d\n", 2230 adapter->netdev->name, err); 2231 goto rq_err; 2232 } 2233 2234 err = vmxnet3_request_irqs(adapter); 2235 if (err) { 2236 printk(KERN_ERR "Failed to setup irq for %s: error %d\n", 2237 adapter->netdev->name, err); 2238 goto irq_err; 2239 } 2240 2241 vmxnet3_setup_driver_shared(adapter); 2242 2243 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO( 2244 adapter->shared_pa)); 2245 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI( 2246 adapter->shared_pa)); 2247 spin_lock_irqsave(&adapter->cmd_lock, flags); 2248 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2249 VMXNET3_CMD_ACTIVATE_DEV); 2250 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2251 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2252 2253 if (ret != 0) { 2254 printk(KERN_ERR "Failed to activate dev %s: error %u\n", 2255 adapter->netdev->name, ret); 2256 err = -EINVAL; 2257 goto activate_err; 2258 } 2259 2260 for (i = 0; i < adapter->num_rx_queues; i++) { 2261 VMXNET3_WRITE_BAR0_REG(adapter, 2262 VMXNET3_REG_RXPROD + i * VMXNET3_REG_ALIGN, 2263 adapter->rx_queue[i].rx_ring[0].next2fill); 2264 VMXNET3_WRITE_BAR0_REG(adapter, (VMXNET3_REG_RXPROD2 + 2265 (i * VMXNET3_REG_ALIGN)), 2266 adapter->rx_queue[i].rx_ring[1].next2fill); 2267 } 2268 2269 /* Apply the rx filter settins last. */ 2270 vmxnet3_set_mc(adapter->netdev); 2271 2272 /* 2273 * Check link state when first activating device. It will start the 2274 * tx queue if the link is up. 2275 */ 2276 vmxnet3_check_link(adapter, true); 2277 for (i = 0; i < adapter->num_rx_queues; i++) 2278 napi_enable(&adapter->rx_queue[i].napi); 2279 vmxnet3_enable_all_intrs(adapter); 2280 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 2281 return 0; 2282 2283 activate_err: 2284 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0); 2285 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0); 2286 vmxnet3_free_irqs(adapter); 2287 irq_err: 2288 rq_err: 2289 /* free up buffers we allocated */ 2290 vmxnet3_rq_cleanup_all(adapter); 2291 return err; 2292 } 2293 2294 2295 void 2296 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter) 2297 { 2298 unsigned long flags; 2299 spin_lock_irqsave(&adapter->cmd_lock, flags); 2300 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 2301 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2302 } 2303 2304 2305 int 2306 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 2307 { 2308 int i; 2309 unsigned long flags; 2310 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 2311 return 0; 2312 2313 2314 spin_lock_irqsave(&adapter->cmd_lock, flags); 2315 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2316 VMXNET3_CMD_QUIESCE_DEV); 2317 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2318 vmxnet3_disable_all_intrs(adapter); 2319 2320 for (i = 0; i < adapter->num_rx_queues; i++) 2321 napi_disable(&adapter->rx_queue[i].napi); 2322 netif_tx_disable(adapter->netdev); 2323 adapter->link_speed = 0; 2324 netif_carrier_off(adapter->netdev); 2325 2326 vmxnet3_tq_cleanup_all(adapter); 2327 vmxnet3_rq_cleanup_all(adapter); 2328 vmxnet3_free_irqs(adapter); 2329 return 0; 2330 } 2331 2332 2333 static void 2334 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) 2335 { 2336 u32 tmp; 2337 2338 tmp = *(u32 *)mac; 2339 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp); 2340 2341 tmp = (mac[5] << 8) | mac[4]; 2342 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp); 2343 } 2344 2345 2346 static int 2347 vmxnet3_set_mac_addr(struct net_device *netdev, void *p) 2348 { 2349 struct sockaddr *addr = p; 2350 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2351 2352 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 2353 vmxnet3_write_mac_addr(adapter, addr->sa_data); 2354 2355 return 0; 2356 } 2357 2358 2359 /* ==================== initialization and cleanup routines ============ */ 2360 2361 static int 2362 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter, bool *dma64) 2363 { 2364 int err; 2365 unsigned long mmio_start, mmio_len; 2366 struct pci_dev *pdev = adapter->pdev; 2367 2368 err = pci_enable_device(pdev); 2369 if (err) { 2370 printk(KERN_ERR "Failed to enable adapter %s: error %d\n", 2371 pci_name(pdev), err); 2372 return err; 2373 } 2374 2375 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64)) == 0) { 2376 if (pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)) != 0) { 2377 printk(KERN_ERR "pci_set_consistent_dma_mask failed " 2378 "for adapter %s\n", pci_name(pdev)); 2379 err = -EIO; 2380 goto err_set_mask; 2381 } 2382 *dma64 = true; 2383 } else { 2384 if (pci_set_dma_mask(pdev, DMA_BIT_MASK(32)) != 0) { 2385 printk(KERN_ERR "pci_set_dma_mask failed for adapter " 2386 "%s\n", pci_name(pdev)); 2387 err = -EIO; 2388 goto err_set_mask; 2389 } 2390 *dma64 = false; 2391 } 2392 2393 err = pci_request_selected_regions(pdev, (1 << 2) - 1, 2394 vmxnet3_driver_name); 2395 if (err) { 2396 printk(KERN_ERR "Failed to request region for adapter %s: " 2397 "error %d\n", pci_name(pdev), err); 2398 goto err_set_mask; 2399 } 2400 2401 pci_set_master(pdev); 2402 2403 mmio_start = pci_resource_start(pdev, 0); 2404 mmio_len = pci_resource_len(pdev, 0); 2405 adapter->hw_addr0 = ioremap(mmio_start, mmio_len); 2406 if (!adapter->hw_addr0) { 2407 printk(KERN_ERR "Failed to map bar0 for adapter %s\n", 2408 pci_name(pdev)); 2409 err = -EIO; 2410 goto err_ioremap; 2411 } 2412 2413 mmio_start = pci_resource_start(pdev, 1); 2414 mmio_len = pci_resource_len(pdev, 1); 2415 adapter->hw_addr1 = ioremap(mmio_start, mmio_len); 2416 if (!adapter->hw_addr1) { 2417 printk(KERN_ERR "Failed to map bar1 for adapter %s\n", 2418 pci_name(pdev)); 2419 err = -EIO; 2420 goto err_bar1; 2421 } 2422 return 0; 2423 2424 err_bar1: 2425 iounmap(adapter->hw_addr0); 2426 err_ioremap: 2427 pci_release_selected_regions(pdev, (1 << 2) - 1); 2428 err_set_mask: 2429 pci_disable_device(pdev); 2430 return err; 2431 } 2432 2433 2434 static void 2435 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter) 2436 { 2437 BUG_ON(!adapter->pdev); 2438 2439 iounmap(adapter->hw_addr0); 2440 iounmap(adapter->hw_addr1); 2441 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1); 2442 pci_disable_device(adapter->pdev); 2443 } 2444 2445 2446 static void 2447 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 2448 { 2449 size_t sz, i, ring0_size, ring1_size, comp_size; 2450 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[0]; 2451 2452 2453 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 2454 VMXNET3_MAX_ETH_HDR_SIZE) { 2455 adapter->skb_buf_size = adapter->netdev->mtu + 2456 VMXNET3_MAX_ETH_HDR_SIZE; 2457 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE) 2458 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE; 2459 2460 adapter->rx_buf_per_pkt = 1; 2461 } else { 2462 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE; 2463 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE + 2464 VMXNET3_MAX_ETH_HDR_SIZE; 2465 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE; 2466 } 2467 2468 /* 2469 * for simplicity, force the ring0 size to be a multiple of 2470 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 2471 */ 2472 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 2473 ring0_size = adapter->rx_queue[0].rx_ring[0].size; 2474 ring0_size = (ring0_size + sz - 1) / sz * sz; 2475 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE / 2476 sz * sz); 2477 ring1_size = adapter->rx_queue[0].rx_ring[1].size; 2478 comp_size = ring0_size + ring1_size; 2479 2480 for (i = 0; i < adapter->num_rx_queues; i++) { 2481 rq = &adapter->rx_queue[i]; 2482 rq->rx_ring[0].size = ring0_size; 2483 rq->rx_ring[1].size = ring1_size; 2484 rq->comp_ring.size = comp_size; 2485 } 2486 } 2487 2488 2489 int 2490 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 2491 u32 rx_ring_size, u32 rx_ring2_size) 2492 { 2493 int err = 0, i; 2494 2495 for (i = 0; i < adapter->num_tx_queues; i++) { 2496 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 2497 tq->tx_ring.size = tx_ring_size; 2498 tq->data_ring.size = tx_ring_size; 2499 tq->comp_ring.size = tx_ring_size; 2500 tq->shared = &adapter->tqd_start[i].ctrl; 2501 tq->stopped = true; 2502 tq->adapter = adapter; 2503 tq->qid = i; 2504 err = vmxnet3_tq_create(tq, adapter); 2505 /* 2506 * Too late to change num_tx_queues. We cannot do away with 2507 * lesser number of queues than what we asked for 2508 */ 2509 if (err) 2510 goto queue_err; 2511 } 2512 2513 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size; 2514 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size; 2515 vmxnet3_adjust_rx_ring_size(adapter); 2516 for (i = 0; i < adapter->num_rx_queues; i++) { 2517 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2518 /* qid and qid2 for rx queues will be assigned later when num 2519 * of rx queues is finalized after allocating intrs */ 2520 rq->shared = &adapter->rqd_start[i].ctrl; 2521 rq->adapter = adapter; 2522 err = vmxnet3_rq_create(rq, adapter); 2523 if (err) { 2524 if (i == 0) { 2525 printk(KERN_ERR "Could not allocate any rx" 2526 "queues. Aborting.\n"); 2527 goto queue_err; 2528 } else { 2529 printk(KERN_INFO "Number of rx queues changed " 2530 "to : %d.\n", i); 2531 adapter->num_rx_queues = i; 2532 err = 0; 2533 break; 2534 } 2535 } 2536 } 2537 return err; 2538 queue_err: 2539 vmxnet3_tq_destroy_all(adapter); 2540 return err; 2541 } 2542 2543 static int 2544 vmxnet3_open(struct net_device *netdev) 2545 { 2546 struct vmxnet3_adapter *adapter; 2547 int err, i; 2548 2549 adapter = netdev_priv(netdev); 2550 2551 for (i = 0; i < adapter->num_tx_queues; i++) 2552 spin_lock_init(&adapter->tx_queue[i].tx_lock); 2553 2554 err = vmxnet3_create_queues(adapter, VMXNET3_DEF_TX_RING_SIZE, 2555 VMXNET3_DEF_RX_RING_SIZE, 2556 VMXNET3_DEF_RX_RING_SIZE); 2557 if (err) 2558 goto queue_err; 2559 2560 err = vmxnet3_activate_dev(adapter); 2561 if (err) 2562 goto activate_err; 2563 2564 return 0; 2565 2566 activate_err: 2567 vmxnet3_rq_destroy_all(adapter); 2568 vmxnet3_tq_destroy_all(adapter); 2569 queue_err: 2570 return err; 2571 } 2572 2573 2574 static int 2575 vmxnet3_close(struct net_device *netdev) 2576 { 2577 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2578 2579 /* 2580 * Reset_work may be in the middle of resetting the device, wait for its 2581 * completion. 2582 */ 2583 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 2584 msleep(1); 2585 2586 vmxnet3_quiesce_dev(adapter); 2587 2588 vmxnet3_rq_destroy_all(adapter); 2589 vmxnet3_tq_destroy_all(adapter); 2590 2591 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 2592 2593 2594 return 0; 2595 } 2596 2597 2598 void 2599 vmxnet3_force_close(struct vmxnet3_adapter *adapter) 2600 { 2601 int i; 2602 2603 /* 2604 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 2605 * vmxnet3_close() will deadlock. 2606 */ 2607 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 2608 2609 /* we need to enable NAPI, otherwise dev_close will deadlock */ 2610 for (i = 0; i < adapter->num_rx_queues; i++) 2611 napi_enable(&adapter->rx_queue[i].napi); 2612 dev_close(adapter->netdev); 2613 } 2614 2615 2616 static int 2617 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) 2618 { 2619 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2620 int err = 0; 2621 2622 if (new_mtu < VMXNET3_MIN_MTU || new_mtu > VMXNET3_MAX_MTU) 2623 return -EINVAL; 2624 2625 netdev->mtu = new_mtu; 2626 2627 /* 2628 * Reset_work may be in the middle of resetting the device, wait for its 2629 * completion. 2630 */ 2631 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 2632 msleep(1); 2633 2634 if (netif_running(netdev)) { 2635 vmxnet3_quiesce_dev(adapter); 2636 vmxnet3_reset_dev(adapter); 2637 2638 /* we need to re-create the rx queue based on the new mtu */ 2639 vmxnet3_rq_destroy_all(adapter); 2640 vmxnet3_adjust_rx_ring_size(adapter); 2641 err = vmxnet3_rq_create_all(adapter); 2642 if (err) { 2643 printk(KERN_ERR "%s: failed to re-create rx queues," 2644 " error %d. Closing it.\n", netdev->name, err); 2645 goto out; 2646 } 2647 2648 err = vmxnet3_activate_dev(adapter); 2649 if (err) { 2650 printk(KERN_ERR "%s: failed to re-activate, error %d. " 2651 "Closing it\n", netdev->name, err); 2652 goto out; 2653 } 2654 } 2655 2656 out: 2657 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 2658 if (err) 2659 vmxnet3_force_close(adapter); 2660 2661 return err; 2662 } 2663 2664 2665 static void 2666 vmxnet3_declare_features(struct vmxnet3_adapter *adapter, bool dma64) 2667 { 2668 struct net_device *netdev = adapter->netdev; 2669 2670 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 2671 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_TX | 2672 NETIF_F_HW_VLAN_RX | NETIF_F_TSO | NETIF_F_TSO6 | 2673 NETIF_F_LRO; 2674 if (dma64) 2675 netdev->hw_features |= NETIF_F_HIGHDMA; 2676 netdev->vlan_features = netdev->hw_features & 2677 ~(NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX); 2678 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_FILTER; 2679 2680 netdev_info(adapter->netdev, 2681 "features: sg csum vlan jf tso tsoIPv6 lro%s\n", 2682 dma64 ? " highDMA" : ""); 2683 } 2684 2685 2686 static void 2687 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) 2688 { 2689 u32 tmp; 2690 2691 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL); 2692 *(u32 *)mac = tmp; 2693 2694 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH); 2695 mac[4] = tmp & 0xff; 2696 mac[5] = (tmp >> 8) & 0xff; 2697 } 2698 2699 #ifdef CONFIG_PCI_MSI 2700 2701 /* 2702 * Enable MSIx vectors. 2703 * Returns : 2704 * 0 on successful enabling of required vectors, 2705 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required 2706 * could be enabled. 2707 * number of vectors which can be enabled otherwise (this number is smaller 2708 * than VMXNET3_LINUX_MIN_MSIX_VECT) 2709 */ 2710 2711 static int 2712 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, 2713 int vectors) 2714 { 2715 int err = 0, vector_threshold; 2716 vector_threshold = VMXNET3_LINUX_MIN_MSIX_VECT; 2717 2718 while (vectors >= vector_threshold) { 2719 err = pci_enable_msix(adapter->pdev, adapter->intr.msix_entries, 2720 vectors); 2721 if (!err) { 2722 adapter->intr.num_intrs = vectors; 2723 return 0; 2724 } else if (err < 0) { 2725 netdev_err(adapter->netdev, 2726 "Failed to enable MSI-X, error: %d\n", err); 2727 vectors = 0; 2728 } else if (err < vector_threshold) { 2729 break; 2730 } else { 2731 /* If fails to enable required number of MSI-x vectors 2732 * try enabling minimum number of vectors required. 2733 */ 2734 netdev_err(adapter->netdev, 2735 "Failed to enable %d MSI-X, trying %d instead\n", 2736 vectors, vector_threshold); 2737 vectors = vector_threshold; 2738 } 2739 } 2740 2741 netdev_info(adapter->netdev, 2742 "Number of MSI-X interrupts which can be allocated are lower than min threshold required.\n"); 2743 return err; 2744 } 2745 2746 2747 #endif /* CONFIG_PCI_MSI */ 2748 2749 static void 2750 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 2751 { 2752 u32 cfg; 2753 unsigned long flags; 2754 2755 /* intr settings */ 2756 spin_lock_irqsave(&adapter->cmd_lock, flags); 2757 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2758 VMXNET3_CMD_GET_CONF_INTR); 2759 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2760 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2761 adapter->intr.type = cfg & 0x3; 2762 adapter->intr.mask_mode = (cfg >> 2) & 0x3; 2763 2764 if (adapter->intr.type == VMXNET3_IT_AUTO) { 2765 adapter->intr.type = VMXNET3_IT_MSIX; 2766 } 2767 2768 #ifdef CONFIG_PCI_MSI 2769 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2770 int vector, err = 0; 2771 2772 adapter->intr.num_intrs = (adapter->share_intr == 2773 VMXNET3_INTR_TXSHARE) ? 1 : 2774 adapter->num_tx_queues; 2775 adapter->intr.num_intrs += (adapter->share_intr == 2776 VMXNET3_INTR_BUDDYSHARE) ? 0 : 2777 adapter->num_rx_queues; 2778 adapter->intr.num_intrs += 1; /* for link event */ 2779 2780 adapter->intr.num_intrs = (adapter->intr.num_intrs > 2781 VMXNET3_LINUX_MIN_MSIX_VECT 2782 ? adapter->intr.num_intrs : 2783 VMXNET3_LINUX_MIN_MSIX_VECT); 2784 2785 for (vector = 0; vector < adapter->intr.num_intrs; vector++) 2786 adapter->intr.msix_entries[vector].entry = vector; 2787 2788 err = vmxnet3_acquire_msix_vectors(adapter, 2789 adapter->intr.num_intrs); 2790 /* If we cannot allocate one MSIx vector per queue 2791 * then limit the number of rx queues to 1 2792 */ 2793 if (err == VMXNET3_LINUX_MIN_MSIX_VECT) { 2794 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE 2795 || adapter->num_rx_queues != 1) { 2796 adapter->share_intr = VMXNET3_INTR_TXSHARE; 2797 printk(KERN_ERR "Number of rx queues : 1\n"); 2798 adapter->num_rx_queues = 1; 2799 adapter->intr.num_intrs = 2800 VMXNET3_LINUX_MIN_MSIX_VECT; 2801 } 2802 return; 2803 } 2804 if (!err) 2805 return; 2806 2807 /* If we cannot allocate MSIx vectors use only one rx queue */ 2808 netdev_info(adapter->netdev, 2809 "Failed to enable MSI-X, error %d . Limiting #rx queues to 1, try MSI.\n", 2810 err); 2811 2812 adapter->intr.type = VMXNET3_IT_MSI; 2813 } 2814 2815 if (adapter->intr.type == VMXNET3_IT_MSI) { 2816 int err; 2817 err = pci_enable_msi(adapter->pdev); 2818 if (!err) { 2819 adapter->num_rx_queues = 1; 2820 adapter->intr.num_intrs = 1; 2821 return; 2822 } 2823 } 2824 #endif /* CONFIG_PCI_MSI */ 2825 2826 adapter->num_rx_queues = 1; 2827 printk(KERN_INFO "Using INTx interrupt, #Rx queues: 1.\n"); 2828 adapter->intr.type = VMXNET3_IT_INTX; 2829 2830 /* INT-X related setting */ 2831 adapter->intr.num_intrs = 1; 2832 } 2833 2834 2835 static void 2836 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter) 2837 { 2838 if (adapter->intr.type == VMXNET3_IT_MSIX) 2839 pci_disable_msix(adapter->pdev); 2840 else if (adapter->intr.type == VMXNET3_IT_MSI) 2841 pci_disable_msi(adapter->pdev); 2842 else 2843 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX); 2844 } 2845 2846 2847 static void 2848 vmxnet3_tx_timeout(struct net_device *netdev) 2849 { 2850 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2851 adapter->tx_timeout_count++; 2852 2853 printk(KERN_ERR "%s: tx hang\n", adapter->netdev->name); 2854 schedule_work(&adapter->work); 2855 netif_wake_queue(adapter->netdev); 2856 } 2857 2858 2859 static void 2860 vmxnet3_reset_work(struct work_struct *data) 2861 { 2862 struct vmxnet3_adapter *adapter; 2863 2864 adapter = container_of(data, struct vmxnet3_adapter, work); 2865 2866 /* if another thread is resetting the device, no need to proceed */ 2867 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 2868 return; 2869 2870 /* if the device is closed, we must leave it alone */ 2871 rtnl_lock(); 2872 if (netif_running(adapter->netdev)) { 2873 printk(KERN_INFO "%s: resetting\n", adapter->netdev->name); 2874 vmxnet3_quiesce_dev(adapter); 2875 vmxnet3_reset_dev(adapter); 2876 vmxnet3_activate_dev(adapter); 2877 } else { 2878 printk(KERN_INFO "%s: already closed\n", adapter->netdev->name); 2879 } 2880 rtnl_unlock(); 2881 2882 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 2883 } 2884 2885 2886 static int 2887 vmxnet3_probe_device(struct pci_dev *pdev, 2888 const struct pci_device_id *id) 2889 { 2890 static const struct net_device_ops vmxnet3_netdev_ops = { 2891 .ndo_open = vmxnet3_open, 2892 .ndo_stop = vmxnet3_close, 2893 .ndo_start_xmit = vmxnet3_xmit_frame, 2894 .ndo_set_mac_address = vmxnet3_set_mac_addr, 2895 .ndo_change_mtu = vmxnet3_change_mtu, 2896 .ndo_set_features = vmxnet3_set_features, 2897 .ndo_get_stats64 = vmxnet3_get_stats64, 2898 .ndo_tx_timeout = vmxnet3_tx_timeout, 2899 .ndo_set_rx_mode = vmxnet3_set_mc, 2900 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid, 2901 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid, 2902 #ifdef CONFIG_NET_POLL_CONTROLLER 2903 .ndo_poll_controller = vmxnet3_netpoll, 2904 #endif 2905 }; 2906 int err; 2907 bool dma64 = false; /* stupid gcc */ 2908 u32 ver; 2909 struct net_device *netdev; 2910 struct vmxnet3_adapter *adapter; 2911 u8 mac[ETH_ALEN]; 2912 int size; 2913 int num_tx_queues; 2914 int num_rx_queues; 2915 2916 if (!pci_msi_enabled()) 2917 enable_mq = 0; 2918 2919 #ifdef VMXNET3_RSS 2920 if (enable_mq) 2921 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 2922 (int)num_online_cpus()); 2923 else 2924 #endif 2925 num_rx_queues = 1; 2926 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 2927 2928 if (enable_mq) 2929 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES, 2930 (int)num_online_cpus()); 2931 else 2932 num_tx_queues = 1; 2933 2934 num_tx_queues = rounddown_pow_of_two(num_tx_queues); 2935 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter), 2936 max(num_tx_queues, num_rx_queues)); 2937 printk(KERN_INFO "# of Tx queues : %d, # of Rx queues : %d\n", 2938 num_tx_queues, num_rx_queues); 2939 2940 if (!netdev) 2941 return -ENOMEM; 2942 2943 pci_set_drvdata(pdev, netdev); 2944 adapter = netdev_priv(netdev); 2945 adapter->netdev = netdev; 2946 adapter->pdev = pdev; 2947 2948 spin_lock_init(&adapter->cmd_lock); 2949 adapter->shared = pci_alloc_consistent(adapter->pdev, 2950 sizeof(struct Vmxnet3_DriverShared), 2951 &adapter->shared_pa); 2952 if (!adapter->shared) { 2953 printk(KERN_ERR "Failed to allocate memory for %s\n", 2954 pci_name(pdev)); 2955 err = -ENOMEM; 2956 goto err_alloc_shared; 2957 } 2958 2959 adapter->num_rx_queues = num_rx_queues; 2960 adapter->num_tx_queues = num_tx_queues; 2961 2962 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 2963 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues; 2964 adapter->tqd_start = pci_alloc_consistent(adapter->pdev, size, 2965 &adapter->queue_desc_pa); 2966 2967 if (!adapter->tqd_start) { 2968 printk(KERN_ERR "Failed to allocate memory for %s\n", 2969 pci_name(pdev)); 2970 err = -ENOMEM; 2971 goto err_alloc_queue_desc; 2972 } 2973 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start + 2974 adapter->num_tx_queues); 2975 2976 adapter->pm_conf = kmalloc(sizeof(struct Vmxnet3_PMConf), GFP_KERNEL); 2977 if (adapter->pm_conf == NULL) { 2978 err = -ENOMEM; 2979 goto err_alloc_pm; 2980 } 2981 2982 #ifdef VMXNET3_RSS 2983 2984 adapter->rss_conf = kmalloc(sizeof(struct UPT1_RSSConf), GFP_KERNEL); 2985 if (adapter->rss_conf == NULL) { 2986 err = -ENOMEM; 2987 goto err_alloc_rss; 2988 } 2989 #endif /* VMXNET3_RSS */ 2990 2991 err = vmxnet3_alloc_pci_resources(adapter, &dma64); 2992 if (err < 0) 2993 goto err_alloc_pci; 2994 2995 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS); 2996 if (ver & 1) { 2997 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1); 2998 } else { 2999 printk(KERN_ERR "Incompatible h/w version (0x%x) for adapter" 3000 " %s\n", ver, pci_name(pdev)); 3001 err = -EBUSY; 3002 goto err_ver; 3003 } 3004 3005 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS); 3006 if (ver & 1) { 3007 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1); 3008 } else { 3009 printk(KERN_ERR "Incompatible upt version (0x%x) for " 3010 "adapter %s\n", ver, pci_name(pdev)); 3011 err = -EBUSY; 3012 goto err_ver; 3013 } 3014 3015 SET_NETDEV_DEV(netdev, &pdev->dev); 3016 vmxnet3_declare_features(adapter, dma64); 3017 3018 adapter->dev_number = atomic_read(&devices_found); 3019 3020 adapter->share_intr = irq_share_mode; 3021 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE && 3022 adapter->num_tx_queues != adapter->num_rx_queues) 3023 adapter->share_intr = VMXNET3_INTR_DONTSHARE; 3024 3025 vmxnet3_alloc_intr_resources(adapter); 3026 3027 #ifdef VMXNET3_RSS 3028 if (adapter->num_rx_queues > 1 && 3029 adapter->intr.type == VMXNET3_IT_MSIX) { 3030 adapter->rss = true; 3031 printk(KERN_INFO "RSS is enabled.\n"); 3032 } else { 3033 adapter->rss = false; 3034 } 3035 #endif 3036 3037 vmxnet3_read_mac_addr(adapter, mac); 3038 memcpy(netdev->dev_addr, mac, netdev->addr_len); 3039 3040 netdev->netdev_ops = &vmxnet3_netdev_ops; 3041 vmxnet3_set_ethtool_ops(netdev); 3042 netdev->watchdog_timeo = 5 * HZ; 3043 3044 INIT_WORK(&adapter->work, vmxnet3_reset_work); 3045 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3046 3047 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3048 int i; 3049 for (i = 0; i < adapter->num_rx_queues; i++) { 3050 netif_napi_add(adapter->netdev, 3051 &adapter->rx_queue[i].napi, 3052 vmxnet3_poll_rx_only, 64); 3053 } 3054 } else { 3055 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi, 3056 vmxnet3_poll, 64); 3057 } 3058 3059 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); 3060 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues); 3061 3062 netif_carrier_off(netdev); 3063 err = register_netdev(netdev); 3064 3065 if (err) { 3066 printk(KERN_ERR "Failed to register adapter %s\n", 3067 pci_name(pdev)); 3068 goto err_register; 3069 } 3070 3071 vmxnet3_check_link(adapter, false); 3072 atomic_inc(&devices_found); 3073 return 0; 3074 3075 err_register: 3076 vmxnet3_free_intr_resources(adapter); 3077 err_ver: 3078 vmxnet3_free_pci_resources(adapter); 3079 err_alloc_pci: 3080 #ifdef VMXNET3_RSS 3081 kfree(adapter->rss_conf); 3082 err_alloc_rss: 3083 #endif 3084 kfree(adapter->pm_conf); 3085 err_alloc_pm: 3086 pci_free_consistent(adapter->pdev, size, adapter->tqd_start, 3087 adapter->queue_desc_pa); 3088 err_alloc_queue_desc: 3089 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared), 3090 adapter->shared, adapter->shared_pa); 3091 err_alloc_shared: 3092 pci_set_drvdata(pdev, NULL); 3093 free_netdev(netdev); 3094 return err; 3095 } 3096 3097 3098 static void 3099 vmxnet3_remove_device(struct pci_dev *pdev) 3100 { 3101 struct net_device *netdev = pci_get_drvdata(pdev); 3102 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3103 int size = 0; 3104 int num_rx_queues; 3105 3106 #ifdef VMXNET3_RSS 3107 if (enable_mq) 3108 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3109 (int)num_online_cpus()); 3110 else 3111 #endif 3112 num_rx_queues = 1; 3113 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 3114 3115 cancel_work_sync(&adapter->work); 3116 3117 unregister_netdev(netdev); 3118 3119 vmxnet3_free_intr_resources(adapter); 3120 vmxnet3_free_pci_resources(adapter); 3121 #ifdef VMXNET3_RSS 3122 kfree(adapter->rss_conf); 3123 #endif 3124 kfree(adapter->pm_conf); 3125 3126 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 3127 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues; 3128 pci_free_consistent(adapter->pdev, size, adapter->tqd_start, 3129 adapter->queue_desc_pa); 3130 pci_free_consistent(adapter->pdev, sizeof(struct Vmxnet3_DriverShared), 3131 adapter->shared, adapter->shared_pa); 3132 free_netdev(netdev); 3133 } 3134 3135 3136 #ifdef CONFIG_PM 3137 3138 static int 3139 vmxnet3_suspend(struct device *device) 3140 { 3141 struct pci_dev *pdev = to_pci_dev(device); 3142 struct net_device *netdev = pci_get_drvdata(pdev); 3143 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3144 struct Vmxnet3_PMConf *pmConf; 3145 struct ethhdr *ehdr; 3146 struct arphdr *ahdr; 3147 u8 *arpreq; 3148 struct in_device *in_dev; 3149 struct in_ifaddr *ifa; 3150 unsigned long flags; 3151 int i = 0; 3152 3153 if (!netif_running(netdev)) 3154 return 0; 3155 3156 for (i = 0; i < adapter->num_rx_queues; i++) 3157 napi_disable(&adapter->rx_queue[i].napi); 3158 3159 vmxnet3_disable_all_intrs(adapter); 3160 vmxnet3_free_irqs(adapter); 3161 vmxnet3_free_intr_resources(adapter); 3162 3163 netif_device_detach(netdev); 3164 netif_tx_stop_all_queues(netdev); 3165 3166 /* Create wake-up filters. */ 3167 pmConf = adapter->pm_conf; 3168 memset(pmConf, 0, sizeof(*pmConf)); 3169 3170 if (adapter->wol & WAKE_UCAST) { 3171 pmConf->filters[i].patternSize = ETH_ALEN; 3172 pmConf->filters[i].maskSize = 1; 3173 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN); 3174 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */ 3175 3176 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 3177 i++; 3178 } 3179 3180 if (adapter->wol & WAKE_ARP) { 3181 in_dev = in_dev_get(netdev); 3182 if (!in_dev) 3183 goto skip_arp; 3184 3185 ifa = (struct in_ifaddr *)in_dev->ifa_list; 3186 if (!ifa) 3187 goto skip_arp; 3188 3189 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/ 3190 sizeof(struct arphdr) + /* ARP header */ 3191 2 * ETH_ALEN + /* 2 Ethernet addresses*/ 3192 2 * sizeof(u32); /*2 IPv4 addresses */ 3193 pmConf->filters[i].maskSize = 3194 (pmConf->filters[i].patternSize - 1) / 8 + 1; 3195 3196 /* ETH_P_ARP in Ethernet header. */ 3197 ehdr = (struct ethhdr *)pmConf->filters[i].pattern; 3198 ehdr->h_proto = htons(ETH_P_ARP); 3199 3200 /* ARPOP_REQUEST in ARP header. */ 3201 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN]; 3202 ahdr->ar_op = htons(ARPOP_REQUEST); 3203 arpreq = (u8 *)(ahdr + 1); 3204 3205 /* The Unicast IPv4 address in 'tip' field. */ 3206 arpreq += 2 * ETH_ALEN + sizeof(u32); 3207 *(u32 *)arpreq = ifa->ifa_address; 3208 3209 /* The mask for the relevant bits. */ 3210 pmConf->filters[i].mask[0] = 0x00; 3211 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */ 3212 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */ 3213 pmConf->filters[i].mask[3] = 0x00; 3214 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */ 3215 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */ 3216 in_dev_put(in_dev); 3217 3218 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 3219 i++; 3220 } 3221 3222 skip_arp: 3223 if (adapter->wol & WAKE_MAGIC) 3224 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC; 3225 3226 pmConf->numFilters = i; 3227 3228 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); 3229 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( 3230 *pmConf)); 3231 adapter->shared->devRead.pmConfDesc.confPA = cpu_to_le64(virt_to_phys( 3232 pmConf)); 3233 3234 spin_lock_irqsave(&adapter->cmd_lock, flags); 3235 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3236 VMXNET3_CMD_UPDATE_PMCFG); 3237 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3238 3239 pci_save_state(pdev); 3240 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND), 3241 adapter->wol); 3242 pci_disable_device(pdev); 3243 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND)); 3244 3245 return 0; 3246 } 3247 3248 3249 static int 3250 vmxnet3_resume(struct device *device) 3251 { 3252 int err, i = 0; 3253 unsigned long flags; 3254 struct pci_dev *pdev = to_pci_dev(device); 3255 struct net_device *netdev = pci_get_drvdata(pdev); 3256 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3257 struct Vmxnet3_PMConf *pmConf; 3258 3259 if (!netif_running(netdev)) 3260 return 0; 3261 3262 /* Destroy wake-up filters. */ 3263 pmConf = adapter->pm_conf; 3264 memset(pmConf, 0, sizeof(*pmConf)); 3265 3266 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); 3267 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( 3268 *pmConf)); 3269 adapter->shared->devRead.pmConfDesc.confPA = cpu_to_le64(virt_to_phys( 3270 pmConf)); 3271 3272 netif_device_attach(netdev); 3273 pci_set_power_state(pdev, PCI_D0); 3274 pci_restore_state(pdev); 3275 err = pci_enable_device_mem(pdev); 3276 if (err != 0) 3277 return err; 3278 3279 pci_enable_wake(pdev, PCI_D0, 0); 3280 3281 spin_lock_irqsave(&adapter->cmd_lock, flags); 3282 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3283 VMXNET3_CMD_UPDATE_PMCFG); 3284 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3285 vmxnet3_alloc_intr_resources(adapter); 3286 vmxnet3_request_irqs(adapter); 3287 for (i = 0; i < adapter->num_rx_queues; i++) 3288 napi_enable(&adapter->rx_queue[i].napi); 3289 vmxnet3_enable_all_intrs(adapter); 3290 3291 return 0; 3292 } 3293 3294 static const struct dev_pm_ops vmxnet3_pm_ops = { 3295 .suspend = vmxnet3_suspend, 3296 .resume = vmxnet3_resume, 3297 }; 3298 #endif 3299 3300 static struct pci_driver vmxnet3_driver = { 3301 .name = vmxnet3_driver_name, 3302 .id_table = vmxnet3_pciid_table, 3303 .probe = vmxnet3_probe_device, 3304 .remove = vmxnet3_remove_device, 3305 #ifdef CONFIG_PM 3306 .driver.pm = &vmxnet3_pm_ops, 3307 #endif 3308 }; 3309 3310 3311 static int __init 3312 vmxnet3_init_module(void) 3313 { 3314 printk(KERN_INFO "%s - version %s\n", VMXNET3_DRIVER_DESC, 3315 VMXNET3_DRIVER_VERSION_REPORT); 3316 return pci_register_driver(&vmxnet3_driver); 3317 } 3318 3319 module_init(vmxnet3_init_module); 3320 3321 3322 static void 3323 vmxnet3_exit_module(void) 3324 { 3325 pci_unregister_driver(&vmxnet3_driver); 3326 } 3327 3328 module_exit(vmxnet3_exit_module); 3329 3330 MODULE_AUTHOR("VMware, Inc."); 3331 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC); 3332 MODULE_LICENSE("GPL v2"); 3333 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING); 3334