1 /* 2 * Linux driver for VMware's vmxnet3 ethernet NIC. 3 * 4 * Copyright (C) 2008-2024, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * The full GNU General Public License is included in this distribution in 21 * the file called "COPYING". 22 * 23 * Maintained by: pv-drivers@vmware.com 24 * 25 */ 26 27 #include <linux/module.h> 28 #include <net/ip6_checksum.h> 29 30 #ifdef CONFIG_X86 31 #include <asm/msr.h> 32 #endif 33 34 #include "vmxnet3_int.h" 35 #include "vmxnet3_xdp.h" 36 37 char vmxnet3_driver_name[] = "vmxnet3"; 38 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver" 39 40 /* 41 * PCI Device ID Table 42 * Last entry must be all 0s 43 */ 44 static const struct pci_device_id vmxnet3_pciid_table[] = { 45 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)}, 46 {0} 47 }; 48 49 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table); 50 51 static int enable_mq = 1; 52 53 static void 54 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac); 55 56 /* 57 * Enable/Disable the given intr 58 */ 59 static void 60 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 61 { 62 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0); 63 } 64 65 66 static void 67 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 68 { 69 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1); 70 } 71 72 73 /* 74 * Enable/Disable all intrs used by the device 75 */ 76 static void 77 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) 78 { 79 int i; 80 81 for (i = 0; i < adapter->intr.num_intrs; i++) 82 vmxnet3_enable_intr(adapter, i); 83 if (!VMXNET3_VERSION_GE_6(adapter) || 84 !adapter->queuesExtEnabled) { 85 adapter->shared->devRead.intrConf.intrCtrl &= 86 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 87 } else { 88 adapter->shared->devReadExt.intrConfExt.intrCtrl &= 89 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 90 } 91 } 92 93 94 static void 95 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) 96 { 97 int i; 98 99 if (!VMXNET3_VERSION_GE_6(adapter) || 100 !adapter->queuesExtEnabled) { 101 adapter->shared->devRead.intrConf.intrCtrl |= 102 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 103 } else { 104 adapter->shared->devReadExt.intrConfExt.intrCtrl |= 105 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 106 } 107 for (i = 0; i < adapter->intr.num_intrs; i++) 108 vmxnet3_disable_intr(adapter, i); 109 } 110 111 112 static void 113 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events) 114 { 115 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events); 116 } 117 118 119 static bool 120 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 121 { 122 return tq->stopped; 123 } 124 125 126 static void 127 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 128 { 129 tq->stopped = false; 130 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue); 131 } 132 133 134 static void 135 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 136 { 137 tq->stopped = false; 138 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 139 } 140 141 142 static void 143 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 144 { 145 tq->stopped = true; 146 tq->num_stop++; 147 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 148 } 149 150 static u64 151 vmxnet3_get_cycles(int pmc) 152 { 153 #ifdef CONFIG_X86 154 return native_read_pmc(pmc); 155 #else 156 return 0; 157 #endif 158 } 159 160 static bool 161 vmxnet3_apply_timestamp(struct vmxnet3_tx_queue *tq, u16 rate) 162 { 163 #ifdef CONFIG_X86 164 if (rate > 0) { 165 if (tq->tsPktCount == 1) { 166 if (rate != 1) 167 tq->tsPktCount = rate; 168 return true; 169 } 170 tq->tsPktCount--; 171 } 172 #endif 173 return false; 174 } 175 176 /* Check if capability is supported by UPT device or 177 * UPT is even requested 178 */ 179 bool 180 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap) 181 { 182 if (cap_supported & (1UL << VMXNET3_DCR_ERROR) || 183 cap_supported & (1UL << cap)) { 184 return true; 185 } 186 187 return false; 188 } 189 190 191 /* 192 * Check the link state. This may start or stop the tx queue. 193 */ 194 static void 195 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 196 { 197 u32 ret; 198 int i; 199 unsigned long flags; 200 201 spin_lock_irqsave(&adapter->cmd_lock, flags); 202 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 203 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 204 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 205 206 adapter->link_speed = ret >> 16; 207 if (ret & 1) { /* Link is up. */ 208 /* 209 * From vmxnet3 v9, the hypervisor reports the speed in Gbps. 210 * Convert the speed to Mbps before rporting it to the kernel. 211 * Max link speed supported is 10000G. 212 */ 213 if (VMXNET3_VERSION_GE_9(adapter) && 214 adapter->link_speed < 10000) 215 adapter->link_speed = adapter->link_speed * 1000; 216 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n", 217 adapter->link_speed); 218 netif_carrier_on(adapter->netdev); 219 220 if (affectTxQueue) { 221 for (i = 0; i < adapter->num_tx_queues; i++) 222 vmxnet3_tq_start(&adapter->tx_queue[i], 223 adapter); 224 } 225 } else { 226 netdev_info(adapter->netdev, "NIC Link is Down\n"); 227 netif_carrier_off(adapter->netdev); 228 229 if (affectTxQueue) { 230 for (i = 0; i < adapter->num_tx_queues; i++) 231 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter); 232 } 233 } 234 } 235 236 static void 237 vmxnet3_process_events(struct vmxnet3_adapter *adapter) 238 { 239 int i; 240 unsigned long flags; 241 u32 events = le32_to_cpu(adapter->shared->ecr); 242 if (!events) 243 return; 244 245 vmxnet3_ack_events(adapter, events); 246 247 /* Check if link state has changed */ 248 if (events & VMXNET3_ECR_LINK) 249 vmxnet3_check_link(adapter, true); 250 251 /* Check if there is an error on xmit/recv queues */ 252 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 253 spin_lock_irqsave(&adapter->cmd_lock, flags); 254 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 255 VMXNET3_CMD_GET_QUEUE_STATUS); 256 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 257 258 for (i = 0; i < adapter->num_tx_queues; i++) 259 if (adapter->tqd_start[i].status.stopped) 260 dev_err(&adapter->netdev->dev, 261 "%s: tq[%d] error 0x%x\n", 262 adapter->netdev->name, i, le32_to_cpu( 263 adapter->tqd_start[i].status.error)); 264 for (i = 0; i < adapter->num_rx_queues; i++) 265 if (adapter->rqd_start[i].status.stopped) 266 dev_err(&adapter->netdev->dev, 267 "%s: rq[%d] error 0x%x\n", 268 adapter->netdev->name, i, 269 adapter->rqd_start[i].status.error); 270 271 schedule_work(&adapter->work); 272 } 273 } 274 275 #ifdef __BIG_ENDIAN_BITFIELD 276 /* 277 * The device expects the bitfields in shared structures to be written in 278 * little endian. When CPU is big endian, the following routines are used to 279 * correctly read and write into ABI. 280 * The general technique used here is : double word bitfields are defined in 281 * opposite order for big endian architecture. Then before reading them in 282 * driver the complete double word is translated using le32_to_cpu. Similarly 283 * After the driver writes into bitfields, cpu_to_le32 is used to translate the 284 * double words into required format. 285 * In order to avoid touching bits in shared structure more than once, temporary 286 * descriptors are used. These are passed as srcDesc to following functions. 287 */ 288 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc, 289 struct Vmxnet3_RxDesc *dstDesc) 290 { 291 u32 *src = (u32 *)srcDesc + 2; 292 u32 *dst = (u32 *)dstDesc + 2; 293 dstDesc->addr = le64_to_cpu(srcDesc->addr); 294 *dst = le32_to_cpu(*src); 295 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1); 296 } 297 298 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc, 299 struct Vmxnet3_TxDesc *dstDesc) 300 { 301 int i; 302 u32 *src = (u32 *)(srcDesc + 1); 303 u32 *dst = (u32 *)(dstDesc + 1); 304 305 /* Working backwards so that the gen bit is set at the end. */ 306 for (i = 2; i > 0; i--) { 307 src--; 308 dst--; 309 *dst = cpu_to_le32(*src); 310 } 311 } 312 313 314 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc, 315 struct Vmxnet3_RxCompDesc *dstDesc) 316 { 317 int i = 0; 318 u32 *src = (u32 *)srcDesc; 319 u32 *dst = (u32 *)dstDesc; 320 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) { 321 *dst = le32_to_cpu(*src); 322 src++; 323 dst++; 324 } 325 } 326 327 328 /* Used to read bitfield values from double words. */ 329 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size) 330 { 331 u32 temp = le32_to_cpu(*bitfield); 332 u32 mask = ((1 << size) - 1) << pos; 333 temp &= mask; 334 temp >>= pos; 335 return temp; 336 } 337 338 339 340 #endif /* __BIG_ENDIAN_BITFIELD */ 341 342 #ifdef __BIG_ENDIAN_BITFIELD 343 344 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \ 345 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \ 346 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE) 347 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \ 348 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \ 349 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE) 350 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \ 351 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \ 352 VMXNET3_TCD_GEN_SIZE) 353 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \ 354 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE) 355 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \ 356 (dstrcd) = (tmp); \ 357 vmxnet3_RxCompToCPU((rcd), (tmp)); \ 358 } while (0) 359 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \ 360 (dstrxd) = (tmp); \ 361 vmxnet3_RxDescToCPU((rxd), (tmp)); \ 362 } while (0) 363 364 #else 365 366 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen) 367 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop) 368 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen) 369 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx) 370 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd) 371 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd) 372 373 #endif /* __BIG_ENDIAN_BITFIELD */ 374 375 376 static void 377 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi, 378 struct pci_dev *pdev) 379 { 380 u32 map_type = tbi->map_type; 381 382 if (map_type & VMXNET3_MAP_SINGLE) 383 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len, 384 DMA_TO_DEVICE); 385 else if (map_type & VMXNET3_MAP_PAGE) 386 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len, 387 DMA_TO_DEVICE); 388 else 389 BUG_ON(map_type & ~VMXNET3_MAP_XDP); 390 391 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */ 392 } 393 394 395 static int 396 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq, 397 struct pci_dev *pdev, struct vmxnet3_adapter *adapter, 398 struct xdp_frame_bulk *bq) 399 { 400 struct vmxnet3_tx_buf_info *tbi; 401 int entries = 0; 402 u32 map_type; 403 404 /* no out of order completion */ 405 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp); 406 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1); 407 408 tbi = &tq->buf_info[eop_idx]; 409 BUG_ON(!tbi->skb); 410 map_type = tbi->map_type; 411 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size); 412 413 while (tq->tx_ring.next2comp != eop_idx) { 414 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp, 415 pdev); 416 417 /* update next2comp w/o tx_lock. Since we are marking more, 418 * instead of less, tx ring entries avail, the worst case is 419 * that the tx routine incorrectly re-queues a pkt due to 420 * insufficient tx ring entries. 421 */ 422 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 423 entries++; 424 } 425 426 if (map_type & VMXNET3_MAP_XDP) 427 xdp_return_frame_bulk(tbi->xdpf, bq); 428 else 429 dev_kfree_skb_any(tbi->skb); 430 431 /* xdpf and skb are in an anonymous union. */ 432 tbi->skb = NULL; 433 434 return entries; 435 } 436 437 438 static int 439 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq, 440 struct vmxnet3_adapter *adapter) 441 { 442 union Vmxnet3_GenericDesc *gdesc; 443 struct xdp_frame_bulk bq; 444 int completed = 0; 445 446 xdp_frame_bulk_init(&bq); 447 rcu_read_lock(); 448 449 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 450 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) { 451 /* Prevent any &gdesc->tcd field from being (speculatively) 452 * read before (&gdesc->tcd)->gen is read. 453 */ 454 dma_rmb(); 455 456 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX( 457 &gdesc->tcd), tq, adapter->pdev, 458 adapter, &bq); 459 460 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring); 461 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 462 } 463 xdp_flush_frame_bulk(&bq); 464 rcu_read_unlock(); 465 466 if (completed) { 467 spin_lock(&tq->tx_lock); 468 if (unlikely(vmxnet3_tq_stopped(tq, adapter) && 469 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) > 470 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) && 471 netif_carrier_ok(adapter->netdev))) { 472 vmxnet3_tq_wake(tq, adapter); 473 } 474 spin_unlock(&tq->tx_lock); 475 } 476 return completed; 477 } 478 479 480 static void 481 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq, 482 struct vmxnet3_adapter *adapter) 483 { 484 struct xdp_frame_bulk bq; 485 u32 map_type; 486 int i; 487 488 xdp_frame_bulk_init(&bq); 489 rcu_read_lock(); 490 491 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) { 492 struct vmxnet3_tx_buf_info *tbi; 493 494 tbi = tq->buf_info + tq->tx_ring.next2comp; 495 map_type = tbi->map_type; 496 497 vmxnet3_unmap_tx_buf(tbi, adapter->pdev); 498 if (tbi->skb) { 499 if (map_type & VMXNET3_MAP_XDP) 500 xdp_return_frame_bulk(tbi->xdpf, &bq); 501 else 502 dev_kfree_skb_any(tbi->skb); 503 tbi->skb = NULL; 504 } 505 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 506 } 507 508 xdp_flush_frame_bulk(&bq); 509 rcu_read_unlock(); 510 511 /* sanity check, verify all buffers are indeed unmapped */ 512 for (i = 0; i < tq->tx_ring.size; i++) 513 BUG_ON(tq->buf_info[i].map_type != VMXNET3_MAP_NONE); 514 515 tq->tx_ring.gen = VMXNET3_INIT_GEN; 516 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 517 518 tq->comp_ring.gen = VMXNET3_INIT_GEN; 519 tq->comp_ring.next2proc = 0; 520 } 521 522 523 static void 524 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 525 struct vmxnet3_adapter *adapter) 526 { 527 if (tq->tx_ring.base) { 528 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size * 529 sizeof(struct Vmxnet3_TxDesc), 530 tq->tx_ring.base, tq->tx_ring.basePA); 531 tq->tx_ring.base = NULL; 532 } 533 if (tq->data_ring.base) { 534 dma_free_coherent(&adapter->pdev->dev, 535 tq->data_ring.size * tq->txdata_desc_size, 536 tq->data_ring.base, tq->data_ring.basePA); 537 tq->data_ring.base = NULL; 538 } 539 if (tq->ts_ring.base) { 540 dma_free_coherent(&adapter->pdev->dev, 541 tq->tx_ring.size * tq->tx_ts_desc_size, 542 tq->ts_ring.base, tq->ts_ring.basePA); 543 tq->ts_ring.base = NULL; 544 } 545 if (tq->comp_ring.base) { 546 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size * 547 sizeof(struct Vmxnet3_TxCompDesc), 548 tq->comp_ring.base, tq->comp_ring.basePA); 549 tq->comp_ring.base = NULL; 550 } 551 kfree(tq->buf_info); 552 tq->buf_info = NULL; 553 } 554 555 556 /* Destroy all tx queues */ 557 void 558 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter) 559 { 560 int i; 561 562 for (i = 0; i < adapter->num_tx_queues; i++) 563 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter); 564 } 565 566 567 static void 568 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 569 struct vmxnet3_adapter *adapter) 570 { 571 int i; 572 573 /* reset the tx ring contents to 0 and reset the tx ring states */ 574 memset(tq->tx_ring.base, 0, tq->tx_ring.size * 575 sizeof(struct Vmxnet3_TxDesc)); 576 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 577 tq->tx_ring.gen = VMXNET3_INIT_GEN; 578 579 memset(tq->data_ring.base, 0, 580 tq->data_ring.size * tq->txdata_desc_size); 581 582 if (tq->ts_ring.base) 583 memset(tq->ts_ring.base, 0, 584 tq->tx_ring.size * tq->tx_ts_desc_size); 585 586 /* reset the tx comp ring contents to 0 and reset comp ring states */ 587 memset(tq->comp_ring.base, 0, tq->comp_ring.size * 588 sizeof(struct Vmxnet3_TxCompDesc)); 589 tq->comp_ring.next2proc = 0; 590 tq->comp_ring.gen = VMXNET3_INIT_GEN; 591 592 /* reset the bookkeeping data */ 593 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size); 594 for (i = 0; i < tq->tx_ring.size; i++) 595 tq->buf_info[i].map_type = VMXNET3_MAP_NONE; 596 597 /* stats are not reset */ 598 } 599 600 601 static int 602 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq, 603 struct vmxnet3_adapter *adapter) 604 { 605 BUG_ON(tq->tx_ring.base || tq->data_ring.base || 606 tq->comp_ring.base || tq->buf_info); 607 608 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 609 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc), 610 &tq->tx_ring.basePA, GFP_KERNEL); 611 if (!tq->tx_ring.base) { 612 netdev_err(adapter->netdev, "failed to allocate tx ring\n"); 613 goto err; 614 } 615 616 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 617 tq->data_ring.size * tq->txdata_desc_size, 618 &tq->data_ring.basePA, GFP_KERNEL); 619 if (!tq->data_ring.base) { 620 netdev_err(adapter->netdev, "failed to allocate tx data ring\n"); 621 goto err; 622 } 623 624 if (tq->tx_ts_desc_size != 0) { 625 tq->ts_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 626 tq->tx_ring.size * tq->tx_ts_desc_size, 627 &tq->ts_ring.basePA, GFP_KERNEL); 628 if (!tq->ts_ring.base) { 629 netdev_err(adapter->netdev, "failed to allocate tx ts ring\n"); 630 tq->tx_ts_desc_size = 0; 631 } 632 } else { 633 tq->ts_ring.base = NULL; 634 } 635 636 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 637 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc), 638 &tq->comp_ring.basePA, GFP_KERNEL); 639 if (!tq->comp_ring.base) { 640 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n"); 641 goto err; 642 } 643 644 tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]), 645 GFP_KERNEL, 646 dev_to_node(&adapter->pdev->dev)); 647 if (!tq->buf_info) 648 goto err; 649 650 return 0; 651 652 err: 653 vmxnet3_tq_destroy(tq, adapter); 654 return -ENOMEM; 655 } 656 657 static void 658 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter) 659 { 660 int i; 661 662 for (i = 0; i < adapter->num_tx_queues; i++) 663 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter); 664 } 665 666 /* 667 * starting from ring->next2fill, allocate rx buffers for the given ring 668 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers 669 * are allocated or allocation fails 670 */ 671 672 static int 673 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx, 674 int num_to_alloc, struct vmxnet3_adapter *adapter) 675 { 676 int num_allocated = 0; 677 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx]; 678 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx]; 679 u32 val; 680 681 while (num_allocated <= num_to_alloc) { 682 struct vmxnet3_rx_buf_info *rbi; 683 union Vmxnet3_GenericDesc *gd; 684 685 rbi = rbi_base + ring->next2fill; 686 gd = ring->base + ring->next2fill; 687 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 688 689 if (rbi->buf_type == VMXNET3_RX_BUF_XDP) { 690 void *data = vmxnet3_pp_get_buff(rq->page_pool, 691 &rbi->dma_addr, 692 GFP_KERNEL); 693 if (!data) { 694 rq->stats.rx_buf_alloc_failure++; 695 break; 696 } 697 rbi->page = virt_to_page(data); 698 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 699 } else if (rbi->buf_type == VMXNET3_RX_BUF_SKB) { 700 if (rbi->skb == NULL) { 701 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev, 702 rbi->len, 703 GFP_KERNEL); 704 if (unlikely(rbi->skb == NULL)) { 705 rq->stats.rx_buf_alloc_failure++; 706 break; 707 } 708 709 rbi->dma_addr = dma_map_single( 710 &adapter->pdev->dev, 711 rbi->skb->data, rbi->len, 712 DMA_FROM_DEVICE); 713 if (dma_mapping_error(&adapter->pdev->dev, 714 rbi->dma_addr)) { 715 dev_kfree_skb_any(rbi->skb); 716 rbi->skb = NULL; 717 rq->stats.rx_buf_alloc_failure++; 718 break; 719 } 720 } else { 721 /* rx buffer skipped by the device */ 722 } 723 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 724 } else { 725 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE || 726 rbi->len != PAGE_SIZE); 727 728 if (rbi->page == NULL) { 729 rbi->page = alloc_page(GFP_ATOMIC); 730 if (unlikely(rbi->page == NULL)) { 731 rq->stats.rx_buf_alloc_failure++; 732 break; 733 } 734 rbi->dma_addr = dma_map_page( 735 &adapter->pdev->dev, 736 rbi->page, 0, PAGE_SIZE, 737 DMA_FROM_DEVICE); 738 if (dma_mapping_error(&adapter->pdev->dev, 739 rbi->dma_addr)) { 740 put_page(rbi->page); 741 rbi->page = NULL; 742 rq->stats.rx_buf_alloc_failure++; 743 break; 744 } 745 } else { 746 /* rx buffers skipped by the device */ 747 } 748 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT; 749 } 750 751 gd->rxd.addr = cpu_to_le64(rbi->dma_addr); 752 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT) 753 | val | rbi->len); 754 755 /* Fill the last buffer but dont mark it ready, or else the 756 * device will think that the queue is full */ 757 if (num_allocated == num_to_alloc) { 758 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 759 break; 760 } 761 762 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT); 763 num_allocated++; 764 vmxnet3_cmd_ring_adv_next2fill(ring); 765 } 766 767 netdev_dbg(adapter->netdev, 768 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n", 769 num_allocated, ring->next2fill, ring->next2comp); 770 771 /* so that the device can distinguish a full ring and an empty ring */ 772 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp); 773 774 return num_allocated; 775 } 776 777 778 static void 779 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd, 780 struct vmxnet3_rx_buf_info *rbi) 781 { 782 skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags; 783 784 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); 785 786 skb_frag_fill_page_desc(frag, rbi->page, 0, rcd->len); 787 skb->data_len += rcd->len; 788 skb->truesize += PAGE_SIZE; 789 skb_shinfo(skb)->nr_frags++; 790 } 791 792 793 static int 794 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx, 795 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev, 796 struct vmxnet3_adapter *adapter) 797 { 798 u32 dw2, len; 799 unsigned long buf_offset; 800 int i; 801 union Vmxnet3_GenericDesc *gdesc; 802 struct vmxnet3_tx_buf_info *tbi = NULL; 803 804 BUG_ON(ctx->copy_size > skb_headlen(skb)); 805 806 /* use the previous gen bit for the SOP desc */ 807 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 808 809 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; 810 gdesc = ctx->sop_txd; /* both loops below can be skipped */ 811 812 /* no need to map the buffer if headers are copied */ 813 if (ctx->copy_size) { 814 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA + 815 tq->tx_ring.next2fill * 816 tq->txdata_desc_size); 817 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size); 818 ctx->sop_txd->dword[3] = 0; 819 820 tbi = tq->buf_info + tq->tx_ring.next2fill; 821 tbi->map_type = VMXNET3_MAP_NONE; 822 823 netdev_dbg(adapter->netdev, 824 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 825 tq->tx_ring.next2fill, 826 le64_to_cpu(ctx->sop_txd->txd.addr), 827 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]); 828 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 829 830 /* use the right gen for non-SOP desc */ 831 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 832 } 833 834 /* linear part can use multiple tx desc if it's big */ 835 len = skb_headlen(skb) - ctx->copy_size; 836 buf_offset = ctx->copy_size; 837 while (len) { 838 u32 buf_size; 839 840 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 841 buf_size = len; 842 dw2 |= len; 843 } else { 844 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 845 /* spec says that for TxDesc.len, 0 == 2^14 */ 846 } 847 848 tbi = tq->buf_info + tq->tx_ring.next2fill; 849 tbi->map_type = VMXNET3_MAP_SINGLE; 850 tbi->dma_addr = dma_map_single(&adapter->pdev->dev, 851 skb->data + buf_offset, buf_size, 852 DMA_TO_DEVICE); 853 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 854 return -EFAULT; 855 856 tbi->len = buf_size; 857 858 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 859 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 860 861 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 862 gdesc->dword[2] = cpu_to_le32(dw2); 863 gdesc->dword[3] = 0; 864 865 netdev_dbg(adapter->netdev, 866 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 867 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 868 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 869 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 870 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 871 872 len -= buf_size; 873 buf_offset += buf_size; 874 } 875 876 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 877 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 878 u32 buf_size; 879 880 buf_offset = 0; 881 len = skb_frag_size(frag); 882 while (len) { 883 tbi = tq->buf_info + tq->tx_ring.next2fill; 884 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 885 buf_size = len; 886 dw2 |= len; 887 } else { 888 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 889 /* spec says that for TxDesc.len, 0 == 2^14 */ 890 } 891 tbi->map_type = VMXNET3_MAP_PAGE; 892 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag, 893 buf_offset, buf_size, 894 DMA_TO_DEVICE); 895 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 896 return -EFAULT; 897 898 tbi->len = buf_size; 899 900 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 901 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 902 903 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 904 gdesc->dword[2] = cpu_to_le32(dw2); 905 gdesc->dword[3] = 0; 906 907 netdev_dbg(adapter->netdev, 908 "txd[%u]: 0x%llx %u %u\n", 909 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 910 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 911 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 912 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 913 914 len -= buf_size; 915 buf_offset += buf_size; 916 } 917 } 918 919 ctx->eop_txd = gdesc; 920 921 /* set the last buf_info for the pkt */ 922 tbi->skb = skb; 923 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base; 924 if (tq->tx_ts_desc_size != 0) { 925 ctx->ts_txd = (struct Vmxnet3_TxTSDesc *)((u8 *)tq->ts_ring.base + 926 tbi->sop_idx * tq->tx_ts_desc_size); 927 ctx->ts_txd->ts.tsi = 0; 928 } 929 930 return 0; 931 } 932 933 934 /* Init all tx queues */ 935 static void 936 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter) 937 { 938 int i; 939 940 for (i = 0; i < adapter->num_tx_queues; i++) 941 vmxnet3_tq_init(&adapter->tx_queue[i], adapter); 942 } 943 944 945 /* 946 * parse relevant protocol headers: 947 * For a tso pkt, relevant headers are L2/3/4 including options 948 * For a pkt requesting csum offloading, they are L2/3 and may include L4 949 * if it's a TCP/UDP pkt 950 * 951 * Returns: 952 * -1: error happens during parsing 953 * 0: protocol headers parsed, but too big to be copied 954 * 1: protocol headers parsed and copied 955 * 956 * Other effects: 957 * 1. related *ctx fields are updated. 958 * 2. ctx->copy_size is # of bytes copied 959 * 3. the portion to be copied is guaranteed to be in the linear part 960 * 961 */ 962 static int 963 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 964 struct vmxnet3_tx_ctx *ctx, 965 struct vmxnet3_adapter *adapter) 966 { 967 u8 protocol = 0; 968 969 if (ctx->mss) { /* TSO */ 970 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 971 ctx->l4_offset = skb_inner_transport_offset(skb); 972 ctx->l4_hdr_size = inner_tcp_hdrlen(skb); 973 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 974 } else { 975 ctx->l4_offset = skb_transport_offset(skb); 976 ctx->l4_hdr_size = tcp_hdrlen(skb); 977 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 978 } 979 } else { 980 if (skb->ip_summed == CHECKSUM_PARTIAL) { 981 /* For encap packets, skb_checksum_start_offset refers 982 * to inner L4 offset. Thus, below works for encap as 983 * well as non-encap case 984 */ 985 ctx->l4_offset = skb_checksum_start_offset(skb); 986 987 if (VMXNET3_VERSION_GE_4(adapter) && 988 skb->encapsulation) { 989 struct iphdr *iph = inner_ip_hdr(skb); 990 991 if (iph->version == 4) { 992 protocol = iph->protocol; 993 } else { 994 const struct ipv6hdr *ipv6h; 995 996 ipv6h = inner_ipv6_hdr(skb); 997 protocol = ipv6h->nexthdr; 998 } 999 } else { 1000 if (ctx->ipv4) { 1001 const struct iphdr *iph = ip_hdr(skb); 1002 1003 protocol = iph->protocol; 1004 } else if (ctx->ipv6) { 1005 const struct ipv6hdr *ipv6h; 1006 1007 ipv6h = ipv6_hdr(skb); 1008 protocol = ipv6h->nexthdr; 1009 } 1010 } 1011 1012 switch (protocol) { 1013 case IPPROTO_TCP: 1014 ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) : 1015 tcp_hdrlen(skb); 1016 break; 1017 case IPPROTO_UDP: 1018 ctx->l4_hdr_size = sizeof(struct udphdr); 1019 break; 1020 default: 1021 ctx->l4_hdr_size = 0; 1022 break; 1023 } 1024 1025 ctx->copy_size = min(ctx->l4_offset + 1026 ctx->l4_hdr_size, skb->len); 1027 } else { 1028 ctx->l4_offset = 0; 1029 ctx->l4_hdr_size = 0; 1030 /* copy as much as allowed */ 1031 ctx->copy_size = min_t(unsigned int, 1032 tq->txdata_desc_size, 1033 skb_headlen(skb)); 1034 } 1035 1036 if (skb->len <= tq->txdata_desc_size) 1037 ctx->copy_size = skb->len; 1038 1039 /* make sure headers are accessible directly */ 1040 if (unlikely(!pskb_may_pull(skb, ctx->copy_size))) 1041 goto err; 1042 } 1043 1044 if (unlikely(ctx->copy_size > tq->txdata_desc_size)) { 1045 tq->stats.oversized_hdr++; 1046 ctx->copy_size = 0; 1047 return 0; 1048 } 1049 1050 return 1; 1051 err: 1052 return -1; 1053 } 1054 1055 /* 1056 * copy relevant protocol headers to the transmit ring: 1057 * For a tso pkt, relevant headers are L2/3/4 including options 1058 * For a pkt requesting csum offloading, they are L2/3 and may include L4 1059 * if it's a TCP/UDP pkt 1060 * 1061 * 1062 * Note that this requires that vmxnet3_parse_hdr be called first to set the 1063 * appropriate bits in ctx first 1064 */ 1065 static void 1066 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1067 struct vmxnet3_tx_ctx *ctx, 1068 struct vmxnet3_adapter *adapter) 1069 { 1070 struct Vmxnet3_TxDataDesc *tdd; 1071 1072 tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base + 1073 tq->tx_ring.next2fill * 1074 tq->txdata_desc_size); 1075 1076 memcpy(tdd->data, skb->data, ctx->copy_size); 1077 netdev_dbg(adapter->netdev, 1078 "copy %u bytes to dataRing[%u]\n", 1079 ctx->copy_size, tq->tx_ring.next2fill); 1080 } 1081 1082 1083 static void 1084 vmxnet3_prepare_inner_tso(struct sk_buff *skb, 1085 struct vmxnet3_tx_ctx *ctx) 1086 { 1087 struct tcphdr *tcph = inner_tcp_hdr(skb); 1088 struct iphdr *iph = inner_ip_hdr(skb); 1089 1090 if (iph->version == 4) { 1091 iph->check = 0; 1092 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1093 IPPROTO_TCP, 0); 1094 } else { 1095 struct ipv6hdr *iph = inner_ipv6_hdr(skb); 1096 1097 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0, 1098 IPPROTO_TCP, 0); 1099 } 1100 } 1101 1102 static void 1103 vmxnet3_prepare_tso(struct sk_buff *skb, 1104 struct vmxnet3_tx_ctx *ctx) 1105 { 1106 struct tcphdr *tcph = tcp_hdr(skb); 1107 1108 if (ctx->ipv4) { 1109 struct iphdr *iph = ip_hdr(skb); 1110 1111 iph->check = 0; 1112 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1113 IPPROTO_TCP, 0); 1114 } else if (ctx->ipv6) { 1115 tcp_v6_gso_csum_prep(skb); 1116 } 1117 } 1118 1119 static int txd_estimate(const struct sk_buff *skb) 1120 { 1121 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1122 int i; 1123 1124 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1125 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1126 1127 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag)); 1128 } 1129 return count; 1130 } 1131 1132 /* 1133 * Transmits a pkt thru a given tq 1134 * Returns: 1135 * NETDEV_TX_OK: descriptors are setup successfully 1136 * NETDEV_TX_OK: error occurred, the pkt is dropped 1137 * NETDEV_TX_BUSY: tx ring is full, queue is stopped 1138 * 1139 * Side-effects: 1140 * 1. tx ring may be changed 1141 * 2. tq stats may be updated accordingly 1142 * 3. shared->txNumDeferred may be updated 1143 */ 1144 1145 static int 1146 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1147 struct vmxnet3_adapter *adapter, struct net_device *netdev) 1148 { 1149 int ret; 1150 u32 count; 1151 int num_pkts; 1152 int tx_num_deferred; 1153 unsigned long flags; 1154 struct vmxnet3_tx_ctx ctx; 1155 union Vmxnet3_GenericDesc *gdesc; 1156 #ifdef __BIG_ENDIAN_BITFIELD 1157 /* Use temporary descriptor to avoid touching bits multiple times */ 1158 union Vmxnet3_GenericDesc tempTxDesc; 1159 #endif 1160 1161 count = txd_estimate(skb); 1162 1163 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP)); 1164 ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6)); 1165 1166 ctx.mss = skb_shinfo(skb)->gso_size; 1167 if (ctx.mss) { 1168 if (skb_header_cloned(skb)) { 1169 if (unlikely(pskb_expand_head(skb, 0, 0, 1170 GFP_ATOMIC) != 0)) { 1171 tq->stats.drop_tso++; 1172 goto drop_pkt; 1173 } 1174 tq->stats.copy_skb_header++; 1175 } 1176 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1177 /* tso pkts must not use more than 1178 * VMXNET3_MAX_TSO_TXD_PER_PKT entries 1179 */ 1180 if (skb_linearize(skb) != 0) { 1181 tq->stats.drop_too_many_frags++; 1182 goto drop_pkt; 1183 } 1184 tq->stats.linearized++; 1185 1186 /* recalculate the # of descriptors to use */ 1187 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1188 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1189 tq->stats.drop_too_many_frags++; 1190 goto drop_pkt; 1191 } 1192 } 1193 if (skb->encapsulation) { 1194 vmxnet3_prepare_inner_tso(skb, &ctx); 1195 } else { 1196 vmxnet3_prepare_tso(skb, &ctx); 1197 } 1198 } else { 1199 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) { 1200 1201 /* non-tso pkts must not use more than 1202 * VMXNET3_MAX_TXD_PER_PKT entries 1203 */ 1204 if (skb_linearize(skb) != 0) { 1205 tq->stats.drop_too_many_frags++; 1206 goto drop_pkt; 1207 } 1208 tq->stats.linearized++; 1209 1210 /* recalculate the # of descriptors to use */ 1211 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1212 } 1213 } 1214 1215 ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter); 1216 if (ret >= 0) { 1217 BUG_ON(ret <= 0 && ctx.copy_size != 0); 1218 /* hdrs parsed, check against other limits */ 1219 if (ctx.mss) { 1220 if (unlikely(ctx.l4_offset + ctx.l4_hdr_size > 1221 VMXNET3_MAX_TX_BUF_SIZE)) { 1222 tq->stats.drop_oversized_hdr++; 1223 goto drop_pkt; 1224 } 1225 } else { 1226 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1227 if (unlikely(ctx.l4_offset + 1228 skb->csum_offset > 1229 VMXNET3_MAX_CSUM_OFFSET)) { 1230 tq->stats.drop_oversized_hdr++; 1231 goto drop_pkt; 1232 } 1233 } 1234 } 1235 } else { 1236 tq->stats.drop_hdr_inspect_err++; 1237 goto drop_pkt; 1238 } 1239 1240 spin_lock_irqsave(&tq->tx_lock, flags); 1241 1242 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) { 1243 tq->stats.tx_ring_full++; 1244 netdev_dbg(adapter->netdev, 1245 "tx queue stopped on %s, next2comp %u" 1246 " next2fill %u\n", adapter->netdev->name, 1247 tq->tx_ring.next2comp, tq->tx_ring.next2fill); 1248 1249 vmxnet3_tq_stop(tq, adapter); 1250 spin_unlock_irqrestore(&tq->tx_lock, flags); 1251 return NETDEV_TX_BUSY; 1252 } 1253 1254 1255 vmxnet3_copy_hdr(skb, tq, &ctx, adapter); 1256 1257 /* fill tx descs related to addr & len */ 1258 if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter)) 1259 goto unlock_drop_pkt; 1260 1261 /* setup the EOP desc */ 1262 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); 1263 1264 /* setup the SOP desc */ 1265 #ifdef __BIG_ENDIAN_BITFIELD 1266 gdesc = &tempTxDesc; 1267 gdesc->dword[2] = ctx.sop_txd->dword[2]; 1268 gdesc->dword[3] = ctx.sop_txd->dword[3]; 1269 #else 1270 gdesc = ctx.sop_txd; 1271 #endif 1272 tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred); 1273 if (ctx.mss) { 1274 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 1275 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1276 if (VMXNET3_VERSION_GE_7(adapter)) { 1277 gdesc->txd.om = VMXNET3_OM_TSO; 1278 gdesc->txd.ext1 = 1; 1279 } else { 1280 gdesc->txd.om = VMXNET3_OM_ENCAP; 1281 } 1282 gdesc->txd.msscof = ctx.mss; 1283 1284 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) 1285 gdesc->txd.oco = 1; 1286 } else { 1287 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1288 gdesc->txd.om = VMXNET3_OM_TSO; 1289 gdesc->txd.msscof = ctx.mss; 1290 } 1291 num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss; 1292 } else { 1293 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1294 if (VMXNET3_VERSION_GE_4(adapter) && 1295 skb->encapsulation) { 1296 gdesc->txd.hlen = ctx.l4_offset + 1297 ctx.l4_hdr_size; 1298 if (VMXNET3_VERSION_GE_7(adapter)) { 1299 gdesc->txd.om = VMXNET3_OM_CSUM; 1300 gdesc->txd.msscof = ctx.l4_offset + 1301 skb->csum_offset; 1302 gdesc->txd.ext1 = 1; 1303 } else { 1304 gdesc->txd.om = VMXNET3_OM_ENCAP; 1305 gdesc->txd.msscof = 0; /* Reserved */ 1306 } 1307 } else { 1308 gdesc->txd.hlen = ctx.l4_offset; 1309 gdesc->txd.om = VMXNET3_OM_CSUM; 1310 gdesc->txd.msscof = ctx.l4_offset + 1311 skb->csum_offset; 1312 } 1313 } else { 1314 gdesc->txd.om = 0; 1315 gdesc->txd.msscof = 0; 1316 } 1317 num_pkts = 1; 1318 } 1319 le32_add_cpu(&tq->shared->txNumDeferred, num_pkts); 1320 tx_num_deferred += num_pkts; 1321 1322 if (skb_vlan_tag_present(skb)) { 1323 gdesc->txd.ti = 1; 1324 gdesc->txd.tci = skb_vlan_tag_get(skb); 1325 } 1326 1327 if (tq->tx_ts_desc_size != 0 && 1328 adapter->latencyConf->sampleRate != 0) { 1329 if (vmxnet3_apply_timestamp(tq, adapter->latencyConf->sampleRate)) { 1330 ctx.ts_txd->ts.tsData = vmxnet3_get_cycles(VMXNET3_PMC_PSEUDO_TSC); 1331 ctx.ts_txd->ts.tsi = 1; 1332 } 1333 } 1334 1335 /* Ensure that the write to (&gdesc->txd)->gen will be observed after 1336 * all other writes to &gdesc->txd. 1337 */ 1338 dma_wmb(); 1339 1340 /* finally flips the GEN bit of the SOP desc. */ 1341 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ 1342 VMXNET3_TXD_GEN); 1343 #ifdef __BIG_ENDIAN_BITFIELD 1344 /* Finished updating in bitfields of Tx Desc, so write them in original 1345 * place. 1346 */ 1347 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc, 1348 (struct Vmxnet3_TxDesc *)ctx.sop_txd); 1349 gdesc = ctx.sop_txd; 1350 #endif 1351 netdev_dbg(adapter->netdev, 1352 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n", 1353 (u32)(ctx.sop_txd - 1354 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr), 1355 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3])); 1356 1357 spin_unlock_irqrestore(&tq->tx_lock, flags); 1358 1359 if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) { 1360 tq->shared->txNumDeferred = 0; 1361 VMXNET3_WRITE_BAR0_REG(adapter, 1362 adapter->tx_prod_offset + tq->qid * 8, 1363 tq->tx_ring.next2fill); 1364 } 1365 1366 return NETDEV_TX_OK; 1367 1368 unlock_drop_pkt: 1369 spin_unlock_irqrestore(&tq->tx_lock, flags); 1370 drop_pkt: 1371 tq->stats.drop_total++; 1372 dev_kfree_skb_any(skb); 1373 return NETDEV_TX_OK; 1374 } 1375 1376 static int 1377 vmxnet3_create_pp(struct vmxnet3_adapter *adapter, 1378 struct vmxnet3_rx_queue *rq, int size) 1379 { 1380 bool xdp_prog = vmxnet3_xdp_enabled(adapter); 1381 const struct page_pool_params pp_params = { 1382 .order = 0, 1383 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 1384 .pool_size = size, 1385 .nid = NUMA_NO_NODE, 1386 .dev = &adapter->pdev->dev, 1387 .offset = VMXNET3_XDP_RX_OFFSET, 1388 .max_len = VMXNET3_XDP_MAX_FRSIZE, 1389 .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE, 1390 }; 1391 struct page_pool *pp; 1392 int err; 1393 1394 pp = page_pool_create(&pp_params); 1395 if (IS_ERR(pp)) 1396 return PTR_ERR(pp); 1397 1398 err = xdp_rxq_info_reg(&rq->xdp_rxq, adapter->netdev, rq->qid, 1399 rq->napi.napi_id); 1400 if (err < 0) 1401 goto err_free_pp; 1402 1403 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_PAGE_POOL, pp); 1404 if (err) 1405 goto err_unregister_rxq; 1406 1407 rq->page_pool = pp; 1408 1409 return 0; 1410 1411 err_unregister_rxq: 1412 xdp_rxq_info_unreg(&rq->xdp_rxq); 1413 err_free_pp: 1414 page_pool_destroy(pp); 1415 1416 return err; 1417 } 1418 1419 void * 1420 vmxnet3_pp_get_buff(struct page_pool *pp, dma_addr_t *dma_addr, 1421 gfp_t gfp_mask) 1422 { 1423 struct page *page; 1424 1425 page = page_pool_alloc_pages(pp, gfp_mask | __GFP_NOWARN); 1426 if (unlikely(!page)) 1427 return NULL; 1428 1429 *dma_addr = page_pool_get_dma_addr(page) + pp->p.offset; 1430 1431 return page_address(page); 1432 } 1433 1434 static netdev_tx_t 1435 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1436 { 1437 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1438 1439 BUG_ON(skb->queue_mapping > adapter->num_tx_queues); 1440 return vmxnet3_tq_xmit(skb, 1441 &adapter->tx_queue[skb->queue_mapping], 1442 adapter, netdev); 1443 } 1444 1445 1446 static void 1447 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, 1448 struct sk_buff *skb, 1449 union Vmxnet3_GenericDesc *gdesc) 1450 { 1451 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) { 1452 if (gdesc->rcd.v4 && 1453 (le32_to_cpu(gdesc->dword[3]) & 1454 VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) { 1455 skb->ip_summed = CHECKSUM_UNNECESSARY; 1456 if ((le32_to_cpu(gdesc->dword[0]) & 1457 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1458 skb->csum_level = 1; 1459 } 1460 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1461 !(le32_to_cpu(gdesc->dword[0]) & 1462 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1463 WARN_ON_ONCE(gdesc->rcd.frg && 1464 !(le32_to_cpu(gdesc->dword[0]) & 1465 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1466 } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) & 1467 (1 << VMXNET3_RCD_TUC_SHIFT))) { 1468 skb->ip_summed = CHECKSUM_UNNECESSARY; 1469 if ((le32_to_cpu(gdesc->dword[0]) & 1470 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1471 skb->csum_level = 1; 1472 } 1473 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1474 !(le32_to_cpu(gdesc->dword[0]) & 1475 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1476 WARN_ON_ONCE(gdesc->rcd.frg && 1477 !(le32_to_cpu(gdesc->dword[0]) & 1478 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1479 } else { 1480 if (gdesc->rcd.csum) { 1481 skb->csum = htons(gdesc->rcd.csum); 1482 skb->ip_summed = CHECKSUM_PARTIAL; 1483 } else { 1484 skb_checksum_none_assert(skb); 1485 } 1486 } 1487 } else { 1488 skb_checksum_none_assert(skb); 1489 } 1490 } 1491 1492 1493 static void 1494 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd, 1495 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter) 1496 { 1497 rq->stats.drop_err++; 1498 if (!rcd->fcs) 1499 rq->stats.drop_fcs++; 1500 1501 rq->stats.drop_total++; 1502 1503 /* 1504 * We do not unmap and chain the rx buffer to the skb. 1505 * We basically pretend this buffer is not used and will be recycled 1506 * by vmxnet3_rq_alloc_rx_buf() 1507 */ 1508 1509 /* 1510 * ctx->skb may be NULL if this is the first and the only one 1511 * desc for the pkt 1512 */ 1513 if (ctx->skb) 1514 dev_kfree_skb_irq(ctx->skb); 1515 1516 ctx->skb = NULL; 1517 } 1518 1519 1520 static u32 1521 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, 1522 union Vmxnet3_GenericDesc *gdesc) 1523 { 1524 u32 hlen, maplen; 1525 union { 1526 void *ptr; 1527 struct ethhdr *eth; 1528 struct vlan_ethhdr *veth; 1529 struct iphdr *ipv4; 1530 struct ipv6hdr *ipv6; 1531 struct tcphdr *tcp; 1532 } hdr; 1533 BUG_ON(gdesc->rcd.tcp == 0); 1534 1535 maplen = skb_headlen(skb); 1536 if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen)) 1537 return 0; 1538 1539 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 1540 skb->protocol == cpu_to_be16(ETH_P_8021AD)) 1541 hlen = sizeof(struct vlan_ethhdr); 1542 else 1543 hlen = sizeof(struct ethhdr); 1544 1545 hdr.eth = eth_hdr(skb); 1546 if (gdesc->rcd.v4) { 1547 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) && 1548 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)); 1549 hdr.ptr += hlen; 1550 BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP); 1551 hlen = hdr.ipv4->ihl << 2; 1552 hdr.ptr += hdr.ipv4->ihl << 2; 1553 } else if (gdesc->rcd.v6) { 1554 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) && 1555 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)); 1556 hdr.ptr += hlen; 1557 /* Use an estimated value, since we also need to handle 1558 * TSO case. 1559 */ 1560 if (hdr.ipv6->nexthdr != IPPROTO_TCP) 1561 return sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 1562 hlen = sizeof(struct ipv6hdr); 1563 hdr.ptr += sizeof(struct ipv6hdr); 1564 } else { 1565 /* Non-IP pkt, dont estimate header length */ 1566 return 0; 1567 } 1568 1569 if (hlen + sizeof(struct tcphdr) > maplen) 1570 return 0; 1571 1572 return (hlen + (hdr.tcp->doff << 2)); 1573 } 1574 1575 static int 1576 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, 1577 struct vmxnet3_adapter *adapter, int quota) 1578 { 1579 u32 rxprod_reg[2] = { 1580 adapter->rx_prod_offset, adapter->rx_prod2_offset 1581 }; 1582 u32 num_pkts = 0; 1583 bool skip_page_frags = false; 1584 bool encap_lro = false; 1585 struct Vmxnet3_RxCompDesc *rcd; 1586 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; 1587 u16 segCnt = 0, mss = 0; 1588 int comp_offset, fill_offset; 1589 #ifdef __BIG_ENDIAN_BITFIELD 1590 struct Vmxnet3_RxDesc rxCmdDesc; 1591 struct Vmxnet3_RxCompDesc rxComp; 1592 #endif 1593 bool need_flush = false; 1594 1595 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, 1596 &rxComp); 1597 while (rcd->gen == rq->comp_ring.gen) { 1598 struct vmxnet3_rx_buf_info *rbi; 1599 struct sk_buff *skb, *new_skb = NULL; 1600 struct page *new_page = NULL; 1601 dma_addr_t new_dma_addr; 1602 int num_to_alloc; 1603 struct Vmxnet3_RxDesc *rxd; 1604 u32 idx, ring_idx; 1605 struct vmxnet3_cmd_ring *ring = NULL; 1606 if (num_pkts >= quota) { 1607 /* we may stop even before we see the EOP desc of 1608 * the current pkt 1609 */ 1610 break; 1611 } 1612 1613 /* Prevent any rcd field from being (speculatively) read before 1614 * rcd->gen is read. 1615 */ 1616 dma_rmb(); 1617 1618 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 && 1619 rcd->rqID != rq->dataRingQid); 1620 idx = rcd->rxdIdx; 1621 ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID); 1622 ring = rq->rx_ring + ring_idx; 1623 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1624 &rxCmdDesc); 1625 rbi = rq->buf_info[ring_idx] + idx; 1626 1627 BUG_ON(rxd->addr != rbi->dma_addr || 1628 rxd->len != rbi->len); 1629 1630 if (unlikely(rcd->eop && rcd->err)) { 1631 vmxnet3_rx_error(rq, rcd, ctx, adapter); 1632 goto rcd_done; 1633 } 1634 1635 if (rcd->sop && rcd->eop && vmxnet3_xdp_enabled(adapter)) { 1636 struct sk_buff *skb_xdp_pass; 1637 int act; 1638 1639 if (VMXNET3_RX_DATA_RING(adapter, rcd->rqID)) { 1640 ctx->skb = NULL; 1641 goto skip_xdp; /* Handle it later. */ 1642 } 1643 1644 if (rbi->buf_type != VMXNET3_RX_BUF_XDP) 1645 goto rcd_done; 1646 1647 act = vmxnet3_process_xdp(adapter, rq, rcd, rbi, rxd, 1648 &skb_xdp_pass); 1649 if (act == XDP_PASS) { 1650 ctx->skb = skb_xdp_pass; 1651 goto sop_done; 1652 } 1653 ctx->skb = NULL; 1654 need_flush |= act == XDP_REDIRECT; 1655 1656 goto rcd_done; 1657 } 1658 skip_xdp: 1659 1660 if (rcd->sop) { /* first buf of the pkt */ 1661 bool rxDataRingUsed; 1662 u16 len; 1663 1664 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD || 1665 (rcd->rqID != rq->qid && 1666 rcd->rqID != rq->dataRingQid)); 1667 1668 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB && 1669 rbi->buf_type != VMXNET3_RX_BUF_XDP); 1670 BUG_ON(ctx->skb != NULL || rbi->skb == NULL); 1671 1672 if (unlikely(rcd->len == 0)) { 1673 /* Pretend the rx buffer is skipped. */ 1674 BUG_ON(!(rcd->sop && rcd->eop)); 1675 netdev_dbg(adapter->netdev, 1676 "rxRing[%u][%u] 0 length\n", 1677 ring_idx, idx); 1678 goto rcd_done; 1679 } 1680 1681 skip_page_frags = false; 1682 ctx->skb = rbi->skb; 1683 1684 if (rq->rx_ts_desc_size != 0 && rcd->ext2) { 1685 struct Vmxnet3_RxTSDesc *ts_rxd; 1686 1687 ts_rxd = (struct Vmxnet3_RxTSDesc *)((u8 *)rq->ts_ring.base + 1688 idx * rq->rx_ts_desc_size); 1689 ts_rxd->ts.tsData = vmxnet3_get_cycles(VMXNET3_PMC_PSEUDO_TSC); 1690 ts_rxd->ts.tsi = 1; 1691 } 1692 1693 rxDataRingUsed = 1694 VMXNET3_RX_DATA_RING(adapter, rcd->rqID); 1695 len = rxDataRingUsed ? rcd->len : rbi->len; 1696 1697 if (rxDataRingUsed && vmxnet3_xdp_enabled(adapter)) { 1698 struct sk_buff *skb_xdp_pass; 1699 size_t sz; 1700 int act; 1701 1702 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1703 act = vmxnet3_process_xdp_small(adapter, rq, 1704 &rq->data_ring.base[sz], 1705 rcd->len, 1706 &skb_xdp_pass); 1707 if (act == XDP_PASS) { 1708 ctx->skb = skb_xdp_pass; 1709 goto sop_done; 1710 } 1711 need_flush |= act == XDP_REDIRECT; 1712 1713 goto rcd_done; 1714 } 1715 new_skb = netdev_alloc_skb_ip_align(adapter->netdev, 1716 len); 1717 if (new_skb == NULL) { 1718 /* Skb allocation failed, do not handover this 1719 * skb to stack. Reuse it. Drop the existing pkt 1720 */ 1721 rq->stats.rx_buf_alloc_failure++; 1722 ctx->skb = NULL; 1723 rq->stats.drop_total++; 1724 skip_page_frags = true; 1725 goto rcd_done; 1726 } 1727 1728 if (rxDataRingUsed && adapter->rxdataring_enabled) { 1729 size_t sz; 1730 1731 BUG_ON(rcd->len > rq->data_ring.desc_size); 1732 1733 ctx->skb = new_skb; 1734 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1735 memcpy(new_skb->data, 1736 &rq->data_ring.base[sz], rcd->len); 1737 } else { 1738 ctx->skb = rbi->skb; 1739 1740 new_dma_addr = 1741 dma_map_single(&adapter->pdev->dev, 1742 new_skb->data, rbi->len, 1743 DMA_FROM_DEVICE); 1744 if (dma_mapping_error(&adapter->pdev->dev, 1745 new_dma_addr)) { 1746 dev_kfree_skb(new_skb); 1747 /* Skb allocation failed, do not 1748 * handover this skb to stack. Reuse 1749 * it. Drop the existing pkt. 1750 */ 1751 rq->stats.rx_buf_alloc_failure++; 1752 ctx->skb = NULL; 1753 rq->stats.drop_total++; 1754 skip_page_frags = true; 1755 goto rcd_done; 1756 } 1757 1758 dma_unmap_single(&adapter->pdev->dev, 1759 rbi->dma_addr, 1760 rbi->len, 1761 DMA_FROM_DEVICE); 1762 1763 /* Immediate refill */ 1764 rbi->skb = new_skb; 1765 rbi->dma_addr = new_dma_addr; 1766 rxd->addr = cpu_to_le64(rbi->dma_addr); 1767 rxd->len = rbi->len; 1768 } 1769 1770 skb_record_rx_queue(ctx->skb, rq->qid); 1771 skb_put(ctx->skb, rcd->len); 1772 1773 if (VMXNET3_VERSION_GE_2(adapter) && 1774 rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) { 1775 struct Vmxnet3_RxCompDescExt *rcdlro; 1776 union Vmxnet3_GenericDesc *gdesc; 1777 1778 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd; 1779 gdesc = (union Vmxnet3_GenericDesc *)rcd; 1780 1781 segCnt = rcdlro->segCnt; 1782 WARN_ON_ONCE(segCnt == 0); 1783 mss = rcdlro->mss; 1784 if (unlikely(segCnt <= 1)) 1785 segCnt = 0; 1786 encap_lro = (le32_to_cpu(gdesc->dword[0]) & 1787 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)); 1788 } else { 1789 segCnt = 0; 1790 } 1791 } else { 1792 BUG_ON(ctx->skb == NULL && !skip_page_frags); 1793 1794 /* non SOP buffer must be type 1 in most cases */ 1795 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE); 1796 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY); 1797 1798 /* If an sop buffer was dropped, skip all 1799 * following non-sop fragments. They will be reused. 1800 */ 1801 if (skip_page_frags) 1802 goto rcd_done; 1803 1804 if (rcd->len) { 1805 new_page = alloc_page(GFP_ATOMIC); 1806 /* Replacement page frag could not be allocated. 1807 * Reuse this page. Drop the pkt and free the 1808 * skb which contained this page as a frag. Skip 1809 * processing all the following non-sop frags. 1810 */ 1811 if (unlikely(!new_page)) { 1812 rq->stats.rx_buf_alloc_failure++; 1813 dev_kfree_skb(ctx->skb); 1814 ctx->skb = NULL; 1815 skip_page_frags = true; 1816 goto rcd_done; 1817 } 1818 new_dma_addr = dma_map_page(&adapter->pdev->dev, 1819 new_page, 1820 0, PAGE_SIZE, 1821 DMA_FROM_DEVICE); 1822 if (dma_mapping_error(&adapter->pdev->dev, 1823 new_dma_addr)) { 1824 put_page(new_page); 1825 rq->stats.rx_buf_alloc_failure++; 1826 dev_kfree_skb(ctx->skb); 1827 ctx->skb = NULL; 1828 skip_page_frags = true; 1829 goto rcd_done; 1830 } 1831 1832 dma_unmap_page(&adapter->pdev->dev, 1833 rbi->dma_addr, rbi->len, 1834 DMA_FROM_DEVICE); 1835 1836 vmxnet3_append_frag(ctx->skb, rcd, rbi); 1837 1838 /* Immediate refill */ 1839 rbi->page = new_page; 1840 rbi->dma_addr = new_dma_addr; 1841 rxd->addr = cpu_to_le64(rbi->dma_addr); 1842 rxd->len = rbi->len; 1843 } 1844 } 1845 1846 1847 sop_done: 1848 skb = ctx->skb; 1849 if (rcd->eop) { 1850 u32 mtu = adapter->netdev->mtu; 1851 skb->len += skb->data_len; 1852 1853 #ifdef VMXNET3_RSS 1854 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE && 1855 (adapter->netdev->features & NETIF_F_RXHASH)) { 1856 enum pkt_hash_types hash_type; 1857 1858 switch (rcd->rssType) { 1859 case VMXNET3_RCD_RSS_TYPE_IPV4: 1860 case VMXNET3_RCD_RSS_TYPE_IPV6: 1861 hash_type = PKT_HASH_TYPE_L3; 1862 break; 1863 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 1864 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 1865 case VMXNET3_RCD_RSS_TYPE_UDPIPV4: 1866 case VMXNET3_RCD_RSS_TYPE_UDPIPV6: 1867 hash_type = PKT_HASH_TYPE_L4; 1868 break; 1869 default: 1870 hash_type = PKT_HASH_TYPE_L3; 1871 break; 1872 } 1873 skb_set_hash(skb, 1874 le32_to_cpu(rcd->rssHash), 1875 hash_type); 1876 } 1877 #endif 1878 vmxnet3_rx_csum(adapter, skb, 1879 (union Vmxnet3_GenericDesc *)rcd); 1880 skb->protocol = eth_type_trans(skb, adapter->netdev); 1881 if ((!rcd->tcp && !encap_lro) || 1882 !(adapter->netdev->features & NETIF_F_LRO)) 1883 goto not_lro; 1884 1885 if (segCnt != 0 && mss != 0) { 1886 skb_shinfo(skb)->gso_type = rcd->v4 ? 1887 SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1888 skb_shinfo(skb)->gso_size = mss; 1889 skb_shinfo(skb)->gso_segs = segCnt; 1890 } else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) { 1891 u32 hlen; 1892 1893 hlen = vmxnet3_get_hdr_len(adapter, skb, 1894 (union Vmxnet3_GenericDesc *)rcd); 1895 if (hlen == 0) 1896 goto not_lro; 1897 1898 skb_shinfo(skb)->gso_type = 1899 rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1900 if (segCnt != 0) { 1901 skb_shinfo(skb)->gso_segs = segCnt; 1902 skb_shinfo(skb)->gso_size = 1903 DIV_ROUND_UP(skb->len - 1904 hlen, segCnt); 1905 } else { 1906 skb_shinfo(skb)->gso_size = mtu - hlen; 1907 } 1908 } 1909 not_lro: 1910 if (unlikely(rcd->ts)) 1911 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci); 1912 1913 /* Use GRO callback if UPT is enabled */ 1914 if ((adapter->netdev->features & NETIF_F_LRO) && 1915 !rq->shared->updateRxProd) 1916 netif_receive_skb(skb); 1917 else 1918 napi_gro_receive(&rq->napi, skb); 1919 1920 ctx->skb = NULL; 1921 encap_lro = false; 1922 num_pkts++; 1923 } 1924 1925 rcd_done: 1926 /* device may have skipped some rx descs */ 1927 ring = rq->rx_ring + ring_idx; 1928 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 1929 1930 comp_offset = vmxnet3_cmd_ring_desc_avail(ring); 1931 fill_offset = (idx > ring->next2fill ? 0 : ring->size) + 1932 idx - ring->next2fill - 1; 1933 if (!ring->isOutOfOrder || fill_offset >= comp_offset) 1934 ring->next2comp = idx; 1935 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring); 1936 1937 /* Ensure that the writes to rxd->gen bits will be observed 1938 * after all other writes to rxd objects. 1939 */ 1940 dma_wmb(); 1941 1942 while (num_to_alloc) { 1943 rbi = rq->buf_info[ring_idx] + ring->next2fill; 1944 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP))) 1945 goto refill_buf; 1946 if (ring_idx == 0) { 1947 /* ring0 Type1 buffers can get skipped; re-fill them */ 1948 if (rbi->buf_type != VMXNET3_RX_BUF_SKB) 1949 goto refill_buf; 1950 } 1951 if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) { 1952 refill_buf: 1953 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd, 1954 &rxCmdDesc); 1955 WARN_ON(!rxd->addr); 1956 1957 /* Recv desc is ready to be used by the device */ 1958 rxd->gen = ring->gen; 1959 vmxnet3_cmd_ring_adv_next2fill(ring); 1960 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 1961 num_to_alloc--; 1962 } else { 1963 /* rx completion hasn't occurred */ 1964 ring->isOutOfOrder = 1; 1965 break; 1966 } 1967 } 1968 1969 if (num_to_alloc == 0) { 1970 ring->isOutOfOrder = 0; 1971 } 1972 1973 /* if needed, update the register */ 1974 if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) { 1975 VMXNET3_WRITE_BAR0_REG(adapter, 1976 rxprod_reg[ring_idx] + rq->qid * 8, 1977 ring->next2fill); 1978 } 1979 1980 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring); 1981 vmxnet3_getRxComp(rcd, 1982 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp); 1983 } 1984 if (need_flush) 1985 xdp_do_flush(); 1986 1987 return num_pkts; 1988 } 1989 1990 1991 static void 1992 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, 1993 struct vmxnet3_adapter *adapter) 1994 { 1995 u32 i, ring_idx; 1996 struct Vmxnet3_RxDesc *rxd; 1997 1998 /* ring has already been cleaned up */ 1999 if (!rq->rx_ring[0].base) 2000 return; 2001 2002 for (ring_idx = 0; ring_idx < 2; ring_idx++) { 2003 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) { 2004 struct vmxnet3_rx_buf_info *rbi; 2005 #ifdef __BIG_ENDIAN_BITFIELD 2006 struct Vmxnet3_RxDesc rxDesc; 2007 #endif 2008 2009 rbi = &rq->buf_info[ring_idx][i]; 2010 vmxnet3_getRxDesc(rxd, 2011 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc); 2012 2013 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 2014 rbi->page && rbi->buf_type == VMXNET3_RX_BUF_XDP) { 2015 page_pool_recycle_direct(rq->page_pool, 2016 rbi->page); 2017 rbi->page = NULL; 2018 } else if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 2019 rbi->skb) { 2020 dma_unmap_single(&adapter->pdev->dev, rxd->addr, 2021 rxd->len, DMA_FROM_DEVICE); 2022 dev_kfree_skb(rbi->skb); 2023 rbi->skb = NULL; 2024 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY && 2025 rbi->page) { 2026 dma_unmap_page(&adapter->pdev->dev, rxd->addr, 2027 rxd->len, DMA_FROM_DEVICE); 2028 put_page(rbi->page); 2029 rbi->page = NULL; 2030 } 2031 } 2032 2033 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN; 2034 rq->rx_ring[ring_idx].next2fill = 2035 rq->rx_ring[ring_idx].next2comp = 0; 2036 } 2037 2038 rq->comp_ring.gen = VMXNET3_INIT_GEN; 2039 rq->comp_ring.next2proc = 0; 2040 2041 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) 2042 xdp_rxq_info_unreg(&rq->xdp_rxq); 2043 page_pool_destroy(rq->page_pool); 2044 rq->page_pool = NULL; 2045 } 2046 2047 2048 static void 2049 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter) 2050 { 2051 int i; 2052 2053 for (i = 0; i < adapter->num_rx_queues; i++) 2054 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter); 2055 rcu_assign_pointer(adapter->xdp_bpf_prog, NULL); 2056 } 2057 2058 2059 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 2060 struct vmxnet3_adapter *adapter) 2061 { 2062 int i; 2063 int j; 2064 2065 /* all rx buffers must have already been freed */ 2066 for (i = 0; i < 2; i++) { 2067 if (rq->buf_info[i]) { 2068 for (j = 0; j < rq->rx_ring[i].size; j++) 2069 BUG_ON(rq->buf_info[i][j].page != NULL); 2070 } 2071 } 2072 2073 2074 for (i = 0; i < 2; i++) { 2075 if (rq->rx_ring[i].base) { 2076 dma_free_coherent(&adapter->pdev->dev, 2077 rq->rx_ring[i].size 2078 * sizeof(struct Vmxnet3_RxDesc), 2079 rq->rx_ring[i].base, 2080 rq->rx_ring[i].basePA); 2081 rq->rx_ring[i].base = NULL; 2082 } 2083 } 2084 2085 if (rq->data_ring.base) { 2086 dma_free_coherent(&adapter->pdev->dev, 2087 rq->rx_ring[0].size * rq->data_ring.desc_size, 2088 rq->data_ring.base, rq->data_ring.basePA); 2089 rq->data_ring.base = NULL; 2090 } 2091 2092 if (rq->ts_ring.base) { 2093 dma_free_coherent(&adapter->pdev->dev, 2094 rq->rx_ring[0].size * rq->rx_ts_desc_size, 2095 rq->ts_ring.base, rq->ts_ring.basePA); 2096 rq->ts_ring.base = NULL; 2097 } 2098 2099 if (rq->comp_ring.base) { 2100 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size 2101 * sizeof(struct Vmxnet3_RxCompDesc), 2102 rq->comp_ring.base, rq->comp_ring.basePA); 2103 rq->comp_ring.base = NULL; 2104 } 2105 2106 kfree(rq->buf_info[0]); 2107 rq->buf_info[0] = NULL; 2108 rq->buf_info[1] = NULL; 2109 } 2110 2111 static void 2112 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter) 2113 { 2114 int i; 2115 2116 for (i = 0; i < adapter->num_rx_queues; i++) { 2117 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2118 2119 if (rq->data_ring.base) { 2120 dma_free_coherent(&adapter->pdev->dev, 2121 (rq->rx_ring[0].size * 2122 rq->data_ring.desc_size), 2123 rq->data_ring.base, 2124 rq->data_ring.basePA); 2125 rq->data_ring.base = NULL; 2126 } 2127 rq->data_ring.desc_size = 0; 2128 } 2129 } 2130 2131 static int 2132 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq, 2133 struct vmxnet3_adapter *adapter) 2134 { 2135 int i, err; 2136 2137 /* initialize buf_info */ 2138 for (i = 0; i < rq->rx_ring[0].size; i++) { 2139 2140 /* 1st buf for a pkt is skbuff or xdp page */ 2141 if (i % adapter->rx_buf_per_pkt == 0) { 2142 rq->buf_info[0][i].buf_type = vmxnet3_xdp_enabled(adapter) ? 2143 VMXNET3_RX_BUF_XDP : 2144 VMXNET3_RX_BUF_SKB; 2145 rq->buf_info[0][i].len = adapter->skb_buf_size; 2146 } else { /* subsequent bufs for a pkt is frag */ 2147 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE; 2148 rq->buf_info[0][i].len = PAGE_SIZE; 2149 } 2150 } 2151 for (i = 0; i < rq->rx_ring[1].size; i++) { 2152 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE; 2153 rq->buf_info[1][i].len = PAGE_SIZE; 2154 } 2155 2156 /* reset internal state and allocate buffers for both rings */ 2157 for (i = 0; i < 2; i++) { 2158 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0; 2159 2160 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size * 2161 sizeof(struct Vmxnet3_RxDesc)); 2162 rq->rx_ring[i].gen = VMXNET3_INIT_GEN; 2163 rq->rx_ring[i].isOutOfOrder = 0; 2164 } 2165 2166 err = vmxnet3_create_pp(adapter, rq, 2167 rq->rx_ring[0].size + rq->rx_ring[1].size); 2168 if (err) 2169 return err; 2170 2171 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1, 2172 adapter) == 0) { 2173 xdp_rxq_info_unreg(&rq->xdp_rxq); 2174 page_pool_destroy(rq->page_pool); 2175 rq->page_pool = NULL; 2176 2177 /* at least has 1 rx buffer for the 1st ring */ 2178 return -ENOMEM; 2179 } 2180 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter); 2181 2182 if (rq->ts_ring.base) 2183 memset(rq->ts_ring.base, 0, 2184 rq->rx_ring[0].size * rq->rx_ts_desc_size); 2185 2186 /* reset the comp ring */ 2187 rq->comp_ring.next2proc = 0; 2188 memset(rq->comp_ring.base, 0, rq->comp_ring.size * 2189 sizeof(struct Vmxnet3_RxCompDesc)); 2190 rq->comp_ring.gen = VMXNET3_INIT_GEN; 2191 2192 /* reset rxctx */ 2193 rq->rx_ctx.skb = NULL; 2194 2195 /* stats are not reset */ 2196 return 0; 2197 } 2198 2199 2200 static int 2201 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter) 2202 { 2203 int i, err = 0; 2204 2205 for (i = 0; i < adapter->num_rx_queues; i++) { 2206 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter); 2207 if (unlikely(err)) { 2208 dev_err(&adapter->netdev->dev, "%s: failed to " 2209 "initialize rx queue%i\n", 2210 adapter->netdev->name, i); 2211 break; 2212 } 2213 } 2214 return err; 2215 2216 } 2217 2218 2219 static int 2220 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 2221 { 2222 int i; 2223 size_t sz; 2224 struct vmxnet3_rx_buf_info *bi; 2225 2226 for (i = 0; i < 2; i++) { 2227 2228 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc); 2229 rq->rx_ring[i].base = dma_alloc_coherent( 2230 &adapter->pdev->dev, sz, 2231 &rq->rx_ring[i].basePA, 2232 GFP_KERNEL); 2233 if (!rq->rx_ring[i].base) { 2234 netdev_err(adapter->netdev, 2235 "failed to allocate rx ring %d\n", i); 2236 goto err; 2237 } 2238 } 2239 2240 if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) { 2241 sz = rq->rx_ring[0].size * rq->data_ring.desc_size; 2242 rq->data_ring.base = 2243 dma_alloc_coherent(&adapter->pdev->dev, sz, 2244 &rq->data_ring.basePA, 2245 GFP_KERNEL); 2246 if (!rq->data_ring.base) { 2247 netdev_err(adapter->netdev, 2248 "rx data ring will be disabled\n"); 2249 adapter->rxdataring_enabled = false; 2250 } 2251 } else { 2252 rq->data_ring.base = NULL; 2253 rq->data_ring.desc_size = 0; 2254 } 2255 2256 if (rq->rx_ts_desc_size != 0) { 2257 sz = rq->rx_ring[0].size * rq->rx_ts_desc_size; 2258 rq->ts_ring.base = 2259 dma_alloc_coherent(&adapter->pdev->dev, sz, 2260 &rq->ts_ring.basePA, 2261 GFP_KERNEL); 2262 if (!rq->ts_ring.base) { 2263 netdev_err(adapter->netdev, 2264 "rx ts ring will be disabled\n"); 2265 rq->rx_ts_desc_size = 0; 2266 } 2267 } else { 2268 rq->ts_ring.base = NULL; 2269 } 2270 2271 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc); 2272 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz, 2273 &rq->comp_ring.basePA, 2274 GFP_KERNEL); 2275 if (!rq->comp_ring.base) { 2276 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n"); 2277 goto err; 2278 } 2279 2280 bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size, 2281 sizeof(rq->buf_info[0][0]), GFP_KERNEL, 2282 dev_to_node(&adapter->pdev->dev)); 2283 if (!bi) 2284 goto err; 2285 2286 rq->buf_info[0] = bi; 2287 rq->buf_info[1] = bi + rq->rx_ring[0].size; 2288 2289 return 0; 2290 2291 err: 2292 vmxnet3_rq_destroy(rq, adapter); 2293 return -ENOMEM; 2294 } 2295 2296 2297 int 2298 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter) 2299 { 2300 int i, err = 0; 2301 2302 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 2303 2304 for (i = 0; i < adapter->num_rx_queues; i++) { 2305 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter); 2306 if (unlikely(err)) { 2307 dev_err(&adapter->netdev->dev, 2308 "%s: failed to create rx queue%i\n", 2309 adapter->netdev->name, i); 2310 goto err_out; 2311 } 2312 } 2313 2314 if (!adapter->rxdataring_enabled) 2315 vmxnet3_rq_destroy_all_rxdataring(adapter); 2316 2317 return err; 2318 err_out: 2319 vmxnet3_rq_destroy_all(adapter); 2320 return err; 2321 2322 } 2323 2324 /* Multiple queue aware polling function for tx and rx */ 2325 2326 static int 2327 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 2328 { 2329 int rcd_done = 0, i; 2330 if (unlikely(adapter->shared->ecr)) 2331 vmxnet3_process_events(adapter); 2332 for (i = 0; i < adapter->num_tx_queues; i++) 2333 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter); 2334 2335 for (i = 0; i < adapter->num_rx_queues; i++) 2336 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i], 2337 adapter, budget); 2338 return rcd_done; 2339 } 2340 2341 2342 static int 2343 vmxnet3_poll(struct napi_struct *napi, int budget) 2344 { 2345 struct vmxnet3_rx_queue *rx_queue = container_of(napi, 2346 struct vmxnet3_rx_queue, napi); 2347 int rxd_done; 2348 2349 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget); 2350 2351 if (rxd_done < budget) { 2352 napi_complete_done(napi, rxd_done); 2353 vmxnet3_enable_all_intrs(rx_queue->adapter); 2354 } 2355 return rxd_done; 2356 } 2357 2358 /* 2359 * NAPI polling function for MSI-X mode with multiple Rx queues 2360 * Returns the # of the NAPI credit consumed (# of rx descriptors processed) 2361 */ 2362 2363 static int 2364 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget) 2365 { 2366 struct vmxnet3_rx_queue *rq = container_of(napi, 2367 struct vmxnet3_rx_queue, napi); 2368 struct vmxnet3_adapter *adapter = rq->adapter; 2369 int rxd_done; 2370 2371 /* When sharing interrupt with corresponding tx queue, process 2372 * tx completions in that queue as well 2373 */ 2374 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { 2375 struct vmxnet3_tx_queue *tq = 2376 &adapter->tx_queue[rq - adapter->rx_queue]; 2377 vmxnet3_tq_tx_complete(tq, adapter); 2378 } 2379 2380 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget); 2381 2382 if (rxd_done < budget) { 2383 napi_complete_done(napi, rxd_done); 2384 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx); 2385 } 2386 return rxd_done; 2387 } 2388 2389 2390 #ifdef CONFIG_PCI_MSI 2391 2392 /* 2393 * Handle completion interrupts on tx queues 2394 * Returns whether or not the intr is handled 2395 */ 2396 2397 static irqreturn_t 2398 vmxnet3_msix_tx(int irq, void *data) 2399 { 2400 struct vmxnet3_tx_queue *tq = data; 2401 struct vmxnet3_adapter *adapter = tq->adapter; 2402 2403 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2404 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx); 2405 2406 /* Handle the case where only one irq is allocate for all tx queues */ 2407 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2408 int i; 2409 for (i = 0; i < adapter->num_tx_queues; i++) { 2410 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i]; 2411 vmxnet3_tq_tx_complete(txq, adapter); 2412 } 2413 } else { 2414 vmxnet3_tq_tx_complete(tq, adapter); 2415 } 2416 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx); 2417 2418 return IRQ_HANDLED; 2419 } 2420 2421 2422 /* 2423 * Handle completion interrupts on rx queues. Returns whether or not the 2424 * intr is handled 2425 */ 2426 2427 static irqreturn_t 2428 vmxnet3_msix_rx(int irq, void *data) 2429 { 2430 struct vmxnet3_rx_queue *rq = data; 2431 struct vmxnet3_adapter *adapter = rq->adapter; 2432 2433 /* disable intr if needed */ 2434 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2435 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx); 2436 napi_schedule(&rq->napi); 2437 2438 return IRQ_HANDLED; 2439 } 2440 2441 /* 2442 *---------------------------------------------------------------------------- 2443 * 2444 * vmxnet3_msix_event -- 2445 * 2446 * vmxnet3 msix event intr handler 2447 * 2448 * Result: 2449 * whether or not the intr is handled 2450 * 2451 *---------------------------------------------------------------------------- 2452 */ 2453 2454 static irqreturn_t 2455 vmxnet3_msix_event(int irq, void *data) 2456 { 2457 struct net_device *dev = data; 2458 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2459 2460 /* disable intr if needed */ 2461 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2462 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx); 2463 2464 if (adapter->shared->ecr) 2465 vmxnet3_process_events(adapter); 2466 2467 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx); 2468 2469 return IRQ_HANDLED; 2470 } 2471 2472 #endif /* CONFIG_PCI_MSI */ 2473 2474 2475 /* Interrupt handler for vmxnet3 */ 2476 static irqreturn_t 2477 vmxnet3_intr(int irq, void *dev_id) 2478 { 2479 struct net_device *dev = dev_id; 2480 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2481 2482 if (adapter->intr.type == VMXNET3_IT_INTX) { 2483 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 2484 if (unlikely(icr == 0)) 2485 /* not ours */ 2486 return IRQ_NONE; 2487 } 2488 2489 2490 /* disable intr if needed */ 2491 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2492 vmxnet3_disable_all_intrs(adapter); 2493 2494 napi_schedule(&adapter->rx_queue[0].napi); 2495 2496 return IRQ_HANDLED; 2497 } 2498 2499 #ifdef CONFIG_NET_POLL_CONTROLLER 2500 2501 /* netpoll callback. */ 2502 static void 2503 vmxnet3_netpoll(struct net_device *netdev) 2504 { 2505 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2506 2507 switch (adapter->intr.type) { 2508 #ifdef CONFIG_PCI_MSI 2509 case VMXNET3_IT_MSIX: { 2510 int i; 2511 for (i = 0; i < adapter->num_rx_queues; i++) 2512 vmxnet3_msix_rx(0, &adapter->rx_queue[i]); 2513 break; 2514 } 2515 #endif 2516 case VMXNET3_IT_MSI: 2517 default: 2518 vmxnet3_intr(0, adapter->netdev); 2519 break; 2520 } 2521 2522 } 2523 #endif /* CONFIG_NET_POLL_CONTROLLER */ 2524 2525 static int 2526 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 2527 { 2528 struct vmxnet3_intr *intr = &adapter->intr; 2529 int err = 0, i; 2530 int vector = 0; 2531 2532 #ifdef CONFIG_PCI_MSI 2533 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2534 for (i = 0; i < adapter->num_tx_queues; i++) { 2535 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2536 sprintf(adapter->tx_queue[i].name, "%s-tx-%d", 2537 adapter->netdev->name, vector); 2538 err = request_irq( 2539 intr->msix_entries[vector].vector, 2540 vmxnet3_msix_tx, 0, 2541 adapter->tx_queue[i].name, 2542 &adapter->tx_queue[i]); 2543 } else { 2544 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d", 2545 adapter->netdev->name, vector); 2546 } 2547 if (err) { 2548 dev_err(&adapter->netdev->dev, 2549 "Failed to request irq for MSIX, %s, " 2550 "error %d\n", 2551 adapter->tx_queue[i].name, err); 2552 return err; 2553 } 2554 2555 /* Handle the case where only 1 MSIx was allocated for 2556 * all tx queues */ 2557 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2558 for (; i < adapter->num_tx_queues; i++) 2559 adapter->tx_queue[i].comp_ring.intr_idx 2560 = vector; 2561 vector++; 2562 break; 2563 } else { 2564 adapter->tx_queue[i].comp_ring.intr_idx 2565 = vector++; 2566 } 2567 } 2568 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) 2569 vector = 0; 2570 2571 for (i = 0; i < adapter->num_rx_queues; i++) { 2572 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) 2573 sprintf(adapter->rx_queue[i].name, "%s-rx-%d", 2574 adapter->netdev->name, vector); 2575 else 2576 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d", 2577 adapter->netdev->name, vector); 2578 err = request_irq(intr->msix_entries[vector].vector, 2579 vmxnet3_msix_rx, 0, 2580 adapter->rx_queue[i].name, 2581 &(adapter->rx_queue[i])); 2582 if (err) { 2583 netdev_err(adapter->netdev, 2584 "Failed to request irq for MSIX, " 2585 "%s, error %d\n", 2586 adapter->rx_queue[i].name, err); 2587 return err; 2588 } 2589 2590 adapter->rx_queue[i].comp_ring.intr_idx = vector++; 2591 } 2592 2593 sprintf(intr->event_msi_vector_name, "%s-event-%d", 2594 adapter->netdev->name, vector); 2595 err = request_irq(intr->msix_entries[vector].vector, 2596 vmxnet3_msix_event, 0, 2597 intr->event_msi_vector_name, adapter->netdev); 2598 intr->event_intr_idx = vector; 2599 2600 } else if (intr->type == VMXNET3_IT_MSI) { 2601 adapter->num_rx_queues = 1; 2602 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 2603 adapter->netdev->name, adapter->netdev); 2604 } else { 2605 #endif 2606 adapter->num_rx_queues = 1; 2607 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 2608 IRQF_SHARED, adapter->netdev->name, 2609 adapter->netdev); 2610 #ifdef CONFIG_PCI_MSI 2611 } 2612 #endif 2613 intr->num_intrs = vector + 1; 2614 if (err) { 2615 netdev_err(adapter->netdev, 2616 "Failed to request irq (intr type:%d), error %d\n", 2617 intr->type, err); 2618 } else { 2619 /* Number of rx queues will not change after this */ 2620 for (i = 0; i < adapter->num_rx_queues; i++) { 2621 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2622 rq->qid = i; 2623 rq->qid2 = i + adapter->num_rx_queues; 2624 rq->dataRingQid = i + 2 * adapter->num_rx_queues; 2625 } 2626 2627 /* init our intr settings */ 2628 for (i = 0; i < intr->num_intrs; i++) 2629 intr->mod_levels[i] = UPT1_IML_ADAPTIVE; 2630 if (adapter->intr.type != VMXNET3_IT_MSIX) { 2631 adapter->intr.event_intr_idx = 0; 2632 for (i = 0; i < adapter->num_tx_queues; i++) 2633 adapter->tx_queue[i].comp_ring.intr_idx = 0; 2634 adapter->rx_queue[0].comp_ring.intr_idx = 0; 2635 } 2636 2637 netdev_info(adapter->netdev, 2638 "intr type %u, mode %u, %u vectors allocated\n", 2639 intr->type, intr->mask_mode, intr->num_intrs); 2640 } 2641 2642 return err; 2643 } 2644 2645 2646 static void 2647 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 2648 { 2649 struct vmxnet3_intr *intr = &adapter->intr; 2650 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0); 2651 2652 switch (intr->type) { 2653 #ifdef CONFIG_PCI_MSI 2654 case VMXNET3_IT_MSIX: 2655 { 2656 int i, vector = 0; 2657 2658 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2659 for (i = 0; i < adapter->num_tx_queues; i++) { 2660 free_irq(intr->msix_entries[vector++].vector, 2661 &(adapter->tx_queue[i])); 2662 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) 2663 break; 2664 } 2665 } 2666 2667 for (i = 0; i < adapter->num_rx_queues; i++) { 2668 free_irq(intr->msix_entries[vector++].vector, 2669 &(adapter->rx_queue[i])); 2670 } 2671 2672 free_irq(intr->msix_entries[vector].vector, 2673 adapter->netdev); 2674 BUG_ON(vector >= intr->num_intrs); 2675 break; 2676 } 2677 #endif 2678 case VMXNET3_IT_MSI: 2679 free_irq(adapter->pdev->irq, adapter->netdev); 2680 break; 2681 case VMXNET3_IT_INTX: 2682 free_irq(adapter->pdev->irq, adapter->netdev); 2683 break; 2684 default: 2685 BUG(); 2686 } 2687 } 2688 2689 2690 static void 2691 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter) 2692 { 2693 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2694 u16 vid; 2695 2696 /* allow untagged pkts */ 2697 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0); 2698 2699 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2700 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2701 } 2702 2703 2704 static int 2705 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 2706 { 2707 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2708 2709 if (!(netdev->flags & IFF_PROMISC)) { 2710 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2711 unsigned long flags; 2712 2713 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2714 spin_lock_irqsave(&adapter->cmd_lock, flags); 2715 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2716 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2717 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2718 } 2719 2720 set_bit(vid, adapter->active_vlans); 2721 2722 return 0; 2723 } 2724 2725 2726 static int 2727 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 2728 { 2729 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2730 2731 if (!(netdev->flags & IFF_PROMISC)) { 2732 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2733 unsigned long flags; 2734 2735 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid); 2736 spin_lock_irqsave(&adapter->cmd_lock, flags); 2737 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2738 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2739 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2740 } 2741 2742 clear_bit(vid, adapter->active_vlans); 2743 2744 return 0; 2745 } 2746 2747 2748 static u8 * 2749 vmxnet3_copy_mc(struct net_device *netdev) 2750 { 2751 u8 *buf = NULL; 2752 u32 sz = netdev_mc_count(netdev) * ETH_ALEN; 2753 2754 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */ 2755 if (sz <= 0xffff) { 2756 /* We may be called with BH disabled */ 2757 buf = kmalloc(sz, GFP_ATOMIC); 2758 if (buf) { 2759 struct netdev_hw_addr *ha; 2760 int i = 0; 2761 2762 netdev_for_each_mc_addr(ha, netdev) 2763 memcpy(buf + i++ * ETH_ALEN, ha->addr, 2764 ETH_ALEN); 2765 } 2766 } 2767 return buf; 2768 } 2769 2770 2771 static void 2772 vmxnet3_set_mc(struct net_device *netdev) 2773 { 2774 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2775 unsigned long flags; 2776 struct Vmxnet3_RxFilterConf *rxConf = 2777 &adapter->shared->devRead.rxFilterConf; 2778 u8 *new_table = NULL; 2779 dma_addr_t new_table_pa = 0; 2780 bool new_table_pa_valid = false; 2781 u32 new_mode = VMXNET3_RXM_UCAST; 2782 2783 if (netdev->flags & IFF_PROMISC) { 2784 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2785 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable)); 2786 2787 new_mode |= VMXNET3_RXM_PROMISC; 2788 } else { 2789 vmxnet3_restore_vlan(adapter); 2790 } 2791 2792 if (netdev->flags & IFF_BROADCAST) 2793 new_mode |= VMXNET3_RXM_BCAST; 2794 2795 if (netdev->flags & IFF_ALLMULTI) 2796 new_mode |= VMXNET3_RXM_ALL_MULTI; 2797 else 2798 if (!netdev_mc_empty(netdev)) { 2799 new_table = vmxnet3_copy_mc(netdev); 2800 if (new_table) { 2801 size_t sz = netdev_mc_count(netdev) * ETH_ALEN; 2802 2803 rxConf->mfTableLen = cpu_to_le16(sz); 2804 new_table_pa = dma_map_single( 2805 &adapter->pdev->dev, 2806 new_table, 2807 sz, 2808 DMA_TO_DEVICE); 2809 if (!dma_mapping_error(&adapter->pdev->dev, 2810 new_table_pa)) { 2811 new_mode |= VMXNET3_RXM_MCAST; 2812 new_table_pa_valid = true; 2813 rxConf->mfTablePA = cpu_to_le64( 2814 new_table_pa); 2815 } 2816 } 2817 if (!new_table_pa_valid) { 2818 netdev_info(netdev, 2819 "failed to copy mcast list, setting ALL_MULTI\n"); 2820 new_mode |= VMXNET3_RXM_ALL_MULTI; 2821 } 2822 } 2823 2824 if (!(new_mode & VMXNET3_RXM_MCAST)) { 2825 rxConf->mfTableLen = 0; 2826 rxConf->mfTablePA = 0; 2827 } 2828 2829 spin_lock_irqsave(&adapter->cmd_lock, flags); 2830 if (new_mode != rxConf->rxMode) { 2831 rxConf->rxMode = cpu_to_le32(new_mode); 2832 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2833 VMXNET3_CMD_UPDATE_RX_MODE); 2834 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2835 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2836 } 2837 2838 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2839 VMXNET3_CMD_UPDATE_MAC_FILTERS); 2840 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2841 2842 if (new_table_pa_valid) 2843 dma_unmap_single(&adapter->pdev->dev, new_table_pa, 2844 rxConf->mfTableLen, DMA_TO_DEVICE); 2845 kfree(new_table); 2846 } 2847 2848 void 2849 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter) 2850 { 2851 int i; 2852 2853 for (i = 0; i < adapter->num_rx_queues; i++) 2854 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter); 2855 } 2856 2857 2858 /* 2859 * Set up driver_shared based on settings in adapter. 2860 */ 2861 2862 static void 2863 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter) 2864 { 2865 struct Vmxnet3_DriverShared *shared = adapter->shared; 2866 struct Vmxnet3_DSDevRead *devRead = &shared->devRead; 2867 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 2868 struct Vmxnet3_TxQueueConf *tqc; 2869 struct Vmxnet3_RxQueueConf *rqc; 2870 struct Vmxnet3_TxQueueTSConf *tqtsc; 2871 struct Vmxnet3_RxQueueTSConf *rqtsc; 2872 int i; 2873 2874 memset(shared, 0, sizeof(*shared)); 2875 2876 /* driver settings */ 2877 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC); 2878 devRead->misc.driverInfo.version = cpu_to_le32( 2879 VMXNET3_DRIVER_VERSION_NUM); 2880 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ? 2881 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64); 2882 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 2883 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32( 2884 *((u32 *)&devRead->misc.driverInfo.gos)); 2885 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1); 2886 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1); 2887 2888 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa); 2889 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter)); 2890 2891 /* set up feature flags */ 2892 if (adapter->netdev->features & NETIF_F_RXCSUM) 2893 devRead->misc.uptFeatures |= UPT1_F_RXCSUM; 2894 2895 if (adapter->netdev->features & NETIF_F_LRO) { 2896 devRead->misc.uptFeatures |= UPT1_F_LRO; 2897 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS); 2898 } 2899 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2900 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 2901 2902 if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL | 2903 NETIF_F_GSO_UDP_TUNNEL_CSUM)) 2904 devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD; 2905 2906 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2907 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2908 devRead->misc.queueDescLen = cpu_to_le32( 2909 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 2910 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc)); 2911 2912 /* tx queue settings */ 2913 devRead->misc.numTxQueues = adapter->num_tx_queues; 2914 for (i = 0; i < adapter->num_tx_queues; i++) { 2915 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 2916 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL); 2917 tqc = &adapter->tqd_start[i].conf; 2918 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA); 2919 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA); 2920 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA); 2921 tqc->ddPA = cpu_to_le64(~0ULL); 2922 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size); 2923 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size); 2924 tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size); 2925 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size); 2926 tqc->ddLen = cpu_to_le32(0); 2927 tqc->intrIdx = tq->comp_ring.intr_idx; 2928 if (VMXNET3_VERSION_GE_9(adapter)) { 2929 tqtsc = &adapter->tqd_start[i].tsConf; 2930 tqtsc->txTSRingBasePA = cpu_to_le64(tq->ts_ring.basePA); 2931 tqtsc->txTSRingDescSize = cpu_to_le16(tq->tx_ts_desc_size); 2932 } 2933 } 2934 2935 /* rx queue settings */ 2936 devRead->misc.numRxQueues = adapter->num_rx_queues; 2937 for (i = 0; i < adapter->num_rx_queues; i++) { 2938 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2939 rqc = &adapter->rqd_start[i].conf; 2940 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA); 2941 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA); 2942 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA); 2943 rqc->ddPA = cpu_to_le64(~0ULL); 2944 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size); 2945 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size); 2946 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size); 2947 rqc->ddLen = cpu_to_le32(0); 2948 rqc->intrIdx = rq->comp_ring.intr_idx; 2949 if (VMXNET3_VERSION_GE_3(adapter)) { 2950 rqc->rxDataRingBasePA = 2951 cpu_to_le64(rq->data_ring.basePA); 2952 rqc->rxDataRingDescSize = 2953 cpu_to_le16(rq->data_ring.desc_size); 2954 } 2955 if (VMXNET3_VERSION_GE_9(adapter)) { 2956 rqtsc = &adapter->rqd_start[i].tsConf; 2957 rqtsc->rxTSRingBasePA = cpu_to_le64(rq->ts_ring.basePA); 2958 rqtsc->rxTSRingDescSize = cpu_to_le16(rq->rx_ts_desc_size); 2959 } 2960 } 2961 2962 #ifdef VMXNET3_RSS 2963 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf)); 2964 2965 if (adapter->rss) { 2966 struct UPT1_RSSConf *rssConf = adapter->rss_conf; 2967 2968 devRead->misc.uptFeatures |= UPT1_F_RSS; 2969 devRead->misc.numRxQueues = adapter->num_rx_queues; 2970 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 | 2971 UPT1_RSS_HASH_TYPE_IPV4 | 2972 UPT1_RSS_HASH_TYPE_TCP_IPV6 | 2973 UPT1_RSS_HASH_TYPE_IPV6; 2974 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ; 2975 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE; 2976 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE; 2977 netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey)); 2978 2979 for (i = 0; i < rssConf->indTableSize; i++) 2980 rssConf->indTable[i] = ethtool_rxfh_indir_default( 2981 i, adapter->num_rx_queues); 2982 2983 devRead->rssConfDesc.confVer = 1; 2984 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf)); 2985 devRead->rssConfDesc.confPA = 2986 cpu_to_le64(adapter->rss_conf_pa); 2987 } 2988 2989 #endif /* VMXNET3_RSS */ 2990 2991 /* intr settings */ 2992 if (!VMXNET3_VERSION_GE_6(adapter) || 2993 !adapter->queuesExtEnabled) { 2994 devRead->intrConf.autoMask = adapter->intr.mask_mode == 2995 VMXNET3_IMM_AUTO; 2996 devRead->intrConf.numIntrs = adapter->intr.num_intrs; 2997 for (i = 0; i < adapter->intr.num_intrs; i++) 2998 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i]; 2999 3000 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx; 3001 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 3002 } else { 3003 devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode == 3004 VMXNET3_IMM_AUTO; 3005 devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs; 3006 for (i = 0; i < adapter->intr.num_intrs; i++) 3007 devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i]; 3008 3009 devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx; 3010 devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 3011 } 3012 3013 /* rx filter settings */ 3014 devRead->rxFilterConf.rxMode = 0; 3015 vmxnet3_restore_vlan(adapter); 3016 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr); 3017 3018 /* the rest are already zeroed */ 3019 } 3020 3021 static void 3022 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter) 3023 { 3024 struct Vmxnet3_DriverShared *shared = adapter->shared; 3025 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 3026 unsigned long flags; 3027 3028 if (!VMXNET3_VERSION_GE_7(adapter)) 3029 return; 3030 3031 cmdInfo->ringBufSize = adapter->ringBufSize; 3032 spin_lock_irqsave(&adapter->cmd_lock, flags); 3033 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3034 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 3035 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3036 } 3037 3038 static void 3039 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter) 3040 { 3041 struct Vmxnet3_DriverShared *shared = adapter->shared; 3042 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 3043 unsigned long flags; 3044 3045 if (!VMXNET3_VERSION_GE_3(adapter)) 3046 return; 3047 3048 spin_lock_irqsave(&adapter->cmd_lock, flags); 3049 cmdInfo->varConf.confVer = 1; 3050 cmdInfo->varConf.confLen = 3051 cpu_to_le32(sizeof(*adapter->coal_conf)); 3052 cmdInfo->varConf.confPA = cpu_to_le64(adapter->coal_conf_pa); 3053 3054 if (adapter->default_coal_mode) { 3055 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3056 VMXNET3_CMD_GET_COALESCE); 3057 } else { 3058 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3059 VMXNET3_CMD_SET_COALESCE); 3060 } 3061 3062 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3063 } 3064 3065 static void 3066 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter) 3067 { 3068 struct Vmxnet3_DriverShared *shared = adapter->shared; 3069 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 3070 unsigned long flags; 3071 3072 if (!VMXNET3_VERSION_GE_4(adapter)) 3073 return; 3074 3075 spin_lock_irqsave(&adapter->cmd_lock, flags); 3076 3077 if (adapter->default_rss_fields) { 3078 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3079 VMXNET3_CMD_GET_RSS_FIELDS); 3080 adapter->rss_fields = 3081 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3082 } else { 3083 if (VMXNET3_VERSION_GE_7(adapter)) { 3084 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 || 3085 adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) && 3086 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3087 VMXNET3_CAP_UDP_RSS)) { 3088 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS; 3089 } else { 3090 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS); 3091 } 3092 3093 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) && 3094 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3095 VMXNET3_CAP_ESP_RSS_IPV4)) { 3096 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4; 3097 } else { 3098 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4); 3099 } 3100 3101 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) && 3102 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3103 VMXNET3_CAP_ESP_RSS_IPV6)) { 3104 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6; 3105 } else { 3106 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6); 3107 } 3108 3109 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3110 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3111 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3112 } 3113 cmdInfo->setRssFields = adapter->rss_fields; 3114 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3115 VMXNET3_CMD_SET_RSS_FIELDS); 3116 /* Not all requested RSS may get applied, so get and 3117 * cache what was actually applied. 3118 */ 3119 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3120 VMXNET3_CMD_GET_RSS_FIELDS); 3121 adapter->rss_fields = 3122 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3123 } 3124 3125 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3126 } 3127 3128 int 3129 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 3130 { 3131 int err, i; 3132 u32 ret; 3133 unsigned long flags; 3134 3135 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d," 3136 " ring sizes %u %u %u\n", adapter->netdev->name, 3137 adapter->skb_buf_size, adapter->rx_buf_per_pkt, 3138 adapter->tx_queue[0].tx_ring.size, 3139 adapter->rx_queue[0].rx_ring[0].size, 3140 adapter->rx_queue[0].rx_ring[1].size); 3141 3142 vmxnet3_tq_init_all(adapter); 3143 err = vmxnet3_rq_init_all(adapter); 3144 if (err) { 3145 netdev_err(adapter->netdev, 3146 "Failed to init rx queue error %d\n", err); 3147 goto rq_err; 3148 } 3149 3150 err = vmxnet3_request_irqs(adapter); 3151 if (err) { 3152 netdev_err(adapter->netdev, 3153 "Failed to setup irq for error %d\n", err); 3154 goto irq_err; 3155 } 3156 3157 vmxnet3_setup_driver_shared(adapter); 3158 3159 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO( 3160 adapter->shared_pa)); 3161 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI( 3162 adapter->shared_pa)); 3163 spin_lock_irqsave(&adapter->cmd_lock, flags); 3164 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3165 VMXNET3_CMD_ACTIVATE_DEV); 3166 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3167 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3168 3169 if (ret != 0) { 3170 netdev_err(adapter->netdev, 3171 "Failed to activate dev: error %u\n", ret); 3172 err = -EINVAL; 3173 goto activate_err; 3174 } 3175 3176 vmxnet3_init_bufsize(adapter); 3177 vmxnet3_init_coalesce(adapter); 3178 vmxnet3_init_rssfields(adapter); 3179 3180 for (i = 0; i < adapter->num_rx_queues; i++) { 3181 VMXNET3_WRITE_BAR0_REG(adapter, 3182 adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN, 3183 adapter->rx_queue[i].rx_ring[0].next2fill); 3184 VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset + 3185 (i * VMXNET3_REG_ALIGN)), 3186 adapter->rx_queue[i].rx_ring[1].next2fill); 3187 } 3188 3189 /* Apply the rx filter settins last. */ 3190 vmxnet3_set_mc(adapter->netdev); 3191 3192 /* 3193 * Check link state when first activating device. It will start the 3194 * tx queue if the link is up. 3195 */ 3196 vmxnet3_check_link(adapter, true); 3197 netif_tx_wake_all_queues(adapter->netdev); 3198 for (i = 0; i < adapter->num_rx_queues; i++) 3199 napi_enable(&adapter->rx_queue[i].napi); 3200 vmxnet3_enable_all_intrs(adapter); 3201 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3202 return 0; 3203 3204 activate_err: 3205 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0); 3206 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0); 3207 vmxnet3_free_irqs(adapter); 3208 irq_err: 3209 rq_err: 3210 /* free up buffers we allocated */ 3211 vmxnet3_rq_cleanup_all(adapter); 3212 return err; 3213 } 3214 3215 3216 void 3217 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter) 3218 { 3219 unsigned long flags; 3220 spin_lock_irqsave(&adapter->cmd_lock, flags); 3221 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 3222 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3223 } 3224 3225 3226 int 3227 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 3228 { 3229 int i; 3230 unsigned long flags; 3231 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 3232 return 0; 3233 3234 3235 spin_lock_irqsave(&adapter->cmd_lock, flags); 3236 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3237 VMXNET3_CMD_QUIESCE_DEV); 3238 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3239 vmxnet3_disable_all_intrs(adapter); 3240 3241 for (i = 0; i < adapter->num_rx_queues; i++) 3242 napi_disable(&adapter->rx_queue[i].napi); 3243 netif_tx_disable(adapter->netdev); 3244 adapter->link_speed = 0; 3245 netif_carrier_off(adapter->netdev); 3246 3247 vmxnet3_tq_cleanup_all(adapter); 3248 vmxnet3_rq_cleanup_all(adapter); 3249 vmxnet3_free_irqs(adapter); 3250 return 0; 3251 } 3252 3253 3254 static void 3255 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac) 3256 { 3257 u32 tmp; 3258 3259 tmp = *(u32 *)mac; 3260 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp); 3261 3262 tmp = (mac[5] << 8) | mac[4]; 3263 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp); 3264 } 3265 3266 3267 static int 3268 vmxnet3_set_mac_addr(struct net_device *netdev, void *p) 3269 { 3270 struct sockaddr *addr = p; 3271 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3272 3273 dev_addr_set(netdev, addr->sa_data); 3274 vmxnet3_write_mac_addr(adapter, addr->sa_data); 3275 3276 return 0; 3277 } 3278 3279 3280 /* ==================== initialization and cleanup routines ============ */ 3281 3282 static int 3283 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter) 3284 { 3285 int err; 3286 unsigned long mmio_start, mmio_len; 3287 struct pci_dev *pdev = adapter->pdev; 3288 3289 err = pci_enable_device(pdev); 3290 if (err) { 3291 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err); 3292 return err; 3293 } 3294 3295 err = pci_request_selected_regions(pdev, (1 << 2) - 1, 3296 vmxnet3_driver_name); 3297 if (err) { 3298 dev_err(&pdev->dev, 3299 "Failed to request region for adapter: error %d\n", err); 3300 goto err_enable_device; 3301 } 3302 3303 pci_set_master(pdev); 3304 3305 mmio_start = pci_resource_start(pdev, 0); 3306 mmio_len = pci_resource_len(pdev, 0); 3307 adapter->hw_addr0 = ioremap(mmio_start, mmio_len); 3308 if (!adapter->hw_addr0) { 3309 dev_err(&pdev->dev, "Failed to map bar0\n"); 3310 err = -EIO; 3311 goto err_ioremap; 3312 } 3313 3314 mmio_start = pci_resource_start(pdev, 1); 3315 mmio_len = pci_resource_len(pdev, 1); 3316 adapter->hw_addr1 = ioremap(mmio_start, mmio_len); 3317 if (!adapter->hw_addr1) { 3318 dev_err(&pdev->dev, "Failed to map bar1\n"); 3319 err = -EIO; 3320 goto err_bar1; 3321 } 3322 return 0; 3323 3324 err_bar1: 3325 iounmap(adapter->hw_addr0); 3326 err_ioremap: 3327 pci_release_selected_regions(pdev, (1 << 2) - 1); 3328 err_enable_device: 3329 pci_disable_device(pdev); 3330 return err; 3331 } 3332 3333 3334 static void 3335 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter) 3336 { 3337 BUG_ON(!adapter->pdev); 3338 3339 iounmap(adapter->hw_addr0); 3340 iounmap(adapter->hw_addr1); 3341 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1); 3342 pci_disable_device(adapter->pdev); 3343 } 3344 3345 3346 void 3347 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 3348 { 3349 size_t sz, i, ring0_size, ring1_size, comp_size; 3350 /* With version7 ring1 will have only T0 buffers */ 3351 if (!VMXNET3_VERSION_GE_7(adapter)) { 3352 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 3353 VMXNET3_MAX_ETH_HDR_SIZE) { 3354 adapter->skb_buf_size = adapter->netdev->mtu + 3355 VMXNET3_MAX_ETH_HDR_SIZE; 3356 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE) 3357 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE; 3358 3359 adapter->rx_buf_per_pkt = 1; 3360 } else { 3361 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE; 3362 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE + 3363 VMXNET3_MAX_ETH_HDR_SIZE; 3364 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE; 3365 } 3366 } else { 3367 adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE, 3368 VMXNET3_MAX_SKB_BUF_SIZE); 3369 adapter->rx_buf_per_pkt = 1; 3370 adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size); 3371 adapter->ringBufSize.ring1BufSizeType1 = 0; 3372 adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE); 3373 } 3374 3375 /* 3376 * for simplicity, force the ring0 size to be a multiple of 3377 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 3378 */ 3379 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 3380 ring0_size = adapter->rx_queue[0].rx_ring[0].size; 3381 ring0_size = (ring0_size + sz - 1) / sz * sz; 3382 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE / 3383 sz * sz); 3384 ring1_size = adapter->rx_queue[0].rx_ring[1].size; 3385 ring1_size = (ring1_size + sz - 1) / sz * sz; 3386 ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE / 3387 sz * sz); 3388 /* For v7 and later, keep ring size power of 2 for UPT */ 3389 if (VMXNET3_VERSION_GE_7(adapter)) { 3390 ring0_size = rounddown_pow_of_two(ring0_size); 3391 ring1_size = rounddown_pow_of_two(ring1_size); 3392 } 3393 comp_size = ring0_size + ring1_size; 3394 3395 for (i = 0; i < adapter->num_rx_queues; i++) { 3396 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3397 3398 rq->rx_ring[0].size = ring0_size; 3399 rq->rx_ring[1].size = ring1_size; 3400 rq->comp_ring.size = comp_size; 3401 } 3402 } 3403 3404 3405 int 3406 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 3407 u32 rx_ring_size, u32 rx_ring2_size, 3408 u16 txdata_desc_size, u16 rxdata_desc_size) 3409 { 3410 int err = 0, i; 3411 3412 for (i = 0; i < adapter->num_tx_queues; i++) { 3413 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 3414 tq->tx_ring.size = tx_ring_size; 3415 tq->data_ring.size = tx_ring_size; 3416 tq->comp_ring.size = tx_ring_size; 3417 tq->txdata_desc_size = txdata_desc_size; 3418 tq->shared = &adapter->tqd_start[i].ctrl; 3419 tq->stopped = true; 3420 tq->adapter = adapter; 3421 tq->qid = i; 3422 tq->tx_ts_desc_size = adapter->tx_ts_desc_size; 3423 tq->tsPktCount = 1; 3424 err = vmxnet3_tq_create(tq, adapter); 3425 /* 3426 * Too late to change num_tx_queues. We cannot do away with 3427 * lesser number of queues than what we asked for 3428 */ 3429 if (err) 3430 goto queue_err; 3431 } 3432 3433 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size; 3434 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size; 3435 vmxnet3_adjust_rx_ring_size(adapter); 3436 3437 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 3438 for (i = 0; i < adapter->num_rx_queues; i++) { 3439 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3440 /* qid and qid2 for rx queues will be assigned later when num 3441 * of rx queues is finalized after allocating intrs */ 3442 rq->shared = &adapter->rqd_start[i].ctrl; 3443 rq->adapter = adapter; 3444 rq->data_ring.desc_size = rxdata_desc_size; 3445 rq->rx_ts_desc_size = adapter->rx_ts_desc_size; 3446 err = vmxnet3_rq_create(rq, adapter); 3447 if (err) { 3448 if (i == 0) { 3449 netdev_err(adapter->netdev, 3450 "Could not allocate any rx queues. " 3451 "Aborting.\n"); 3452 goto queue_err; 3453 } else { 3454 netdev_info(adapter->netdev, 3455 "Number of rx queues changed " 3456 "to : %d.\n", i); 3457 adapter->num_rx_queues = i; 3458 err = 0; 3459 break; 3460 } 3461 } 3462 } 3463 3464 if (!adapter->rxdataring_enabled) 3465 vmxnet3_rq_destroy_all_rxdataring(adapter); 3466 3467 return err; 3468 queue_err: 3469 vmxnet3_tq_destroy_all(adapter); 3470 return err; 3471 } 3472 3473 static int 3474 vmxnet3_open(struct net_device *netdev) 3475 { 3476 struct vmxnet3_adapter *adapter; 3477 int err, i; 3478 3479 adapter = netdev_priv(netdev); 3480 3481 for (i = 0; i < adapter->num_tx_queues; i++) 3482 spin_lock_init(&adapter->tx_queue[i].tx_lock); 3483 3484 if (VMXNET3_VERSION_GE_3(adapter)) { 3485 unsigned long flags; 3486 u16 txdata_desc_size; 3487 u32 ret; 3488 3489 spin_lock_irqsave(&adapter->cmd_lock, flags); 3490 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3491 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 3492 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3493 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3494 3495 txdata_desc_size = ret & 0xffff; 3496 if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) || 3497 (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) || 3498 (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) { 3499 adapter->txdata_desc_size = 3500 sizeof(struct Vmxnet3_TxDataDesc); 3501 } else { 3502 adapter->txdata_desc_size = txdata_desc_size; 3503 } 3504 if (VMXNET3_VERSION_GE_9(adapter)) 3505 adapter->rxdata_desc_size = (ret >> 16) & 0xffff; 3506 } else { 3507 adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc); 3508 } 3509 3510 if (VMXNET3_VERSION_GE_9(adapter)) { 3511 unsigned long flags; 3512 u16 tx_ts_desc_size = 0; 3513 u16 rx_ts_desc_size = 0; 3514 u32 ret; 3515 3516 spin_lock_irqsave(&adapter->cmd_lock, flags); 3517 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3518 VMXNET3_CMD_GET_TSRING_DESC_SIZE); 3519 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3520 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3521 if (ret > 0) { 3522 tx_ts_desc_size = (ret & 0xff); 3523 rx_ts_desc_size = ((ret >> 16) & 0xff); 3524 } 3525 if (tx_ts_desc_size > VMXNET3_TXTS_DESC_MAX_SIZE || 3526 tx_ts_desc_size & VMXNET3_TXTS_DESC_SIZE_MASK) 3527 tx_ts_desc_size = 0; 3528 if (rx_ts_desc_size > VMXNET3_RXTS_DESC_MAX_SIZE || 3529 rx_ts_desc_size & VMXNET3_RXTS_DESC_SIZE_MASK) 3530 rx_ts_desc_size = 0; 3531 adapter->tx_ts_desc_size = tx_ts_desc_size; 3532 adapter->rx_ts_desc_size = rx_ts_desc_size; 3533 } else { 3534 adapter->tx_ts_desc_size = 0; 3535 adapter->rx_ts_desc_size = 0; 3536 } 3537 3538 err = vmxnet3_create_queues(adapter, 3539 adapter->tx_ring_size, 3540 adapter->rx_ring_size, 3541 adapter->rx_ring2_size, 3542 adapter->txdata_desc_size, 3543 adapter->rxdata_desc_size); 3544 if (err) 3545 goto queue_err; 3546 3547 err = vmxnet3_activate_dev(adapter); 3548 if (err) 3549 goto activate_err; 3550 3551 return 0; 3552 3553 activate_err: 3554 vmxnet3_rq_destroy_all(adapter); 3555 vmxnet3_tq_destroy_all(adapter); 3556 queue_err: 3557 return err; 3558 } 3559 3560 3561 static int 3562 vmxnet3_close(struct net_device *netdev) 3563 { 3564 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3565 3566 /* 3567 * Reset_work may be in the middle of resetting the device, wait for its 3568 * completion. 3569 */ 3570 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3571 usleep_range(1000, 2000); 3572 3573 vmxnet3_quiesce_dev(adapter); 3574 3575 vmxnet3_rq_destroy_all(adapter); 3576 vmxnet3_tq_destroy_all(adapter); 3577 3578 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3579 3580 3581 return 0; 3582 } 3583 3584 3585 void 3586 vmxnet3_force_close(struct vmxnet3_adapter *adapter) 3587 { 3588 int i; 3589 3590 /* 3591 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 3592 * vmxnet3_close() will deadlock. 3593 */ 3594 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 3595 3596 /* we need to enable NAPI, otherwise dev_close will deadlock */ 3597 for (i = 0; i < adapter->num_rx_queues; i++) 3598 napi_enable(&adapter->rx_queue[i].napi); 3599 /* 3600 * Need to clear the quiesce bit to ensure that vmxnet3_close 3601 * can quiesce the device properly 3602 */ 3603 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3604 dev_close(adapter->netdev); 3605 } 3606 3607 3608 static int 3609 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) 3610 { 3611 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3612 int err = 0; 3613 3614 /* 3615 * Reset_work may be in the middle of resetting the device, wait for its 3616 * completion. 3617 */ 3618 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3619 usleep_range(1000, 2000); 3620 3621 if (netif_running(netdev)) { 3622 vmxnet3_quiesce_dev(adapter); 3623 vmxnet3_reset_dev(adapter); 3624 3625 /* we need to re-create the rx queue based on the new mtu */ 3626 vmxnet3_rq_destroy_all(adapter); 3627 WRITE_ONCE(netdev->mtu, new_mtu); 3628 vmxnet3_adjust_rx_ring_size(adapter); 3629 err = vmxnet3_rq_create_all(adapter); 3630 if (err) { 3631 netdev_err(netdev, 3632 "failed to re-create rx queues, " 3633 " error %d. Closing it.\n", err); 3634 goto out; 3635 } 3636 3637 err = vmxnet3_activate_dev(adapter); 3638 if (err) { 3639 netdev_err(netdev, 3640 "failed to re-activate, error %d. " 3641 "Closing it\n", err); 3642 goto out; 3643 } 3644 } else { 3645 WRITE_ONCE(netdev->mtu, new_mtu); 3646 } 3647 3648 out: 3649 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3650 if (err) 3651 vmxnet3_force_close(adapter); 3652 3653 return err; 3654 } 3655 3656 3657 static void 3658 vmxnet3_declare_features(struct vmxnet3_adapter *adapter) 3659 { 3660 struct net_device *netdev = adapter->netdev; 3661 unsigned long flags; 3662 3663 if (VMXNET3_VERSION_GE_9(adapter)) { 3664 spin_lock_irqsave(&adapter->cmd_lock, flags); 3665 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3666 VMXNET3_CMD_GET_DISABLED_OFFLOADS); 3667 adapter->disabledOffloads = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3668 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3669 } 3670 3671 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 3672 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3673 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3674 NETIF_F_LRO | NETIF_F_HIGHDMA; 3675 3676 if (VMXNET3_VERSION_GE_4(adapter)) { 3677 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3678 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3679 3680 netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM | 3681 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3682 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3683 NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL | 3684 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3685 } 3686 3687 if (adapter->disabledOffloads & VMXNET3_OFFLOAD_TSO) { 3688 netdev->hw_features &= ~(NETIF_F_TSO | NETIF_F_TSO6); 3689 netdev->hw_enc_features &= ~(NETIF_F_TSO | NETIF_F_TSO6); 3690 } 3691 3692 if (adapter->disabledOffloads & VMXNET3_OFFLOAD_LRO) { 3693 netdev->hw_features &= ~(NETIF_F_LRO); 3694 netdev->hw_enc_features &= ~(NETIF_F_LRO); 3695 } 3696 3697 if (VMXNET3_VERSION_GE_7(adapter)) { 3698 unsigned long flags; 3699 3700 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3701 VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) { 3702 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD; 3703 } 3704 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3705 VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) { 3706 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD; 3707 } 3708 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3709 VMXNET3_CAP_GENEVE_TSO)) { 3710 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO; 3711 } 3712 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3713 VMXNET3_CAP_VXLAN_TSO)) { 3714 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO; 3715 } 3716 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3717 VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) { 3718 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD; 3719 } 3720 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3721 VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) { 3722 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD; 3723 } 3724 3725 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3726 spin_lock_irqsave(&adapter->cmd_lock, flags); 3727 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3728 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3729 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3730 3731 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) && 3732 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) && 3733 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) && 3734 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) { 3735 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3736 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3737 } 3738 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) && 3739 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) { 3740 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3741 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3742 } 3743 } 3744 3745 netdev->vlan_features = netdev->hw_features & 3746 ~(NETIF_F_HW_VLAN_CTAG_TX | 3747 NETIF_F_HW_VLAN_CTAG_RX); 3748 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 3749 } 3750 3751 3752 static void 3753 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) 3754 { 3755 u32 tmp; 3756 3757 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL); 3758 *(u32 *)mac = tmp; 3759 3760 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH); 3761 mac[4] = tmp & 0xff; 3762 mac[5] = (tmp >> 8) & 0xff; 3763 } 3764 3765 #ifdef CONFIG_PCI_MSI 3766 3767 /* 3768 * Enable MSIx vectors. 3769 * Returns : 3770 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required 3771 * were enabled. 3772 * number of vectors which were enabled otherwise (this number is greater 3773 * than VMXNET3_LINUX_MIN_MSIX_VECT) 3774 */ 3775 3776 static int 3777 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec) 3778 { 3779 int ret = pci_enable_msix_range(adapter->pdev, 3780 adapter->intr.msix_entries, nvec, nvec); 3781 3782 if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) { 3783 dev_err(&adapter->netdev->dev, 3784 "Failed to enable %d MSI-X, trying %d\n", 3785 nvec, VMXNET3_LINUX_MIN_MSIX_VECT); 3786 3787 ret = pci_enable_msix_range(adapter->pdev, 3788 adapter->intr.msix_entries, 3789 VMXNET3_LINUX_MIN_MSIX_VECT, 3790 VMXNET3_LINUX_MIN_MSIX_VECT); 3791 } 3792 3793 if (ret < 0) { 3794 dev_err(&adapter->netdev->dev, 3795 "Failed to enable MSI-X, error: %d\n", ret); 3796 } 3797 3798 return ret; 3799 } 3800 3801 3802 #endif /* CONFIG_PCI_MSI */ 3803 3804 static void 3805 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 3806 { 3807 u32 cfg; 3808 unsigned long flags; 3809 3810 /* intr settings */ 3811 spin_lock_irqsave(&adapter->cmd_lock, flags); 3812 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3813 VMXNET3_CMD_GET_CONF_INTR); 3814 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3815 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3816 adapter->intr.type = cfg & 0x3; 3817 adapter->intr.mask_mode = (cfg >> 2) & 0x3; 3818 3819 if (adapter->intr.type == VMXNET3_IT_AUTO) { 3820 adapter->intr.type = VMXNET3_IT_MSIX; 3821 } 3822 3823 #ifdef CONFIG_PCI_MSI 3824 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3825 int i, nvec, nvec_allocated; 3826 3827 nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ? 3828 1 : adapter->num_tx_queues; 3829 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ? 3830 0 : adapter->num_rx_queues; 3831 nvec += 1; /* for link event */ 3832 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ? 3833 nvec : VMXNET3_LINUX_MIN_MSIX_VECT; 3834 3835 for (i = 0; i < nvec; i++) 3836 adapter->intr.msix_entries[i].entry = i; 3837 3838 nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec); 3839 if (nvec_allocated < 0) 3840 goto msix_err; 3841 3842 /* If we cannot allocate one MSIx vector per queue 3843 * then limit the number of rx queues to 1 3844 */ 3845 if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT && 3846 nvec != VMXNET3_LINUX_MIN_MSIX_VECT) { 3847 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE 3848 || adapter->num_rx_queues != 1) { 3849 adapter->share_intr = VMXNET3_INTR_TXSHARE; 3850 netdev_err(adapter->netdev, 3851 "Number of rx queues : 1\n"); 3852 adapter->num_rx_queues = 1; 3853 } 3854 } 3855 3856 adapter->intr.num_intrs = nvec_allocated; 3857 return; 3858 3859 msix_err: 3860 /* If we cannot allocate MSIx vectors use only one rx queue */ 3861 dev_info(&adapter->pdev->dev, 3862 "Failed to enable MSI-X, error %d. " 3863 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated); 3864 3865 adapter->intr.type = VMXNET3_IT_MSI; 3866 } 3867 3868 if (adapter->intr.type == VMXNET3_IT_MSI) { 3869 if (!pci_enable_msi(adapter->pdev)) { 3870 adapter->num_rx_queues = 1; 3871 adapter->intr.num_intrs = 1; 3872 return; 3873 } 3874 } 3875 #endif /* CONFIG_PCI_MSI */ 3876 3877 adapter->num_rx_queues = 1; 3878 dev_info(&adapter->netdev->dev, 3879 "Using INTx interrupt, #Rx queues: 1.\n"); 3880 adapter->intr.type = VMXNET3_IT_INTX; 3881 3882 /* INT-X related setting */ 3883 adapter->intr.num_intrs = 1; 3884 } 3885 3886 3887 static void 3888 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter) 3889 { 3890 if (adapter->intr.type == VMXNET3_IT_MSIX) 3891 pci_disable_msix(adapter->pdev); 3892 else if (adapter->intr.type == VMXNET3_IT_MSI) 3893 pci_disable_msi(adapter->pdev); 3894 else 3895 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX); 3896 } 3897 3898 3899 static void 3900 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue) 3901 { 3902 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3903 adapter->tx_timeout_count++; 3904 3905 netdev_err(adapter->netdev, "tx hang\n"); 3906 schedule_work(&adapter->work); 3907 } 3908 3909 3910 static void 3911 vmxnet3_reset_work(struct work_struct *data) 3912 { 3913 struct vmxnet3_adapter *adapter; 3914 3915 adapter = container_of(data, struct vmxnet3_adapter, work); 3916 3917 /* if another thread is resetting the device, no need to proceed */ 3918 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3919 return; 3920 3921 /* if the device is closed, we must leave it alone */ 3922 rtnl_lock(); 3923 if (netif_running(adapter->netdev)) { 3924 netdev_notice(adapter->netdev, "resetting\n"); 3925 vmxnet3_quiesce_dev(adapter); 3926 vmxnet3_reset_dev(adapter); 3927 vmxnet3_activate_dev(adapter); 3928 } else { 3929 netdev_info(adapter->netdev, "already closed\n"); 3930 } 3931 rtnl_unlock(); 3932 3933 netif_wake_queue(adapter->netdev); 3934 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3935 } 3936 3937 3938 static int 3939 vmxnet3_probe_device(struct pci_dev *pdev, 3940 const struct pci_device_id *id) 3941 { 3942 static const struct net_device_ops vmxnet3_netdev_ops = { 3943 .ndo_open = vmxnet3_open, 3944 .ndo_stop = vmxnet3_close, 3945 .ndo_start_xmit = vmxnet3_xmit_frame, 3946 .ndo_set_mac_address = vmxnet3_set_mac_addr, 3947 .ndo_change_mtu = vmxnet3_change_mtu, 3948 .ndo_fix_features = vmxnet3_fix_features, 3949 .ndo_set_features = vmxnet3_set_features, 3950 .ndo_features_check = vmxnet3_features_check, 3951 .ndo_get_stats64 = vmxnet3_get_stats64, 3952 .ndo_tx_timeout = vmxnet3_tx_timeout, 3953 .ndo_set_rx_mode = vmxnet3_set_mc, 3954 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid, 3955 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid, 3956 #ifdef CONFIG_NET_POLL_CONTROLLER 3957 .ndo_poll_controller = vmxnet3_netpoll, 3958 #endif 3959 .ndo_bpf = vmxnet3_xdp, 3960 .ndo_xdp_xmit = vmxnet3_xdp_xmit, 3961 }; 3962 int err; 3963 u32 ver; 3964 struct net_device *netdev; 3965 struct vmxnet3_adapter *adapter; 3966 u8 mac[ETH_ALEN]; 3967 int size, i; 3968 int num_tx_queues; 3969 int num_rx_queues; 3970 int queues; 3971 unsigned long flags; 3972 3973 if (!pci_msi_enabled()) 3974 enable_mq = 0; 3975 3976 #ifdef VMXNET3_RSS 3977 if (enable_mq) 3978 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3979 (int)num_online_cpus()); 3980 else 3981 #endif 3982 num_rx_queues = 1; 3983 3984 if (enable_mq) 3985 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES, 3986 (int)num_online_cpus()); 3987 else 3988 num_tx_queues = 1; 3989 3990 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter), 3991 max(num_tx_queues, num_rx_queues)); 3992 if (!netdev) 3993 return -ENOMEM; 3994 3995 pci_set_drvdata(pdev, netdev); 3996 adapter = netdev_priv(netdev); 3997 adapter->netdev = netdev; 3998 adapter->pdev = pdev; 3999 4000 adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE; 4001 adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE; 4002 adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE; 4003 4004 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4005 if (err) { 4006 dev_err(&pdev->dev, "dma_set_mask failed\n"); 4007 goto err_set_mask; 4008 } 4009 4010 spin_lock_init(&adapter->cmd_lock); 4011 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter, 4012 sizeof(struct vmxnet3_adapter), 4013 DMA_TO_DEVICE); 4014 if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) { 4015 dev_err(&pdev->dev, "Failed to map dma\n"); 4016 err = -EFAULT; 4017 goto err_set_mask; 4018 } 4019 adapter->shared = dma_alloc_coherent( 4020 &adapter->pdev->dev, 4021 sizeof(struct Vmxnet3_DriverShared), 4022 &adapter->shared_pa, GFP_KERNEL); 4023 if (!adapter->shared) { 4024 dev_err(&pdev->dev, "Failed to allocate memory\n"); 4025 err = -ENOMEM; 4026 goto err_alloc_shared; 4027 } 4028 4029 err = vmxnet3_alloc_pci_resources(adapter); 4030 if (err < 0) 4031 goto err_alloc_pci; 4032 4033 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS); 4034 for (i = VMXNET3_REV_9; i >= VMXNET3_REV_1; i--) { 4035 if (ver & (1 << i)) { 4036 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1 << i); 4037 adapter->version = i + 1; 4038 break; 4039 } 4040 } 4041 if (i < VMXNET3_REV_1) { 4042 dev_err(&pdev->dev, 4043 "Incompatible h/w version (0x%x) for adapter\n", ver); 4044 err = -EBUSY; 4045 goto err_ver; 4046 } 4047 dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version); 4048 4049 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS); 4050 if (ver & 1) { 4051 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1); 4052 } else { 4053 dev_err(&pdev->dev, 4054 "Incompatible upt version (0x%x) for adapter\n", ver); 4055 err = -EBUSY; 4056 goto err_ver; 4057 } 4058 4059 if (VMXNET3_VERSION_GE_7(adapter)) { 4060 adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR); 4061 adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR); 4062 if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 4063 adapter->dev_caps[0] = adapter->devcap_supported[0] & 4064 (1UL << VMXNET3_CAP_LARGE_BAR); 4065 } 4066 if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) && 4067 adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 4068 adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 4069 adapter->dev_caps[0] |= adapter->devcap_supported[0] & 4070 (1UL << VMXNET3_CAP_OOORX_COMP); 4071 } 4072 if (adapter->dev_caps[0]) 4073 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 4074 4075 spin_lock_irqsave(&adapter->cmd_lock, flags); 4076 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 4077 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4078 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4079 } 4080 4081 if (VMXNET3_VERSION_GE_7(adapter) && 4082 adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 4083 adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 4084 adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD; 4085 adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2; 4086 } else { 4087 adapter->tx_prod_offset = VMXNET3_REG_TXPROD; 4088 adapter->rx_prod_offset = VMXNET3_REG_RXPROD; 4089 adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2; 4090 } 4091 4092 if (VMXNET3_VERSION_GE_6(adapter)) { 4093 spin_lock_irqsave(&adapter->cmd_lock, flags); 4094 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4095 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 4096 queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4097 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4098 if (queues > 0) { 4099 adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff)); 4100 adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff)); 4101 } else { 4102 adapter->num_rx_queues = min(num_rx_queues, 4103 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4104 adapter->num_tx_queues = min(num_tx_queues, 4105 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 4106 } 4107 if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES || 4108 adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) { 4109 adapter->queuesExtEnabled = true; 4110 } else { 4111 adapter->queuesExtEnabled = false; 4112 } 4113 } else { 4114 adapter->queuesExtEnabled = false; 4115 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 4116 num_tx_queues = rounddown_pow_of_two(num_tx_queues); 4117 adapter->num_rx_queues = min(num_rx_queues, 4118 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4119 adapter->num_tx_queues = min(num_tx_queues, 4120 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 4121 } 4122 dev_info(&pdev->dev, 4123 "# of Tx queues : %d, # of Rx queues : %d\n", 4124 adapter->num_tx_queues, adapter->num_rx_queues); 4125 4126 adapter->rx_buf_per_pkt = 1; 4127 4128 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 4129 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues; 4130 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size, 4131 &adapter->queue_desc_pa, 4132 GFP_KERNEL); 4133 4134 if (!adapter->tqd_start) { 4135 dev_err(&pdev->dev, "Failed to allocate memory\n"); 4136 err = -ENOMEM; 4137 goto err_ver; 4138 } 4139 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start + 4140 adapter->num_tx_queues); 4141 if (VMXNET3_VERSION_GE_9(adapter)) 4142 adapter->latencyConf = &adapter->tqd_start->tsConf.latencyConf; 4143 4144 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev, 4145 sizeof(struct Vmxnet3_PMConf), 4146 &adapter->pm_conf_pa, 4147 GFP_KERNEL); 4148 if (adapter->pm_conf == NULL) { 4149 err = -ENOMEM; 4150 goto err_alloc_pm; 4151 } 4152 4153 #ifdef VMXNET3_RSS 4154 4155 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev, 4156 sizeof(struct UPT1_RSSConf), 4157 &adapter->rss_conf_pa, 4158 GFP_KERNEL); 4159 if (adapter->rss_conf == NULL) { 4160 err = -ENOMEM; 4161 goto err_alloc_rss; 4162 } 4163 #endif /* VMXNET3_RSS */ 4164 4165 if (VMXNET3_VERSION_GE_3(adapter)) { 4166 adapter->coal_conf = 4167 dma_alloc_coherent(&adapter->pdev->dev, 4168 sizeof(struct Vmxnet3_CoalesceScheme) 4169 , 4170 &adapter->coal_conf_pa, 4171 GFP_KERNEL); 4172 if (!adapter->coal_conf) { 4173 err = -ENOMEM; 4174 goto err_coal_conf; 4175 } 4176 adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED; 4177 adapter->default_coal_mode = true; 4178 } 4179 4180 if (VMXNET3_VERSION_GE_4(adapter)) { 4181 adapter->default_rss_fields = true; 4182 adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT; 4183 } 4184 4185 SET_NETDEV_DEV(netdev, &pdev->dev); 4186 vmxnet3_declare_features(adapter); 4187 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 4188 NETDEV_XDP_ACT_NDO_XMIT; 4189 4190 adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ? 4191 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 4192 4193 if (adapter->num_tx_queues == adapter->num_rx_queues) 4194 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE; 4195 else 4196 adapter->share_intr = VMXNET3_INTR_DONTSHARE; 4197 4198 vmxnet3_alloc_intr_resources(adapter); 4199 4200 #ifdef VMXNET3_RSS 4201 if (adapter->num_rx_queues > 1 && 4202 adapter->intr.type == VMXNET3_IT_MSIX) { 4203 adapter->rss = true; 4204 netdev->hw_features |= NETIF_F_RXHASH; 4205 netdev->features |= NETIF_F_RXHASH; 4206 dev_dbg(&pdev->dev, "RSS is enabled.\n"); 4207 } else { 4208 adapter->rss = false; 4209 } 4210 #endif 4211 4212 vmxnet3_read_mac_addr(adapter, mac); 4213 dev_addr_set(netdev, mac); 4214 4215 netdev->netdev_ops = &vmxnet3_netdev_ops; 4216 vmxnet3_set_ethtool_ops(netdev); 4217 netdev->watchdog_timeo = 5 * HZ; 4218 4219 /* MTU range: 60 - 9190 */ 4220 netdev->min_mtu = VMXNET3_MIN_MTU; 4221 if (VMXNET3_VERSION_GE_6(adapter)) 4222 netdev->max_mtu = VMXNET3_V6_MAX_MTU; 4223 else 4224 netdev->max_mtu = VMXNET3_MAX_MTU; 4225 4226 INIT_WORK(&adapter->work, vmxnet3_reset_work); 4227 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 4228 4229 if (adapter->intr.type == VMXNET3_IT_MSIX) { 4230 int i; 4231 for (i = 0; i < adapter->num_rx_queues; i++) { 4232 netif_napi_add(adapter->netdev, 4233 &adapter->rx_queue[i].napi, 4234 vmxnet3_poll_rx_only); 4235 } 4236 } else { 4237 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi, 4238 vmxnet3_poll); 4239 } 4240 4241 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); 4242 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues); 4243 4244 netif_carrier_off(netdev); 4245 err = register_netdev(netdev); 4246 4247 if (err) { 4248 dev_err(&pdev->dev, "Failed to register adapter\n"); 4249 goto err_register; 4250 } 4251 4252 vmxnet3_check_link(adapter, false); 4253 return 0; 4254 4255 err_register: 4256 if (VMXNET3_VERSION_GE_3(adapter)) { 4257 dma_free_coherent(&adapter->pdev->dev, 4258 sizeof(struct Vmxnet3_CoalesceScheme), 4259 adapter->coal_conf, adapter->coal_conf_pa); 4260 } 4261 vmxnet3_free_intr_resources(adapter); 4262 err_coal_conf: 4263 #ifdef VMXNET3_RSS 4264 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 4265 adapter->rss_conf, adapter->rss_conf_pa); 4266 err_alloc_rss: 4267 #endif 4268 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 4269 adapter->pm_conf, adapter->pm_conf_pa); 4270 err_alloc_pm: 4271 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 4272 adapter->queue_desc_pa); 4273 err_ver: 4274 vmxnet3_free_pci_resources(adapter); 4275 err_alloc_pci: 4276 dma_free_coherent(&adapter->pdev->dev, 4277 sizeof(struct Vmxnet3_DriverShared), 4278 adapter->shared, adapter->shared_pa); 4279 err_alloc_shared: 4280 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4281 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4282 err_set_mask: 4283 free_netdev(netdev); 4284 return err; 4285 } 4286 4287 4288 static void 4289 vmxnet3_remove_device(struct pci_dev *pdev) 4290 { 4291 struct net_device *netdev = pci_get_drvdata(pdev); 4292 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4293 int size = 0; 4294 int num_rx_queues, rx_queues; 4295 unsigned long flags; 4296 4297 #ifdef VMXNET3_RSS 4298 if (enable_mq) 4299 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 4300 (int)num_online_cpus()); 4301 else 4302 #endif 4303 num_rx_queues = 1; 4304 if (!VMXNET3_VERSION_GE_6(adapter)) { 4305 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 4306 } 4307 if (VMXNET3_VERSION_GE_6(adapter)) { 4308 spin_lock_irqsave(&adapter->cmd_lock, flags); 4309 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4310 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 4311 rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4312 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4313 if (rx_queues > 0) 4314 rx_queues = (rx_queues >> 8) & 0xff; 4315 else 4316 rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4317 num_rx_queues = min(num_rx_queues, rx_queues); 4318 } else { 4319 num_rx_queues = min(num_rx_queues, 4320 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4321 } 4322 4323 cancel_work_sync(&adapter->work); 4324 4325 unregister_netdev(netdev); 4326 4327 vmxnet3_free_intr_resources(adapter); 4328 vmxnet3_free_pci_resources(adapter); 4329 if (VMXNET3_VERSION_GE_3(adapter)) { 4330 dma_free_coherent(&adapter->pdev->dev, 4331 sizeof(struct Vmxnet3_CoalesceScheme), 4332 adapter->coal_conf, adapter->coal_conf_pa); 4333 } 4334 #ifdef VMXNET3_RSS 4335 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 4336 adapter->rss_conf, adapter->rss_conf_pa); 4337 #endif 4338 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 4339 adapter->pm_conf, adapter->pm_conf_pa); 4340 4341 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 4342 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues; 4343 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 4344 adapter->queue_desc_pa); 4345 dma_free_coherent(&adapter->pdev->dev, 4346 sizeof(struct Vmxnet3_DriverShared), 4347 adapter->shared, adapter->shared_pa); 4348 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4349 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4350 free_netdev(netdev); 4351 } 4352 4353 static void vmxnet3_shutdown_device(struct pci_dev *pdev) 4354 { 4355 struct net_device *netdev = pci_get_drvdata(pdev); 4356 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4357 unsigned long flags; 4358 4359 /* Reset_work may be in the middle of resetting the device, wait for its 4360 * completion. 4361 */ 4362 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 4363 usleep_range(1000, 2000); 4364 4365 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, 4366 &adapter->state)) { 4367 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4368 return; 4369 } 4370 spin_lock_irqsave(&adapter->cmd_lock, flags); 4371 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4372 VMXNET3_CMD_QUIESCE_DEV); 4373 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4374 vmxnet3_disable_all_intrs(adapter); 4375 4376 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4377 } 4378 4379 4380 #ifdef CONFIG_PM 4381 4382 static int 4383 vmxnet3_suspend(struct device *device) 4384 { 4385 struct pci_dev *pdev = to_pci_dev(device); 4386 struct net_device *netdev = pci_get_drvdata(pdev); 4387 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4388 struct Vmxnet3_PMConf *pmConf; 4389 struct ethhdr *ehdr; 4390 struct arphdr *ahdr; 4391 u8 *arpreq; 4392 struct in_device *in_dev; 4393 struct in_ifaddr *ifa; 4394 unsigned long flags; 4395 int i = 0; 4396 4397 if (!netif_running(netdev)) 4398 return 0; 4399 4400 for (i = 0; i < adapter->num_rx_queues; i++) 4401 napi_disable(&adapter->rx_queue[i].napi); 4402 4403 vmxnet3_disable_all_intrs(adapter); 4404 vmxnet3_free_irqs(adapter); 4405 vmxnet3_free_intr_resources(adapter); 4406 4407 netif_device_detach(netdev); 4408 4409 /* Create wake-up filters. */ 4410 pmConf = adapter->pm_conf; 4411 memset(pmConf, 0, sizeof(*pmConf)); 4412 4413 if (adapter->wol & WAKE_UCAST) { 4414 pmConf->filters[i].patternSize = ETH_ALEN; 4415 pmConf->filters[i].maskSize = 1; 4416 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN); 4417 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */ 4418 4419 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4420 i++; 4421 } 4422 4423 if (adapter->wol & WAKE_ARP) { 4424 rcu_read_lock(); 4425 4426 in_dev = __in_dev_get_rcu(netdev); 4427 if (!in_dev) { 4428 rcu_read_unlock(); 4429 goto skip_arp; 4430 } 4431 4432 ifa = rcu_dereference(in_dev->ifa_list); 4433 if (!ifa) { 4434 rcu_read_unlock(); 4435 goto skip_arp; 4436 } 4437 4438 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/ 4439 sizeof(struct arphdr) + /* ARP header */ 4440 2 * ETH_ALEN + /* 2 Ethernet addresses*/ 4441 2 * sizeof(u32); /*2 IPv4 addresses */ 4442 pmConf->filters[i].maskSize = 4443 (pmConf->filters[i].patternSize - 1) / 8 + 1; 4444 4445 /* ETH_P_ARP in Ethernet header. */ 4446 ehdr = (struct ethhdr *)pmConf->filters[i].pattern; 4447 ehdr->h_proto = htons(ETH_P_ARP); 4448 4449 /* ARPOP_REQUEST in ARP header. */ 4450 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN]; 4451 ahdr->ar_op = htons(ARPOP_REQUEST); 4452 arpreq = (u8 *)(ahdr + 1); 4453 4454 /* The Unicast IPv4 address in 'tip' field. */ 4455 arpreq += 2 * ETH_ALEN + sizeof(u32); 4456 *(__be32 *)arpreq = ifa->ifa_address; 4457 4458 rcu_read_unlock(); 4459 4460 /* The mask for the relevant bits. */ 4461 pmConf->filters[i].mask[0] = 0x00; 4462 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */ 4463 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */ 4464 pmConf->filters[i].mask[3] = 0x00; 4465 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */ 4466 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */ 4467 4468 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4469 i++; 4470 } 4471 4472 skip_arp: 4473 if (adapter->wol & WAKE_MAGIC) 4474 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC; 4475 4476 pmConf->numFilters = i; 4477 4478 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); 4479 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( 4480 *pmConf)); 4481 adapter->shared->devRead.pmConfDesc.confPA = 4482 cpu_to_le64(adapter->pm_conf_pa); 4483 4484 spin_lock_irqsave(&adapter->cmd_lock, flags); 4485 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4486 VMXNET3_CMD_UPDATE_PMCFG); 4487 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4488 4489 pci_save_state(pdev); 4490 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND), 4491 adapter->wol); 4492 pci_disable_device(pdev); 4493 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND)); 4494 4495 return 0; 4496 } 4497 4498 4499 static int 4500 vmxnet3_resume(struct device *device) 4501 { 4502 int err; 4503 unsigned long flags; 4504 struct pci_dev *pdev = to_pci_dev(device); 4505 struct net_device *netdev = pci_get_drvdata(pdev); 4506 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4507 4508 if (!netif_running(netdev)) 4509 return 0; 4510 4511 pci_set_power_state(pdev, PCI_D0); 4512 pci_restore_state(pdev); 4513 err = pci_enable_device_mem(pdev); 4514 if (err != 0) 4515 return err; 4516 4517 pci_enable_wake(pdev, PCI_D0, 0); 4518 4519 vmxnet3_alloc_intr_resources(adapter); 4520 4521 /* During hibernate and suspend, device has to be reinitialized as the 4522 * device state need not be preserved. 4523 */ 4524 4525 /* Need not check adapter state as other reset tasks cannot run during 4526 * device resume. 4527 */ 4528 spin_lock_irqsave(&adapter->cmd_lock, flags); 4529 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4530 VMXNET3_CMD_QUIESCE_DEV); 4531 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4532 vmxnet3_tq_cleanup_all(adapter); 4533 vmxnet3_rq_cleanup_all(adapter); 4534 4535 vmxnet3_reset_dev(adapter); 4536 err = vmxnet3_activate_dev(adapter); 4537 if (err != 0) { 4538 netdev_err(netdev, 4539 "failed to re-activate on resume, error: %d", err); 4540 vmxnet3_force_close(adapter); 4541 return err; 4542 } 4543 netif_device_attach(netdev); 4544 4545 return 0; 4546 } 4547 4548 static const struct dev_pm_ops vmxnet3_pm_ops = { 4549 .suspend = vmxnet3_suspend, 4550 .resume = vmxnet3_resume, 4551 .freeze = vmxnet3_suspend, 4552 .restore = vmxnet3_resume, 4553 }; 4554 #endif 4555 4556 static struct pci_driver vmxnet3_driver = { 4557 .name = vmxnet3_driver_name, 4558 .id_table = vmxnet3_pciid_table, 4559 .probe = vmxnet3_probe_device, 4560 .remove = vmxnet3_remove_device, 4561 .shutdown = vmxnet3_shutdown_device, 4562 #ifdef CONFIG_PM 4563 .driver.pm = &vmxnet3_pm_ops, 4564 #endif 4565 }; 4566 4567 4568 static int __init 4569 vmxnet3_init_module(void) 4570 { 4571 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC, 4572 VMXNET3_DRIVER_VERSION_REPORT); 4573 return pci_register_driver(&vmxnet3_driver); 4574 } 4575 4576 module_init(vmxnet3_init_module); 4577 4578 4579 static void 4580 vmxnet3_exit_module(void) 4581 { 4582 pci_unregister_driver(&vmxnet3_driver); 4583 } 4584 4585 module_exit(vmxnet3_exit_module); 4586 4587 MODULE_AUTHOR("VMware, Inc."); 4588 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC); 4589 MODULE_LICENSE("GPL v2"); 4590 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING); 4591