1 /* 2 * Linux driver for VMware's vmxnet3 ethernet NIC. 3 * 4 * Copyright (C) 2008-2024, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * The full GNU General Public License is included in this distribution in 21 * the file called "COPYING". 22 * 23 * Maintained by: pv-drivers@vmware.com 24 * 25 */ 26 27 #include <linux/module.h> 28 #include <net/ip6_checksum.h> 29 30 #include "vmxnet3_int.h" 31 #include "vmxnet3_xdp.h" 32 33 char vmxnet3_driver_name[] = "vmxnet3"; 34 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver" 35 36 /* 37 * PCI Device ID Table 38 * Last entry must be all 0s 39 */ 40 static const struct pci_device_id vmxnet3_pciid_table[] = { 41 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)}, 42 {0} 43 }; 44 45 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table); 46 47 static int enable_mq = 1; 48 49 static void 50 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac); 51 52 /* 53 * Enable/Disable the given intr 54 */ 55 static void 56 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 57 { 58 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0); 59 } 60 61 62 static void 63 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 64 { 65 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1); 66 } 67 68 69 /* 70 * Enable/Disable all intrs used by the device 71 */ 72 static void 73 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) 74 { 75 int i; 76 77 for (i = 0; i < adapter->intr.num_intrs; i++) 78 vmxnet3_enable_intr(adapter, i); 79 if (!VMXNET3_VERSION_GE_6(adapter) || 80 !adapter->queuesExtEnabled) { 81 adapter->shared->devRead.intrConf.intrCtrl &= 82 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 83 } else { 84 adapter->shared->devReadExt.intrConfExt.intrCtrl &= 85 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 86 } 87 } 88 89 90 static void 91 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) 92 { 93 int i; 94 95 if (!VMXNET3_VERSION_GE_6(adapter) || 96 !adapter->queuesExtEnabled) { 97 adapter->shared->devRead.intrConf.intrCtrl |= 98 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 99 } else { 100 adapter->shared->devReadExt.intrConfExt.intrCtrl |= 101 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 102 } 103 for (i = 0; i < adapter->intr.num_intrs; i++) 104 vmxnet3_disable_intr(adapter, i); 105 } 106 107 108 static void 109 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events) 110 { 111 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events); 112 } 113 114 115 static bool 116 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 117 { 118 return tq->stopped; 119 } 120 121 122 static void 123 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 124 { 125 tq->stopped = false; 126 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue); 127 } 128 129 130 static void 131 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 132 { 133 tq->stopped = false; 134 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 135 } 136 137 138 static void 139 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 140 { 141 tq->stopped = true; 142 tq->num_stop++; 143 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 144 } 145 146 static u64 147 vmxnet3_get_cycles(int pmc) 148 { 149 #ifdef CONFIG_X86 150 return native_read_pmc(pmc); 151 #else 152 return 0; 153 #endif 154 } 155 156 static bool 157 vmxnet3_apply_timestamp(struct vmxnet3_tx_queue *tq, u16 rate) 158 { 159 #ifdef CONFIG_X86 160 if (rate > 0) { 161 if (tq->tsPktCount == 1) { 162 if (rate != 1) 163 tq->tsPktCount = rate; 164 return true; 165 } 166 tq->tsPktCount--; 167 } 168 #endif 169 return false; 170 } 171 172 /* Check if capability is supported by UPT device or 173 * UPT is even requested 174 */ 175 bool 176 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap) 177 { 178 if (cap_supported & (1UL << VMXNET3_DCR_ERROR) || 179 cap_supported & (1UL << cap)) { 180 return true; 181 } 182 183 return false; 184 } 185 186 187 /* 188 * Check the link state. This may start or stop the tx queue. 189 */ 190 static void 191 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 192 { 193 u32 ret; 194 int i; 195 unsigned long flags; 196 197 spin_lock_irqsave(&adapter->cmd_lock, flags); 198 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 199 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 200 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 201 202 adapter->link_speed = ret >> 16; 203 if (ret & 1) { /* Link is up. */ 204 /* 205 * From vmxnet3 v9, the hypervisor reports the speed in Gbps. 206 * Convert the speed to Mbps before rporting it to the kernel. 207 * Max link speed supported is 10000G. 208 */ 209 if (VMXNET3_VERSION_GE_9(adapter) && 210 adapter->link_speed < 10000) 211 adapter->link_speed = adapter->link_speed * 1000; 212 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n", 213 adapter->link_speed); 214 netif_carrier_on(adapter->netdev); 215 216 if (affectTxQueue) { 217 for (i = 0; i < adapter->num_tx_queues; i++) 218 vmxnet3_tq_start(&adapter->tx_queue[i], 219 adapter); 220 } 221 } else { 222 netdev_info(adapter->netdev, "NIC Link is Down\n"); 223 netif_carrier_off(adapter->netdev); 224 225 if (affectTxQueue) { 226 for (i = 0; i < adapter->num_tx_queues; i++) 227 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter); 228 } 229 } 230 } 231 232 static void 233 vmxnet3_process_events(struct vmxnet3_adapter *adapter) 234 { 235 int i; 236 unsigned long flags; 237 u32 events = le32_to_cpu(adapter->shared->ecr); 238 if (!events) 239 return; 240 241 vmxnet3_ack_events(adapter, events); 242 243 /* Check if link state has changed */ 244 if (events & VMXNET3_ECR_LINK) 245 vmxnet3_check_link(adapter, true); 246 247 /* Check if there is an error on xmit/recv queues */ 248 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 249 spin_lock_irqsave(&adapter->cmd_lock, flags); 250 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 251 VMXNET3_CMD_GET_QUEUE_STATUS); 252 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 253 254 for (i = 0; i < adapter->num_tx_queues; i++) 255 if (adapter->tqd_start[i].status.stopped) 256 dev_err(&adapter->netdev->dev, 257 "%s: tq[%d] error 0x%x\n", 258 adapter->netdev->name, i, le32_to_cpu( 259 adapter->tqd_start[i].status.error)); 260 for (i = 0; i < adapter->num_rx_queues; i++) 261 if (adapter->rqd_start[i].status.stopped) 262 dev_err(&adapter->netdev->dev, 263 "%s: rq[%d] error 0x%x\n", 264 adapter->netdev->name, i, 265 adapter->rqd_start[i].status.error); 266 267 schedule_work(&adapter->work); 268 } 269 } 270 271 #ifdef __BIG_ENDIAN_BITFIELD 272 /* 273 * The device expects the bitfields in shared structures to be written in 274 * little endian. When CPU is big endian, the following routines are used to 275 * correctly read and write into ABI. 276 * The general technique used here is : double word bitfields are defined in 277 * opposite order for big endian architecture. Then before reading them in 278 * driver the complete double word is translated using le32_to_cpu. Similarly 279 * After the driver writes into bitfields, cpu_to_le32 is used to translate the 280 * double words into required format. 281 * In order to avoid touching bits in shared structure more than once, temporary 282 * descriptors are used. These are passed as srcDesc to following functions. 283 */ 284 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc, 285 struct Vmxnet3_RxDesc *dstDesc) 286 { 287 u32 *src = (u32 *)srcDesc + 2; 288 u32 *dst = (u32 *)dstDesc + 2; 289 dstDesc->addr = le64_to_cpu(srcDesc->addr); 290 *dst = le32_to_cpu(*src); 291 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1); 292 } 293 294 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc, 295 struct Vmxnet3_TxDesc *dstDesc) 296 { 297 int i; 298 u32 *src = (u32 *)(srcDesc + 1); 299 u32 *dst = (u32 *)(dstDesc + 1); 300 301 /* Working backwards so that the gen bit is set at the end. */ 302 for (i = 2; i > 0; i--) { 303 src--; 304 dst--; 305 *dst = cpu_to_le32(*src); 306 } 307 } 308 309 310 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc, 311 struct Vmxnet3_RxCompDesc *dstDesc) 312 { 313 int i = 0; 314 u32 *src = (u32 *)srcDesc; 315 u32 *dst = (u32 *)dstDesc; 316 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) { 317 *dst = le32_to_cpu(*src); 318 src++; 319 dst++; 320 } 321 } 322 323 324 /* Used to read bitfield values from double words. */ 325 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size) 326 { 327 u32 temp = le32_to_cpu(*bitfield); 328 u32 mask = ((1 << size) - 1) << pos; 329 temp &= mask; 330 temp >>= pos; 331 return temp; 332 } 333 334 335 336 #endif /* __BIG_ENDIAN_BITFIELD */ 337 338 #ifdef __BIG_ENDIAN_BITFIELD 339 340 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \ 341 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \ 342 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE) 343 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \ 344 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \ 345 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE) 346 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \ 347 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \ 348 VMXNET3_TCD_GEN_SIZE) 349 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \ 350 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE) 351 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \ 352 (dstrcd) = (tmp); \ 353 vmxnet3_RxCompToCPU((rcd), (tmp)); \ 354 } while (0) 355 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \ 356 (dstrxd) = (tmp); \ 357 vmxnet3_RxDescToCPU((rxd), (tmp)); \ 358 } while (0) 359 360 #else 361 362 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen) 363 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop) 364 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen) 365 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx) 366 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd) 367 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd) 368 369 #endif /* __BIG_ENDIAN_BITFIELD */ 370 371 372 static void 373 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi, 374 struct pci_dev *pdev) 375 { 376 u32 map_type = tbi->map_type; 377 378 if (map_type & VMXNET3_MAP_SINGLE) 379 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len, 380 DMA_TO_DEVICE); 381 else if (map_type & VMXNET3_MAP_PAGE) 382 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len, 383 DMA_TO_DEVICE); 384 else 385 BUG_ON(map_type & ~VMXNET3_MAP_XDP); 386 387 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */ 388 } 389 390 391 static int 392 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq, 393 struct pci_dev *pdev, struct vmxnet3_adapter *adapter, 394 struct xdp_frame_bulk *bq) 395 { 396 struct vmxnet3_tx_buf_info *tbi; 397 int entries = 0; 398 u32 map_type; 399 400 /* no out of order completion */ 401 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp); 402 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1); 403 404 tbi = &tq->buf_info[eop_idx]; 405 BUG_ON(!tbi->skb); 406 map_type = tbi->map_type; 407 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size); 408 409 while (tq->tx_ring.next2comp != eop_idx) { 410 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp, 411 pdev); 412 413 /* update next2comp w/o tx_lock. Since we are marking more, 414 * instead of less, tx ring entries avail, the worst case is 415 * that the tx routine incorrectly re-queues a pkt due to 416 * insufficient tx ring entries. 417 */ 418 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 419 entries++; 420 } 421 422 if (map_type & VMXNET3_MAP_XDP) 423 xdp_return_frame_bulk(tbi->xdpf, bq); 424 else 425 dev_kfree_skb_any(tbi->skb); 426 427 /* xdpf and skb are in an anonymous union. */ 428 tbi->skb = NULL; 429 430 return entries; 431 } 432 433 434 static int 435 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq, 436 struct vmxnet3_adapter *adapter) 437 { 438 union Vmxnet3_GenericDesc *gdesc; 439 struct xdp_frame_bulk bq; 440 int completed = 0; 441 442 xdp_frame_bulk_init(&bq); 443 rcu_read_lock(); 444 445 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 446 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) { 447 /* Prevent any &gdesc->tcd field from being (speculatively) 448 * read before (&gdesc->tcd)->gen is read. 449 */ 450 dma_rmb(); 451 452 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX( 453 &gdesc->tcd), tq, adapter->pdev, 454 adapter, &bq); 455 456 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring); 457 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 458 } 459 xdp_flush_frame_bulk(&bq); 460 rcu_read_unlock(); 461 462 if (completed) { 463 spin_lock(&tq->tx_lock); 464 if (unlikely(vmxnet3_tq_stopped(tq, adapter) && 465 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) > 466 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) && 467 netif_carrier_ok(adapter->netdev))) { 468 vmxnet3_tq_wake(tq, adapter); 469 } 470 spin_unlock(&tq->tx_lock); 471 } 472 return completed; 473 } 474 475 476 static void 477 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq, 478 struct vmxnet3_adapter *adapter) 479 { 480 struct xdp_frame_bulk bq; 481 u32 map_type; 482 int i; 483 484 xdp_frame_bulk_init(&bq); 485 rcu_read_lock(); 486 487 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) { 488 struct vmxnet3_tx_buf_info *tbi; 489 490 tbi = tq->buf_info + tq->tx_ring.next2comp; 491 map_type = tbi->map_type; 492 493 vmxnet3_unmap_tx_buf(tbi, adapter->pdev); 494 if (tbi->skb) { 495 if (map_type & VMXNET3_MAP_XDP) 496 xdp_return_frame_bulk(tbi->xdpf, &bq); 497 else 498 dev_kfree_skb_any(tbi->skb); 499 tbi->skb = NULL; 500 } 501 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 502 } 503 504 xdp_flush_frame_bulk(&bq); 505 rcu_read_unlock(); 506 507 /* sanity check, verify all buffers are indeed unmapped */ 508 for (i = 0; i < tq->tx_ring.size; i++) 509 BUG_ON(tq->buf_info[i].map_type != VMXNET3_MAP_NONE); 510 511 tq->tx_ring.gen = VMXNET3_INIT_GEN; 512 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 513 514 tq->comp_ring.gen = VMXNET3_INIT_GEN; 515 tq->comp_ring.next2proc = 0; 516 } 517 518 519 static void 520 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 521 struct vmxnet3_adapter *adapter) 522 { 523 if (tq->tx_ring.base) { 524 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size * 525 sizeof(struct Vmxnet3_TxDesc), 526 tq->tx_ring.base, tq->tx_ring.basePA); 527 tq->tx_ring.base = NULL; 528 } 529 if (tq->data_ring.base) { 530 dma_free_coherent(&adapter->pdev->dev, 531 tq->data_ring.size * tq->txdata_desc_size, 532 tq->data_ring.base, tq->data_ring.basePA); 533 tq->data_ring.base = NULL; 534 } 535 if (tq->ts_ring.base) { 536 dma_free_coherent(&adapter->pdev->dev, 537 tq->tx_ring.size * tq->tx_ts_desc_size, 538 tq->ts_ring.base, tq->ts_ring.basePA); 539 tq->ts_ring.base = NULL; 540 } 541 if (tq->comp_ring.base) { 542 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size * 543 sizeof(struct Vmxnet3_TxCompDesc), 544 tq->comp_ring.base, tq->comp_ring.basePA); 545 tq->comp_ring.base = NULL; 546 } 547 kfree(tq->buf_info); 548 tq->buf_info = NULL; 549 } 550 551 552 /* Destroy all tx queues */ 553 void 554 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter) 555 { 556 int i; 557 558 for (i = 0; i < adapter->num_tx_queues; i++) 559 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter); 560 } 561 562 563 static void 564 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 565 struct vmxnet3_adapter *adapter) 566 { 567 int i; 568 569 /* reset the tx ring contents to 0 and reset the tx ring states */ 570 memset(tq->tx_ring.base, 0, tq->tx_ring.size * 571 sizeof(struct Vmxnet3_TxDesc)); 572 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 573 tq->tx_ring.gen = VMXNET3_INIT_GEN; 574 575 memset(tq->data_ring.base, 0, 576 tq->data_ring.size * tq->txdata_desc_size); 577 578 if (tq->ts_ring.base) 579 memset(tq->ts_ring.base, 0, 580 tq->tx_ring.size * tq->tx_ts_desc_size); 581 582 /* reset the tx comp ring contents to 0 and reset comp ring states */ 583 memset(tq->comp_ring.base, 0, tq->comp_ring.size * 584 sizeof(struct Vmxnet3_TxCompDesc)); 585 tq->comp_ring.next2proc = 0; 586 tq->comp_ring.gen = VMXNET3_INIT_GEN; 587 588 /* reset the bookkeeping data */ 589 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size); 590 for (i = 0; i < tq->tx_ring.size; i++) 591 tq->buf_info[i].map_type = VMXNET3_MAP_NONE; 592 593 /* stats are not reset */ 594 } 595 596 597 static int 598 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq, 599 struct vmxnet3_adapter *adapter) 600 { 601 BUG_ON(tq->tx_ring.base || tq->data_ring.base || 602 tq->comp_ring.base || tq->buf_info); 603 604 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 605 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc), 606 &tq->tx_ring.basePA, GFP_KERNEL); 607 if (!tq->tx_ring.base) { 608 netdev_err(adapter->netdev, "failed to allocate tx ring\n"); 609 goto err; 610 } 611 612 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 613 tq->data_ring.size * tq->txdata_desc_size, 614 &tq->data_ring.basePA, GFP_KERNEL); 615 if (!tq->data_ring.base) { 616 netdev_err(adapter->netdev, "failed to allocate tx data ring\n"); 617 goto err; 618 } 619 620 if (tq->tx_ts_desc_size != 0) { 621 tq->ts_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 622 tq->tx_ring.size * tq->tx_ts_desc_size, 623 &tq->ts_ring.basePA, GFP_KERNEL); 624 if (!tq->ts_ring.base) { 625 netdev_err(adapter->netdev, "failed to allocate tx ts ring\n"); 626 tq->tx_ts_desc_size = 0; 627 } 628 } else { 629 tq->ts_ring.base = NULL; 630 } 631 632 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 633 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc), 634 &tq->comp_ring.basePA, GFP_KERNEL); 635 if (!tq->comp_ring.base) { 636 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n"); 637 goto err; 638 } 639 640 tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]), 641 GFP_KERNEL, 642 dev_to_node(&adapter->pdev->dev)); 643 if (!tq->buf_info) 644 goto err; 645 646 return 0; 647 648 err: 649 vmxnet3_tq_destroy(tq, adapter); 650 return -ENOMEM; 651 } 652 653 static void 654 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter) 655 { 656 int i; 657 658 for (i = 0; i < adapter->num_tx_queues; i++) 659 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter); 660 } 661 662 /* 663 * starting from ring->next2fill, allocate rx buffers for the given ring 664 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers 665 * are allocated or allocation fails 666 */ 667 668 static int 669 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx, 670 int num_to_alloc, struct vmxnet3_adapter *adapter) 671 { 672 int num_allocated = 0; 673 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx]; 674 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx]; 675 u32 val; 676 677 while (num_allocated <= num_to_alloc) { 678 struct vmxnet3_rx_buf_info *rbi; 679 union Vmxnet3_GenericDesc *gd; 680 681 rbi = rbi_base + ring->next2fill; 682 gd = ring->base + ring->next2fill; 683 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 684 685 if (rbi->buf_type == VMXNET3_RX_BUF_XDP) { 686 void *data = vmxnet3_pp_get_buff(rq->page_pool, 687 &rbi->dma_addr, 688 GFP_KERNEL); 689 if (!data) { 690 rq->stats.rx_buf_alloc_failure++; 691 break; 692 } 693 rbi->page = virt_to_page(data); 694 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 695 } else if (rbi->buf_type == VMXNET3_RX_BUF_SKB) { 696 if (rbi->skb == NULL) { 697 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev, 698 rbi->len, 699 GFP_KERNEL); 700 if (unlikely(rbi->skb == NULL)) { 701 rq->stats.rx_buf_alloc_failure++; 702 break; 703 } 704 705 rbi->dma_addr = dma_map_single( 706 &adapter->pdev->dev, 707 rbi->skb->data, rbi->len, 708 DMA_FROM_DEVICE); 709 if (dma_mapping_error(&adapter->pdev->dev, 710 rbi->dma_addr)) { 711 dev_kfree_skb_any(rbi->skb); 712 rbi->skb = NULL; 713 rq->stats.rx_buf_alloc_failure++; 714 break; 715 } 716 } else { 717 /* rx buffer skipped by the device */ 718 } 719 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 720 } else { 721 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE || 722 rbi->len != PAGE_SIZE); 723 724 if (rbi->page == NULL) { 725 rbi->page = alloc_page(GFP_ATOMIC); 726 if (unlikely(rbi->page == NULL)) { 727 rq->stats.rx_buf_alloc_failure++; 728 break; 729 } 730 rbi->dma_addr = dma_map_page( 731 &adapter->pdev->dev, 732 rbi->page, 0, PAGE_SIZE, 733 DMA_FROM_DEVICE); 734 if (dma_mapping_error(&adapter->pdev->dev, 735 rbi->dma_addr)) { 736 put_page(rbi->page); 737 rbi->page = NULL; 738 rq->stats.rx_buf_alloc_failure++; 739 break; 740 } 741 } else { 742 /* rx buffers skipped by the device */ 743 } 744 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT; 745 } 746 747 gd->rxd.addr = cpu_to_le64(rbi->dma_addr); 748 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT) 749 | val | rbi->len); 750 751 /* Fill the last buffer but dont mark it ready, or else the 752 * device will think that the queue is full */ 753 if (num_allocated == num_to_alloc) { 754 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 755 break; 756 } 757 758 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT); 759 num_allocated++; 760 vmxnet3_cmd_ring_adv_next2fill(ring); 761 } 762 763 netdev_dbg(adapter->netdev, 764 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n", 765 num_allocated, ring->next2fill, ring->next2comp); 766 767 /* so that the device can distinguish a full ring and an empty ring */ 768 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp); 769 770 return num_allocated; 771 } 772 773 774 static void 775 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd, 776 struct vmxnet3_rx_buf_info *rbi) 777 { 778 skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags; 779 780 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); 781 782 skb_frag_fill_page_desc(frag, rbi->page, 0, rcd->len); 783 skb->data_len += rcd->len; 784 skb->truesize += PAGE_SIZE; 785 skb_shinfo(skb)->nr_frags++; 786 } 787 788 789 static int 790 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx, 791 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev, 792 struct vmxnet3_adapter *adapter) 793 { 794 u32 dw2, len; 795 unsigned long buf_offset; 796 int i; 797 union Vmxnet3_GenericDesc *gdesc; 798 struct vmxnet3_tx_buf_info *tbi = NULL; 799 800 BUG_ON(ctx->copy_size > skb_headlen(skb)); 801 802 /* use the previous gen bit for the SOP desc */ 803 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 804 805 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; 806 gdesc = ctx->sop_txd; /* both loops below can be skipped */ 807 808 /* no need to map the buffer if headers are copied */ 809 if (ctx->copy_size) { 810 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA + 811 tq->tx_ring.next2fill * 812 tq->txdata_desc_size); 813 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size); 814 ctx->sop_txd->dword[3] = 0; 815 816 tbi = tq->buf_info + tq->tx_ring.next2fill; 817 tbi->map_type = VMXNET3_MAP_NONE; 818 819 netdev_dbg(adapter->netdev, 820 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 821 tq->tx_ring.next2fill, 822 le64_to_cpu(ctx->sop_txd->txd.addr), 823 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]); 824 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 825 826 /* use the right gen for non-SOP desc */ 827 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 828 } 829 830 /* linear part can use multiple tx desc if it's big */ 831 len = skb_headlen(skb) - ctx->copy_size; 832 buf_offset = ctx->copy_size; 833 while (len) { 834 u32 buf_size; 835 836 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 837 buf_size = len; 838 dw2 |= len; 839 } else { 840 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 841 /* spec says that for TxDesc.len, 0 == 2^14 */ 842 } 843 844 tbi = tq->buf_info + tq->tx_ring.next2fill; 845 tbi->map_type = VMXNET3_MAP_SINGLE; 846 tbi->dma_addr = dma_map_single(&adapter->pdev->dev, 847 skb->data + buf_offset, buf_size, 848 DMA_TO_DEVICE); 849 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 850 return -EFAULT; 851 852 tbi->len = buf_size; 853 854 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 855 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 856 857 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 858 gdesc->dword[2] = cpu_to_le32(dw2); 859 gdesc->dword[3] = 0; 860 861 netdev_dbg(adapter->netdev, 862 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 863 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 864 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 865 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 866 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 867 868 len -= buf_size; 869 buf_offset += buf_size; 870 } 871 872 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 873 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 874 u32 buf_size; 875 876 buf_offset = 0; 877 len = skb_frag_size(frag); 878 while (len) { 879 tbi = tq->buf_info + tq->tx_ring.next2fill; 880 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 881 buf_size = len; 882 dw2 |= len; 883 } else { 884 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 885 /* spec says that for TxDesc.len, 0 == 2^14 */ 886 } 887 tbi->map_type = VMXNET3_MAP_PAGE; 888 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag, 889 buf_offset, buf_size, 890 DMA_TO_DEVICE); 891 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 892 return -EFAULT; 893 894 tbi->len = buf_size; 895 896 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 897 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 898 899 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 900 gdesc->dword[2] = cpu_to_le32(dw2); 901 gdesc->dword[3] = 0; 902 903 netdev_dbg(adapter->netdev, 904 "txd[%u]: 0x%llx %u %u\n", 905 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 906 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 907 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 908 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 909 910 len -= buf_size; 911 buf_offset += buf_size; 912 } 913 } 914 915 ctx->eop_txd = gdesc; 916 917 /* set the last buf_info for the pkt */ 918 tbi->skb = skb; 919 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base; 920 if (tq->tx_ts_desc_size != 0) { 921 ctx->ts_txd = (struct Vmxnet3_TxTSDesc *)((u8 *)tq->ts_ring.base + 922 tbi->sop_idx * tq->tx_ts_desc_size); 923 ctx->ts_txd->ts.tsi = 0; 924 } 925 926 return 0; 927 } 928 929 930 /* Init all tx queues */ 931 static void 932 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter) 933 { 934 int i; 935 936 for (i = 0; i < adapter->num_tx_queues; i++) 937 vmxnet3_tq_init(&adapter->tx_queue[i], adapter); 938 } 939 940 941 /* 942 * parse relevant protocol headers: 943 * For a tso pkt, relevant headers are L2/3/4 including options 944 * For a pkt requesting csum offloading, they are L2/3 and may include L4 945 * if it's a TCP/UDP pkt 946 * 947 * Returns: 948 * -1: error happens during parsing 949 * 0: protocol headers parsed, but too big to be copied 950 * 1: protocol headers parsed and copied 951 * 952 * Other effects: 953 * 1. related *ctx fields are updated. 954 * 2. ctx->copy_size is # of bytes copied 955 * 3. the portion to be copied is guaranteed to be in the linear part 956 * 957 */ 958 static int 959 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 960 struct vmxnet3_tx_ctx *ctx, 961 struct vmxnet3_adapter *adapter) 962 { 963 u8 protocol = 0; 964 965 if (ctx->mss) { /* TSO */ 966 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 967 ctx->l4_offset = skb_inner_transport_offset(skb); 968 ctx->l4_hdr_size = inner_tcp_hdrlen(skb); 969 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 970 } else { 971 ctx->l4_offset = skb_transport_offset(skb); 972 ctx->l4_hdr_size = tcp_hdrlen(skb); 973 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 974 } 975 } else { 976 if (skb->ip_summed == CHECKSUM_PARTIAL) { 977 /* For encap packets, skb_checksum_start_offset refers 978 * to inner L4 offset. Thus, below works for encap as 979 * well as non-encap case 980 */ 981 ctx->l4_offset = skb_checksum_start_offset(skb); 982 983 if (VMXNET3_VERSION_GE_4(adapter) && 984 skb->encapsulation) { 985 struct iphdr *iph = inner_ip_hdr(skb); 986 987 if (iph->version == 4) { 988 protocol = iph->protocol; 989 } else { 990 const struct ipv6hdr *ipv6h; 991 992 ipv6h = inner_ipv6_hdr(skb); 993 protocol = ipv6h->nexthdr; 994 } 995 } else { 996 if (ctx->ipv4) { 997 const struct iphdr *iph = ip_hdr(skb); 998 999 protocol = iph->protocol; 1000 } else if (ctx->ipv6) { 1001 const struct ipv6hdr *ipv6h; 1002 1003 ipv6h = ipv6_hdr(skb); 1004 protocol = ipv6h->nexthdr; 1005 } 1006 } 1007 1008 switch (protocol) { 1009 case IPPROTO_TCP: 1010 ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) : 1011 tcp_hdrlen(skb); 1012 break; 1013 case IPPROTO_UDP: 1014 ctx->l4_hdr_size = sizeof(struct udphdr); 1015 break; 1016 default: 1017 ctx->l4_hdr_size = 0; 1018 break; 1019 } 1020 1021 ctx->copy_size = min(ctx->l4_offset + 1022 ctx->l4_hdr_size, skb->len); 1023 } else { 1024 ctx->l4_offset = 0; 1025 ctx->l4_hdr_size = 0; 1026 /* copy as much as allowed */ 1027 ctx->copy_size = min_t(unsigned int, 1028 tq->txdata_desc_size, 1029 skb_headlen(skb)); 1030 } 1031 1032 if (skb->len <= tq->txdata_desc_size) 1033 ctx->copy_size = skb->len; 1034 1035 /* make sure headers are accessible directly */ 1036 if (unlikely(!pskb_may_pull(skb, ctx->copy_size))) 1037 goto err; 1038 } 1039 1040 if (unlikely(ctx->copy_size > tq->txdata_desc_size)) { 1041 tq->stats.oversized_hdr++; 1042 ctx->copy_size = 0; 1043 return 0; 1044 } 1045 1046 return 1; 1047 err: 1048 return -1; 1049 } 1050 1051 /* 1052 * copy relevant protocol headers to the transmit ring: 1053 * For a tso pkt, relevant headers are L2/3/4 including options 1054 * For a pkt requesting csum offloading, they are L2/3 and may include L4 1055 * if it's a TCP/UDP pkt 1056 * 1057 * 1058 * Note that this requires that vmxnet3_parse_hdr be called first to set the 1059 * appropriate bits in ctx first 1060 */ 1061 static void 1062 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1063 struct vmxnet3_tx_ctx *ctx, 1064 struct vmxnet3_adapter *adapter) 1065 { 1066 struct Vmxnet3_TxDataDesc *tdd; 1067 1068 tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base + 1069 tq->tx_ring.next2fill * 1070 tq->txdata_desc_size); 1071 1072 memcpy(tdd->data, skb->data, ctx->copy_size); 1073 netdev_dbg(adapter->netdev, 1074 "copy %u bytes to dataRing[%u]\n", 1075 ctx->copy_size, tq->tx_ring.next2fill); 1076 } 1077 1078 1079 static void 1080 vmxnet3_prepare_inner_tso(struct sk_buff *skb, 1081 struct vmxnet3_tx_ctx *ctx) 1082 { 1083 struct tcphdr *tcph = inner_tcp_hdr(skb); 1084 struct iphdr *iph = inner_ip_hdr(skb); 1085 1086 if (iph->version == 4) { 1087 iph->check = 0; 1088 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1089 IPPROTO_TCP, 0); 1090 } else { 1091 struct ipv6hdr *iph = inner_ipv6_hdr(skb); 1092 1093 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0, 1094 IPPROTO_TCP, 0); 1095 } 1096 } 1097 1098 static void 1099 vmxnet3_prepare_tso(struct sk_buff *skb, 1100 struct vmxnet3_tx_ctx *ctx) 1101 { 1102 struct tcphdr *tcph = tcp_hdr(skb); 1103 1104 if (ctx->ipv4) { 1105 struct iphdr *iph = ip_hdr(skb); 1106 1107 iph->check = 0; 1108 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1109 IPPROTO_TCP, 0); 1110 } else if (ctx->ipv6) { 1111 tcp_v6_gso_csum_prep(skb); 1112 } 1113 } 1114 1115 static int txd_estimate(const struct sk_buff *skb) 1116 { 1117 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1118 int i; 1119 1120 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1121 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1122 1123 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag)); 1124 } 1125 return count; 1126 } 1127 1128 /* 1129 * Transmits a pkt thru a given tq 1130 * Returns: 1131 * NETDEV_TX_OK: descriptors are setup successfully 1132 * NETDEV_TX_OK: error occurred, the pkt is dropped 1133 * NETDEV_TX_BUSY: tx ring is full, queue is stopped 1134 * 1135 * Side-effects: 1136 * 1. tx ring may be changed 1137 * 2. tq stats may be updated accordingly 1138 * 3. shared->txNumDeferred may be updated 1139 */ 1140 1141 static int 1142 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1143 struct vmxnet3_adapter *adapter, struct net_device *netdev) 1144 { 1145 int ret; 1146 u32 count; 1147 int num_pkts; 1148 int tx_num_deferred; 1149 unsigned long flags; 1150 struct vmxnet3_tx_ctx ctx; 1151 union Vmxnet3_GenericDesc *gdesc; 1152 #ifdef __BIG_ENDIAN_BITFIELD 1153 /* Use temporary descriptor to avoid touching bits multiple times */ 1154 union Vmxnet3_GenericDesc tempTxDesc; 1155 #endif 1156 1157 count = txd_estimate(skb); 1158 1159 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP)); 1160 ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6)); 1161 1162 ctx.mss = skb_shinfo(skb)->gso_size; 1163 if (ctx.mss) { 1164 if (skb_header_cloned(skb)) { 1165 if (unlikely(pskb_expand_head(skb, 0, 0, 1166 GFP_ATOMIC) != 0)) { 1167 tq->stats.drop_tso++; 1168 goto drop_pkt; 1169 } 1170 tq->stats.copy_skb_header++; 1171 } 1172 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1173 /* tso pkts must not use more than 1174 * VMXNET3_MAX_TSO_TXD_PER_PKT entries 1175 */ 1176 if (skb_linearize(skb) != 0) { 1177 tq->stats.drop_too_many_frags++; 1178 goto drop_pkt; 1179 } 1180 tq->stats.linearized++; 1181 1182 /* recalculate the # of descriptors to use */ 1183 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1184 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1185 tq->stats.drop_too_many_frags++; 1186 goto drop_pkt; 1187 } 1188 } 1189 if (skb->encapsulation) { 1190 vmxnet3_prepare_inner_tso(skb, &ctx); 1191 } else { 1192 vmxnet3_prepare_tso(skb, &ctx); 1193 } 1194 } else { 1195 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) { 1196 1197 /* non-tso pkts must not use more than 1198 * VMXNET3_MAX_TXD_PER_PKT entries 1199 */ 1200 if (skb_linearize(skb) != 0) { 1201 tq->stats.drop_too_many_frags++; 1202 goto drop_pkt; 1203 } 1204 tq->stats.linearized++; 1205 1206 /* recalculate the # of descriptors to use */ 1207 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1208 } 1209 } 1210 1211 ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter); 1212 if (ret >= 0) { 1213 BUG_ON(ret <= 0 && ctx.copy_size != 0); 1214 /* hdrs parsed, check against other limits */ 1215 if (ctx.mss) { 1216 if (unlikely(ctx.l4_offset + ctx.l4_hdr_size > 1217 VMXNET3_MAX_TX_BUF_SIZE)) { 1218 tq->stats.drop_oversized_hdr++; 1219 goto drop_pkt; 1220 } 1221 } else { 1222 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1223 if (unlikely(ctx.l4_offset + 1224 skb->csum_offset > 1225 VMXNET3_MAX_CSUM_OFFSET)) { 1226 tq->stats.drop_oversized_hdr++; 1227 goto drop_pkt; 1228 } 1229 } 1230 } 1231 } else { 1232 tq->stats.drop_hdr_inspect_err++; 1233 goto drop_pkt; 1234 } 1235 1236 spin_lock_irqsave(&tq->tx_lock, flags); 1237 1238 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) { 1239 tq->stats.tx_ring_full++; 1240 netdev_dbg(adapter->netdev, 1241 "tx queue stopped on %s, next2comp %u" 1242 " next2fill %u\n", adapter->netdev->name, 1243 tq->tx_ring.next2comp, tq->tx_ring.next2fill); 1244 1245 vmxnet3_tq_stop(tq, adapter); 1246 spin_unlock_irqrestore(&tq->tx_lock, flags); 1247 return NETDEV_TX_BUSY; 1248 } 1249 1250 1251 vmxnet3_copy_hdr(skb, tq, &ctx, adapter); 1252 1253 /* fill tx descs related to addr & len */ 1254 if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter)) 1255 goto unlock_drop_pkt; 1256 1257 /* setup the EOP desc */ 1258 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); 1259 1260 /* setup the SOP desc */ 1261 #ifdef __BIG_ENDIAN_BITFIELD 1262 gdesc = &tempTxDesc; 1263 gdesc->dword[2] = ctx.sop_txd->dword[2]; 1264 gdesc->dword[3] = ctx.sop_txd->dword[3]; 1265 #else 1266 gdesc = ctx.sop_txd; 1267 #endif 1268 tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred); 1269 if (ctx.mss) { 1270 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 1271 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1272 if (VMXNET3_VERSION_GE_7(adapter)) { 1273 gdesc->txd.om = VMXNET3_OM_TSO; 1274 gdesc->txd.ext1 = 1; 1275 } else { 1276 gdesc->txd.om = VMXNET3_OM_ENCAP; 1277 } 1278 gdesc->txd.msscof = ctx.mss; 1279 1280 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) 1281 gdesc->txd.oco = 1; 1282 } else { 1283 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1284 gdesc->txd.om = VMXNET3_OM_TSO; 1285 gdesc->txd.msscof = ctx.mss; 1286 } 1287 num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss; 1288 } else { 1289 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1290 if (VMXNET3_VERSION_GE_4(adapter) && 1291 skb->encapsulation) { 1292 gdesc->txd.hlen = ctx.l4_offset + 1293 ctx.l4_hdr_size; 1294 if (VMXNET3_VERSION_GE_7(adapter)) { 1295 gdesc->txd.om = VMXNET3_OM_CSUM; 1296 gdesc->txd.msscof = ctx.l4_offset + 1297 skb->csum_offset; 1298 gdesc->txd.ext1 = 1; 1299 } else { 1300 gdesc->txd.om = VMXNET3_OM_ENCAP; 1301 gdesc->txd.msscof = 0; /* Reserved */ 1302 } 1303 } else { 1304 gdesc->txd.hlen = ctx.l4_offset; 1305 gdesc->txd.om = VMXNET3_OM_CSUM; 1306 gdesc->txd.msscof = ctx.l4_offset + 1307 skb->csum_offset; 1308 } 1309 } else { 1310 gdesc->txd.om = 0; 1311 gdesc->txd.msscof = 0; 1312 } 1313 num_pkts = 1; 1314 } 1315 le32_add_cpu(&tq->shared->txNumDeferred, num_pkts); 1316 tx_num_deferred += num_pkts; 1317 1318 if (skb_vlan_tag_present(skb)) { 1319 gdesc->txd.ti = 1; 1320 gdesc->txd.tci = skb_vlan_tag_get(skb); 1321 } 1322 1323 if (tq->tx_ts_desc_size != 0 && 1324 adapter->latencyConf->sampleRate != 0) { 1325 if (vmxnet3_apply_timestamp(tq, adapter->latencyConf->sampleRate)) { 1326 ctx.ts_txd->ts.tsData = vmxnet3_get_cycles(VMXNET3_PMC_PSEUDO_TSC); 1327 ctx.ts_txd->ts.tsi = 1; 1328 } 1329 } 1330 1331 /* Ensure that the write to (&gdesc->txd)->gen will be observed after 1332 * all other writes to &gdesc->txd. 1333 */ 1334 dma_wmb(); 1335 1336 /* finally flips the GEN bit of the SOP desc. */ 1337 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ 1338 VMXNET3_TXD_GEN); 1339 #ifdef __BIG_ENDIAN_BITFIELD 1340 /* Finished updating in bitfields of Tx Desc, so write them in original 1341 * place. 1342 */ 1343 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc, 1344 (struct Vmxnet3_TxDesc *)ctx.sop_txd); 1345 gdesc = ctx.sop_txd; 1346 #endif 1347 netdev_dbg(adapter->netdev, 1348 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n", 1349 (u32)(ctx.sop_txd - 1350 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr), 1351 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3])); 1352 1353 spin_unlock_irqrestore(&tq->tx_lock, flags); 1354 1355 if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) { 1356 tq->shared->txNumDeferred = 0; 1357 VMXNET3_WRITE_BAR0_REG(adapter, 1358 adapter->tx_prod_offset + tq->qid * 8, 1359 tq->tx_ring.next2fill); 1360 } 1361 1362 return NETDEV_TX_OK; 1363 1364 unlock_drop_pkt: 1365 spin_unlock_irqrestore(&tq->tx_lock, flags); 1366 drop_pkt: 1367 tq->stats.drop_total++; 1368 dev_kfree_skb_any(skb); 1369 return NETDEV_TX_OK; 1370 } 1371 1372 static int 1373 vmxnet3_create_pp(struct vmxnet3_adapter *adapter, 1374 struct vmxnet3_rx_queue *rq, int size) 1375 { 1376 bool xdp_prog = vmxnet3_xdp_enabled(adapter); 1377 const struct page_pool_params pp_params = { 1378 .order = 0, 1379 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 1380 .pool_size = size, 1381 .nid = NUMA_NO_NODE, 1382 .dev = &adapter->pdev->dev, 1383 .offset = VMXNET3_XDP_RX_OFFSET, 1384 .max_len = VMXNET3_XDP_MAX_FRSIZE, 1385 .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE, 1386 }; 1387 struct page_pool *pp; 1388 int err; 1389 1390 pp = page_pool_create(&pp_params); 1391 if (IS_ERR(pp)) 1392 return PTR_ERR(pp); 1393 1394 err = xdp_rxq_info_reg(&rq->xdp_rxq, adapter->netdev, rq->qid, 1395 rq->napi.napi_id); 1396 if (err < 0) 1397 goto err_free_pp; 1398 1399 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_PAGE_POOL, pp); 1400 if (err) 1401 goto err_unregister_rxq; 1402 1403 rq->page_pool = pp; 1404 1405 return 0; 1406 1407 err_unregister_rxq: 1408 xdp_rxq_info_unreg(&rq->xdp_rxq); 1409 err_free_pp: 1410 page_pool_destroy(pp); 1411 1412 return err; 1413 } 1414 1415 void * 1416 vmxnet3_pp_get_buff(struct page_pool *pp, dma_addr_t *dma_addr, 1417 gfp_t gfp_mask) 1418 { 1419 struct page *page; 1420 1421 page = page_pool_alloc_pages(pp, gfp_mask | __GFP_NOWARN); 1422 if (unlikely(!page)) 1423 return NULL; 1424 1425 *dma_addr = page_pool_get_dma_addr(page) + pp->p.offset; 1426 1427 return page_address(page); 1428 } 1429 1430 static netdev_tx_t 1431 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1432 { 1433 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1434 1435 BUG_ON(skb->queue_mapping > adapter->num_tx_queues); 1436 return vmxnet3_tq_xmit(skb, 1437 &adapter->tx_queue[skb->queue_mapping], 1438 adapter, netdev); 1439 } 1440 1441 1442 static void 1443 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, 1444 struct sk_buff *skb, 1445 union Vmxnet3_GenericDesc *gdesc) 1446 { 1447 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) { 1448 if (gdesc->rcd.v4 && 1449 (le32_to_cpu(gdesc->dword[3]) & 1450 VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) { 1451 skb->ip_summed = CHECKSUM_UNNECESSARY; 1452 if ((le32_to_cpu(gdesc->dword[0]) & 1453 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1454 skb->csum_level = 1; 1455 } 1456 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1457 !(le32_to_cpu(gdesc->dword[0]) & 1458 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1459 WARN_ON_ONCE(gdesc->rcd.frg && 1460 !(le32_to_cpu(gdesc->dword[0]) & 1461 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1462 } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) & 1463 (1 << VMXNET3_RCD_TUC_SHIFT))) { 1464 skb->ip_summed = CHECKSUM_UNNECESSARY; 1465 if ((le32_to_cpu(gdesc->dword[0]) & 1466 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1467 skb->csum_level = 1; 1468 } 1469 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1470 !(le32_to_cpu(gdesc->dword[0]) & 1471 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1472 WARN_ON_ONCE(gdesc->rcd.frg && 1473 !(le32_to_cpu(gdesc->dword[0]) & 1474 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1475 } else { 1476 if (gdesc->rcd.csum) { 1477 skb->csum = htons(gdesc->rcd.csum); 1478 skb->ip_summed = CHECKSUM_PARTIAL; 1479 } else { 1480 skb_checksum_none_assert(skb); 1481 } 1482 } 1483 } else { 1484 skb_checksum_none_assert(skb); 1485 } 1486 } 1487 1488 1489 static void 1490 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd, 1491 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter) 1492 { 1493 rq->stats.drop_err++; 1494 if (!rcd->fcs) 1495 rq->stats.drop_fcs++; 1496 1497 rq->stats.drop_total++; 1498 1499 /* 1500 * We do not unmap and chain the rx buffer to the skb. 1501 * We basically pretend this buffer is not used and will be recycled 1502 * by vmxnet3_rq_alloc_rx_buf() 1503 */ 1504 1505 /* 1506 * ctx->skb may be NULL if this is the first and the only one 1507 * desc for the pkt 1508 */ 1509 if (ctx->skb) 1510 dev_kfree_skb_irq(ctx->skb); 1511 1512 ctx->skb = NULL; 1513 } 1514 1515 1516 static u32 1517 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, 1518 union Vmxnet3_GenericDesc *gdesc) 1519 { 1520 u32 hlen, maplen; 1521 union { 1522 void *ptr; 1523 struct ethhdr *eth; 1524 struct vlan_ethhdr *veth; 1525 struct iphdr *ipv4; 1526 struct ipv6hdr *ipv6; 1527 struct tcphdr *tcp; 1528 } hdr; 1529 BUG_ON(gdesc->rcd.tcp == 0); 1530 1531 maplen = skb_headlen(skb); 1532 if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen)) 1533 return 0; 1534 1535 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 1536 skb->protocol == cpu_to_be16(ETH_P_8021AD)) 1537 hlen = sizeof(struct vlan_ethhdr); 1538 else 1539 hlen = sizeof(struct ethhdr); 1540 1541 hdr.eth = eth_hdr(skb); 1542 if (gdesc->rcd.v4) { 1543 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) && 1544 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)); 1545 hdr.ptr += hlen; 1546 BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP); 1547 hlen = hdr.ipv4->ihl << 2; 1548 hdr.ptr += hdr.ipv4->ihl << 2; 1549 } else if (gdesc->rcd.v6) { 1550 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) && 1551 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)); 1552 hdr.ptr += hlen; 1553 /* Use an estimated value, since we also need to handle 1554 * TSO case. 1555 */ 1556 if (hdr.ipv6->nexthdr != IPPROTO_TCP) 1557 return sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 1558 hlen = sizeof(struct ipv6hdr); 1559 hdr.ptr += sizeof(struct ipv6hdr); 1560 } else { 1561 /* Non-IP pkt, dont estimate header length */ 1562 return 0; 1563 } 1564 1565 if (hlen + sizeof(struct tcphdr) > maplen) 1566 return 0; 1567 1568 return (hlen + (hdr.tcp->doff << 2)); 1569 } 1570 1571 static int 1572 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, 1573 struct vmxnet3_adapter *adapter, int quota) 1574 { 1575 u32 rxprod_reg[2] = { 1576 adapter->rx_prod_offset, adapter->rx_prod2_offset 1577 }; 1578 u32 num_pkts = 0; 1579 bool skip_page_frags = false; 1580 bool encap_lro = false; 1581 struct Vmxnet3_RxCompDesc *rcd; 1582 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; 1583 u16 segCnt = 0, mss = 0; 1584 int comp_offset, fill_offset; 1585 #ifdef __BIG_ENDIAN_BITFIELD 1586 struct Vmxnet3_RxDesc rxCmdDesc; 1587 struct Vmxnet3_RxCompDesc rxComp; 1588 #endif 1589 bool need_flush = false; 1590 1591 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, 1592 &rxComp); 1593 while (rcd->gen == rq->comp_ring.gen) { 1594 struct vmxnet3_rx_buf_info *rbi; 1595 struct sk_buff *skb, *new_skb = NULL; 1596 struct page *new_page = NULL; 1597 dma_addr_t new_dma_addr; 1598 int num_to_alloc; 1599 struct Vmxnet3_RxDesc *rxd; 1600 u32 idx, ring_idx; 1601 struct vmxnet3_cmd_ring *ring = NULL; 1602 if (num_pkts >= quota) { 1603 /* we may stop even before we see the EOP desc of 1604 * the current pkt 1605 */ 1606 break; 1607 } 1608 1609 /* Prevent any rcd field from being (speculatively) read before 1610 * rcd->gen is read. 1611 */ 1612 dma_rmb(); 1613 1614 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 && 1615 rcd->rqID != rq->dataRingQid); 1616 idx = rcd->rxdIdx; 1617 ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID); 1618 ring = rq->rx_ring + ring_idx; 1619 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1620 &rxCmdDesc); 1621 rbi = rq->buf_info[ring_idx] + idx; 1622 1623 BUG_ON(rxd->addr != rbi->dma_addr || 1624 rxd->len != rbi->len); 1625 1626 if (unlikely(rcd->eop && rcd->err)) { 1627 vmxnet3_rx_error(rq, rcd, ctx, adapter); 1628 goto rcd_done; 1629 } 1630 1631 if (rcd->sop && rcd->eop && vmxnet3_xdp_enabled(adapter)) { 1632 struct sk_buff *skb_xdp_pass; 1633 int act; 1634 1635 if (VMXNET3_RX_DATA_RING(adapter, rcd->rqID)) { 1636 ctx->skb = NULL; 1637 goto skip_xdp; /* Handle it later. */ 1638 } 1639 1640 if (rbi->buf_type != VMXNET3_RX_BUF_XDP) 1641 goto rcd_done; 1642 1643 act = vmxnet3_process_xdp(adapter, rq, rcd, rbi, rxd, 1644 &skb_xdp_pass); 1645 if (act == XDP_PASS) { 1646 ctx->skb = skb_xdp_pass; 1647 goto sop_done; 1648 } 1649 ctx->skb = NULL; 1650 need_flush |= act == XDP_REDIRECT; 1651 1652 goto rcd_done; 1653 } 1654 skip_xdp: 1655 1656 if (rcd->sop) { /* first buf of the pkt */ 1657 bool rxDataRingUsed; 1658 u16 len; 1659 1660 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD || 1661 (rcd->rqID != rq->qid && 1662 rcd->rqID != rq->dataRingQid)); 1663 1664 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB && 1665 rbi->buf_type != VMXNET3_RX_BUF_XDP); 1666 BUG_ON(ctx->skb != NULL || rbi->skb == NULL); 1667 1668 if (unlikely(rcd->len == 0)) { 1669 /* Pretend the rx buffer is skipped. */ 1670 BUG_ON(!(rcd->sop && rcd->eop)); 1671 netdev_dbg(adapter->netdev, 1672 "rxRing[%u][%u] 0 length\n", 1673 ring_idx, idx); 1674 goto rcd_done; 1675 } 1676 1677 skip_page_frags = false; 1678 ctx->skb = rbi->skb; 1679 1680 if (rq->rx_ts_desc_size != 0 && rcd->ext2) { 1681 struct Vmxnet3_RxTSDesc *ts_rxd; 1682 1683 ts_rxd = (struct Vmxnet3_RxTSDesc *)((u8 *)rq->ts_ring.base + 1684 idx * rq->rx_ts_desc_size); 1685 ts_rxd->ts.tsData = vmxnet3_get_cycles(VMXNET3_PMC_PSEUDO_TSC); 1686 ts_rxd->ts.tsi = 1; 1687 } 1688 1689 rxDataRingUsed = 1690 VMXNET3_RX_DATA_RING(adapter, rcd->rqID); 1691 len = rxDataRingUsed ? rcd->len : rbi->len; 1692 1693 if (rxDataRingUsed && vmxnet3_xdp_enabled(adapter)) { 1694 struct sk_buff *skb_xdp_pass; 1695 size_t sz; 1696 int act; 1697 1698 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1699 act = vmxnet3_process_xdp_small(adapter, rq, 1700 &rq->data_ring.base[sz], 1701 rcd->len, 1702 &skb_xdp_pass); 1703 if (act == XDP_PASS) { 1704 ctx->skb = skb_xdp_pass; 1705 goto sop_done; 1706 } 1707 need_flush |= act == XDP_REDIRECT; 1708 1709 goto rcd_done; 1710 } 1711 new_skb = netdev_alloc_skb_ip_align(adapter->netdev, 1712 len); 1713 if (new_skb == NULL) { 1714 /* Skb allocation failed, do not handover this 1715 * skb to stack. Reuse it. Drop the existing pkt 1716 */ 1717 rq->stats.rx_buf_alloc_failure++; 1718 ctx->skb = NULL; 1719 rq->stats.drop_total++; 1720 skip_page_frags = true; 1721 goto rcd_done; 1722 } 1723 1724 if (rxDataRingUsed && adapter->rxdataring_enabled) { 1725 size_t sz; 1726 1727 BUG_ON(rcd->len > rq->data_ring.desc_size); 1728 1729 ctx->skb = new_skb; 1730 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1731 memcpy(new_skb->data, 1732 &rq->data_ring.base[sz], rcd->len); 1733 } else { 1734 ctx->skb = rbi->skb; 1735 1736 new_dma_addr = 1737 dma_map_single(&adapter->pdev->dev, 1738 new_skb->data, rbi->len, 1739 DMA_FROM_DEVICE); 1740 if (dma_mapping_error(&adapter->pdev->dev, 1741 new_dma_addr)) { 1742 dev_kfree_skb(new_skb); 1743 /* Skb allocation failed, do not 1744 * handover this skb to stack. Reuse 1745 * it. Drop the existing pkt. 1746 */ 1747 rq->stats.rx_buf_alloc_failure++; 1748 ctx->skb = NULL; 1749 rq->stats.drop_total++; 1750 skip_page_frags = true; 1751 goto rcd_done; 1752 } 1753 1754 dma_unmap_single(&adapter->pdev->dev, 1755 rbi->dma_addr, 1756 rbi->len, 1757 DMA_FROM_DEVICE); 1758 1759 /* Immediate refill */ 1760 rbi->skb = new_skb; 1761 rbi->dma_addr = new_dma_addr; 1762 rxd->addr = cpu_to_le64(rbi->dma_addr); 1763 rxd->len = rbi->len; 1764 } 1765 1766 skb_record_rx_queue(ctx->skb, rq->qid); 1767 skb_put(ctx->skb, rcd->len); 1768 1769 if (VMXNET3_VERSION_GE_2(adapter) && 1770 rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) { 1771 struct Vmxnet3_RxCompDescExt *rcdlro; 1772 union Vmxnet3_GenericDesc *gdesc; 1773 1774 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd; 1775 gdesc = (union Vmxnet3_GenericDesc *)rcd; 1776 1777 segCnt = rcdlro->segCnt; 1778 WARN_ON_ONCE(segCnt == 0); 1779 mss = rcdlro->mss; 1780 if (unlikely(segCnt <= 1)) 1781 segCnt = 0; 1782 encap_lro = (le32_to_cpu(gdesc->dword[0]) & 1783 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)); 1784 } else { 1785 segCnt = 0; 1786 } 1787 } else { 1788 BUG_ON(ctx->skb == NULL && !skip_page_frags); 1789 1790 /* non SOP buffer must be type 1 in most cases */ 1791 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE); 1792 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY); 1793 1794 /* If an sop buffer was dropped, skip all 1795 * following non-sop fragments. They will be reused. 1796 */ 1797 if (skip_page_frags) 1798 goto rcd_done; 1799 1800 if (rcd->len) { 1801 new_page = alloc_page(GFP_ATOMIC); 1802 /* Replacement page frag could not be allocated. 1803 * Reuse this page. Drop the pkt and free the 1804 * skb which contained this page as a frag. Skip 1805 * processing all the following non-sop frags. 1806 */ 1807 if (unlikely(!new_page)) { 1808 rq->stats.rx_buf_alloc_failure++; 1809 dev_kfree_skb(ctx->skb); 1810 ctx->skb = NULL; 1811 skip_page_frags = true; 1812 goto rcd_done; 1813 } 1814 new_dma_addr = dma_map_page(&adapter->pdev->dev, 1815 new_page, 1816 0, PAGE_SIZE, 1817 DMA_FROM_DEVICE); 1818 if (dma_mapping_error(&adapter->pdev->dev, 1819 new_dma_addr)) { 1820 put_page(new_page); 1821 rq->stats.rx_buf_alloc_failure++; 1822 dev_kfree_skb(ctx->skb); 1823 ctx->skb = NULL; 1824 skip_page_frags = true; 1825 goto rcd_done; 1826 } 1827 1828 dma_unmap_page(&adapter->pdev->dev, 1829 rbi->dma_addr, rbi->len, 1830 DMA_FROM_DEVICE); 1831 1832 vmxnet3_append_frag(ctx->skb, rcd, rbi); 1833 1834 /* Immediate refill */ 1835 rbi->page = new_page; 1836 rbi->dma_addr = new_dma_addr; 1837 rxd->addr = cpu_to_le64(rbi->dma_addr); 1838 rxd->len = rbi->len; 1839 } 1840 } 1841 1842 1843 sop_done: 1844 skb = ctx->skb; 1845 if (rcd->eop) { 1846 u32 mtu = adapter->netdev->mtu; 1847 skb->len += skb->data_len; 1848 1849 #ifdef VMXNET3_RSS 1850 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE && 1851 (adapter->netdev->features & NETIF_F_RXHASH)) { 1852 enum pkt_hash_types hash_type; 1853 1854 switch (rcd->rssType) { 1855 case VMXNET3_RCD_RSS_TYPE_IPV4: 1856 case VMXNET3_RCD_RSS_TYPE_IPV6: 1857 hash_type = PKT_HASH_TYPE_L3; 1858 break; 1859 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 1860 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 1861 case VMXNET3_RCD_RSS_TYPE_UDPIPV4: 1862 case VMXNET3_RCD_RSS_TYPE_UDPIPV6: 1863 hash_type = PKT_HASH_TYPE_L4; 1864 break; 1865 default: 1866 hash_type = PKT_HASH_TYPE_L3; 1867 break; 1868 } 1869 skb_set_hash(skb, 1870 le32_to_cpu(rcd->rssHash), 1871 hash_type); 1872 } 1873 #endif 1874 vmxnet3_rx_csum(adapter, skb, 1875 (union Vmxnet3_GenericDesc *)rcd); 1876 skb->protocol = eth_type_trans(skb, adapter->netdev); 1877 if ((!rcd->tcp && !encap_lro) || 1878 !(adapter->netdev->features & NETIF_F_LRO)) 1879 goto not_lro; 1880 1881 if (segCnt != 0 && mss != 0) { 1882 skb_shinfo(skb)->gso_type = rcd->v4 ? 1883 SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1884 skb_shinfo(skb)->gso_size = mss; 1885 skb_shinfo(skb)->gso_segs = segCnt; 1886 } else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) { 1887 u32 hlen; 1888 1889 hlen = vmxnet3_get_hdr_len(adapter, skb, 1890 (union Vmxnet3_GenericDesc *)rcd); 1891 if (hlen == 0) 1892 goto not_lro; 1893 1894 skb_shinfo(skb)->gso_type = 1895 rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1896 if (segCnt != 0) { 1897 skb_shinfo(skb)->gso_segs = segCnt; 1898 skb_shinfo(skb)->gso_size = 1899 DIV_ROUND_UP(skb->len - 1900 hlen, segCnt); 1901 } else { 1902 skb_shinfo(skb)->gso_size = mtu - hlen; 1903 } 1904 } 1905 not_lro: 1906 if (unlikely(rcd->ts)) 1907 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci); 1908 1909 /* Use GRO callback if UPT is enabled */ 1910 if ((adapter->netdev->features & NETIF_F_LRO) && 1911 !rq->shared->updateRxProd) 1912 netif_receive_skb(skb); 1913 else 1914 napi_gro_receive(&rq->napi, skb); 1915 1916 ctx->skb = NULL; 1917 encap_lro = false; 1918 num_pkts++; 1919 } 1920 1921 rcd_done: 1922 /* device may have skipped some rx descs */ 1923 ring = rq->rx_ring + ring_idx; 1924 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 1925 1926 comp_offset = vmxnet3_cmd_ring_desc_avail(ring); 1927 fill_offset = (idx > ring->next2fill ? 0 : ring->size) + 1928 idx - ring->next2fill - 1; 1929 if (!ring->isOutOfOrder || fill_offset >= comp_offset) 1930 ring->next2comp = idx; 1931 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring); 1932 1933 /* Ensure that the writes to rxd->gen bits will be observed 1934 * after all other writes to rxd objects. 1935 */ 1936 dma_wmb(); 1937 1938 while (num_to_alloc) { 1939 rbi = rq->buf_info[ring_idx] + ring->next2fill; 1940 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP))) 1941 goto refill_buf; 1942 if (ring_idx == 0) { 1943 /* ring0 Type1 buffers can get skipped; re-fill them */ 1944 if (rbi->buf_type != VMXNET3_RX_BUF_SKB) 1945 goto refill_buf; 1946 } 1947 if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) { 1948 refill_buf: 1949 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd, 1950 &rxCmdDesc); 1951 WARN_ON(!rxd->addr); 1952 1953 /* Recv desc is ready to be used by the device */ 1954 rxd->gen = ring->gen; 1955 vmxnet3_cmd_ring_adv_next2fill(ring); 1956 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 1957 num_to_alloc--; 1958 } else { 1959 /* rx completion hasn't occurred */ 1960 ring->isOutOfOrder = 1; 1961 break; 1962 } 1963 } 1964 1965 if (num_to_alloc == 0) { 1966 ring->isOutOfOrder = 0; 1967 } 1968 1969 /* if needed, update the register */ 1970 if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) { 1971 VMXNET3_WRITE_BAR0_REG(adapter, 1972 rxprod_reg[ring_idx] + rq->qid * 8, 1973 ring->next2fill); 1974 } 1975 1976 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring); 1977 vmxnet3_getRxComp(rcd, 1978 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp); 1979 } 1980 if (need_flush) 1981 xdp_do_flush(); 1982 1983 return num_pkts; 1984 } 1985 1986 1987 static void 1988 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, 1989 struct vmxnet3_adapter *adapter) 1990 { 1991 u32 i, ring_idx; 1992 struct Vmxnet3_RxDesc *rxd; 1993 1994 /* ring has already been cleaned up */ 1995 if (!rq->rx_ring[0].base) 1996 return; 1997 1998 for (ring_idx = 0; ring_idx < 2; ring_idx++) { 1999 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) { 2000 struct vmxnet3_rx_buf_info *rbi; 2001 #ifdef __BIG_ENDIAN_BITFIELD 2002 struct Vmxnet3_RxDesc rxDesc; 2003 #endif 2004 2005 rbi = &rq->buf_info[ring_idx][i]; 2006 vmxnet3_getRxDesc(rxd, 2007 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc); 2008 2009 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 2010 rbi->page && rbi->buf_type == VMXNET3_RX_BUF_XDP) { 2011 page_pool_recycle_direct(rq->page_pool, 2012 rbi->page); 2013 rbi->page = NULL; 2014 } else if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 2015 rbi->skb) { 2016 dma_unmap_single(&adapter->pdev->dev, rxd->addr, 2017 rxd->len, DMA_FROM_DEVICE); 2018 dev_kfree_skb(rbi->skb); 2019 rbi->skb = NULL; 2020 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY && 2021 rbi->page) { 2022 dma_unmap_page(&adapter->pdev->dev, rxd->addr, 2023 rxd->len, DMA_FROM_DEVICE); 2024 put_page(rbi->page); 2025 rbi->page = NULL; 2026 } 2027 } 2028 2029 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN; 2030 rq->rx_ring[ring_idx].next2fill = 2031 rq->rx_ring[ring_idx].next2comp = 0; 2032 } 2033 2034 rq->comp_ring.gen = VMXNET3_INIT_GEN; 2035 rq->comp_ring.next2proc = 0; 2036 } 2037 2038 2039 static void 2040 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter) 2041 { 2042 int i; 2043 2044 for (i = 0; i < adapter->num_rx_queues; i++) 2045 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter); 2046 rcu_assign_pointer(adapter->xdp_bpf_prog, NULL); 2047 } 2048 2049 2050 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 2051 struct vmxnet3_adapter *adapter) 2052 { 2053 int i; 2054 int j; 2055 2056 /* all rx buffers must have already been freed */ 2057 for (i = 0; i < 2; i++) { 2058 if (rq->buf_info[i]) { 2059 for (j = 0; j < rq->rx_ring[i].size; j++) 2060 BUG_ON(rq->buf_info[i][j].page != NULL); 2061 } 2062 } 2063 2064 2065 for (i = 0; i < 2; i++) { 2066 if (rq->rx_ring[i].base) { 2067 dma_free_coherent(&adapter->pdev->dev, 2068 rq->rx_ring[i].size 2069 * sizeof(struct Vmxnet3_RxDesc), 2070 rq->rx_ring[i].base, 2071 rq->rx_ring[i].basePA); 2072 rq->rx_ring[i].base = NULL; 2073 } 2074 } 2075 2076 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) 2077 xdp_rxq_info_unreg(&rq->xdp_rxq); 2078 page_pool_destroy(rq->page_pool); 2079 rq->page_pool = NULL; 2080 2081 if (rq->data_ring.base) { 2082 dma_free_coherent(&adapter->pdev->dev, 2083 rq->rx_ring[0].size * rq->data_ring.desc_size, 2084 rq->data_ring.base, rq->data_ring.basePA); 2085 rq->data_ring.base = NULL; 2086 } 2087 2088 if (rq->ts_ring.base) { 2089 dma_free_coherent(&adapter->pdev->dev, 2090 rq->rx_ring[0].size * rq->rx_ts_desc_size, 2091 rq->ts_ring.base, rq->ts_ring.basePA); 2092 rq->ts_ring.base = NULL; 2093 } 2094 2095 if (rq->comp_ring.base) { 2096 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size 2097 * sizeof(struct Vmxnet3_RxCompDesc), 2098 rq->comp_ring.base, rq->comp_ring.basePA); 2099 rq->comp_ring.base = NULL; 2100 } 2101 2102 kfree(rq->buf_info[0]); 2103 rq->buf_info[0] = NULL; 2104 rq->buf_info[1] = NULL; 2105 } 2106 2107 static void 2108 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter) 2109 { 2110 int i; 2111 2112 for (i = 0; i < adapter->num_rx_queues; i++) { 2113 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2114 2115 if (rq->data_ring.base) { 2116 dma_free_coherent(&adapter->pdev->dev, 2117 (rq->rx_ring[0].size * 2118 rq->data_ring.desc_size), 2119 rq->data_ring.base, 2120 rq->data_ring.basePA); 2121 rq->data_ring.base = NULL; 2122 } 2123 rq->data_ring.desc_size = 0; 2124 } 2125 } 2126 2127 static int 2128 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq, 2129 struct vmxnet3_adapter *adapter) 2130 { 2131 int i, err; 2132 2133 /* initialize buf_info */ 2134 for (i = 0; i < rq->rx_ring[0].size; i++) { 2135 2136 /* 1st buf for a pkt is skbuff or xdp page */ 2137 if (i % adapter->rx_buf_per_pkt == 0) { 2138 rq->buf_info[0][i].buf_type = vmxnet3_xdp_enabled(adapter) ? 2139 VMXNET3_RX_BUF_XDP : 2140 VMXNET3_RX_BUF_SKB; 2141 rq->buf_info[0][i].len = adapter->skb_buf_size; 2142 } else { /* subsequent bufs for a pkt is frag */ 2143 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE; 2144 rq->buf_info[0][i].len = PAGE_SIZE; 2145 } 2146 } 2147 for (i = 0; i < rq->rx_ring[1].size; i++) { 2148 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE; 2149 rq->buf_info[1][i].len = PAGE_SIZE; 2150 } 2151 2152 /* reset internal state and allocate buffers for both rings */ 2153 for (i = 0; i < 2; i++) { 2154 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0; 2155 2156 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size * 2157 sizeof(struct Vmxnet3_RxDesc)); 2158 rq->rx_ring[i].gen = VMXNET3_INIT_GEN; 2159 rq->rx_ring[i].isOutOfOrder = 0; 2160 } 2161 2162 err = vmxnet3_create_pp(adapter, rq, 2163 rq->rx_ring[0].size + rq->rx_ring[1].size); 2164 if (err) 2165 return err; 2166 2167 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1, 2168 adapter) == 0) { 2169 xdp_rxq_info_unreg(&rq->xdp_rxq); 2170 page_pool_destroy(rq->page_pool); 2171 rq->page_pool = NULL; 2172 2173 /* at least has 1 rx buffer for the 1st ring */ 2174 return -ENOMEM; 2175 } 2176 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter); 2177 2178 if (rq->ts_ring.base) 2179 memset(rq->ts_ring.base, 0, 2180 rq->rx_ring[0].size * rq->rx_ts_desc_size); 2181 2182 /* reset the comp ring */ 2183 rq->comp_ring.next2proc = 0; 2184 memset(rq->comp_ring.base, 0, rq->comp_ring.size * 2185 sizeof(struct Vmxnet3_RxCompDesc)); 2186 rq->comp_ring.gen = VMXNET3_INIT_GEN; 2187 2188 /* reset rxctx */ 2189 rq->rx_ctx.skb = NULL; 2190 2191 /* stats are not reset */ 2192 return 0; 2193 } 2194 2195 2196 static int 2197 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter) 2198 { 2199 int i, err = 0; 2200 2201 for (i = 0; i < adapter->num_rx_queues; i++) { 2202 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter); 2203 if (unlikely(err)) { 2204 dev_err(&adapter->netdev->dev, "%s: failed to " 2205 "initialize rx queue%i\n", 2206 adapter->netdev->name, i); 2207 break; 2208 } 2209 } 2210 return err; 2211 2212 } 2213 2214 2215 static int 2216 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 2217 { 2218 int i; 2219 size_t sz; 2220 struct vmxnet3_rx_buf_info *bi; 2221 2222 for (i = 0; i < 2; i++) { 2223 2224 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc); 2225 rq->rx_ring[i].base = dma_alloc_coherent( 2226 &adapter->pdev->dev, sz, 2227 &rq->rx_ring[i].basePA, 2228 GFP_KERNEL); 2229 if (!rq->rx_ring[i].base) { 2230 netdev_err(adapter->netdev, 2231 "failed to allocate rx ring %d\n", i); 2232 goto err; 2233 } 2234 } 2235 2236 if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) { 2237 sz = rq->rx_ring[0].size * rq->data_ring.desc_size; 2238 rq->data_ring.base = 2239 dma_alloc_coherent(&adapter->pdev->dev, sz, 2240 &rq->data_ring.basePA, 2241 GFP_KERNEL); 2242 if (!rq->data_ring.base) { 2243 netdev_err(adapter->netdev, 2244 "rx data ring will be disabled\n"); 2245 adapter->rxdataring_enabled = false; 2246 } 2247 } else { 2248 rq->data_ring.base = NULL; 2249 rq->data_ring.desc_size = 0; 2250 } 2251 2252 if (rq->rx_ts_desc_size != 0) { 2253 sz = rq->rx_ring[0].size * rq->rx_ts_desc_size; 2254 rq->ts_ring.base = 2255 dma_alloc_coherent(&adapter->pdev->dev, sz, 2256 &rq->ts_ring.basePA, 2257 GFP_KERNEL); 2258 if (!rq->ts_ring.base) { 2259 netdev_err(adapter->netdev, 2260 "rx ts ring will be disabled\n"); 2261 rq->rx_ts_desc_size = 0; 2262 } 2263 } else { 2264 rq->ts_ring.base = NULL; 2265 } 2266 2267 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc); 2268 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz, 2269 &rq->comp_ring.basePA, 2270 GFP_KERNEL); 2271 if (!rq->comp_ring.base) { 2272 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n"); 2273 goto err; 2274 } 2275 2276 bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size, 2277 sizeof(rq->buf_info[0][0]), GFP_KERNEL, 2278 dev_to_node(&adapter->pdev->dev)); 2279 if (!bi) 2280 goto err; 2281 2282 rq->buf_info[0] = bi; 2283 rq->buf_info[1] = bi + rq->rx_ring[0].size; 2284 2285 return 0; 2286 2287 err: 2288 vmxnet3_rq_destroy(rq, adapter); 2289 return -ENOMEM; 2290 } 2291 2292 2293 int 2294 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter) 2295 { 2296 int i, err = 0; 2297 2298 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 2299 2300 for (i = 0; i < adapter->num_rx_queues; i++) { 2301 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter); 2302 if (unlikely(err)) { 2303 dev_err(&adapter->netdev->dev, 2304 "%s: failed to create rx queue%i\n", 2305 adapter->netdev->name, i); 2306 goto err_out; 2307 } 2308 } 2309 2310 if (!adapter->rxdataring_enabled) 2311 vmxnet3_rq_destroy_all_rxdataring(adapter); 2312 2313 return err; 2314 err_out: 2315 vmxnet3_rq_destroy_all(adapter); 2316 return err; 2317 2318 } 2319 2320 /* Multiple queue aware polling function for tx and rx */ 2321 2322 static int 2323 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 2324 { 2325 int rcd_done = 0, i; 2326 if (unlikely(adapter->shared->ecr)) 2327 vmxnet3_process_events(adapter); 2328 for (i = 0; i < adapter->num_tx_queues; i++) 2329 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter); 2330 2331 for (i = 0; i < adapter->num_rx_queues; i++) 2332 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i], 2333 adapter, budget); 2334 return rcd_done; 2335 } 2336 2337 2338 static int 2339 vmxnet3_poll(struct napi_struct *napi, int budget) 2340 { 2341 struct vmxnet3_rx_queue *rx_queue = container_of(napi, 2342 struct vmxnet3_rx_queue, napi); 2343 int rxd_done; 2344 2345 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget); 2346 2347 if (rxd_done < budget) { 2348 napi_complete_done(napi, rxd_done); 2349 vmxnet3_enable_all_intrs(rx_queue->adapter); 2350 } 2351 return rxd_done; 2352 } 2353 2354 /* 2355 * NAPI polling function for MSI-X mode with multiple Rx queues 2356 * Returns the # of the NAPI credit consumed (# of rx descriptors processed) 2357 */ 2358 2359 static int 2360 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget) 2361 { 2362 struct vmxnet3_rx_queue *rq = container_of(napi, 2363 struct vmxnet3_rx_queue, napi); 2364 struct vmxnet3_adapter *adapter = rq->adapter; 2365 int rxd_done; 2366 2367 /* When sharing interrupt with corresponding tx queue, process 2368 * tx completions in that queue as well 2369 */ 2370 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { 2371 struct vmxnet3_tx_queue *tq = 2372 &adapter->tx_queue[rq - adapter->rx_queue]; 2373 vmxnet3_tq_tx_complete(tq, adapter); 2374 } 2375 2376 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget); 2377 2378 if (rxd_done < budget) { 2379 napi_complete_done(napi, rxd_done); 2380 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx); 2381 } 2382 return rxd_done; 2383 } 2384 2385 2386 #ifdef CONFIG_PCI_MSI 2387 2388 /* 2389 * Handle completion interrupts on tx queues 2390 * Returns whether or not the intr is handled 2391 */ 2392 2393 static irqreturn_t 2394 vmxnet3_msix_tx(int irq, void *data) 2395 { 2396 struct vmxnet3_tx_queue *tq = data; 2397 struct vmxnet3_adapter *adapter = tq->adapter; 2398 2399 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2400 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx); 2401 2402 /* Handle the case where only one irq is allocate for all tx queues */ 2403 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2404 int i; 2405 for (i = 0; i < adapter->num_tx_queues; i++) { 2406 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i]; 2407 vmxnet3_tq_tx_complete(txq, adapter); 2408 } 2409 } else { 2410 vmxnet3_tq_tx_complete(tq, adapter); 2411 } 2412 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx); 2413 2414 return IRQ_HANDLED; 2415 } 2416 2417 2418 /* 2419 * Handle completion interrupts on rx queues. Returns whether or not the 2420 * intr is handled 2421 */ 2422 2423 static irqreturn_t 2424 vmxnet3_msix_rx(int irq, void *data) 2425 { 2426 struct vmxnet3_rx_queue *rq = data; 2427 struct vmxnet3_adapter *adapter = rq->adapter; 2428 2429 /* disable intr if needed */ 2430 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2431 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx); 2432 napi_schedule(&rq->napi); 2433 2434 return IRQ_HANDLED; 2435 } 2436 2437 /* 2438 *---------------------------------------------------------------------------- 2439 * 2440 * vmxnet3_msix_event -- 2441 * 2442 * vmxnet3 msix event intr handler 2443 * 2444 * Result: 2445 * whether or not the intr is handled 2446 * 2447 *---------------------------------------------------------------------------- 2448 */ 2449 2450 static irqreturn_t 2451 vmxnet3_msix_event(int irq, void *data) 2452 { 2453 struct net_device *dev = data; 2454 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2455 2456 /* disable intr if needed */ 2457 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2458 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx); 2459 2460 if (adapter->shared->ecr) 2461 vmxnet3_process_events(adapter); 2462 2463 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx); 2464 2465 return IRQ_HANDLED; 2466 } 2467 2468 #endif /* CONFIG_PCI_MSI */ 2469 2470 2471 /* Interrupt handler for vmxnet3 */ 2472 static irqreturn_t 2473 vmxnet3_intr(int irq, void *dev_id) 2474 { 2475 struct net_device *dev = dev_id; 2476 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2477 2478 if (adapter->intr.type == VMXNET3_IT_INTX) { 2479 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 2480 if (unlikely(icr == 0)) 2481 /* not ours */ 2482 return IRQ_NONE; 2483 } 2484 2485 2486 /* disable intr if needed */ 2487 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2488 vmxnet3_disable_all_intrs(adapter); 2489 2490 napi_schedule(&adapter->rx_queue[0].napi); 2491 2492 return IRQ_HANDLED; 2493 } 2494 2495 #ifdef CONFIG_NET_POLL_CONTROLLER 2496 2497 /* netpoll callback. */ 2498 static void 2499 vmxnet3_netpoll(struct net_device *netdev) 2500 { 2501 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2502 2503 switch (adapter->intr.type) { 2504 #ifdef CONFIG_PCI_MSI 2505 case VMXNET3_IT_MSIX: { 2506 int i; 2507 for (i = 0; i < adapter->num_rx_queues; i++) 2508 vmxnet3_msix_rx(0, &adapter->rx_queue[i]); 2509 break; 2510 } 2511 #endif 2512 case VMXNET3_IT_MSI: 2513 default: 2514 vmxnet3_intr(0, adapter->netdev); 2515 break; 2516 } 2517 2518 } 2519 #endif /* CONFIG_NET_POLL_CONTROLLER */ 2520 2521 static int 2522 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 2523 { 2524 struct vmxnet3_intr *intr = &adapter->intr; 2525 int err = 0, i; 2526 int vector = 0; 2527 2528 #ifdef CONFIG_PCI_MSI 2529 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2530 for (i = 0; i < adapter->num_tx_queues; i++) { 2531 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2532 sprintf(adapter->tx_queue[i].name, "%s-tx-%d", 2533 adapter->netdev->name, vector); 2534 err = request_irq( 2535 intr->msix_entries[vector].vector, 2536 vmxnet3_msix_tx, 0, 2537 adapter->tx_queue[i].name, 2538 &adapter->tx_queue[i]); 2539 } else { 2540 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d", 2541 adapter->netdev->name, vector); 2542 } 2543 if (err) { 2544 dev_err(&adapter->netdev->dev, 2545 "Failed to request irq for MSIX, %s, " 2546 "error %d\n", 2547 adapter->tx_queue[i].name, err); 2548 return err; 2549 } 2550 2551 /* Handle the case where only 1 MSIx was allocated for 2552 * all tx queues */ 2553 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2554 for (; i < adapter->num_tx_queues; i++) 2555 adapter->tx_queue[i].comp_ring.intr_idx 2556 = vector; 2557 vector++; 2558 break; 2559 } else { 2560 adapter->tx_queue[i].comp_ring.intr_idx 2561 = vector++; 2562 } 2563 } 2564 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) 2565 vector = 0; 2566 2567 for (i = 0; i < adapter->num_rx_queues; i++) { 2568 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) 2569 sprintf(adapter->rx_queue[i].name, "%s-rx-%d", 2570 adapter->netdev->name, vector); 2571 else 2572 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d", 2573 adapter->netdev->name, vector); 2574 err = request_irq(intr->msix_entries[vector].vector, 2575 vmxnet3_msix_rx, 0, 2576 adapter->rx_queue[i].name, 2577 &(adapter->rx_queue[i])); 2578 if (err) { 2579 netdev_err(adapter->netdev, 2580 "Failed to request irq for MSIX, " 2581 "%s, error %d\n", 2582 adapter->rx_queue[i].name, err); 2583 return err; 2584 } 2585 2586 adapter->rx_queue[i].comp_ring.intr_idx = vector++; 2587 } 2588 2589 sprintf(intr->event_msi_vector_name, "%s-event-%d", 2590 adapter->netdev->name, vector); 2591 err = request_irq(intr->msix_entries[vector].vector, 2592 vmxnet3_msix_event, 0, 2593 intr->event_msi_vector_name, adapter->netdev); 2594 intr->event_intr_idx = vector; 2595 2596 } else if (intr->type == VMXNET3_IT_MSI) { 2597 adapter->num_rx_queues = 1; 2598 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 2599 adapter->netdev->name, adapter->netdev); 2600 } else { 2601 #endif 2602 adapter->num_rx_queues = 1; 2603 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 2604 IRQF_SHARED, adapter->netdev->name, 2605 adapter->netdev); 2606 #ifdef CONFIG_PCI_MSI 2607 } 2608 #endif 2609 intr->num_intrs = vector + 1; 2610 if (err) { 2611 netdev_err(adapter->netdev, 2612 "Failed to request irq (intr type:%d), error %d\n", 2613 intr->type, err); 2614 } else { 2615 /* Number of rx queues will not change after this */ 2616 for (i = 0; i < adapter->num_rx_queues; i++) { 2617 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2618 rq->qid = i; 2619 rq->qid2 = i + adapter->num_rx_queues; 2620 rq->dataRingQid = i + 2 * adapter->num_rx_queues; 2621 } 2622 2623 /* init our intr settings */ 2624 for (i = 0; i < intr->num_intrs; i++) 2625 intr->mod_levels[i] = UPT1_IML_ADAPTIVE; 2626 if (adapter->intr.type != VMXNET3_IT_MSIX) { 2627 adapter->intr.event_intr_idx = 0; 2628 for (i = 0; i < adapter->num_tx_queues; i++) 2629 adapter->tx_queue[i].comp_ring.intr_idx = 0; 2630 adapter->rx_queue[0].comp_ring.intr_idx = 0; 2631 } 2632 2633 netdev_info(adapter->netdev, 2634 "intr type %u, mode %u, %u vectors allocated\n", 2635 intr->type, intr->mask_mode, intr->num_intrs); 2636 } 2637 2638 return err; 2639 } 2640 2641 2642 static void 2643 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 2644 { 2645 struct vmxnet3_intr *intr = &adapter->intr; 2646 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0); 2647 2648 switch (intr->type) { 2649 #ifdef CONFIG_PCI_MSI 2650 case VMXNET3_IT_MSIX: 2651 { 2652 int i, vector = 0; 2653 2654 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2655 for (i = 0; i < adapter->num_tx_queues; i++) { 2656 free_irq(intr->msix_entries[vector++].vector, 2657 &(adapter->tx_queue[i])); 2658 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) 2659 break; 2660 } 2661 } 2662 2663 for (i = 0; i < adapter->num_rx_queues; i++) { 2664 free_irq(intr->msix_entries[vector++].vector, 2665 &(adapter->rx_queue[i])); 2666 } 2667 2668 free_irq(intr->msix_entries[vector].vector, 2669 adapter->netdev); 2670 BUG_ON(vector >= intr->num_intrs); 2671 break; 2672 } 2673 #endif 2674 case VMXNET3_IT_MSI: 2675 free_irq(adapter->pdev->irq, adapter->netdev); 2676 break; 2677 case VMXNET3_IT_INTX: 2678 free_irq(adapter->pdev->irq, adapter->netdev); 2679 break; 2680 default: 2681 BUG(); 2682 } 2683 } 2684 2685 2686 static void 2687 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter) 2688 { 2689 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2690 u16 vid; 2691 2692 /* allow untagged pkts */ 2693 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0); 2694 2695 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2696 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2697 } 2698 2699 2700 static int 2701 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 2702 { 2703 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2704 2705 if (!(netdev->flags & IFF_PROMISC)) { 2706 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2707 unsigned long flags; 2708 2709 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2710 spin_lock_irqsave(&adapter->cmd_lock, flags); 2711 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2712 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2713 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2714 } 2715 2716 set_bit(vid, adapter->active_vlans); 2717 2718 return 0; 2719 } 2720 2721 2722 static int 2723 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 2724 { 2725 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2726 2727 if (!(netdev->flags & IFF_PROMISC)) { 2728 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2729 unsigned long flags; 2730 2731 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid); 2732 spin_lock_irqsave(&adapter->cmd_lock, flags); 2733 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2734 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2735 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2736 } 2737 2738 clear_bit(vid, adapter->active_vlans); 2739 2740 return 0; 2741 } 2742 2743 2744 static u8 * 2745 vmxnet3_copy_mc(struct net_device *netdev) 2746 { 2747 u8 *buf = NULL; 2748 u32 sz = netdev_mc_count(netdev) * ETH_ALEN; 2749 2750 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */ 2751 if (sz <= 0xffff) { 2752 /* We may be called with BH disabled */ 2753 buf = kmalloc(sz, GFP_ATOMIC); 2754 if (buf) { 2755 struct netdev_hw_addr *ha; 2756 int i = 0; 2757 2758 netdev_for_each_mc_addr(ha, netdev) 2759 memcpy(buf + i++ * ETH_ALEN, ha->addr, 2760 ETH_ALEN); 2761 } 2762 } 2763 return buf; 2764 } 2765 2766 2767 static void 2768 vmxnet3_set_mc(struct net_device *netdev) 2769 { 2770 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2771 unsigned long flags; 2772 struct Vmxnet3_RxFilterConf *rxConf = 2773 &adapter->shared->devRead.rxFilterConf; 2774 u8 *new_table = NULL; 2775 dma_addr_t new_table_pa = 0; 2776 bool new_table_pa_valid = false; 2777 u32 new_mode = VMXNET3_RXM_UCAST; 2778 2779 if (netdev->flags & IFF_PROMISC) { 2780 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2781 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable)); 2782 2783 new_mode |= VMXNET3_RXM_PROMISC; 2784 } else { 2785 vmxnet3_restore_vlan(adapter); 2786 } 2787 2788 if (netdev->flags & IFF_BROADCAST) 2789 new_mode |= VMXNET3_RXM_BCAST; 2790 2791 if (netdev->flags & IFF_ALLMULTI) 2792 new_mode |= VMXNET3_RXM_ALL_MULTI; 2793 else 2794 if (!netdev_mc_empty(netdev)) { 2795 new_table = vmxnet3_copy_mc(netdev); 2796 if (new_table) { 2797 size_t sz = netdev_mc_count(netdev) * ETH_ALEN; 2798 2799 rxConf->mfTableLen = cpu_to_le16(sz); 2800 new_table_pa = dma_map_single( 2801 &adapter->pdev->dev, 2802 new_table, 2803 sz, 2804 DMA_TO_DEVICE); 2805 if (!dma_mapping_error(&adapter->pdev->dev, 2806 new_table_pa)) { 2807 new_mode |= VMXNET3_RXM_MCAST; 2808 new_table_pa_valid = true; 2809 rxConf->mfTablePA = cpu_to_le64( 2810 new_table_pa); 2811 } 2812 } 2813 if (!new_table_pa_valid) { 2814 netdev_info(netdev, 2815 "failed to copy mcast list, setting ALL_MULTI\n"); 2816 new_mode |= VMXNET3_RXM_ALL_MULTI; 2817 } 2818 } 2819 2820 if (!(new_mode & VMXNET3_RXM_MCAST)) { 2821 rxConf->mfTableLen = 0; 2822 rxConf->mfTablePA = 0; 2823 } 2824 2825 spin_lock_irqsave(&adapter->cmd_lock, flags); 2826 if (new_mode != rxConf->rxMode) { 2827 rxConf->rxMode = cpu_to_le32(new_mode); 2828 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2829 VMXNET3_CMD_UPDATE_RX_MODE); 2830 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2831 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2832 } 2833 2834 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2835 VMXNET3_CMD_UPDATE_MAC_FILTERS); 2836 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2837 2838 if (new_table_pa_valid) 2839 dma_unmap_single(&adapter->pdev->dev, new_table_pa, 2840 rxConf->mfTableLen, DMA_TO_DEVICE); 2841 kfree(new_table); 2842 } 2843 2844 void 2845 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter) 2846 { 2847 int i; 2848 2849 for (i = 0; i < adapter->num_rx_queues; i++) 2850 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter); 2851 } 2852 2853 2854 /* 2855 * Set up driver_shared based on settings in adapter. 2856 */ 2857 2858 static void 2859 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter) 2860 { 2861 struct Vmxnet3_DriverShared *shared = adapter->shared; 2862 struct Vmxnet3_DSDevRead *devRead = &shared->devRead; 2863 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 2864 struct Vmxnet3_TxQueueConf *tqc; 2865 struct Vmxnet3_RxQueueConf *rqc; 2866 struct Vmxnet3_TxQueueTSConf *tqtsc; 2867 struct Vmxnet3_RxQueueTSConf *rqtsc; 2868 int i; 2869 2870 memset(shared, 0, sizeof(*shared)); 2871 2872 /* driver settings */ 2873 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC); 2874 devRead->misc.driverInfo.version = cpu_to_le32( 2875 VMXNET3_DRIVER_VERSION_NUM); 2876 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ? 2877 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64); 2878 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 2879 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32( 2880 *((u32 *)&devRead->misc.driverInfo.gos)); 2881 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1); 2882 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1); 2883 2884 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa); 2885 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter)); 2886 2887 /* set up feature flags */ 2888 if (adapter->netdev->features & NETIF_F_RXCSUM) 2889 devRead->misc.uptFeatures |= UPT1_F_RXCSUM; 2890 2891 if (adapter->netdev->features & NETIF_F_LRO) { 2892 devRead->misc.uptFeatures |= UPT1_F_LRO; 2893 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS); 2894 } 2895 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2896 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 2897 2898 if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL | 2899 NETIF_F_GSO_UDP_TUNNEL_CSUM)) 2900 devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD; 2901 2902 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2903 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2904 devRead->misc.queueDescLen = cpu_to_le32( 2905 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 2906 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc)); 2907 2908 /* tx queue settings */ 2909 devRead->misc.numTxQueues = adapter->num_tx_queues; 2910 for (i = 0; i < adapter->num_tx_queues; i++) { 2911 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 2912 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL); 2913 tqc = &adapter->tqd_start[i].conf; 2914 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA); 2915 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA); 2916 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA); 2917 tqc->ddPA = cpu_to_le64(~0ULL); 2918 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size); 2919 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size); 2920 tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size); 2921 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size); 2922 tqc->ddLen = cpu_to_le32(0); 2923 tqc->intrIdx = tq->comp_ring.intr_idx; 2924 if (VMXNET3_VERSION_GE_9(adapter)) { 2925 tqtsc = &adapter->tqd_start[i].tsConf; 2926 tqtsc->txTSRingBasePA = cpu_to_le64(tq->ts_ring.basePA); 2927 tqtsc->txTSRingDescSize = cpu_to_le16(tq->tx_ts_desc_size); 2928 } 2929 } 2930 2931 /* rx queue settings */ 2932 devRead->misc.numRxQueues = adapter->num_rx_queues; 2933 for (i = 0; i < adapter->num_rx_queues; i++) { 2934 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2935 rqc = &adapter->rqd_start[i].conf; 2936 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA); 2937 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA); 2938 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA); 2939 rqc->ddPA = cpu_to_le64(~0ULL); 2940 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size); 2941 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size); 2942 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size); 2943 rqc->ddLen = cpu_to_le32(0); 2944 rqc->intrIdx = rq->comp_ring.intr_idx; 2945 if (VMXNET3_VERSION_GE_3(adapter)) { 2946 rqc->rxDataRingBasePA = 2947 cpu_to_le64(rq->data_ring.basePA); 2948 rqc->rxDataRingDescSize = 2949 cpu_to_le16(rq->data_ring.desc_size); 2950 } 2951 if (VMXNET3_VERSION_GE_9(adapter)) { 2952 rqtsc = &adapter->rqd_start[i].tsConf; 2953 rqtsc->rxTSRingBasePA = cpu_to_le64(rq->ts_ring.basePA); 2954 rqtsc->rxTSRingDescSize = cpu_to_le16(rq->rx_ts_desc_size); 2955 } 2956 } 2957 2958 #ifdef VMXNET3_RSS 2959 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf)); 2960 2961 if (adapter->rss) { 2962 struct UPT1_RSSConf *rssConf = adapter->rss_conf; 2963 2964 devRead->misc.uptFeatures |= UPT1_F_RSS; 2965 devRead->misc.numRxQueues = adapter->num_rx_queues; 2966 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 | 2967 UPT1_RSS_HASH_TYPE_IPV4 | 2968 UPT1_RSS_HASH_TYPE_TCP_IPV6 | 2969 UPT1_RSS_HASH_TYPE_IPV6; 2970 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ; 2971 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE; 2972 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE; 2973 netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey)); 2974 2975 for (i = 0; i < rssConf->indTableSize; i++) 2976 rssConf->indTable[i] = ethtool_rxfh_indir_default( 2977 i, adapter->num_rx_queues); 2978 2979 devRead->rssConfDesc.confVer = 1; 2980 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf)); 2981 devRead->rssConfDesc.confPA = 2982 cpu_to_le64(adapter->rss_conf_pa); 2983 } 2984 2985 #endif /* VMXNET3_RSS */ 2986 2987 /* intr settings */ 2988 if (!VMXNET3_VERSION_GE_6(adapter) || 2989 !adapter->queuesExtEnabled) { 2990 devRead->intrConf.autoMask = adapter->intr.mask_mode == 2991 VMXNET3_IMM_AUTO; 2992 devRead->intrConf.numIntrs = adapter->intr.num_intrs; 2993 for (i = 0; i < adapter->intr.num_intrs; i++) 2994 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i]; 2995 2996 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx; 2997 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2998 } else { 2999 devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode == 3000 VMXNET3_IMM_AUTO; 3001 devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs; 3002 for (i = 0; i < adapter->intr.num_intrs; i++) 3003 devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i]; 3004 3005 devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx; 3006 devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 3007 } 3008 3009 /* rx filter settings */ 3010 devRead->rxFilterConf.rxMode = 0; 3011 vmxnet3_restore_vlan(adapter); 3012 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr); 3013 3014 /* the rest are already zeroed */ 3015 } 3016 3017 static void 3018 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter) 3019 { 3020 struct Vmxnet3_DriverShared *shared = adapter->shared; 3021 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 3022 unsigned long flags; 3023 3024 if (!VMXNET3_VERSION_GE_7(adapter)) 3025 return; 3026 3027 cmdInfo->ringBufSize = adapter->ringBufSize; 3028 spin_lock_irqsave(&adapter->cmd_lock, flags); 3029 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3030 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 3031 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3032 } 3033 3034 static void 3035 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter) 3036 { 3037 struct Vmxnet3_DriverShared *shared = adapter->shared; 3038 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 3039 unsigned long flags; 3040 3041 if (!VMXNET3_VERSION_GE_3(adapter)) 3042 return; 3043 3044 spin_lock_irqsave(&adapter->cmd_lock, flags); 3045 cmdInfo->varConf.confVer = 1; 3046 cmdInfo->varConf.confLen = 3047 cpu_to_le32(sizeof(*adapter->coal_conf)); 3048 cmdInfo->varConf.confPA = cpu_to_le64(adapter->coal_conf_pa); 3049 3050 if (adapter->default_coal_mode) { 3051 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3052 VMXNET3_CMD_GET_COALESCE); 3053 } else { 3054 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3055 VMXNET3_CMD_SET_COALESCE); 3056 } 3057 3058 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3059 } 3060 3061 static void 3062 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter) 3063 { 3064 struct Vmxnet3_DriverShared *shared = adapter->shared; 3065 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 3066 unsigned long flags; 3067 3068 if (!VMXNET3_VERSION_GE_4(adapter)) 3069 return; 3070 3071 spin_lock_irqsave(&adapter->cmd_lock, flags); 3072 3073 if (adapter->default_rss_fields) { 3074 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3075 VMXNET3_CMD_GET_RSS_FIELDS); 3076 adapter->rss_fields = 3077 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3078 } else { 3079 if (VMXNET3_VERSION_GE_7(adapter)) { 3080 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 || 3081 adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) && 3082 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3083 VMXNET3_CAP_UDP_RSS)) { 3084 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS; 3085 } else { 3086 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS); 3087 } 3088 3089 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) && 3090 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3091 VMXNET3_CAP_ESP_RSS_IPV4)) { 3092 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4; 3093 } else { 3094 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4); 3095 } 3096 3097 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) && 3098 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3099 VMXNET3_CAP_ESP_RSS_IPV6)) { 3100 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6; 3101 } else { 3102 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6); 3103 } 3104 3105 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3106 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3107 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3108 } 3109 cmdInfo->setRssFields = adapter->rss_fields; 3110 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3111 VMXNET3_CMD_SET_RSS_FIELDS); 3112 /* Not all requested RSS may get applied, so get and 3113 * cache what was actually applied. 3114 */ 3115 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3116 VMXNET3_CMD_GET_RSS_FIELDS); 3117 adapter->rss_fields = 3118 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3119 } 3120 3121 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3122 } 3123 3124 int 3125 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 3126 { 3127 int err, i; 3128 u32 ret; 3129 unsigned long flags; 3130 3131 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d," 3132 " ring sizes %u %u %u\n", adapter->netdev->name, 3133 adapter->skb_buf_size, adapter->rx_buf_per_pkt, 3134 adapter->tx_queue[0].tx_ring.size, 3135 adapter->rx_queue[0].rx_ring[0].size, 3136 adapter->rx_queue[0].rx_ring[1].size); 3137 3138 vmxnet3_tq_init_all(adapter); 3139 err = vmxnet3_rq_init_all(adapter); 3140 if (err) { 3141 netdev_err(adapter->netdev, 3142 "Failed to init rx queue error %d\n", err); 3143 goto rq_err; 3144 } 3145 3146 err = vmxnet3_request_irqs(adapter); 3147 if (err) { 3148 netdev_err(adapter->netdev, 3149 "Failed to setup irq for error %d\n", err); 3150 goto irq_err; 3151 } 3152 3153 vmxnet3_setup_driver_shared(adapter); 3154 3155 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO( 3156 adapter->shared_pa)); 3157 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI( 3158 adapter->shared_pa)); 3159 spin_lock_irqsave(&adapter->cmd_lock, flags); 3160 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3161 VMXNET3_CMD_ACTIVATE_DEV); 3162 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3163 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3164 3165 if (ret != 0) { 3166 netdev_err(adapter->netdev, 3167 "Failed to activate dev: error %u\n", ret); 3168 err = -EINVAL; 3169 goto activate_err; 3170 } 3171 3172 vmxnet3_init_bufsize(adapter); 3173 vmxnet3_init_coalesce(adapter); 3174 vmxnet3_init_rssfields(adapter); 3175 3176 for (i = 0; i < adapter->num_rx_queues; i++) { 3177 VMXNET3_WRITE_BAR0_REG(adapter, 3178 adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN, 3179 adapter->rx_queue[i].rx_ring[0].next2fill); 3180 VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset + 3181 (i * VMXNET3_REG_ALIGN)), 3182 adapter->rx_queue[i].rx_ring[1].next2fill); 3183 } 3184 3185 /* Apply the rx filter settins last. */ 3186 vmxnet3_set_mc(adapter->netdev); 3187 3188 /* 3189 * Check link state when first activating device. It will start the 3190 * tx queue if the link is up. 3191 */ 3192 vmxnet3_check_link(adapter, true); 3193 netif_tx_wake_all_queues(adapter->netdev); 3194 for (i = 0; i < adapter->num_rx_queues; i++) 3195 napi_enable(&adapter->rx_queue[i].napi); 3196 vmxnet3_enable_all_intrs(adapter); 3197 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3198 return 0; 3199 3200 activate_err: 3201 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0); 3202 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0); 3203 vmxnet3_free_irqs(adapter); 3204 irq_err: 3205 rq_err: 3206 /* free up buffers we allocated */ 3207 vmxnet3_rq_cleanup_all(adapter); 3208 return err; 3209 } 3210 3211 3212 void 3213 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter) 3214 { 3215 unsigned long flags; 3216 spin_lock_irqsave(&adapter->cmd_lock, flags); 3217 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 3218 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3219 } 3220 3221 3222 int 3223 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 3224 { 3225 int i; 3226 unsigned long flags; 3227 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 3228 return 0; 3229 3230 3231 spin_lock_irqsave(&adapter->cmd_lock, flags); 3232 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3233 VMXNET3_CMD_QUIESCE_DEV); 3234 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3235 vmxnet3_disable_all_intrs(adapter); 3236 3237 for (i = 0; i < adapter->num_rx_queues; i++) 3238 napi_disable(&adapter->rx_queue[i].napi); 3239 netif_tx_disable(adapter->netdev); 3240 adapter->link_speed = 0; 3241 netif_carrier_off(adapter->netdev); 3242 3243 vmxnet3_tq_cleanup_all(adapter); 3244 vmxnet3_rq_cleanup_all(adapter); 3245 vmxnet3_free_irqs(adapter); 3246 return 0; 3247 } 3248 3249 3250 static void 3251 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac) 3252 { 3253 u32 tmp; 3254 3255 tmp = *(u32 *)mac; 3256 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp); 3257 3258 tmp = (mac[5] << 8) | mac[4]; 3259 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp); 3260 } 3261 3262 3263 static int 3264 vmxnet3_set_mac_addr(struct net_device *netdev, void *p) 3265 { 3266 struct sockaddr *addr = p; 3267 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3268 3269 dev_addr_set(netdev, addr->sa_data); 3270 vmxnet3_write_mac_addr(adapter, addr->sa_data); 3271 3272 return 0; 3273 } 3274 3275 3276 /* ==================== initialization and cleanup routines ============ */ 3277 3278 static int 3279 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter) 3280 { 3281 int err; 3282 unsigned long mmio_start, mmio_len; 3283 struct pci_dev *pdev = adapter->pdev; 3284 3285 err = pci_enable_device(pdev); 3286 if (err) { 3287 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err); 3288 return err; 3289 } 3290 3291 err = pci_request_selected_regions(pdev, (1 << 2) - 1, 3292 vmxnet3_driver_name); 3293 if (err) { 3294 dev_err(&pdev->dev, 3295 "Failed to request region for adapter: error %d\n", err); 3296 goto err_enable_device; 3297 } 3298 3299 pci_set_master(pdev); 3300 3301 mmio_start = pci_resource_start(pdev, 0); 3302 mmio_len = pci_resource_len(pdev, 0); 3303 adapter->hw_addr0 = ioremap(mmio_start, mmio_len); 3304 if (!adapter->hw_addr0) { 3305 dev_err(&pdev->dev, "Failed to map bar0\n"); 3306 err = -EIO; 3307 goto err_ioremap; 3308 } 3309 3310 mmio_start = pci_resource_start(pdev, 1); 3311 mmio_len = pci_resource_len(pdev, 1); 3312 adapter->hw_addr1 = ioremap(mmio_start, mmio_len); 3313 if (!adapter->hw_addr1) { 3314 dev_err(&pdev->dev, "Failed to map bar1\n"); 3315 err = -EIO; 3316 goto err_bar1; 3317 } 3318 return 0; 3319 3320 err_bar1: 3321 iounmap(adapter->hw_addr0); 3322 err_ioremap: 3323 pci_release_selected_regions(pdev, (1 << 2) - 1); 3324 err_enable_device: 3325 pci_disable_device(pdev); 3326 return err; 3327 } 3328 3329 3330 static void 3331 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter) 3332 { 3333 BUG_ON(!adapter->pdev); 3334 3335 iounmap(adapter->hw_addr0); 3336 iounmap(adapter->hw_addr1); 3337 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1); 3338 pci_disable_device(adapter->pdev); 3339 } 3340 3341 3342 void 3343 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 3344 { 3345 size_t sz, i, ring0_size, ring1_size, comp_size; 3346 /* With version7 ring1 will have only T0 buffers */ 3347 if (!VMXNET3_VERSION_GE_7(adapter)) { 3348 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 3349 VMXNET3_MAX_ETH_HDR_SIZE) { 3350 adapter->skb_buf_size = adapter->netdev->mtu + 3351 VMXNET3_MAX_ETH_HDR_SIZE; 3352 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE) 3353 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE; 3354 3355 adapter->rx_buf_per_pkt = 1; 3356 } else { 3357 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE; 3358 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE + 3359 VMXNET3_MAX_ETH_HDR_SIZE; 3360 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE; 3361 } 3362 } else { 3363 adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE, 3364 VMXNET3_MAX_SKB_BUF_SIZE); 3365 adapter->rx_buf_per_pkt = 1; 3366 adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size); 3367 adapter->ringBufSize.ring1BufSizeType1 = 0; 3368 adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE); 3369 } 3370 3371 /* 3372 * for simplicity, force the ring0 size to be a multiple of 3373 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 3374 */ 3375 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 3376 ring0_size = adapter->rx_queue[0].rx_ring[0].size; 3377 ring0_size = (ring0_size + sz - 1) / sz * sz; 3378 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE / 3379 sz * sz); 3380 ring1_size = adapter->rx_queue[0].rx_ring[1].size; 3381 ring1_size = (ring1_size + sz - 1) / sz * sz; 3382 ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE / 3383 sz * sz); 3384 /* For v7 and later, keep ring size power of 2 for UPT */ 3385 if (VMXNET3_VERSION_GE_7(adapter)) { 3386 ring0_size = rounddown_pow_of_two(ring0_size); 3387 ring1_size = rounddown_pow_of_two(ring1_size); 3388 } 3389 comp_size = ring0_size + ring1_size; 3390 3391 for (i = 0; i < adapter->num_rx_queues; i++) { 3392 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3393 3394 rq->rx_ring[0].size = ring0_size; 3395 rq->rx_ring[1].size = ring1_size; 3396 rq->comp_ring.size = comp_size; 3397 } 3398 } 3399 3400 3401 int 3402 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 3403 u32 rx_ring_size, u32 rx_ring2_size, 3404 u16 txdata_desc_size, u16 rxdata_desc_size) 3405 { 3406 int err = 0, i; 3407 3408 for (i = 0; i < adapter->num_tx_queues; i++) { 3409 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 3410 tq->tx_ring.size = tx_ring_size; 3411 tq->data_ring.size = tx_ring_size; 3412 tq->comp_ring.size = tx_ring_size; 3413 tq->txdata_desc_size = txdata_desc_size; 3414 tq->shared = &adapter->tqd_start[i].ctrl; 3415 tq->stopped = true; 3416 tq->adapter = adapter; 3417 tq->qid = i; 3418 tq->tx_ts_desc_size = adapter->tx_ts_desc_size; 3419 tq->tsPktCount = 1; 3420 err = vmxnet3_tq_create(tq, adapter); 3421 /* 3422 * Too late to change num_tx_queues. We cannot do away with 3423 * lesser number of queues than what we asked for 3424 */ 3425 if (err) 3426 goto queue_err; 3427 } 3428 3429 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size; 3430 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size; 3431 vmxnet3_adjust_rx_ring_size(adapter); 3432 3433 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 3434 for (i = 0; i < adapter->num_rx_queues; i++) { 3435 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3436 /* qid and qid2 for rx queues will be assigned later when num 3437 * of rx queues is finalized after allocating intrs */ 3438 rq->shared = &adapter->rqd_start[i].ctrl; 3439 rq->adapter = adapter; 3440 rq->data_ring.desc_size = rxdata_desc_size; 3441 rq->rx_ts_desc_size = adapter->rx_ts_desc_size; 3442 err = vmxnet3_rq_create(rq, adapter); 3443 if (err) { 3444 if (i == 0) { 3445 netdev_err(adapter->netdev, 3446 "Could not allocate any rx queues. " 3447 "Aborting.\n"); 3448 goto queue_err; 3449 } else { 3450 netdev_info(adapter->netdev, 3451 "Number of rx queues changed " 3452 "to : %d.\n", i); 3453 adapter->num_rx_queues = i; 3454 err = 0; 3455 break; 3456 } 3457 } 3458 } 3459 3460 if (!adapter->rxdataring_enabled) 3461 vmxnet3_rq_destroy_all_rxdataring(adapter); 3462 3463 return err; 3464 queue_err: 3465 vmxnet3_tq_destroy_all(adapter); 3466 return err; 3467 } 3468 3469 static int 3470 vmxnet3_open(struct net_device *netdev) 3471 { 3472 struct vmxnet3_adapter *adapter; 3473 int err, i; 3474 3475 adapter = netdev_priv(netdev); 3476 3477 for (i = 0; i < adapter->num_tx_queues; i++) 3478 spin_lock_init(&adapter->tx_queue[i].tx_lock); 3479 3480 if (VMXNET3_VERSION_GE_3(adapter)) { 3481 unsigned long flags; 3482 u16 txdata_desc_size; 3483 u32 ret; 3484 3485 spin_lock_irqsave(&adapter->cmd_lock, flags); 3486 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3487 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 3488 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3489 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3490 3491 txdata_desc_size = ret & 0xffff; 3492 if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) || 3493 (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) || 3494 (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) { 3495 adapter->txdata_desc_size = 3496 sizeof(struct Vmxnet3_TxDataDesc); 3497 } else { 3498 adapter->txdata_desc_size = txdata_desc_size; 3499 } 3500 if (VMXNET3_VERSION_GE_9(adapter)) 3501 adapter->rxdata_desc_size = (ret >> 16) & 0xffff; 3502 } else { 3503 adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc); 3504 } 3505 3506 if (VMXNET3_VERSION_GE_9(adapter)) { 3507 unsigned long flags; 3508 u16 tx_ts_desc_size = 0; 3509 u16 rx_ts_desc_size = 0; 3510 u32 ret; 3511 3512 spin_lock_irqsave(&adapter->cmd_lock, flags); 3513 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3514 VMXNET3_CMD_GET_TSRING_DESC_SIZE); 3515 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3516 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3517 if (ret > 0) { 3518 tx_ts_desc_size = (ret & 0xff); 3519 rx_ts_desc_size = ((ret >> 16) & 0xff); 3520 } 3521 if (tx_ts_desc_size > VMXNET3_TXTS_DESC_MAX_SIZE || 3522 tx_ts_desc_size & VMXNET3_TXTS_DESC_SIZE_MASK) 3523 tx_ts_desc_size = 0; 3524 if (rx_ts_desc_size > VMXNET3_RXTS_DESC_MAX_SIZE || 3525 rx_ts_desc_size & VMXNET3_RXTS_DESC_SIZE_MASK) 3526 rx_ts_desc_size = 0; 3527 adapter->tx_ts_desc_size = tx_ts_desc_size; 3528 adapter->rx_ts_desc_size = rx_ts_desc_size; 3529 } else { 3530 adapter->tx_ts_desc_size = 0; 3531 adapter->rx_ts_desc_size = 0; 3532 } 3533 3534 err = vmxnet3_create_queues(adapter, 3535 adapter->tx_ring_size, 3536 adapter->rx_ring_size, 3537 adapter->rx_ring2_size, 3538 adapter->txdata_desc_size, 3539 adapter->rxdata_desc_size); 3540 if (err) 3541 goto queue_err; 3542 3543 err = vmxnet3_activate_dev(adapter); 3544 if (err) 3545 goto activate_err; 3546 3547 return 0; 3548 3549 activate_err: 3550 vmxnet3_rq_destroy_all(adapter); 3551 vmxnet3_tq_destroy_all(adapter); 3552 queue_err: 3553 return err; 3554 } 3555 3556 3557 static int 3558 vmxnet3_close(struct net_device *netdev) 3559 { 3560 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3561 3562 /* 3563 * Reset_work may be in the middle of resetting the device, wait for its 3564 * completion. 3565 */ 3566 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3567 usleep_range(1000, 2000); 3568 3569 vmxnet3_quiesce_dev(adapter); 3570 3571 vmxnet3_rq_destroy_all(adapter); 3572 vmxnet3_tq_destroy_all(adapter); 3573 3574 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3575 3576 3577 return 0; 3578 } 3579 3580 3581 void 3582 vmxnet3_force_close(struct vmxnet3_adapter *adapter) 3583 { 3584 int i; 3585 3586 /* 3587 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 3588 * vmxnet3_close() will deadlock. 3589 */ 3590 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 3591 3592 /* we need to enable NAPI, otherwise dev_close will deadlock */ 3593 for (i = 0; i < adapter->num_rx_queues; i++) 3594 napi_enable(&adapter->rx_queue[i].napi); 3595 /* 3596 * Need to clear the quiesce bit to ensure that vmxnet3_close 3597 * can quiesce the device properly 3598 */ 3599 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3600 dev_close(adapter->netdev); 3601 } 3602 3603 3604 static int 3605 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) 3606 { 3607 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3608 int err = 0; 3609 3610 WRITE_ONCE(netdev->mtu, new_mtu); 3611 3612 /* 3613 * Reset_work may be in the middle of resetting the device, wait for its 3614 * completion. 3615 */ 3616 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3617 usleep_range(1000, 2000); 3618 3619 if (netif_running(netdev)) { 3620 vmxnet3_quiesce_dev(adapter); 3621 vmxnet3_reset_dev(adapter); 3622 3623 /* we need to re-create the rx queue based on the new mtu */ 3624 vmxnet3_rq_destroy_all(adapter); 3625 vmxnet3_adjust_rx_ring_size(adapter); 3626 err = vmxnet3_rq_create_all(adapter); 3627 if (err) { 3628 netdev_err(netdev, 3629 "failed to re-create rx queues, " 3630 " error %d. Closing it.\n", err); 3631 goto out; 3632 } 3633 3634 err = vmxnet3_activate_dev(adapter); 3635 if (err) { 3636 netdev_err(netdev, 3637 "failed to re-activate, error %d. " 3638 "Closing it\n", err); 3639 goto out; 3640 } 3641 } 3642 3643 out: 3644 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3645 if (err) 3646 vmxnet3_force_close(adapter); 3647 3648 return err; 3649 } 3650 3651 3652 static void 3653 vmxnet3_declare_features(struct vmxnet3_adapter *adapter) 3654 { 3655 struct net_device *netdev = adapter->netdev; 3656 unsigned long flags; 3657 3658 if (VMXNET3_VERSION_GE_9(adapter)) { 3659 spin_lock_irqsave(&adapter->cmd_lock, flags); 3660 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3661 VMXNET3_CMD_GET_DISABLED_OFFLOADS); 3662 adapter->disabledOffloads = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3663 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3664 } 3665 3666 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 3667 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3668 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3669 NETIF_F_LRO | NETIF_F_HIGHDMA; 3670 3671 if (VMXNET3_VERSION_GE_4(adapter)) { 3672 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3673 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3674 3675 netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM | 3676 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3677 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3678 NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL | 3679 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3680 } 3681 3682 if (adapter->disabledOffloads & VMXNET3_OFFLOAD_TSO) { 3683 netdev->hw_features &= ~(NETIF_F_TSO | NETIF_F_TSO6); 3684 netdev->hw_enc_features &= ~(NETIF_F_TSO | NETIF_F_TSO6); 3685 } 3686 3687 if (adapter->disabledOffloads & VMXNET3_OFFLOAD_LRO) { 3688 netdev->hw_features &= ~(NETIF_F_LRO); 3689 netdev->hw_enc_features &= ~(NETIF_F_LRO); 3690 } 3691 3692 if (VMXNET3_VERSION_GE_7(adapter)) { 3693 unsigned long flags; 3694 3695 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3696 VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) { 3697 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD; 3698 } 3699 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3700 VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) { 3701 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD; 3702 } 3703 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3704 VMXNET3_CAP_GENEVE_TSO)) { 3705 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO; 3706 } 3707 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3708 VMXNET3_CAP_VXLAN_TSO)) { 3709 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO; 3710 } 3711 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3712 VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) { 3713 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD; 3714 } 3715 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3716 VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) { 3717 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD; 3718 } 3719 3720 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3721 spin_lock_irqsave(&adapter->cmd_lock, flags); 3722 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3723 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3724 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3725 3726 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) && 3727 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) && 3728 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) && 3729 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) { 3730 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3731 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3732 } 3733 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) && 3734 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) { 3735 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3736 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3737 } 3738 } 3739 3740 netdev->vlan_features = netdev->hw_features & 3741 ~(NETIF_F_HW_VLAN_CTAG_TX | 3742 NETIF_F_HW_VLAN_CTAG_RX); 3743 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 3744 } 3745 3746 3747 static void 3748 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) 3749 { 3750 u32 tmp; 3751 3752 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL); 3753 *(u32 *)mac = tmp; 3754 3755 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH); 3756 mac[4] = tmp & 0xff; 3757 mac[5] = (tmp >> 8) & 0xff; 3758 } 3759 3760 #ifdef CONFIG_PCI_MSI 3761 3762 /* 3763 * Enable MSIx vectors. 3764 * Returns : 3765 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required 3766 * were enabled. 3767 * number of vectors which were enabled otherwise (this number is greater 3768 * than VMXNET3_LINUX_MIN_MSIX_VECT) 3769 */ 3770 3771 static int 3772 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec) 3773 { 3774 int ret = pci_enable_msix_range(adapter->pdev, 3775 adapter->intr.msix_entries, nvec, nvec); 3776 3777 if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) { 3778 dev_err(&adapter->netdev->dev, 3779 "Failed to enable %d MSI-X, trying %d\n", 3780 nvec, VMXNET3_LINUX_MIN_MSIX_VECT); 3781 3782 ret = pci_enable_msix_range(adapter->pdev, 3783 adapter->intr.msix_entries, 3784 VMXNET3_LINUX_MIN_MSIX_VECT, 3785 VMXNET3_LINUX_MIN_MSIX_VECT); 3786 } 3787 3788 if (ret < 0) { 3789 dev_err(&adapter->netdev->dev, 3790 "Failed to enable MSI-X, error: %d\n", ret); 3791 } 3792 3793 return ret; 3794 } 3795 3796 3797 #endif /* CONFIG_PCI_MSI */ 3798 3799 static void 3800 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 3801 { 3802 u32 cfg; 3803 unsigned long flags; 3804 3805 /* intr settings */ 3806 spin_lock_irqsave(&adapter->cmd_lock, flags); 3807 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3808 VMXNET3_CMD_GET_CONF_INTR); 3809 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3810 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3811 adapter->intr.type = cfg & 0x3; 3812 adapter->intr.mask_mode = (cfg >> 2) & 0x3; 3813 3814 if (adapter->intr.type == VMXNET3_IT_AUTO) { 3815 adapter->intr.type = VMXNET3_IT_MSIX; 3816 } 3817 3818 #ifdef CONFIG_PCI_MSI 3819 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3820 int i, nvec, nvec_allocated; 3821 3822 nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ? 3823 1 : adapter->num_tx_queues; 3824 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ? 3825 0 : adapter->num_rx_queues; 3826 nvec += 1; /* for link event */ 3827 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ? 3828 nvec : VMXNET3_LINUX_MIN_MSIX_VECT; 3829 3830 for (i = 0; i < nvec; i++) 3831 adapter->intr.msix_entries[i].entry = i; 3832 3833 nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec); 3834 if (nvec_allocated < 0) 3835 goto msix_err; 3836 3837 /* If we cannot allocate one MSIx vector per queue 3838 * then limit the number of rx queues to 1 3839 */ 3840 if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT && 3841 nvec != VMXNET3_LINUX_MIN_MSIX_VECT) { 3842 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE 3843 || adapter->num_rx_queues != 1) { 3844 adapter->share_intr = VMXNET3_INTR_TXSHARE; 3845 netdev_err(adapter->netdev, 3846 "Number of rx queues : 1\n"); 3847 adapter->num_rx_queues = 1; 3848 } 3849 } 3850 3851 adapter->intr.num_intrs = nvec_allocated; 3852 return; 3853 3854 msix_err: 3855 /* If we cannot allocate MSIx vectors use only one rx queue */ 3856 dev_info(&adapter->pdev->dev, 3857 "Failed to enable MSI-X, error %d. " 3858 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated); 3859 3860 adapter->intr.type = VMXNET3_IT_MSI; 3861 } 3862 3863 if (adapter->intr.type == VMXNET3_IT_MSI) { 3864 if (!pci_enable_msi(adapter->pdev)) { 3865 adapter->num_rx_queues = 1; 3866 adapter->intr.num_intrs = 1; 3867 return; 3868 } 3869 } 3870 #endif /* CONFIG_PCI_MSI */ 3871 3872 adapter->num_rx_queues = 1; 3873 dev_info(&adapter->netdev->dev, 3874 "Using INTx interrupt, #Rx queues: 1.\n"); 3875 adapter->intr.type = VMXNET3_IT_INTX; 3876 3877 /* INT-X related setting */ 3878 adapter->intr.num_intrs = 1; 3879 } 3880 3881 3882 static void 3883 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter) 3884 { 3885 if (adapter->intr.type == VMXNET3_IT_MSIX) 3886 pci_disable_msix(adapter->pdev); 3887 else if (adapter->intr.type == VMXNET3_IT_MSI) 3888 pci_disable_msi(adapter->pdev); 3889 else 3890 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX); 3891 } 3892 3893 3894 static void 3895 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue) 3896 { 3897 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3898 adapter->tx_timeout_count++; 3899 3900 netdev_err(adapter->netdev, "tx hang\n"); 3901 schedule_work(&adapter->work); 3902 } 3903 3904 3905 static void 3906 vmxnet3_reset_work(struct work_struct *data) 3907 { 3908 struct vmxnet3_adapter *adapter; 3909 3910 adapter = container_of(data, struct vmxnet3_adapter, work); 3911 3912 /* if another thread is resetting the device, no need to proceed */ 3913 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3914 return; 3915 3916 /* if the device is closed, we must leave it alone */ 3917 rtnl_lock(); 3918 if (netif_running(adapter->netdev)) { 3919 netdev_notice(adapter->netdev, "resetting\n"); 3920 vmxnet3_quiesce_dev(adapter); 3921 vmxnet3_reset_dev(adapter); 3922 vmxnet3_activate_dev(adapter); 3923 } else { 3924 netdev_info(adapter->netdev, "already closed\n"); 3925 } 3926 rtnl_unlock(); 3927 3928 netif_wake_queue(adapter->netdev); 3929 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3930 } 3931 3932 3933 static int 3934 vmxnet3_probe_device(struct pci_dev *pdev, 3935 const struct pci_device_id *id) 3936 { 3937 static const struct net_device_ops vmxnet3_netdev_ops = { 3938 .ndo_open = vmxnet3_open, 3939 .ndo_stop = vmxnet3_close, 3940 .ndo_start_xmit = vmxnet3_xmit_frame, 3941 .ndo_set_mac_address = vmxnet3_set_mac_addr, 3942 .ndo_change_mtu = vmxnet3_change_mtu, 3943 .ndo_fix_features = vmxnet3_fix_features, 3944 .ndo_set_features = vmxnet3_set_features, 3945 .ndo_features_check = vmxnet3_features_check, 3946 .ndo_get_stats64 = vmxnet3_get_stats64, 3947 .ndo_tx_timeout = vmxnet3_tx_timeout, 3948 .ndo_set_rx_mode = vmxnet3_set_mc, 3949 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid, 3950 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid, 3951 #ifdef CONFIG_NET_POLL_CONTROLLER 3952 .ndo_poll_controller = vmxnet3_netpoll, 3953 #endif 3954 .ndo_bpf = vmxnet3_xdp, 3955 .ndo_xdp_xmit = vmxnet3_xdp_xmit, 3956 }; 3957 int err; 3958 u32 ver; 3959 struct net_device *netdev; 3960 struct vmxnet3_adapter *adapter; 3961 u8 mac[ETH_ALEN]; 3962 int size, i; 3963 int num_tx_queues; 3964 int num_rx_queues; 3965 int queues; 3966 unsigned long flags; 3967 3968 if (!pci_msi_enabled()) 3969 enable_mq = 0; 3970 3971 #ifdef VMXNET3_RSS 3972 if (enable_mq) 3973 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3974 (int)num_online_cpus()); 3975 else 3976 #endif 3977 num_rx_queues = 1; 3978 3979 if (enable_mq) 3980 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES, 3981 (int)num_online_cpus()); 3982 else 3983 num_tx_queues = 1; 3984 3985 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter), 3986 max(num_tx_queues, num_rx_queues)); 3987 if (!netdev) 3988 return -ENOMEM; 3989 3990 pci_set_drvdata(pdev, netdev); 3991 adapter = netdev_priv(netdev); 3992 adapter->netdev = netdev; 3993 adapter->pdev = pdev; 3994 3995 adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE; 3996 adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE; 3997 adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE; 3998 3999 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4000 if (err) { 4001 dev_err(&pdev->dev, "dma_set_mask failed\n"); 4002 goto err_set_mask; 4003 } 4004 4005 spin_lock_init(&adapter->cmd_lock); 4006 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter, 4007 sizeof(struct vmxnet3_adapter), 4008 DMA_TO_DEVICE); 4009 if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) { 4010 dev_err(&pdev->dev, "Failed to map dma\n"); 4011 err = -EFAULT; 4012 goto err_set_mask; 4013 } 4014 adapter->shared = dma_alloc_coherent( 4015 &adapter->pdev->dev, 4016 sizeof(struct Vmxnet3_DriverShared), 4017 &adapter->shared_pa, GFP_KERNEL); 4018 if (!adapter->shared) { 4019 dev_err(&pdev->dev, "Failed to allocate memory\n"); 4020 err = -ENOMEM; 4021 goto err_alloc_shared; 4022 } 4023 4024 err = vmxnet3_alloc_pci_resources(adapter); 4025 if (err < 0) 4026 goto err_alloc_pci; 4027 4028 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS); 4029 for (i = VMXNET3_REV_9; i >= VMXNET3_REV_1; i--) { 4030 if (ver & (1 << i)) { 4031 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1 << i); 4032 adapter->version = i + 1; 4033 break; 4034 } 4035 } 4036 if (i < VMXNET3_REV_1) { 4037 dev_err(&pdev->dev, 4038 "Incompatible h/w version (0x%x) for adapter\n", ver); 4039 err = -EBUSY; 4040 goto err_ver; 4041 } 4042 dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version); 4043 4044 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS); 4045 if (ver & 1) { 4046 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1); 4047 } else { 4048 dev_err(&pdev->dev, 4049 "Incompatible upt version (0x%x) for adapter\n", ver); 4050 err = -EBUSY; 4051 goto err_ver; 4052 } 4053 4054 if (VMXNET3_VERSION_GE_7(adapter)) { 4055 adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR); 4056 adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR); 4057 if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 4058 adapter->dev_caps[0] = adapter->devcap_supported[0] & 4059 (1UL << VMXNET3_CAP_LARGE_BAR); 4060 } 4061 if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) && 4062 adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 4063 adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 4064 adapter->dev_caps[0] |= adapter->devcap_supported[0] & 4065 (1UL << VMXNET3_CAP_OOORX_COMP); 4066 } 4067 if (adapter->dev_caps[0]) 4068 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 4069 4070 spin_lock_irqsave(&adapter->cmd_lock, flags); 4071 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 4072 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4073 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4074 } 4075 4076 if (VMXNET3_VERSION_GE_7(adapter) && 4077 adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 4078 adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 4079 adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD; 4080 adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2; 4081 } else { 4082 adapter->tx_prod_offset = VMXNET3_REG_TXPROD; 4083 adapter->rx_prod_offset = VMXNET3_REG_RXPROD; 4084 adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2; 4085 } 4086 4087 if (VMXNET3_VERSION_GE_6(adapter)) { 4088 spin_lock_irqsave(&adapter->cmd_lock, flags); 4089 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4090 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 4091 queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4092 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4093 if (queues > 0) { 4094 adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff)); 4095 adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff)); 4096 } else { 4097 adapter->num_rx_queues = min(num_rx_queues, 4098 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4099 adapter->num_tx_queues = min(num_tx_queues, 4100 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 4101 } 4102 if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES || 4103 adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) { 4104 adapter->queuesExtEnabled = true; 4105 } else { 4106 adapter->queuesExtEnabled = false; 4107 } 4108 } else { 4109 adapter->queuesExtEnabled = false; 4110 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 4111 num_tx_queues = rounddown_pow_of_two(num_tx_queues); 4112 adapter->num_rx_queues = min(num_rx_queues, 4113 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4114 adapter->num_tx_queues = min(num_tx_queues, 4115 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 4116 } 4117 dev_info(&pdev->dev, 4118 "# of Tx queues : %d, # of Rx queues : %d\n", 4119 adapter->num_tx_queues, adapter->num_rx_queues); 4120 4121 adapter->rx_buf_per_pkt = 1; 4122 4123 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 4124 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues; 4125 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size, 4126 &adapter->queue_desc_pa, 4127 GFP_KERNEL); 4128 4129 if (!adapter->tqd_start) { 4130 dev_err(&pdev->dev, "Failed to allocate memory\n"); 4131 err = -ENOMEM; 4132 goto err_ver; 4133 } 4134 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start + 4135 adapter->num_tx_queues); 4136 if (VMXNET3_VERSION_GE_9(adapter)) 4137 adapter->latencyConf = &adapter->tqd_start->tsConf.latencyConf; 4138 4139 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev, 4140 sizeof(struct Vmxnet3_PMConf), 4141 &adapter->pm_conf_pa, 4142 GFP_KERNEL); 4143 if (adapter->pm_conf == NULL) { 4144 err = -ENOMEM; 4145 goto err_alloc_pm; 4146 } 4147 4148 #ifdef VMXNET3_RSS 4149 4150 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev, 4151 sizeof(struct UPT1_RSSConf), 4152 &adapter->rss_conf_pa, 4153 GFP_KERNEL); 4154 if (adapter->rss_conf == NULL) { 4155 err = -ENOMEM; 4156 goto err_alloc_rss; 4157 } 4158 #endif /* VMXNET3_RSS */ 4159 4160 if (VMXNET3_VERSION_GE_3(adapter)) { 4161 adapter->coal_conf = 4162 dma_alloc_coherent(&adapter->pdev->dev, 4163 sizeof(struct Vmxnet3_CoalesceScheme) 4164 , 4165 &adapter->coal_conf_pa, 4166 GFP_KERNEL); 4167 if (!adapter->coal_conf) { 4168 err = -ENOMEM; 4169 goto err_coal_conf; 4170 } 4171 adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED; 4172 adapter->default_coal_mode = true; 4173 } 4174 4175 if (VMXNET3_VERSION_GE_4(adapter)) { 4176 adapter->default_rss_fields = true; 4177 adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT; 4178 } 4179 4180 SET_NETDEV_DEV(netdev, &pdev->dev); 4181 vmxnet3_declare_features(adapter); 4182 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 4183 NETDEV_XDP_ACT_NDO_XMIT; 4184 4185 adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ? 4186 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 4187 4188 if (adapter->num_tx_queues == adapter->num_rx_queues) 4189 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE; 4190 else 4191 adapter->share_intr = VMXNET3_INTR_DONTSHARE; 4192 4193 vmxnet3_alloc_intr_resources(adapter); 4194 4195 #ifdef VMXNET3_RSS 4196 if (adapter->num_rx_queues > 1 && 4197 adapter->intr.type == VMXNET3_IT_MSIX) { 4198 adapter->rss = true; 4199 netdev->hw_features |= NETIF_F_RXHASH; 4200 netdev->features |= NETIF_F_RXHASH; 4201 dev_dbg(&pdev->dev, "RSS is enabled.\n"); 4202 } else { 4203 adapter->rss = false; 4204 } 4205 #endif 4206 4207 vmxnet3_read_mac_addr(adapter, mac); 4208 dev_addr_set(netdev, mac); 4209 4210 netdev->netdev_ops = &vmxnet3_netdev_ops; 4211 vmxnet3_set_ethtool_ops(netdev); 4212 netdev->watchdog_timeo = 5 * HZ; 4213 4214 /* MTU range: 60 - 9190 */ 4215 netdev->min_mtu = VMXNET3_MIN_MTU; 4216 if (VMXNET3_VERSION_GE_6(adapter)) 4217 netdev->max_mtu = VMXNET3_V6_MAX_MTU; 4218 else 4219 netdev->max_mtu = VMXNET3_MAX_MTU; 4220 4221 INIT_WORK(&adapter->work, vmxnet3_reset_work); 4222 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 4223 4224 if (adapter->intr.type == VMXNET3_IT_MSIX) { 4225 int i; 4226 for (i = 0; i < adapter->num_rx_queues; i++) { 4227 netif_napi_add(adapter->netdev, 4228 &adapter->rx_queue[i].napi, 4229 vmxnet3_poll_rx_only); 4230 } 4231 } else { 4232 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi, 4233 vmxnet3_poll); 4234 } 4235 4236 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); 4237 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues); 4238 4239 netif_carrier_off(netdev); 4240 err = register_netdev(netdev); 4241 4242 if (err) { 4243 dev_err(&pdev->dev, "Failed to register adapter\n"); 4244 goto err_register; 4245 } 4246 4247 vmxnet3_check_link(adapter, false); 4248 return 0; 4249 4250 err_register: 4251 if (VMXNET3_VERSION_GE_3(adapter)) { 4252 dma_free_coherent(&adapter->pdev->dev, 4253 sizeof(struct Vmxnet3_CoalesceScheme), 4254 adapter->coal_conf, adapter->coal_conf_pa); 4255 } 4256 vmxnet3_free_intr_resources(adapter); 4257 err_coal_conf: 4258 #ifdef VMXNET3_RSS 4259 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 4260 adapter->rss_conf, adapter->rss_conf_pa); 4261 err_alloc_rss: 4262 #endif 4263 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 4264 adapter->pm_conf, adapter->pm_conf_pa); 4265 err_alloc_pm: 4266 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 4267 adapter->queue_desc_pa); 4268 err_ver: 4269 vmxnet3_free_pci_resources(adapter); 4270 err_alloc_pci: 4271 dma_free_coherent(&adapter->pdev->dev, 4272 sizeof(struct Vmxnet3_DriverShared), 4273 adapter->shared, adapter->shared_pa); 4274 err_alloc_shared: 4275 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4276 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4277 err_set_mask: 4278 free_netdev(netdev); 4279 return err; 4280 } 4281 4282 4283 static void 4284 vmxnet3_remove_device(struct pci_dev *pdev) 4285 { 4286 struct net_device *netdev = pci_get_drvdata(pdev); 4287 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4288 int size = 0; 4289 int num_rx_queues, rx_queues; 4290 unsigned long flags; 4291 4292 #ifdef VMXNET3_RSS 4293 if (enable_mq) 4294 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 4295 (int)num_online_cpus()); 4296 else 4297 #endif 4298 num_rx_queues = 1; 4299 if (!VMXNET3_VERSION_GE_6(adapter)) { 4300 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 4301 } 4302 if (VMXNET3_VERSION_GE_6(adapter)) { 4303 spin_lock_irqsave(&adapter->cmd_lock, flags); 4304 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4305 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 4306 rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4307 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4308 if (rx_queues > 0) 4309 rx_queues = (rx_queues >> 8) & 0xff; 4310 else 4311 rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4312 num_rx_queues = min(num_rx_queues, rx_queues); 4313 } else { 4314 num_rx_queues = min(num_rx_queues, 4315 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4316 } 4317 4318 cancel_work_sync(&adapter->work); 4319 4320 unregister_netdev(netdev); 4321 4322 vmxnet3_free_intr_resources(adapter); 4323 vmxnet3_free_pci_resources(adapter); 4324 if (VMXNET3_VERSION_GE_3(adapter)) { 4325 dma_free_coherent(&adapter->pdev->dev, 4326 sizeof(struct Vmxnet3_CoalesceScheme), 4327 adapter->coal_conf, adapter->coal_conf_pa); 4328 } 4329 #ifdef VMXNET3_RSS 4330 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 4331 adapter->rss_conf, adapter->rss_conf_pa); 4332 #endif 4333 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 4334 adapter->pm_conf, adapter->pm_conf_pa); 4335 4336 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 4337 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues; 4338 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 4339 adapter->queue_desc_pa); 4340 dma_free_coherent(&adapter->pdev->dev, 4341 sizeof(struct Vmxnet3_DriverShared), 4342 adapter->shared, adapter->shared_pa); 4343 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4344 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4345 free_netdev(netdev); 4346 } 4347 4348 static void vmxnet3_shutdown_device(struct pci_dev *pdev) 4349 { 4350 struct net_device *netdev = pci_get_drvdata(pdev); 4351 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4352 unsigned long flags; 4353 4354 /* Reset_work may be in the middle of resetting the device, wait for its 4355 * completion. 4356 */ 4357 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 4358 usleep_range(1000, 2000); 4359 4360 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, 4361 &adapter->state)) { 4362 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4363 return; 4364 } 4365 spin_lock_irqsave(&adapter->cmd_lock, flags); 4366 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4367 VMXNET3_CMD_QUIESCE_DEV); 4368 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4369 vmxnet3_disable_all_intrs(adapter); 4370 4371 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4372 } 4373 4374 4375 #ifdef CONFIG_PM 4376 4377 static int 4378 vmxnet3_suspend(struct device *device) 4379 { 4380 struct pci_dev *pdev = to_pci_dev(device); 4381 struct net_device *netdev = pci_get_drvdata(pdev); 4382 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4383 struct Vmxnet3_PMConf *pmConf; 4384 struct ethhdr *ehdr; 4385 struct arphdr *ahdr; 4386 u8 *arpreq; 4387 struct in_device *in_dev; 4388 struct in_ifaddr *ifa; 4389 unsigned long flags; 4390 int i = 0; 4391 4392 if (!netif_running(netdev)) 4393 return 0; 4394 4395 for (i = 0; i < adapter->num_rx_queues; i++) 4396 napi_disable(&adapter->rx_queue[i].napi); 4397 4398 vmxnet3_disable_all_intrs(adapter); 4399 vmxnet3_free_irqs(adapter); 4400 vmxnet3_free_intr_resources(adapter); 4401 4402 netif_device_detach(netdev); 4403 4404 /* Create wake-up filters. */ 4405 pmConf = adapter->pm_conf; 4406 memset(pmConf, 0, sizeof(*pmConf)); 4407 4408 if (adapter->wol & WAKE_UCAST) { 4409 pmConf->filters[i].patternSize = ETH_ALEN; 4410 pmConf->filters[i].maskSize = 1; 4411 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN); 4412 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */ 4413 4414 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4415 i++; 4416 } 4417 4418 if (adapter->wol & WAKE_ARP) { 4419 rcu_read_lock(); 4420 4421 in_dev = __in_dev_get_rcu(netdev); 4422 if (!in_dev) { 4423 rcu_read_unlock(); 4424 goto skip_arp; 4425 } 4426 4427 ifa = rcu_dereference(in_dev->ifa_list); 4428 if (!ifa) { 4429 rcu_read_unlock(); 4430 goto skip_arp; 4431 } 4432 4433 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/ 4434 sizeof(struct arphdr) + /* ARP header */ 4435 2 * ETH_ALEN + /* 2 Ethernet addresses*/ 4436 2 * sizeof(u32); /*2 IPv4 addresses */ 4437 pmConf->filters[i].maskSize = 4438 (pmConf->filters[i].patternSize - 1) / 8 + 1; 4439 4440 /* ETH_P_ARP in Ethernet header. */ 4441 ehdr = (struct ethhdr *)pmConf->filters[i].pattern; 4442 ehdr->h_proto = htons(ETH_P_ARP); 4443 4444 /* ARPOP_REQUEST in ARP header. */ 4445 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN]; 4446 ahdr->ar_op = htons(ARPOP_REQUEST); 4447 arpreq = (u8 *)(ahdr + 1); 4448 4449 /* The Unicast IPv4 address in 'tip' field. */ 4450 arpreq += 2 * ETH_ALEN + sizeof(u32); 4451 *(__be32 *)arpreq = ifa->ifa_address; 4452 4453 rcu_read_unlock(); 4454 4455 /* The mask for the relevant bits. */ 4456 pmConf->filters[i].mask[0] = 0x00; 4457 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */ 4458 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */ 4459 pmConf->filters[i].mask[3] = 0x00; 4460 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */ 4461 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */ 4462 4463 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4464 i++; 4465 } 4466 4467 skip_arp: 4468 if (adapter->wol & WAKE_MAGIC) 4469 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC; 4470 4471 pmConf->numFilters = i; 4472 4473 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); 4474 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( 4475 *pmConf)); 4476 adapter->shared->devRead.pmConfDesc.confPA = 4477 cpu_to_le64(adapter->pm_conf_pa); 4478 4479 spin_lock_irqsave(&adapter->cmd_lock, flags); 4480 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4481 VMXNET3_CMD_UPDATE_PMCFG); 4482 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4483 4484 pci_save_state(pdev); 4485 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND), 4486 adapter->wol); 4487 pci_disable_device(pdev); 4488 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND)); 4489 4490 return 0; 4491 } 4492 4493 4494 static int 4495 vmxnet3_resume(struct device *device) 4496 { 4497 int err; 4498 unsigned long flags; 4499 struct pci_dev *pdev = to_pci_dev(device); 4500 struct net_device *netdev = pci_get_drvdata(pdev); 4501 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4502 4503 if (!netif_running(netdev)) 4504 return 0; 4505 4506 pci_set_power_state(pdev, PCI_D0); 4507 pci_restore_state(pdev); 4508 err = pci_enable_device_mem(pdev); 4509 if (err != 0) 4510 return err; 4511 4512 pci_enable_wake(pdev, PCI_D0, 0); 4513 4514 vmxnet3_alloc_intr_resources(adapter); 4515 4516 /* During hibernate and suspend, device has to be reinitialized as the 4517 * device state need not be preserved. 4518 */ 4519 4520 /* Need not check adapter state as other reset tasks cannot run during 4521 * device resume. 4522 */ 4523 spin_lock_irqsave(&adapter->cmd_lock, flags); 4524 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4525 VMXNET3_CMD_QUIESCE_DEV); 4526 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4527 vmxnet3_tq_cleanup_all(adapter); 4528 vmxnet3_rq_cleanup_all(adapter); 4529 4530 vmxnet3_reset_dev(adapter); 4531 err = vmxnet3_activate_dev(adapter); 4532 if (err != 0) { 4533 netdev_err(netdev, 4534 "failed to re-activate on resume, error: %d", err); 4535 vmxnet3_force_close(adapter); 4536 return err; 4537 } 4538 netif_device_attach(netdev); 4539 4540 return 0; 4541 } 4542 4543 static const struct dev_pm_ops vmxnet3_pm_ops = { 4544 .suspend = vmxnet3_suspend, 4545 .resume = vmxnet3_resume, 4546 .freeze = vmxnet3_suspend, 4547 .restore = vmxnet3_resume, 4548 }; 4549 #endif 4550 4551 static struct pci_driver vmxnet3_driver = { 4552 .name = vmxnet3_driver_name, 4553 .id_table = vmxnet3_pciid_table, 4554 .probe = vmxnet3_probe_device, 4555 .remove = vmxnet3_remove_device, 4556 .shutdown = vmxnet3_shutdown_device, 4557 #ifdef CONFIG_PM 4558 .driver.pm = &vmxnet3_pm_ops, 4559 #endif 4560 }; 4561 4562 4563 static int __init 4564 vmxnet3_init_module(void) 4565 { 4566 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC, 4567 VMXNET3_DRIVER_VERSION_REPORT); 4568 return pci_register_driver(&vmxnet3_driver); 4569 } 4570 4571 module_init(vmxnet3_init_module); 4572 4573 4574 static void 4575 vmxnet3_exit_module(void) 4576 { 4577 pci_unregister_driver(&vmxnet3_driver); 4578 } 4579 4580 module_exit(vmxnet3_exit_module); 4581 4582 MODULE_AUTHOR("VMware, Inc."); 4583 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC); 4584 MODULE_LICENSE("GPL v2"); 4585 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING); 4586