1 /* 2 * Linux driver for VMware's vmxnet3 ethernet NIC. 3 * 4 * Copyright (C) 2008-2024, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * The full GNU General Public License is included in this distribution in 21 * the file called "COPYING". 22 * 23 * Maintained by: pv-drivers@vmware.com 24 * 25 */ 26 27 #include <linux/module.h> 28 #include <net/ip6_checksum.h> 29 30 #ifdef CONFIG_X86 31 #include <asm/msr.h> 32 #endif 33 34 #include "vmxnet3_int.h" 35 #include "vmxnet3_xdp.h" 36 37 char vmxnet3_driver_name[] = "vmxnet3"; 38 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver" 39 40 /* 41 * PCI Device ID Table 42 * Last entry must be all 0s 43 */ 44 static const struct pci_device_id vmxnet3_pciid_table[] = { 45 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)}, 46 {0} 47 }; 48 49 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table); 50 51 static int enable_mq = 1; 52 53 static void 54 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac); 55 56 /* 57 * Enable/Disable the given intr 58 */ 59 static void 60 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 61 { 62 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0); 63 } 64 65 66 static void 67 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 68 { 69 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1); 70 } 71 72 73 /* 74 * Enable/Disable all intrs used by the device 75 */ 76 static void 77 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) 78 { 79 int i; 80 81 for (i = 0; i < adapter->intr.num_intrs; i++) 82 vmxnet3_enable_intr(adapter, i); 83 if (!VMXNET3_VERSION_GE_6(adapter) || 84 !adapter->queuesExtEnabled) { 85 adapter->shared->devRead.intrConf.intrCtrl &= 86 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 87 } else { 88 adapter->shared->devReadExt.intrConfExt.intrCtrl &= 89 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 90 } 91 } 92 93 94 static void 95 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) 96 { 97 int i; 98 99 if (!VMXNET3_VERSION_GE_6(adapter) || 100 !adapter->queuesExtEnabled) { 101 adapter->shared->devRead.intrConf.intrCtrl |= 102 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 103 } else { 104 adapter->shared->devReadExt.intrConfExt.intrCtrl |= 105 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 106 } 107 for (i = 0; i < adapter->intr.num_intrs; i++) 108 vmxnet3_disable_intr(adapter, i); 109 } 110 111 112 static void 113 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events) 114 { 115 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events); 116 } 117 118 119 static bool 120 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 121 { 122 return tq->stopped; 123 } 124 125 126 static void 127 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 128 { 129 tq->stopped = false; 130 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue); 131 } 132 133 134 static void 135 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 136 { 137 tq->stopped = false; 138 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 139 } 140 141 142 static void 143 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 144 { 145 tq->stopped = true; 146 tq->num_stop++; 147 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 148 } 149 150 static u64 151 vmxnet3_get_cycles(int pmc) 152 { 153 #ifdef CONFIG_X86 154 return native_read_pmc(pmc); 155 #else 156 return 0; 157 #endif 158 } 159 160 static bool 161 vmxnet3_apply_timestamp(struct vmxnet3_tx_queue *tq, u16 rate) 162 { 163 #ifdef CONFIG_X86 164 if (rate > 0) { 165 if (tq->tsPktCount == 1) { 166 if (rate != 1) 167 tq->tsPktCount = rate; 168 return true; 169 } 170 tq->tsPktCount--; 171 } 172 #endif 173 return false; 174 } 175 176 /* Check if capability is supported by UPT device or 177 * UPT is even requested 178 */ 179 bool 180 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap) 181 { 182 if (cap_supported & (1UL << VMXNET3_DCR_ERROR) || 183 cap_supported & (1UL << cap)) { 184 return true; 185 } 186 187 return false; 188 } 189 190 191 /* 192 * Check the link state. This may start or stop the tx queue. 193 */ 194 static void 195 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 196 { 197 u32 ret; 198 int i; 199 unsigned long flags; 200 201 spin_lock_irqsave(&adapter->cmd_lock, flags); 202 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 203 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 204 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 205 206 adapter->link_speed = ret >> 16; 207 if (ret & 1) { /* Link is up. */ 208 /* 209 * From vmxnet3 v9, the hypervisor reports the speed in Gbps. 210 * Convert the speed to Mbps before rporting it to the kernel. 211 * Max link speed supported is 10000G. 212 */ 213 if (VMXNET3_VERSION_GE_9(adapter) && 214 adapter->link_speed < 10000) 215 adapter->link_speed = adapter->link_speed * 1000; 216 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n", 217 adapter->link_speed); 218 netif_carrier_on(adapter->netdev); 219 220 if (affectTxQueue) { 221 for (i = 0; i < adapter->num_tx_queues; i++) 222 vmxnet3_tq_start(&adapter->tx_queue[i], 223 adapter); 224 } 225 } else { 226 netdev_info(adapter->netdev, "NIC Link is Down\n"); 227 netif_carrier_off(adapter->netdev); 228 229 if (affectTxQueue) { 230 for (i = 0; i < adapter->num_tx_queues; i++) 231 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter); 232 } 233 } 234 } 235 236 static void 237 vmxnet3_process_events(struct vmxnet3_adapter *adapter) 238 { 239 int i; 240 unsigned long flags; 241 u32 events = le32_to_cpu(adapter->shared->ecr); 242 if (!events) 243 return; 244 245 vmxnet3_ack_events(adapter, events); 246 247 /* Check if link state has changed */ 248 if (events & VMXNET3_ECR_LINK) 249 vmxnet3_check_link(adapter, true); 250 251 /* Check if there is an error on xmit/recv queues */ 252 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 253 spin_lock_irqsave(&adapter->cmd_lock, flags); 254 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 255 VMXNET3_CMD_GET_QUEUE_STATUS); 256 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 257 258 for (i = 0; i < adapter->num_tx_queues; i++) 259 if (adapter->tqd_start[i].status.stopped) 260 dev_err(&adapter->netdev->dev, 261 "%s: tq[%d] error 0x%x\n", 262 adapter->netdev->name, i, le32_to_cpu( 263 adapter->tqd_start[i].status.error)); 264 for (i = 0; i < adapter->num_rx_queues; i++) 265 if (adapter->rqd_start[i].status.stopped) 266 dev_err(&adapter->netdev->dev, 267 "%s: rq[%d] error 0x%x\n", 268 adapter->netdev->name, i, 269 adapter->rqd_start[i].status.error); 270 271 schedule_work(&adapter->work); 272 } 273 } 274 275 #ifdef __BIG_ENDIAN_BITFIELD 276 /* 277 * The device expects the bitfields in shared structures to be written in 278 * little endian. When CPU is big endian, the following routines are used to 279 * correctly read and write into ABI. 280 * The general technique used here is : double word bitfields are defined in 281 * opposite order for big endian architecture. Then before reading them in 282 * driver the complete double word is translated using le32_to_cpu. Similarly 283 * After the driver writes into bitfields, cpu_to_le32 is used to translate the 284 * double words into required format. 285 * In order to avoid touching bits in shared structure more than once, temporary 286 * descriptors are used. These are passed as srcDesc to following functions. 287 */ 288 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc, 289 struct Vmxnet3_RxDesc *dstDesc) 290 { 291 u32 *src = (u32 *)srcDesc + 2; 292 u32 *dst = (u32 *)dstDesc + 2; 293 dstDesc->addr = le64_to_cpu(srcDesc->addr); 294 *dst = le32_to_cpu(*src); 295 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1); 296 } 297 298 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc, 299 struct Vmxnet3_TxDesc *dstDesc) 300 { 301 int i; 302 u32 *src = (u32 *)(srcDesc + 1); 303 u32 *dst = (u32 *)(dstDesc + 1); 304 305 /* Working backwards so that the gen bit is set at the end. */ 306 for (i = 2; i > 0; i--) { 307 src--; 308 dst--; 309 *dst = cpu_to_le32(*src); 310 } 311 } 312 313 314 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc, 315 struct Vmxnet3_RxCompDesc *dstDesc) 316 { 317 int i = 0; 318 u32 *src = (u32 *)srcDesc; 319 u32 *dst = (u32 *)dstDesc; 320 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) { 321 *dst = le32_to_cpu(*src); 322 src++; 323 dst++; 324 } 325 } 326 327 328 /* Used to read bitfield values from double words. */ 329 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size) 330 { 331 u32 temp = le32_to_cpu(*bitfield); 332 u32 mask = ((1 << size) - 1) << pos; 333 temp &= mask; 334 temp >>= pos; 335 return temp; 336 } 337 338 339 340 #endif /* __BIG_ENDIAN_BITFIELD */ 341 342 #ifdef __BIG_ENDIAN_BITFIELD 343 344 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \ 345 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \ 346 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE) 347 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \ 348 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \ 349 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE) 350 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \ 351 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \ 352 VMXNET3_TCD_GEN_SIZE) 353 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \ 354 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE) 355 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \ 356 (dstrcd) = (tmp); \ 357 vmxnet3_RxCompToCPU((rcd), (tmp)); \ 358 } while (0) 359 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \ 360 (dstrxd) = (tmp); \ 361 vmxnet3_RxDescToCPU((rxd), (tmp)); \ 362 } while (0) 363 364 #else 365 366 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen) 367 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop) 368 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen) 369 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx) 370 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd) 371 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd) 372 373 #endif /* __BIG_ENDIAN_BITFIELD */ 374 375 376 static void 377 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi, 378 struct pci_dev *pdev) 379 { 380 u32 map_type = tbi->map_type; 381 382 if (map_type & VMXNET3_MAP_SINGLE) 383 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len, 384 DMA_TO_DEVICE); 385 else if (map_type & VMXNET3_MAP_PAGE) 386 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len, 387 DMA_TO_DEVICE); 388 else 389 BUG_ON(map_type & ~VMXNET3_MAP_XDP); 390 391 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */ 392 } 393 394 395 static int 396 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq, 397 struct pci_dev *pdev, struct vmxnet3_adapter *adapter, 398 struct xdp_frame_bulk *bq) 399 { 400 struct vmxnet3_tx_buf_info *tbi; 401 int entries = 0; 402 u32 map_type; 403 404 /* no out of order completion */ 405 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp); 406 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1); 407 408 tbi = &tq->buf_info[eop_idx]; 409 BUG_ON(!tbi->skb); 410 map_type = tbi->map_type; 411 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size); 412 413 while (tq->tx_ring.next2comp != eop_idx) { 414 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp, 415 pdev); 416 417 /* update next2comp w/o tx_lock. Since we are marking more, 418 * instead of less, tx ring entries avail, the worst case is 419 * that the tx routine incorrectly re-queues a pkt due to 420 * insufficient tx ring entries. 421 */ 422 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 423 entries++; 424 } 425 426 if (map_type & VMXNET3_MAP_XDP) 427 xdp_return_frame_bulk(tbi->xdpf, bq); 428 else 429 dev_kfree_skb_any(tbi->skb); 430 431 /* xdpf and skb are in an anonymous union. */ 432 tbi->skb = NULL; 433 434 return entries; 435 } 436 437 438 static int 439 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq, 440 struct vmxnet3_adapter *adapter) 441 { 442 union Vmxnet3_GenericDesc *gdesc; 443 struct xdp_frame_bulk bq; 444 int completed = 0; 445 446 xdp_frame_bulk_init(&bq); 447 rcu_read_lock(); 448 449 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 450 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) { 451 /* Prevent any &gdesc->tcd field from being (speculatively) 452 * read before (&gdesc->tcd)->gen is read. 453 */ 454 dma_rmb(); 455 456 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX( 457 &gdesc->tcd), tq, adapter->pdev, 458 adapter, &bq); 459 460 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring); 461 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 462 } 463 xdp_flush_frame_bulk(&bq); 464 rcu_read_unlock(); 465 466 if (completed) { 467 spin_lock(&tq->tx_lock); 468 if (unlikely(vmxnet3_tq_stopped(tq, adapter) && 469 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) > 470 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) && 471 netif_carrier_ok(adapter->netdev))) { 472 vmxnet3_tq_wake(tq, adapter); 473 } 474 spin_unlock(&tq->tx_lock); 475 } 476 return completed; 477 } 478 479 480 static void 481 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq, 482 struct vmxnet3_adapter *adapter) 483 { 484 struct xdp_frame_bulk bq; 485 u32 map_type; 486 int i; 487 488 xdp_frame_bulk_init(&bq); 489 rcu_read_lock(); 490 491 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) { 492 struct vmxnet3_tx_buf_info *tbi; 493 494 tbi = tq->buf_info + tq->tx_ring.next2comp; 495 map_type = tbi->map_type; 496 497 vmxnet3_unmap_tx_buf(tbi, adapter->pdev); 498 if (tbi->skb) { 499 if (map_type & VMXNET3_MAP_XDP) 500 xdp_return_frame_bulk(tbi->xdpf, &bq); 501 else 502 dev_kfree_skb_any(tbi->skb); 503 tbi->skb = NULL; 504 } 505 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 506 } 507 508 xdp_flush_frame_bulk(&bq); 509 rcu_read_unlock(); 510 511 /* sanity check, verify all buffers are indeed unmapped */ 512 for (i = 0; i < tq->tx_ring.size; i++) 513 BUG_ON(tq->buf_info[i].map_type != VMXNET3_MAP_NONE); 514 515 tq->tx_ring.gen = VMXNET3_INIT_GEN; 516 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 517 518 tq->comp_ring.gen = VMXNET3_INIT_GEN; 519 tq->comp_ring.next2proc = 0; 520 } 521 522 523 static void 524 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 525 struct vmxnet3_adapter *adapter) 526 { 527 if (tq->tx_ring.base) { 528 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size * 529 sizeof(struct Vmxnet3_TxDesc), 530 tq->tx_ring.base, tq->tx_ring.basePA); 531 tq->tx_ring.base = NULL; 532 } 533 if (tq->data_ring.base) { 534 dma_free_coherent(&adapter->pdev->dev, 535 tq->data_ring.size * tq->txdata_desc_size, 536 tq->data_ring.base, tq->data_ring.basePA); 537 tq->data_ring.base = NULL; 538 } 539 if (tq->ts_ring.base) { 540 dma_free_coherent(&adapter->pdev->dev, 541 tq->tx_ring.size * tq->tx_ts_desc_size, 542 tq->ts_ring.base, tq->ts_ring.basePA); 543 tq->ts_ring.base = NULL; 544 } 545 if (tq->comp_ring.base) { 546 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size * 547 sizeof(struct Vmxnet3_TxCompDesc), 548 tq->comp_ring.base, tq->comp_ring.basePA); 549 tq->comp_ring.base = NULL; 550 } 551 kfree(tq->buf_info); 552 tq->buf_info = NULL; 553 } 554 555 556 /* Destroy all tx queues */ 557 void 558 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter) 559 { 560 int i; 561 562 for (i = 0; i < adapter->num_tx_queues; i++) 563 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter); 564 } 565 566 567 static void 568 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 569 struct vmxnet3_adapter *adapter) 570 { 571 int i; 572 573 /* reset the tx ring contents to 0 and reset the tx ring states */ 574 memset(tq->tx_ring.base, 0, tq->tx_ring.size * 575 sizeof(struct Vmxnet3_TxDesc)); 576 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 577 tq->tx_ring.gen = VMXNET3_INIT_GEN; 578 579 memset(tq->data_ring.base, 0, 580 tq->data_ring.size * tq->txdata_desc_size); 581 582 if (tq->ts_ring.base) 583 memset(tq->ts_ring.base, 0, 584 tq->tx_ring.size * tq->tx_ts_desc_size); 585 586 /* reset the tx comp ring contents to 0 and reset comp ring states */ 587 memset(tq->comp_ring.base, 0, tq->comp_ring.size * 588 sizeof(struct Vmxnet3_TxCompDesc)); 589 tq->comp_ring.next2proc = 0; 590 tq->comp_ring.gen = VMXNET3_INIT_GEN; 591 592 /* reset the bookkeeping data */ 593 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size); 594 for (i = 0; i < tq->tx_ring.size; i++) 595 tq->buf_info[i].map_type = VMXNET3_MAP_NONE; 596 597 /* stats are not reset */ 598 } 599 600 601 static int 602 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq, 603 struct vmxnet3_adapter *adapter) 604 { 605 BUG_ON(tq->tx_ring.base || tq->data_ring.base || 606 tq->comp_ring.base || tq->buf_info); 607 608 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 609 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc), 610 &tq->tx_ring.basePA, GFP_KERNEL); 611 if (!tq->tx_ring.base) { 612 netdev_err(adapter->netdev, "failed to allocate tx ring\n"); 613 goto err; 614 } 615 616 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 617 tq->data_ring.size * tq->txdata_desc_size, 618 &tq->data_ring.basePA, GFP_KERNEL); 619 if (!tq->data_ring.base) { 620 netdev_err(adapter->netdev, "failed to allocate tx data ring\n"); 621 goto err; 622 } 623 624 if (tq->tx_ts_desc_size != 0) { 625 tq->ts_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 626 tq->tx_ring.size * tq->tx_ts_desc_size, 627 &tq->ts_ring.basePA, GFP_KERNEL); 628 if (!tq->ts_ring.base) { 629 netdev_err(adapter->netdev, "failed to allocate tx ts ring\n"); 630 tq->tx_ts_desc_size = 0; 631 } 632 } else { 633 tq->ts_ring.base = NULL; 634 } 635 636 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 637 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc), 638 &tq->comp_ring.basePA, GFP_KERNEL); 639 if (!tq->comp_ring.base) { 640 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n"); 641 goto err; 642 } 643 644 tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]), 645 GFP_KERNEL, 646 dev_to_node(&adapter->pdev->dev)); 647 if (!tq->buf_info) 648 goto err; 649 650 return 0; 651 652 err: 653 vmxnet3_tq_destroy(tq, adapter); 654 return -ENOMEM; 655 } 656 657 static void 658 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter) 659 { 660 int i; 661 662 for (i = 0; i < adapter->num_tx_queues; i++) 663 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter); 664 } 665 666 /* 667 * starting from ring->next2fill, allocate rx buffers for the given ring 668 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers 669 * are allocated or allocation fails 670 */ 671 672 static int 673 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx, 674 int num_to_alloc, struct vmxnet3_adapter *adapter) 675 { 676 int num_allocated = 0; 677 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx]; 678 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx]; 679 u32 val; 680 681 while (num_allocated <= num_to_alloc) { 682 struct vmxnet3_rx_buf_info *rbi; 683 union Vmxnet3_GenericDesc *gd; 684 685 rbi = rbi_base + ring->next2fill; 686 gd = ring->base + ring->next2fill; 687 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 688 689 if (rbi->buf_type == VMXNET3_RX_BUF_XDP) { 690 void *data = vmxnet3_pp_get_buff(rq->page_pool, 691 &rbi->dma_addr, 692 GFP_KERNEL); 693 if (!data) { 694 rq->stats.rx_buf_alloc_failure++; 695 break; 696 } 697 rbi->page = virt_to_page(data); 698 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 699 } else if (rbi->buf_type == VMXNET3_RX_BUF_SKB) { 700 if (rbi->skb == NULL) { 701 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev, 702 rbi->len, 703 GFP_KERNEL); 704 if (unlikely(rbi->skb == NULL)) { 705 rq->stats.rx_buf_alloc_failure++; 706 break; 707 } 708 709 rbi->dma_addr = dma_map_single( 710 &adapter->pdev->dev, 711 rbi->skb->data, rbi->len, 712 DMA_FROM_DEVICE); 713 if (dma_mapping_error(&adapter->pdev->dev, 714 rbi->dma_addr)) { 715 dev_kfree_skb_any(rbi->skb); 716 rbi->skb = NULL; 717 rq->stats.rx_buf_alloc_failure++; 718 break; 719 } 720 } else { 721 /* rx buffer skipped by the device */ 722 } 723 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 724 } else { 725 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE || 726 rbi->len != PAGE_SIZE); 727 728 if (rbi->page == NULL) { 729 rbi->page = alloc_page(GFP_ATOMIC); 730 if (unlikely(rbi->page == NULL)) { 731 rq->stats.rx_buf_alloc_failure++; 732 break; 733 } 734 rbi->dma_addr = dma_map_page( 735 &adapter->pdev->dev, 736 rbi->page, 0, PAGE_SIZE, 737 DMA_FROM_DEVICE); 738 if (dma_mapping_error(&adapter->pdev->dev, 739 rbi->dma_addr)) { 740 put_page(rbi->page); 741 rbi->page = NULL; 742 rq->stats.rx_buf_alloc_failure++; 743 break; 744 } 745 } else { 746 /* rx buffers skipped by the device */ 747 } 748 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT; 749 } 750 751 gd->rxd.addr = cpu_to_le64(rbi->dma_addr); 752 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT) 753 | val | rbi->len); 754 755 /* Fill the last buffer but dont mark it ready, or else the 756 * device will think that the queue is full */ 757 if (num_allocated == num_to_alloc) { 758 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 759 break; 760 } 761 762 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT); 763 num_allocated++; 764 vmxnet3_cmd_ring_adv_next2fill(ring); 765 } 766 767 netdev_dbg(adapter->netdev, 768 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n", 769 num_allocated, ring->next2fill, ring->next2comp); 770 771 /* so that the device can distinguish a full ring and an empty ring */ 772 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp); 773 774 return num_allocated; 775 } 776 777 778 static void 779 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd, 780 struct vmxnet3_rx_buf_info *rbi) 781 { 782 skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags; 783 784 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); 785 786 skb_frag_fill_page_desc(frag, rbi->page, 0, rcd->len); 787 skb->data_len += rcd->len; 788 skb->truesize += PAGE_SIZE; 789 skb_shinfo(skb)->nr_frags++; 790 } 791 792 793 static int 794 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx, 795 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev, 796 struct vmxnet3_adapter *adapter) 797 { 798 u32 dw2, len; 799 unsigned long buf_offset; 800 int i; 801 union Vmxnet3_GenericDesc *gdesc; 802 struct vmxnet3_tx_buf_info *tbi = NULL; 803 804 BUG_ON(ctx->copy_size > skb_headlen(skb)); 805 806 /* use the previous gen bit for the SOP desc */ 807 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 808 809 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; 810 gdesc = ctx->sop_txd; /* both loops below can be skipped */ 811 812 /* no need to map the buffer if headers are copied */ 813 if (ctx->copy_size) { 814 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA + 815 tq->tx_ring.next2fill * 816 tq->txdata_desc_size); 817 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size); 818 ctx->sop_txd->dword[3] = 0; 819 820 tbi = tq->buf_info + tq->tx_ring.next2fill; 821 tbi->map_type = VMXNET3_MAP_NONE; 822 823 netdev_dbg(adapter->netdev, 824 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 825 tq->tx_ring.next2fill, 826 le64_to_cpu(ctx->sop_txd->txd.addr), 827 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]); 828 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 829 830 /* use the right gen for non-SOP desc */ 831 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 832 } 833 834 /* linear part can use multiple tx desc if it's big */ 835 len = skb_headlen(skb) - ctx->copy_size; 836 buf_offset = ctx->copy_size; 837 while (len) { 838 u32 buf_size; 839 840 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 841 buf_size = len; 842 dw2 |= len; 843 } else { 844 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 845 /* spec says that for TxDesc.len, 0 == 2^14 */ 846 } 847 848 tbi = tq->buf_info + tq->tx_ring.next2fill; 849 tbi->map_type = VMXNET3_MAP_SINGLE; 850 tbi->dma_addr = dma_map_single(&adapter->pdev->dev, 851 skb->data + buf_offset, buf_size, 852 DMA_TO_DEVICE); 853 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 854 return -EFAULT; 855 856 tbi->len = buf_size; 857 858 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 859 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 860 861 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 862 gdesc->dword[2] = cpu_to_le32(dw2); 863 gdesc->dword[3] = 0; 864 865 netdev_dbg(adapter->netdev, 866 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 867 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 868 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 869 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 870 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 871 872 len -= buf_size; 873 buf_offset += buf_size; 874 } 875 876 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 877 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 878 u32 buf_size; 879 880 buf_offset = 0; 881 len = skb_frag_size(frag); 882 while (len) { 883 tbi = tq->buf_info + tq->tx_ring.next2fill; 884 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 885 buf_size = len; 886 dw2 |= len; 887 } else { 888 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 889 /* spec says that for TxDesc.len, 0 == 2^14 */ 890 } 891 tbi->map_type = VMXNET3_MAP_PAGE; 892 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag, 893 buf_offset, buf_size, 894 DMA_TO_DEVICE); 895 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 896 return -EFAULT; 897 898 tbi->len = buf_size; 899 900 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 901 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 902 903 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 904 gdesc->dword[2] = cpu_to_le32(dw2); 905 gdesc->dword[3] = 0; 906 907 netdev_dbg(adapter->netdev, 908 "txd[%u]: 0x%llx %u %u\n", 909 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 910 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 911 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 912 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 913 914 len -= buf_size; 915 buf_offset += buf_size; 916 } 917 } 918 919 ctx->eop_txd = gdesc; 920 921 /* set the last buf_info for the pkt */ 922 tbi->skb = skb; 923 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base; 924 if (tq->tx_ts_desc_size != 0) { 925 ctx->ts_txd = (struct Vmxnet3_TxTSDesc *)((u8 *)tq->ts_ring.base + 926 tbi->sop_idx * tq->tx_ts_desc_size); 927 ctx->ts_txd->ts.tsi = 0; 928 } 929 930 return 0; 931 } 932 933 934 /* Init all tx queues */ 935 static void 936 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter) 937 { 938 int i; 939 940 for (i = 0; i < adapter->num_tx_queues; i++) 941 vmxnet3_tq_init(&adapter->tx_queue[i], adapter); 942 } 943 944 945 /* 946 * parse relevant protocol headers: 947 * For a tso pkt, relevant headers are L2/3/4 including options 948 * For a pkt requesting csum offloading, they are L2/3 and may include L4 949 * if it's a TCP/UDP pkt 950 * 951 * Returns: 952 * -1: error happens during parsing 953 * 0: protocol headers parsed, but too big to be copied 954 * 1: protocol headers parsed and copied 955 * 956 * Other effects: 957 * 1. related *ctx fields are updated. 958 * 2. ctx->copy_size is # of bytes copied 959 * 3. the portion to be copied is guaranteed to be in the linear part 960 * 961 */ 962 static int 963 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 964 struct vmxnet3_tx_ctx *ctx, 965 struct vmxnet3_adapter *adapter) 966 { 967 u8 protocol = 0; 968 969 if (ctx->mss) { /* TSO */ 970 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 971 ctx->l4_offset = skb_inner_transport_offset(skb); 972 ctx->l4_hdr_size = inner_tcp_hdrlen(skb); 973 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 974 } else { 975 ctx->l4_offset = skb_transport_offset(skb); 976 ctx->l4_hdr_size = tcp_hdrlen(skb); 977 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 978 } 979 } else { 980 if (skb->ip_summed == CHECKSUM_PARTIAL) { 981 /* For encap packets, skb_checksum_start_offset refers 982 * to inner L4 offset. Thus, below works for encap as 983 * well as non-encap case 984 */ 985 ctx->l4_offset = skb_checksum_start_offset(skb); 986 987 if (VMXNET3_VERSION_GE_4(adapter) && 988 skb->encapsulation) { 989 struct iphdr *iph = inner_ip_hdr(skb); 990 991 if (iph->version == 4) { 992 protocol = iph->protocol; 993 } else { 994 const struct ipv6hdr *ipv6h; 995 996 ipv6h = inner_ipv6_hdr(skb); 997 protocol = ipv6h->nexthdr; 998 } 999 } else { 1000 if (ctx->ipv4) { 1001 const struct iphdr *iph = ip_hdr(skb); 1002 1003 protocol = iph->protocol; 1004 } else if (ctx->ipv6) { 1005 const struct ipv6hdr *ipv6h; 1006 1007 ipv6h = ipv6_hdr(skb); 1008 protocol = ipv6h->nexthdr; 1009 } 1010 } 1011 1012 switch (protocol) { 1013 case IPPROTO_TCP: 1014 ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) : 1015 tcp_hdrlen(skb); 1016 break; 1017 case IPPROTO_UDP: 1018 ctx->l4_hdr_size = sizeof(struct udphdr); 1019 break; 1020 default: 1021 ctx->l4_hdr_size = 0; 1022 break; 1023 } 1024 1025 ctx->copy_size = min(ctx->l4_offset + 1026 ctx->l4_hdr_size, skb->len); 1027 } else { 1028 ctx->l4_offset = 0; 1029 ctx->l4_hdr_size = 0; 1030 /* copy as much as allowed */ 1031 ctx->copy_size = min_t(unsigned int, 1032 tq->txdata_desc_size, 1033 skb_headlen(skb)); 1034 } 1035 1036 if (skb->len <= tq->txdata_desc_size) 1037 ctx->copy_size = skb->len; 1038 1039 /* make sure headers are accessible directly */ 1040 if (unlikely(!pskb_may_pull(skb, ctx->copy_size))) 1041 goto err; 1042 } 1043 1044 if (unlikely(ctx->copy_size > tq->txdata_desc_size)) { 1045 tq->stats.oversized_hdr++; 1046 ctx->copy_size = 0; 1047 return 0; 1048 } 1049 1050 return 1; 1051 err: 1052 return -1; 1053 } 1054 1055 /* 1056 * copy relevant protocol headers to the transmit ring: 1057 * For a tso pkt, relevant headers are L2/3/4 including options 1058 * For a pkt requesting csum offloading, they are L2/3 and may include L4 1059 * if it's a TCP/UDP pkt 1060 * 1061 * 1062 * Note that this requires that vmxnet3_parse_hdr be called first to set the 1063 * appropriate bits in ctx first 1064 */ 1065 static void 1066 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1067 struct vmxnet3_tx_ctx *ctx, 1068 struct vmxnet3_adapter *adapter) 1069 { 1070 struct Vmxnet3_TxDataDesc *tdd; 1071 1072 tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base + 1073 tq->tx_ring.next2fill * 1074 tq->txdata_desc_size); 1075 1076 memcpy(tdd->data, skb->data, ctx->copy_size); 1077 netdev_dbg(adapter->netdev, 1078 "copy %u bytes to dataRing[%u]\n", 1079 ctx->copy_size, tq->tx_ring.next2fill); 1080 } 1081 1082 1083 static void 1084 vmxnet3_prepare_inner_tso(struct sk_buff *skb, 1085 struct vmxnet3_tx_ctx *ctx) 1086 { 1087 struct tcphdr *tcph = inner_tcp_hdr(skb); 1088 struct iphdr *iph = inner_ip_hdr(skb); 1089 1090 if (iph->version == 4) { 1091 iph->check = 0; 1092 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1093 IPPROTO_TCP, 0); 1094 } else { 1095 struct ipv6hdr *iph = inner_ipv6_hdr(skb); 1096 1097 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0, 1098 IPPROTO_TCP, 0); 1099 } 1100 } 1101 1102 static void 1103 vmxnet3_prepare_tso(struct sk_buff *skb, 1104 struct vmxnet3_tx_ctx *ctx) 1105 { 1106 struct tcphdr *tcph = tcp_hdr(skb); 1107 1108 if (ctx->ipv4) { 1109 struct iphdr *iph = ip_hdr(skb); 1110 1111 iph->check = 0; 1112 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1113 IPPROTO_TCP, 0); 1114 } else if (ctx->ipv6) { 1115 tcp_v6_gso_csum_prep(skb); 1116 } 1117 } 1118 1119 static int txd_estimate(const struct sk_buff *skb) 1120 { 1121 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1122 int i; 1123 1124 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1125 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1126 1127 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag)); 1128 } 1129 return count; 1130 } 1131 1132 /* 1133 * Transmits a pkt thru a given tq 1134 * Returns: 1135 * NETDEV_TX_OK: descriptors are setup successfully 1136 * NETDEV_TX_OK: error occurred, the pkt is dropped 1137 * NETDEV_TX_BUSY: tx ring is full, queue is stopped 1138 * 1139 * Side-effects: 1140 * 1. tx ring may be changed 1141 * 2. tq stats may be updated accordingly 1142 * 3. shared->txNumDeferred may be updated 1143 */ 1144 1145 static int 1146 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1147 struct vmxnet3_adapter *adapter, struct net_device *netdev) 1148 { 1149 int ret; 1150 u32 count; 1151 int num_pkts; 1152 int tx_num_deferred; 1153 unsigned long flags; 1154 struct vmxnet3_tx_ctx ctx; 1155 union Vmxnet3_GenericDesc *gdesc; 1156 #ifdef __BIG_ENDIAN_BITFIELD 1157 /* Use temporary descriptor to avoid touching bits multiple times */ 1158 union Vmxnet3_GenericDesc tempTxDesc; 1159 #endif 1160 1161 count = txd_estimate(skb); 1162 1163 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP)); 1164 ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6)); 1165 1166 ctx.mss = skb_shinfo(skb)->gso_size; 1167 if (ctx.mss) { 1168 if (skb_header_cloned(skb)) { 1169 if (unlikely(pskb_expand_head(skb, 0, 0, 1170 GFP_ATOMIC) != 0)) { 1171 tq->stats.drop_tso++; 1172 goto drop_pkt; 1173 } 1174 tq->stats.copy_skb_header++; 1175 } 1176 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1177 /* tso pkts must not use more than 1178 * VMXNET3_MAX_TSO_TXD_PER_PKT entries 1179 */ 1180 if (skb_linearize(skb) != 0) { 1181 tq->stats.drop_too_many_frags++; 1182 goto drop_pkt; 1183 } 1184 tq->stats.linearized++; 1185 1186 /* recalculate the # of descriptors to use */ 1187 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1188 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1189 tq->stats.drop_too_many_frags++; 1190 goto drop_pkt; 1191 } 1192 } 1193 if (skb->encapsulation) { 1194 vmxnet3_prepare_inner_tso(skb, &ctx); 1195 } else { 1196 vmxnet3_prepare_tso(skb, &ctx); 1197 } 1198 } else { 1199 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) { 1200 1201 /* non-tso pkts must not use more than 1202 * VMXNET3_MAX_TXD_PER_PKT entries 1203 */ 1204 if (skb_linearize(skb) != 0) { 1205 tq->stats.drop_too_many_frags++; 1206 goto drop_pkt; 1207 } 1208 tq->stats.linearized++; 1209 1210 /* recalculate the # of descriptors to use */ 1211 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1212 } 1213 } 1214 1215 ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter); 1216 if (ret >= 0) { 1217 BUG_ON(ret <= 0 && ctx.copy_size != 0); 1218 /* hdrs parsed, check against other limits */ 1219 if (ctx.mss) { 1220 if (unlikely(ctx.l4_offset + ctx.l4_hdr_size > 1221 VMXNET3_MAX_TX_BUF_SIZE)) { 1222 tq->stats.drop_oversized_hdr++; 1223 goto drop_pkt; 1224 } 1225 } else { 1226 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1227 if (unlikely(ctx.l4_offset + 1228 skb->csum_offset > 1229 VMXNET3_MAX_CSUM_OFFSET)) { 1230 tq->stats.drop_oversized_hdr++; 1231 goto drop_pkt; 1232 } 1233 } 1234 } 1235 } else { 1236 tq->stats.drop_hdr_inspect_err++; 1237 goto drop_pkt; 1238 } 1239 1240 spin_lock_irqsave(&tq->tx_lock, flags); 1241 1242 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) { 1243 tq->stats.tx_ring_full++; 1244 netdev_dbg(adapter->netdev, 1245 "tx queue stopped on %s, next2comp %u" 1246 " next2fill %u\n", adapter->netdev->name, 1247 tq->tx_ring.next2comp, tq->tx_ring.next2fill); 1248 1249 vmxnet3_tq_stop(tq, adapter); 1250 spin_unlock_irqrestore(&tq->tx_lock, flags); 1251 return NETDEV_TX_BUSY; 1252 } 1253 1254 1255 vmxnet3_copy_hdr(skb, tq, &ctx, adapter); 1256 1257 /* fill tx descs related to addr & len */ 1258 if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter)) 1259 goto unlock_drop_pkt; 1260 1261 /* setup the EOP desc */ 1262 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); 1263 1264 /* setup the SOP desc */ 1265 #ifdef __BIG_ENDIAN_BITFIELD 1266 gdesc = &tempTxDesc; 1267 gdesc->dword[2] = ctx.sop_txd->dword[2]; 1268 gdesc->dword[3] = ctx.sop_txd->dword[3]; 1269 #else 1270 gdesc = ctx.sop_txd; 1271 #endif 1272 tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred); 1273 if (ctx.mss) { 1274 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 1275 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1276 if (VMXNET3_VERSION_GE_7(adapter)) { 1277 gdesc->txd.om = VMXNET3_OM_TSO; 1278 gdesc->txd.ext1 = 1; 1279 } else { 1280 gdesc->txd.om = VMXNET3_OM_ENCAP; 1281 } 1282 gdesc->txd.msscof = ctx.mss; 1283 1284 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) 1285 gdesc->txd.oco = 1; 1286 } else { 1287 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1288 gdesc->txd.om = VMXNET3_OM_TSO; 1289 gdesc->txd.msscof = ctx.mss; 1290 } 1291 num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss; 1292 } else { 1293 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1294 if (VMXNET3_VERSION_GE_4(adapter) && 1295 skb->encapsulation) { 1296 gdesc->txd.hlen = ctx.l4_offset + 1297 ctx.l4_hdr_size; 1298 if (VMXNET3_VERSION_GE_7(adapter)) { 1299 gdesc->txd.om = VMXNET3_OM_CSUM; 1300 gdesc->txd.msscof = ctx.l4_offset + 1301 skb->csum_offset; 1302 gdesc->txd.ext1 = 1; 1303 } else { 1304 gdesc->txd.om = VMXNET3_OM_ENCAP; 1305 gdesc->txd.msscof = 0; /* Reserved */ 1306 } 1307 } else { 1308 gdesc->txd.hlen = ctx.l4_offset; 1309 gdesc->txd.om = VMXNET3_OM_CSUM; 1310 gdesc->txd.msscof = ctx.l4_offset + 1311 skb->csum_offset; 1312 } 1313 } else { 1314 gdesc->txd.om = 0; 1315 gdesc->txd.msscof = 0; 1316 } 1317 num_pkts = 1; 1318 } 1319 le32_add_cpu(&tq->shared->txNumDeferred, num_pkts); 1320 tx_num_deferred += num_pkts; 1321 1322 if (skb_vlan_tag_present(skb)) { 1323 gdesc->txd.ti = 1; 1324 gdesc->txd.tci = skb_vlan_tag_get(skb); 1325 } 1326 1327 if (tq->tx_ts_desc_size != 0 && 1328 adapter->latencyConf->sampleRate != 0) { 1329 if (vmxnet3_apply_timestamp(tq, adapter->latencyConf->sampleRate)) { 1330 ctx.ts_txd->ts.tsData = vmxnet3_get_cycles(VMXNET3_PMC_PSEUDO_TSC); 1331 ctx.ts_txd->ts.tsi = 1; 1332 } 1333 } 1334 1335 /* Ensure that the write to (&gdesc->txd)->gen will be observed after 1336 * all other writes to &gdesc->txd. 1337 */ 1338 dma_wmb(); 1339 1340 /* finally flips the GEN bit of the SOP desc. */ 1341 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ 1342 VMXNET3_TXD_GEN); 1343 #ifdef __BIG_ENDIAN_BITFIELD 1344 /* Finished updating in bitfields of Tx Desc, so write them in original 1345 * place. 1346 */ 1347 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc, 1348 (struct Vmxnet3_TxDesc *)ctx.sop_txd); 1349 gdesc = ctx.sop_txd; 1350 #endif 1351 netdev_dbg(adapter->netdev, 1352 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n", 1353 (u32)(ctx.sop_txd - 1354 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr), 1355 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3])); 1356 1357 spin_unlock_irqrestore(&tq->tx_lock, flags); 1358 1359 if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) { 1360 tq->shared->txNumDeferred = 0; 1361 VMXNET3_WRITE_BAR0_REG(adapter, 1362 adapter->tx_prod_offset + tq->qid * 8, 1363 tq->tx_ring.next2fill); 1364 } 1365 1366 return NETDEV_TX_OK; 1367 1368 unlock_drop_pkt: 1369 spin_unlock_irqrestore(&tq->tx_lock, flags); 1370 drop_pkt: 1371 tq->stats.drop_total++; 1372 dev_kfree_skb_any(skb); 1373 return NETDEV_TX_OK; 1374 } 1375 1376 static int 1377 vmxnet3_create_pp(struct vmxnet3_adapter *adapter, 1378 struct vmxnet3_rx_queue *rq, int size) 1379 { 1380 bool xdp_prog = vmxnet3_xdp_enabled(adapter); 1381 const struct page_pool_params pp_params = { 1382 .order = 0, 1383 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 1384 .pool_size = size, 1385 .nid = NUMA_NO_NODE, 1386 .dev = &adapter->pdev->dev, 1387 .offset = VMXNET3_XDP_RX_OFFSET, 1388 .max_len = VMXNET3_XDP_MAX_FRSIZE, 1389 .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE, 1390 }; 1391 struct page_pool *pp; 1392 int err; 1393 1394 pp = page_pool_create(&pp_params); 1395 if (IS_ERR(pp)) 1396 return PTR_ERR(pp); 1397 1398 err = xdp_rxq_info_reg(&rq->xdp_rxq, adapter->netdev, rq->qid, 1399 rq->napi.napi_id); 1400 if (err < 0) 1401 goto err_free_pp; 1402 1403 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_PAGE_POOL, pp); 1404 if (err) 1405 goto err_unregister_rxq; 1406 1407 rq->page_pool = pp; 1408 1409 return 0; 1410 1411 err_unregister_rxq: 1412 xdp_rxq_info_unreg(&rq->xdp_rxq); 1413 err_free_pp: 1414 page_pool_destroy(pp); 1415 1416 return err; 1417 } 1418 1419 void * 1420 vmxnet3_pp_get_buff(struct page_pool *pp, dma_addr_t *dma_addr, 1421 gfp_t gfp_mask) 1422 { 1423 struct page *page; 1424 1425 page = page_pool_alloc_pages(pp, gfp_mask | __GFP_NOWARN); 1426 if (unlikely(!page)) 1427 return NULL; 1428 1429 *dma_addr = page_pool_get_dma_addr(page) + pp->p.offset; 1430 1431 return page_address(page); 1432 } 1433 1434 static netdev_tx_t 1435 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1436 { 1437 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1438 1439 BUG_ON(skb->queue_mapping > adapter->num_tx_queues); 1440 return vmxnet3_tq_xmit(skb, 1441 &adapter->tx_queue[skb->queue_mapping], 1442 adapter, netdev); 1443 } 1444 1445 1446 static void 1447 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, 1448 struct sk_buff *skb, 1449 union Vmxnet3_GenericDesc *gdesc) 1450 { 1451 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) { 1452 if (gdesc->rcd.v4 && 1453 (le32_to_cpu(gdesc->dword[3]) & 1454 VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) { 1455 skb->ip_summed = CHECKSUM_UNNECESSARY; 1456 if ((le32_to_cpu(gdesc->dword[0]) & 1457 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1458 skb->csum_level = 1; 1459 } 1460 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1461 !(le32_to_cpu(gdesc->dword[0]) & 1462 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1463 WARN_ON_ONCE(gdesc->rcd.frg && 1464 !(le32_to_cpu(gdesc->dword[0]) & 1465 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1466 } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) & 1467 (1 << VMXNET3_RCD_TUC_SHIFT))) { 1468 skb->ip_summed = CHECKSUM_UNNECESSARY; 1469 if ((le32_to_cpu(gdesc->dword[0]) & 1470 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1471 skb->csum_level = 1; 1472 } 1473 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1474 !(le32_to_cpu(gdesc->dword[0]) & 1475 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1476 WARN_ON_ONCE(gdesc->rcd.frg && 1477 !(le32_to_cpu(gdesc->dword[0]) & 1478 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1479 } else { 1480 if (gdesc->rcd.csum) { 1481 skb->csum = htons(gdesc->rcd.csum); 1482 skb->ip_summed = CHECKSUM_PARTIAL; 1483 } else { 1484 skb_checksum_none_assert(skb); 1485 } 1486 } 1487 } else { 1488 skb_checksum_none_assert(skb); 1489 } 1490 } 1491 1492 1493 static void 1494 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd, 1495 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter) 1496 { 1497 rq->stats.drop_err++; 1498 if (!rcd->fcs) 1499 rq->stats.drop_fcs++; 1500 1501 rq->stats.drop_total++; 1502 1503 /* 1504 * We do not unmap and chain the rx buffer to the skb. 1505 * We basically pretend this buffer is not used and will be recycled 1506 * by vmxnet3_rq_alloc_rx_buf() 1507 */ 1508 1509 /* 1510 * ctx->skb may be NULL if this is the first and the only one 1511 * desc for the pkt 1512 */ 1513 if (ctx->skb) 1514 dev_kfree_skb_irq(ctx->skb); 1515 1516 ctx->skb = NULL; 1517 } 1518 1519 1520 static u32 1521 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, 1522 union Vmxnet3_GenericDesc *gdesc) 1523 { 1524 u32 hlen, maplen; 1525 union { 1526 void *ptr; 1527 struct ethhdr *eth; 1528 struct vlan_ethhdr *veth; 1529 struct iphdr *ipv4; 1530 struct ipv6hdr *ipv6; 1531 struct tcphdr *tcp; 1532 } hdr; 1533 1534 /* v4/v6/tcp then describe the inner header, which we can't locate. */ 1535 if ((le32_to_cpu(gdesc->dword[0]) & (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)) || 1536 gdesc->rcd.tcp == 0) 1537 return 0; 1538 1539 maplen = skb_headlen(skb); 1540 if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen)) 1541 return 0; 1542 1543 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 1544 skb->protocol == cpu_to_be16(ETH_P_8021AD)) 1545 hlen = sizeof(struct vlan_ethhdr); 1546 else 1547 hlen = sizeof(struct ethhdr); 1548 1549 hdr.eth = eth_hdr(skb); 1550 if (gdesc->rcd.v4) { 1551 if (hdr.eth->h_proto != htons(ETH_P_IP) && 1552 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)) 1553 return 0; 1554 1555 hdr.ptr += hlen; 1556 if (hdr.ipv4->protocol != IPPROTO_TCP) 1557 return 0; 1558 1559 hlen = hdr.ipv4->ihl << 2; 1560 hdr.ptr += hdr.ipv4->ihl << 2; 1561 } else if (gdesc->rcd.v6) { 1562 if (hdr.eth->h_proto != htons(ETH_P_IPV6) && 1563 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)) 1564 return 0; 1565 1566 hdr.ptr += hlen; 1567 /* Use an estimated value, since we also need to handle 1568 * TSO case. 1569 */ 1570 if (hdr.ipv6->nexthdr != IPPROTO_TCP) 1571 return sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 1572 hlen = sizeof(struct ipv6hdr); 1573 hdr.ptr += sizeof(struct ipv6hdr); 1574 } else { 1575 /* Non-IP pkt, dont estimate header length */ 1576 return 0; 1577 } 1578 1579 if (hlen + sizeof(struct tcphdr) > maplen) 1580 return 0; 1581 1582 return (hlen + (hdr.tcp->doff << 2)); 1583 } 1584 1585 static void 1586 vmxnet3_lro_tunnel(struct sk_buff *skb, __be16 ip_proto) 1587 { 1588 struct udphdr *uh = NULL; 1589 1590 if (ip_proto == htons(ETH_P_IP)) { 1591 struct iphdr *iph = (struct iphdr *)skb->data; 1592 1593 if (iph->protocol == IPPROTO_UDP) 1594 uh = (struct udphdr *)(iph + 1); 1595 } else { 1596 struct ipv6hdr *iph = (struct ipv6hdr *)skb->data; 1597 1598 if (iph->nexthdr == IPPROTO_UDP) 1599 uh = (struct udphdr *)(iph + 1); 1600 } 1601 if (uh) { 1602 if (uh->check) 1603 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; 1604 else 1605 skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; 1606 } 1607 } 1608 1609 static int 1610 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, 1611 struct vmxnet3_adapter *adapter, int quota) 1612 { 1613 u32 rxprod_reg[2] = { 1614 adapter->rx_prod_offset, adapter->rx_prod2_offset 1615 }; 1616 u32 num_pkts = 0; 1617 bool skip_page_frags = false; 1618 bool encap_lro = false; 1619 struct Vmxnet3_RxCompDesc *rcd; 1620 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; 1621 u16 segCnt = 0, mss = 0; 1622 int comp_offset, fill_offset; 1623 #ifdef __BIG_ENDIAN_BITFIELD 1624 struct Vmxnet3_RxDesc rxCmdDesc; 1625 struct Vmxnet3_RxCompDesc rxComp; 1626 #endif 1627 bool need_flush = false; 1628 1629 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, 1630 &rxComp); 1631 while (rcd->gen == rq->comp_ring.gen) { 1632 struct vmxnet3_rx_buf_info *rbi; 1633 struct sk_buff *skb, *new_skb = NULL; 1634 struct page *new_page = NULL; 1635 dma_addr_t new_dma_addr; 1636 int num_to_alloc; 1637 struct Vmxnet3_RxDesc *rxd; 1638 u32 idx, ring_idx; 1639 struct vmxnet3_cmd_ring *ring = NULL; 1640 if (num_pkts >= quota) { 1641 /* we may stop even before we see the EOP desc of 1642 * the current pkt 1643 */ 1644 break; 1645 } 1646 1647 /* Prevent any rcd field from being (speculatively) read before 1648 * rcd->gen is read. 1649 */ 1650 dma_rmb(); 1651 1652 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 && 1653 rcd->rqID != rq->dataRingQid); 1654 idx = rcd->rxdIdx; 1655 ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID); 1656 ring = rq->rx_ring + ring_idx; 1657 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1658 &rxCmdDesc); 1659 rbi = rq->buf_info[ring_idx] + idx; 1660 1661 BUG_ON(rxd->addr != rbi->dma_addr || 1662 rxd->len != rbi->len); 1663 1664 if (unlikely(rcd->eop && rcd->err)) { 1665 vmxnet3_rx_error(rq, rcd, ctx, adapter); 1666 goto rcd_done; 1667 } 1668 1669 if (rcd->sop && rcd->eop && vmxnet3_xdp_enabled(adapter)) { 1670 struct sk_buff *skb_xdp_pass; 1671 int act; 1672 1673 if (VMXNET3_RX_DATA_RING(adapter, rcd->rqID)) { 1674 ctx->skb = NULL; 1675 goto skip_xdp; /* Handle it later. */ 1676 } 1677 1678 if (rbi->buf_type != VMXNET3_RX_BUF_XDP) 1679 goto rcd_done; 1680 1681 act = vmxnet3_process_xdp(adapter, rq, rcd, rbi, rxd, 1682 &skb_xdp_pass); 1683 if (act == XDP_PASS) { 1684 ctx->skb = skb_xdp_pass; 1685 goto sop_done; 1686 } 1687 ctx->skb = NULL; 1688 need_flush |= act == XDP_REDIRECT; 1689 1690 goto rcd_done; 1691 } 1692 skip_xdp: 1693 1694 if (rcd->sop) { /* first buf of the pkt */ 1695 bool rxDataRingUsed; 1696 u16 len; 1697 1698 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD || 1699 (rcd->rqID != rq->qid && 1700 rcd->rqID != rq->dataRingQid)); 1701 1702 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB && 1703 rbi->buf_type != VMXNET3_RX_BUF_XDP); 1704 BUG_ON(ctx->skb != NULL || rbi->skb == NULL); 1705 1706 if (unlikely(rcd->len == 0)) { 1707 /* Pretend the rx buffer is skipped. */ 1708 BUG_ON(!(rcd->sop && rcd->eop)); 1709 netdev_dbg(adapter->netdev, 1710 "rxRing[%u][%u] 0 length\n", 1711 ring_idx, idx); 1712 goto rcd_done; 1713 } 1714 1715 skip_page_frags = false; 1716 ctx->skb = rbi->skb; 1717 1718 if (rq->rx_ts_desc_size != 0 && rcd->ext2) { 1719 struct Vmxnet3_RxTSDesc *ts_rxd; 1720 1721 ts_rxd = (struct Vmxnet3_RxTSDesc *)((u8 *)rq->ts_ring.base + 1722 idx * rq->rx_ts_desc_size); 1723 ts_rxd->ts.tsData = vmxnet3_get_cycles(VMXNET3_PMC_PSEUDO_TSC); 1724 ts_rxd->ts.tsi = 1; 1725 } 1726 1727 rxDataRingUsed = 1728 VMXNET3_RX_DATA_RING(adapter, rcd->rqID); 1729 len = rxDataRingUsed ? rcd->len : rbi->len; 1730 1731 if (rxDataRingUsed && vmxnet3_xdp_enabled(adapter)) { 1732 struct sk_buff *skb_xdp_pass; 1733 size_t sz; 1734 int act; 1735 1736 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1737 act = vmxnet3_process_xdp_small(adapter, rq, 1738 &rq->data_ring.base[sz], 1739 rcd->len, 1740 &skb_xdp_pass); 1741 if (act == XDP_PASS) { 1742 ctx->skb = skb_xdp_pass; 1743 goto sop_done; 1744 } 1745 need_flush |= act == XDP_REDIRECT; 1746 1747 goto rcd_done; 1748 } 1749 new_skb = netdev_alloc_skb_ip_align(adapter->netdev, 1750 len); 1751 if (new_skb == NULL) { 1752 /* Skb allocation failed, do not handover this 1753 * skb to stack. Reuse it. Drop the existing pkt 1754 */ 1755 rq->stats.rx_buf_alloc_failure++; 1756 ctx->skb = NULL; 1757 rq->stats.drop_total++; 1758 skip_page_frags = true; 1759 goto rcd_done; 1760 } 1761 1762 if (rxDataRingUsed && adapter->rxdataring_enabled) { 1763 size_t sz; 1764 1765 BUG_ON(rcd->len > rq->data_ring.desc_size); 1766 1767 ctx->skb = new_skb; 1768 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1769 memcpy(new_skb->data, 1770 &rq->data_ring.base[sz], rcd->len); 1771 } else { 1772 ctx->skb = rbi->skb; 1773 1774 new_dma_addr = 1775 dma_map_single(&adapter->pdev->dev, 1776 new_skb->data, rbi->len, 1777 DMA_FROM_DEVICE); 1778 if (dma_mapping_error(&adapter->pdev->dev, 1779 new_dma_addr)) { 1780 dev_kfree_skb(new_skb); 1781 /* Skb allocation failed, do not 1782 * handover this skb to stack. Reuse 1783 * it. Drop the existing pkt. 1784 */ 1785 rq->stats.rx_buf_alloc_failure++; 1786 ctx->skb = NULL; 1787 rq->stats.drop_total++; 1788 skip_page_frags = true; 1789 goto rcd_done; 1790 } 1791 1792 dma_unmap_single(&adapter->pdev->dev, 1793 rbi->dma_addr, 1794 rbi->len, 1795 DMA_FROM_DEVICE); 1796 1797 /* Immediate refill */ 1798 rbi->skb = new_skb; 1799 rbi->dma_addr = new_dma_addr; 1800 rxd->addr = cpu_to_le64(rbi->dma_addr); 1801 rxd->len = rbi->len; 1802 } 1803 1804 skb_record_rx_queue(ctx->skb, rq->qid); 1805 skb_put(ctx->skb, rcd->len); 1806 1807 if (VMXNET3_VERSION_GE_2(adapter) && 1808 rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) { 1809 struct Vmxnet3_RxCompDescExt *rcdlro; 1810 union Vmxnet3_GenericDesc *gdesc; 1811 1812 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd; 1813 gdesc = (union Vmxnet3_GenericDesc *)rcd; 1814 1815 segCnt = rcdlro->segCnt; 1816 WARN_ON_ONCE(segCnt == 0); 1817 mss = rcdlro->mss; 1818 if (unlikely(segCnt <= 1)) 1819 segCnt = 0; 1820 encap_lro = (le32_to_cpu(gdesc->dword[0]) & 1821 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)); 1822 } else { 1823 segCnt = 0; 1824 } 1825 } else { 1826 BUG_ON(ctx->skb == NULL && !skip_page_frags); 1827 1828 /* non SOP buffer must be type 1 in most cases */ 1829 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE); 1830 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY); 1831 1832 /* If an sop buffer was dropped, skip all 1833 * following non-sop fragments. They will be reused. 1834 */ 1835 if (skip_page_frags) 1836 goto rcd_done; 1837 1838 if (rcd->len) { 1839 new_page = alloc_page(GFP_ATOMIC); 1840 /* Replacement page frag could not be allocated. 1841 * Reuse this page. Drop the pkt and free the 1842 * skb which contained this page as a frag. Skip 1843 * processing all the following non-sop frags. 1844 */ 1845 if (unlikely(!new_page)) { 1846 rq->stats.rx_buf_alloc_failure++; 1847 dev_kfree_skb(ctx->skb); 1848 ctx->skb = NULL; 1849 skip_page_frags = true; 1850 goto rcd_done; 1851 } 1852 new_dma_addr = dma_map_page(&adapter->pdev->dev, 1853 new_page, 1854 0, PAGE_SIZE, 1855 DMA_FROM_DEVICE); 1856 if (dma_mapping_error(&adapter->pdev->dev, 1857 new_dma_addr)) { 1858 put_page(new_page); 1859 rq->stats.rx_buf_alloc_failure++; 1860 dev_kfree_skb(ctx->skb); 1861 ctx->skb = NULL; 1862 skip_page_frags = true; 1863 goto rcd_done; 1864 } 1865 1866 dma_unmap_page(&adapter->pdev->dev, 1867 rbi->dma_addr, rbi->len, 1868 DMA_FROM_DEVICE); 1869 1870 vmxnet3_append_frag(ctx->skb, rcd, rbi); 1871 1872 /* Immediate refill */ 1873 rbi->page = new_page; 1874 rbi->dma_addr = new_dma_addr; 1875 rxd->addr = cpu_to_le64(rbi->dma_addr); 1876 rxd->len = rbi->len; 1877 } 1878 } 1879 1880 1881 sop_done: 1882 skb = ctx->skb; 1883 if (rcd->eop) { 1884 u32 mtu = adapter->netdev->mtu; 1885 skb->len += skb->data_len; 1886 1887 #ifdef VMXNET3_RSS 1888 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE && 1889 (adapter->netdev->features & NETIF_F_RXHASH)) { 1890 enum pkt_hash_types hash_type; 1891 1892 switch (rcd->rssType) { 1893 case VMXNET3_RCD_RSS_TYPE_IPV4: 1894 case VMXNET3_RCD_RSS_TYPE_IPV6: 1895 hash_type = PKT_HASH_TYPE_L3; 1896 break; 1897 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 1898 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 1899 case VMXNET3_RCD_RSS_TYPE_UDPIPV4: 1900 case VMXNET3_RCD_RSS_TYPE_UDPIPV6: 1901 hash_type = PKT_HASH_TYPE_L4; 1902 break; 1903 default: 1904 hash_type = PKT_HASH_TYPE_L3; 1905 break; 1906 } 1907 skb_set_hash(skb, 1908 le32_to_cpu(rcd->rssHash), 1909 hash_type); 1910 } 1911 #endif 1912 vmxnet3_rx_csum(adapter, skb, 1913 (union Vmxnet3_GenericDesc *)rcd); 1914 skb->protocol = eth_type_trans(skb, adapter->netdev); 1915 if ((!rcd->tcp && !encap_lro) || 1916 !(adapter->netdev->features & NETIF_F_LRO)) 1917 goto not_lro; 1918 1919 if (segCnt != 0 && mss != 0) { 1920 skb_shinfo(skb)->gso_type = rcd->v4 ? 1921 SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1922 if (encap_lro) 1923 vmxnet3_lro_tunnel(skb, skb->protocol); 1924 skb_shinfo(skb)->gso_size = mss; 1925 skb_shinfo(skb)->gso_segs = segCnt; 1926 } else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) { 1927 u32 hlen; 1928 1929 hlen = vmxnet3_get_hdr_len(adapter, skb, 1930 (union Vmxnet3_GenericDesc *)rcd); 1931 if (hlen == 0) 1932 goto not_lro; 1933 1934 skb_shinfo(skb)->gso_type = 1935 rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1936 if (segCnt != 0) { 1937 skb_shinfo(skb)->gso_segs = segCnt; 1938 skb_shinfo(skb)->gso_size = 1939 DIV_ROUND_UP(skb->len - 1940 hlen, segCnt); 1941 } else { 1942 skb_shinfo(skb)->gso_size = mtu - hlen; 1943 } 1944 } 1945 not_lro: 1946 if (unlikely(rcd->ts)) 1947 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci); 1948 1949 /* Use GRO callback if UPT is enabled */ 1950 if ((adapter->netdev->features & NETIF_F_LRO) && 1951 !rq->shared->updateRxProd) 1952 netif_receive_skb(skb); 1953 else 1954 napi_gro_receive(&rq->napi, skb); 1955 1956 ctx->skb = NULL; 1957 encap_lro = false; 1958 num_pkts++; 1959 } 1960 1961 rcd_done: 1962 /* device may have skipped some rx descs */ 1963 ring = rq->rx_ring + ring_idx; 1964 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 1965 1966 comp_offset = vmxnet3_cmd_ring_desc_avail(ring); 1967 fill_offset = (idx > ring->next2fill ? 0 : ring->size) + 1968 idx - ring->next2fill - 1; 1969 if (!ring->isOutOfOrder || fill_offset >= comp_offset) 1970 ring->next2comp = idx; 1971 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring); 1972 1973 /* Ensure that the writes to rxd->gen bits will be observed 1974 * after all other writes to rxd objects. 1975 */ 1976 dma_wmb(); 1977 1978 while (num_to_alloc) { 1979 rbi = rq->buf_info[ring_idx] + ring->next2fill; 1980 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP))) 1981 goto refill_buf; 1982 if (ring_idx == 0) { 1983 /* ring0 Type1 buffers can get skipped; re-fill them */ 1984 if (rbi->buf_type != VMXNET3_RX_BUF_SKB) 1985 goto refill_buf; 1986 } 1987 if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) { 1988 refill_buf: 1989 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd, 1990 &rxCmdDesc); 1991 WARN_ON(!rxd->addr); 1992 1993 /* Recv desc is ready to be used by the device */ 1994 rxd->gen = ring->gen; 1995 vmxnet3_cmd_ring_adv_next2fill(ring); 1996 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 1997 num_to_alloc--; 1998 } else { 1999 /* rx completion hasn't occurred */ 2000 ring->isOutOfOrder = 1; 2001 break; 2002 } 2003 } 2004 2005 if (num_to_alloc == 0) { 2006 ring->isOutOfOrder = 0; 2007 } 2008 2009 /* if needed, update the register */ 2010 if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) { 2011 VMXNET3_WRITE_BAR0_REG(adapter, 2012 rxprod_reg[ring_idx] + rq->qid * 8, 2013 ring->next2fill); 2014 } 2015 2016 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring); 2017 vmxnet3_getRxComp(rcd, 2018 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp); 2019 } 2020 if (need_flush) 2021 xdp_do_flush(); 2022 2023 return num_pkts; 2024 } 2025 2026 2027 static void 2028 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, 2029 struct vmxnet3_adapter *adapter) 2030 { 2031 u32 i, ring_idx; 2032 struct Vmxnet3_RxDesc *rxd; 2033 2034 /* ring has already been cleaned up */ 2035 if (!rq->rx_ring[0].base) 2036 return; 2037 2038 for (ring_idx = 0; ring_idx < 2; ring_idx++) { 2039 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) { 2040 struct vmxnet3_rx_buf_info *rbi; 2041 #ifdef __BIG_ENDIAN_BITFIELD 2042 struct Vmxnet3_RxDesc rxDesc; 2043 #endif 2044 2045 rbi = &rq->buf_info[ring_idx][i]; 2046 vmxnet3_getRxDesc(rxd, 2047 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc); 2048 2049 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 2050 rbi->page && rbi->buf_type == VMXNET3_RX_BUF_XDP) { 2051 page_pool_recycle_direct(rq->page_pool, 2052 rbi->page); 2053 rbi->page = NULL; 2054 } else if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 2055 rbi->skb) { 2056 dma_unmap_single(&adapter->pdev->dev, rxd->addr, 2057 rxd->len, DMA_FROM_DEVICE); 2058 dev_kfree_skb(rbi->skb); 2059 rbi->skb = NULL; 2060 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY && 2061 rbi->page) { 2062 dma_unmap_page(&adapter->pdev->dev, rxd->addr, 2063 rxd->len, DMA_FROM_DEVICE); 2064 put_page(rbi->page); 2065 rbi->page = NULL; 2066 } 2067 } 2068 2069 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN; 2070 rq->rx_ring[ring_idx].next2fill = 2071 rq->rx_ring[ring_idx].next2comp = 0; 2072 } 2073 2074 rq->comp_ring.gen = VMXNET3_INIT_GEN; 2075 rq->comp_ring.next2proc = 0; 2076 2077 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) 2078 xdp_rxq_info_unreg(&rq->xdp_rxq); 2079 page_pool_destroy(rq->page_pool); 2080 rq->page_pool = NULL; 2081 } 2082 2083 2084 static void 2085 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter) 2086 { 2087 int i; 2088 2089 for (i = 0; i < adapter->num_rx_queues; i++) 2090 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter); 2091 rcu_assign_pointer(adapter->xdp_bpf_prog, NULL); 2092 } 2093 2094 2095 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 2096 struct vmxnet3_adapter *adapter) 2097 { 2098 int i; 2099 int j; 2100 2101 /* all rx buffers must have already been freed */ 2102 for (i = 0; i < 2; i++) { 2103 if (rq->buf_info[i]) { 2104 for (j = 0; j < rq->rx_ring[i].size; j++) 2105 BUG_ON(rq->buf_info[i][j].page != NULL); 2106 } 2107 } 2108 2109 2110 for (i = 0; i < 2; i++) { 2111 if (rq->rx_ring[i].base) { 2112 dma_free_coherent(&adapter->pdev->dev, 2113 rq->rx_ring[i].size 2114 * sizeof(struct Vmxnet3_RxDesc), 2115 rq->rx_ring[i].base, 2116 rq->rx_ring[i].basePA); 2117 rq->rx_ring[i].base = NULL; 2118 } 2119 } 2120 2121 if (rq->data_ring.base) { 2122 dma_free_coherent(&adapter->pdev->dev, 2123 rq->rx_ring[0].size * rq->data_ring.desc_size, 2124 rq->data_ring.base, rq->data_ring.basePA); 2125 rq->data_ring.base = NULL; 2126 } 2127 2128 if (rq->ts_ring.base) { 2129 dma_free_coherent(&adapter->pdev->dev, 2130 rq->rx_ring[0].size * rq->rx_ts_desc_size, 2131 rq->ts_ring.base, rq->ts_ring.basePA); 2132 rq->ts_ring.base = NULL; 2133 } 2134 2135 if (rq->comp_ring.base) { 2136 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size 2137 * sizeof(struct Vmxnet3_RxCompDesc), 2138 rq->comp_ring.base, rq->comp_ring.basePA); 2139 rq->comp_ring.base = NULL; 2140 } 2141 2142 kfree(rq->buf_info[0]); 2143 rq->buf_info[0] = NULL; 2144 rq->buf_info[1] = NULL; 2145 } 2146 2147 static void 2148 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter) 2149 { 2150 int i; 2151 2152 for (i = 0; i < adapter->num_rx_queues; i++) { 2153 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2154 2155 if (rq->data_ring.base) { 2156 dma_free_coherent(&adapter->pdev->dev, 2157 (rq->rx_ring[0].size * 2158 rq->data_ring.desc_size), 2159 rq->data_ring.base, 2160 rq->data_ring.basePA); 2161 rq->data_ring.base = NULL; 2162 } 2163 rq->data_ring.desc_size = 0; 2164 } 2165 } 2166 2167 static int 2168 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq, 2169 struct vmxnet3_adapter *adapter) 2170 { 2171 int i, err; 2172 2173 /* initialize buf_info */ 2174 for (i = 0; i < rq->rx_ring[0].size; i++) { 2175 2176 /* 1st buf for a pkt is skbuff or xdp page */ 2177 if (i % adapter->rx_buf_per_pkt == 0) { 2178 rq->buf_info[0][i].buf_type = vmxnet3_xdp_enabled(adapter) ? 2179 VMXNET3_RX_BUF_XDP : 2180 VMXNET3_RX_BUF_SKB; 2181 rq->buf_info[0][i].len = adapter->skb_buf_size; 2182 } else { /* subsequent bufs for a pkt is frag */ 2183 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE; 2184 rq->buf_info[0][i].len = PAGE_SIZE; 2185 } 2186 } 2187 for (i = 0; i < rq->rx_ring[1].size; i++) { 2188 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE; 2189 rq->buf_info[1][i].len = PAGE_SIZE; 2190 } 2191 2192 /* reset internal state and allocate buffers for both rings */ 2193 for (i = 0; i < 2; i++) { 2194 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0; 2195 2196 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size * 2197 sizeof(struct Vmxnet3_RxDesc)); 2198 rq->rx_ring[i].gen = VMXNET3_INIT_GEN; 2199 rq->rx_ring[i].isOutOfOrder = 0; 2200 } 2201 2202 err = vmxnet3_create_pp(adapter, rq, 2203 rq->rx_ring[0].size + rq->rx_ring[1].size); 2204 if (err) 2205 return err; 2206 2207 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1, 2208 adapter) == 0) { 2209 xdp_rxq_info_unreg(&rq->xdp_rxq); 2210 page_pool_destroy(rq->page_pool); 2211 rq->page_pool = NULL; 2212 2213 /* at least has 1 rx buffer for the 1st ring */ 2214 return -ENOMEM; 2215 } 2216 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter); 2217 2218 if (rq->ts_ring.base) 2219 memset(rq->ts_ring.base, 0, 2220 rq->rx_ring[0].size * rq->rx_ts_desc_size); 2221 2222 /* reset the comp ring */ 2223 rq->comp_ring.next2proc = 0; 2224 memset(rq->comp_ring.base, 0, rq->comp_ring.size * 2225 sizeof(struct Vmxnet3_RxCompDesc)); 2226 rq->comp_ring.gen = VMXNET3_INIT_GEN; 2227 2228 /* reset rxctx */ 2229 rq->rx_ctx.skb = NULL; 2230 2231 /* stats are not reset */ 2232 return 0; 2233 } 2234 2235 2236 static int 2237 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter) 2238 { 2239 int i, err = 0; 2240 2241 for (i = 0; i < adapter->num_rx_queues; i++) { 2242 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter); 2243 if (unlikely(err)) { 2244 dev_err(&adapter->netdev->dev, "%s: failed to " 2245 "initialize rx queue%i\n", 2246 adapter->netdev->name, i); 2247 break; 2248 } 2249 } 2250 return err; 2251 2252 } 2253 2254 2255 static int 2256 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 2257 { 2258 int i; 2259 size_t sz; 2260 struct vmxnet3_rx_buf_info *bi; 2261 2262 for (i = 0; i < 2; i++) { 2263 2264 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc); 2265 rq->rx_ring[i].base = dma_alloc_coherent( 2266 &adapter->pdev->dev, sz, 2267 &rq->rx_ring[i].basePA, 2268 GFP_KERNEL); 2269 if (!rq->rx_ring[i].base) { 2270 netdev_err(adapter->netdev, 2271 "failed to allocate rx ring %d\n", i); 2272 goto err; 2273 } 2274 } 2275 2276 if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) { 2277 sz = rq->rx_ring[0].size * rq->data_ring.desc_size; 2278 rq->data_ring.base = 2279 dma_alloc_coherent(&adapter->pdev->dev, sz, 2280 &rq->data_ring.basePA, 2281 GFP_KERNEL | __GFP_NOWARN); 2282 if (!rq->data_ring.base) { 2283 netdev_err(adapter->netdev, 2284 "failed to allocate %zu bytes, rx data ring will be disabled\n", sz); 2285 adapter->rxdataring_enabled = false; 2286 } 2287 } else { 2288 rq->data_ring.base = NULL; 2289 rq->data_ring.desc_size = 0; 2290 } 2291 2292 if (rq->rx_ts_desc_size != 0) { 2293 sz = rq->rx_ring[0].size * rq->rx_ts_desc_size; 2294 rq->ts_ring.base = 2295 dma_alloc_coherent(&adapter->pdev->dev, sz, 2296 &rq->ts_ring.basePA, 2297 GFP_KERNEL); 2298 if (!rq->ts_ring.base) { 2299 netdev_err(adapter->netdev, 2300 "rx ts ring will be disabled\n"); 2301 rq->rx_ts_desc_size = 0; 2302 } 2303 } else { 2304 rq->ts_ring.base = NULL; 2305 } 2306 2307 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc); 2308 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz, 2309 &rq->comp_ring.basePA, 2310 GFP_KERNEL); 2311 if (!rq->comp_ring.base) { 2312 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n"); 2313 goto err; 2314 } 2315 2316 bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size, 2317 sizeof(rq->buf_info[0][0]), GFP_KERNEL, 2318 dev_to_node(&adapter->pdev->dev)); 2319 if (!bi) 2320 goto err; 2321 2322 rq->buf_info[0] = bi; 2323 rq->buf_info[1] = bi + rq->rx_ring[0].size; 2324 2325 return 0; 2326 2327 err: 2328 vmxnet3_rq_destroy(rq, adapter); 2329 return -ENOMEM; 2330 } 2331 2332 2333 int 2334 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter) 2335 { 2336 int i, err = 0; 2337 2338 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 2339 2340 for (i = 0; i < adapter->num_rx_queues; i++) { 2341 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter); 2342 if (unlikely(err)) { 2343 dev_err(&adapter->netdev->dev, 2344 "%s: failed to create rx queue%i\n", 2345 adapter->netdev->name, i); 2346 goto err_out; 2347 } 2348 } 2349 2350 if (!adapter->rxdataring_enabled) 2351 vmxnet3_rq_destroy_all_rxdataring(adapter); 2352 2353 return err; 2354 err_out: 2355 vmxnet3_rq_destroy_all(adapter); 2356 return err; 2357 2358 } 2359 2360 /* Multiple queue aware polling function for tx and rx */ 2361 2362 static int 2363 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 2364 { 2365 int rcd_done = 0, i; 2366 if (unlikely(adapter->shared->ecr)) 2367 vmxnet3_process_events(adapter); 2368 for (i = 0; i < adapter->num_tx_queues; i++) 2369 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter); 2370 2371 for (i = 0; i < adapter->num_rx_queues; i++) 2372 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i], 2373 adapter, budget); 2374 return rcd_done; 2375 } 2376 2377 2378 static int 2379 vmxnet3_poll(struct napi_struct *napi, int budget) 2380 { 2381 struct vmxnet3_rx_queue *rx_queue = container_of(napi, 2382 struct vmxnet3_rx_queue, napi); 2383 int rxd_done; 2384 2385 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget); 2386 2387 if (rxd_done < budget) { 2388 napi_complete_done(napi, rxd_done); 2389 vmxnet3_enable_all_intrs(rx_queue->adapter); 2390 } 2391 return rxd_done; 2392 } 2393 2394 /* 2395 * NAPI polling function for MSI-X mode with multiple Rx queues 2396 * Returns the # of the NAPI credit consumed (# of rx descriptors processed) 2397 */ 2398 2399 static int 2400 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget) 2401 { 2402 struct vmxnet3_rx_queue *rq = container_of(napi, 2403 struct vmxnet3_rx_queue, napi); 2404 struct vmxnet3_adapter *adapter = rq->adapter; 2405 int rxd_done; 2406 2407 /* When sharing interrupt with corresponding tx queue, process 2408 * tx completions in that queue as well 2409 */ 2410 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { 2411 struct vmxnet3_tx_queue *tq = 2412 &adapter->tx_queue[rq - adapter->rx_queue]; 2413 vmxnet3_tq_tx_complete(tq, adapter); 2414 } 2415 2416 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget); 2417 2418 if (rxd_done < budget) { 2419 napi_complete_done(napi, rxd_done); 2420 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx); 2421 } 2422 return rxd_done; 2423 } 2424 2425 2426 #ifdef CONFIG_PCI_MSI 2427 2428 /* 2429 * Handle completion interrupts on tx queues 2430 * Returns whether or not the intr is handled 2431 */ 2432 2433 static irqreturn_t 2434 vmxnet3_msix_tx(int irq, void *data) 2435 { 2436 struct vmxnet3_tx_queue *tq = data; 2437 struct vmxnet3_adapter *adapter = tq->adapter; 2438 2439 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2440 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx); 2441 2442 /* Handle the case where only one irq is allocate for all tx queues */ 2443 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2444 int i; 2445 for (i = 0; i < adapter->num_tx_queues; i++) { 2446 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i]; 2447 vmxnet3_tq_tx_complete(txq, adapter); 2448 } 2449 } else { 2450 vmxnet3_tq_tx_complete(tq, adapter); 2451 } 2452 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx); 2453 2454 return IRQ_HANDLED; 2455 } 2456 2457 2458 /* 2459 * Handle completion interrupts on rx queues. Returns whether or not the 2460 * intr is handled 2461 */ 2462 2463 static irqreturn_t 2464 vmxnet3_msix_rx(int irq, void *data) 2465 { 2466 struct vmxnet3_rx_queue *rq = data; 2467 struct vmxnet3_adapter *adapter = rq->adapter; 2468 2469 /* disable intr if needed */ 2470 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2471 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx); 2472 napi_schedule(&rq->napi); 2473 2474 return IRQ_HANDLED; 2475 } 2476 2477 /* 2478 *---------------------------------------------------------------------------- 2479 * 2480 * vmxnet3_msix_event -- 2481 * 2482 * vmxnet3 msix event intr handler 2483 * 2484 * Result: 2485 * whether or not the intr is handled 2486 * 2487 *---------------------------------------------------------------------------- 2488 */ 2489 2490 static irqreturn_t 2491 vmxnet3_msix_event(int irq, void *data) 2492 { 2493 struct net_device *dev = data; 2494 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2495 2496 /* disable intr if needed */ 2497 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2498 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx); 2499 2500 if (adapter->shared->ecr) 2501 vmxnet3_process_events(adapter); 2502 2503 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx); 2504 2505 return IRQ_HANDLED; 2506 } 2507 2508 #endif /* CONFIG_PCI_MSI */ 2509 2510 2511 /* Interrupt handler for vmxnet3 */ 2512 static irqreturn_t 2513 vmxnet3_intr(int irq, void *dev_id) 2514 { 2515 struct net_device *dev = dev_id; 2516 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2517 2518 if (adapter->intr.type == VMXNET3_IT_INTX) { 2519 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 2520 if (unlikely(icr == 0)) 2521 /* not ours */ 2522 return IRQ_NONE; 2523 } 2524 2525 2526 /* disable intr if needed */ 2527 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2528 vmxnet3_disable_all_intrs(adapter); 2529 2530 napi_schedule(&adapter->rx_queue[0].napi); 2531 2532 return IRQ_HANDLED; 2533 } 2534 2535 #ifdef CONFIG_NET_POLL_CONTROLLER 2536 2537 /* netpoll callback. */ 2538 static void 2539 vmxnet3_netpoll(struct net_device *netdev) 2540 { 2541 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2542 2543 switch (adapter->intr.type) { 2544 #ifdef CONFIG_PCI_MSI 2545 case VMXNET3_IT_MSIX: { 2546 int i; 2547 for (i = 0; i < adapter->num_rx_queues; i++) 2548 vmxnet3_msix_rx(0, &adapter->rx_queue[i]); 2549 break; 2550 } 2551 #endif 2552 case VMXNET3_IT_MSI: 2553 default: 2554 vmxnet3_intr(0, adapter->netdev); 2555 break; 2556 } 2557 2558 } 2559 #endif /* CONFIG_NET_POLL_CONTROLLER */ 2560 2561 static int 2562 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 2563 { 2564 struct vmxnet3_intr *intr = &adapter->intr; 2565 int err = 0, i; 2566 int vector = 0; 2567 2568 #ifdef CONFIG_PCI_MSI 2569 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2570 for (i = 0; i < adapter->num_tx_queues; i++) { 2571 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2572 sprintf(adapter->tx_queue[i].name, "%s-tx-%d", 2573 adapter->netdev->name, vector); 2574 err = request_irq( 2575 intr->msix_entries[vector].vector, 2576 vmxnet3_msix_tx, 0, 2577 adapter->tx_queue[i].name, 2578 &adapter->tx_queue[i]); 2579 } else { 2580 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d", 2581 adapter->netdev->name, vector); 2582 } 2583 if (err) { 2584 dev_err(&adapter->netdev->dev, 2585 "Failed to request irq for MSIX, %s, " 2586 "error %d\n", 2587 adapter->tx_queue[i].name, err); 2588 return err; 2589 } 2590 2591 /* Handle the case where only 1 MSIx was allocated for 2592 * all tx queues */ 2593 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2594 for (; i < adapter->num_tx_queues; i++) 2595 adapter->tx_queue[i].comp_ring.intr_idx 2596 = vector; 2597 vector++; 2598 break; 2599 } else { 2600 adapter->tx_queue[i].comp_ring.intr_idx 2601 = vector++; 2602 } 2603 } 2604 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) 2605 vector = 0; 2606 2607 for (i = 0; i < adapter->num_rx_queues; i++) { 2608 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) 2609 sprintf(adapter->rx_queue[i].name, "%s-rx-%d", 2610 adapter->netdev->name, vector); 2611 else 2612 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d", 2613 adapter->netdev->name, vector); 2614 err = request_irq(intr->msix_entries[vector].vector, 2615 vmxnet3_msix_rx, 0, 2616 adapter->rx_queue[i].name, 2617 &(adapter->rx_queue[i])); 2618 if (err) { 2619 netdev_err(adapter->netdev, 2620 "Failed to request irq for MSIX, " 2621 "%s, error %d\n", 2622 adapter->rx_queue[i].name, err); 2623 return err; 2624 } 2625 2626 adapter->rx_queue[i].comp_ring.intr_idx = vector++; 2627 } 2628 2629 sprintf(intr->event_msi_vector_name, "%s-event-%d", 2630 adapter->netdev->name, vector); 2631 err = request_irq(intr->msix_entries[vector].vector, 2632 vmxnet3_msix_event, 0, 2633 intr->event_msi_vector_name, adapter->netdev); 2634 intr->event_intr_idx = vector; 2635 2636 } else if (intr->type == VMXNET3_IT_MSI) { 2637 adapter->num_rx_queues = 1; 2638 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 2639 adapter->netdev->name, adapter->netdev); 2640 } else { 2641 #endif 2642 adapter->num_rx_queues = 1; 2643 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 2644 IRQF_SHARED, adapter->netdev->name, 2645 adapter->netdev); 2646 #ifdef CONFIG_PCI_MSI 2647 } 2648 #endif 2649 intr->num_intrs = vector + 1; 2650 if (err) { 2651 netdev_err(adapter->netdev, 2652 "Failed to request irq (intr type:%d), error %d\n", 2653 intr->type, err); 2654 } else { 2655 /* Number of rx queues will not change after this */ 2656 for (i = 0; i < adapter->num_rx_queues; i++) { 2657 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2658 rq->qid = i; 2659 rq->qid2 = i + adapter->num_rx_queues; 2660 rq->dataRingQid = i + 2 * adapter->num_rx_queues; 2661 } 2662 2663 /* init our intr settings */ 2664 for (i = 0; i < intr->num_intrs; i++) 2665 intr->mod_levels[i] = UPT1_IML_ADAPTIVE; 2666 if (adapter->intr.type != VMXNET3_IT_MSIX) { 2667 adapter->intr.event_intr_idx = 0; 2668 for (i = 0; i < adapter->num_tx_queues; i++) 2669 adapter->tx_queue[i].comp_ring.intr_idx = 0; 2670 adapter->rx_queue[0].comp_ring.intr_idx = 0; 2671 } 2672 2673 netdev_info(adapter->netdev, 2674 "intr type %u, mode %u, %u vectors allocated\n", 2675 intr->type, intr->mask_mode, intr->num_intrs); 2676 } 2677 2678 return err; 2679 } 2680 2681 2682 static void 2683 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 2684 { 2685 struct vmxnet3_intr *intr = &adapter->intr; 2686 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0); 2687 2688 switch (intr->type) { 2689 #ifdef CONFIG_PCI_MSI 2690 case VMXNET3_IT_MSIX: 2691 { 2692 int i, vector = 0; 2693 2694 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2695 for (i = 0; i < adapter->num_tx_queues; i++) { 2696 free_irq(intr->msix_entries[vector++].vector, 2697 &(adapter->tx_queue[i])); 2698 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) 2699 break; 2700 } 2701 } 2702 2703 for (i = 0; i < adapter->num_rx_queues; i++) { 2704 free_irq(intr->msix_entries[vector++].vector, 2705 &(adapter->rx_queue[i])); 2706 } 2707 2708 free_irq(intr->msix_entries[vector].vector, 2709 adapter->netdev); 2710 BUG_ON(vector >= intr->num_intrs); 2711 break; 2712 } 2713 #endif 2714 case VMXNET3_IT_MSI: 2715 free_irq(adapter->pdev->irq, adapter->netdev); 2716 break; 2717 case VMXNET3_IT_INTX: 2718 free_irq(adapter->pdev->irq, adapter->netdev); 2719 break; 2720 default: 2721 BUG(); 2722 } 2723 } 2724 2725 2726 static void 2727 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter) 2728 { 2729 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2730 u16 vid; 2731 2732 /* allow untagged pkts */ 2733 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0); 2734 2735 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2736 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2737 } 2738 2739 2740 static int 2741 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 2742 { 2743 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2744 2745 if (!(netdev->flags & IFF_PROMISC)) { 2746 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2747 unsigned long flags; 2748 2749 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2750 spin_lock_irqsave(&adapter->cmd_lock, flags); 2751 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2752 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2753 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2754 } 2755 2756 set_bit(vid, adapter->active_vlans); 2757 2758 return 0; 2759 } 2760 2761 2762 static int 2763 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 2764 { 2765 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2766 2767 if (!(netdev->flags & IFF_PROMISC)) { 2768 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2769 unsigned long flags; 2770 2771 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid); 2772 spin_lock_irqsave(&adapter->cmd_lock, flags); 2773 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2774 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2775 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2776 } 2777 2778 clear_bit(vid, adapter->active_vlans); 2779 2780 return 0; 2781 } 2782 2783 2784 static u8 * 2785 vmxnet3_copy_mc(struct net_device *netdev) 2786 { 2787 u8 *buf = NULL; 2788 u32 sz = netdev_mc_count(netdev) * ETH_ALEN; 2789 2790 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */ 2791 if (sz <= 0xffff) { 2792 /* We may be called with BH disabled */ 2793 buf = kmalloc(sz, GFP_ATOMIC); 2794 if (buf) { 2795 struct netdev_hw_addr *ha; 2796 int i = 0; 2797 2798 netdev_for_each_mc_addr(ha, netdev) 2799 memcpy(buf + i++ * ETH_ALEN, ha->addr, 2800 ETH_ALEN); 2801 } 2802 } 2803 return buf; 2804 } 2805 2806 2807 static void 2808 vmxnet3_set_mc(struct net_device *netdev) 2809 { 2810 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2811 unsigned long flags; 2812 struct Vmxnet3_RxFilterConf *rxConf = 2813 &adapter->shared->devRead.rxFilterConf; 2814 u8 *new_table = NULL; 2815 dma_addr_t new_table_pa = 0; 2816 bool new_table_pa_valid = false; 2817 u32 new_mode = VMXNET3_RXM_UCAST; 2818 2819 if (netdev->flags & IFF_PROMISC) { 2820 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2821 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable)); 2822 2823 new_mode |= VMXNET3_RXM_PROMISC; 2824 } else { 2825 vmxnet3_restore_vlan(adapter); 2826 } 2827 2828 if (netdev->flags & IFF_BROADCAST) 2829 new_mode |= VMXNET3_RXM_BCAST; 2830 2831 if (netdev->flags & IFF_ALLMULTI) 2832 new_mode |= VMXNET3_RXM_ALL_MULTI; 2833 else 2834 if (!netdev_mc_empty(netdev)) { 2835 new_table = vmxnet3_copy_mc(netdev); 2836 if (new_table) { 2837 size_t sz = netdev_mc_count(netdev) * ETH_ALEN; 2838 2839 rxConf->mfTableLen = cpu_to_le16(sz); 2840 new_table_pa = dma_map_single( 2841 &adapter->pdev->dev, 2842 new_table, 2843 sz, 2844 DMA_TO_DEVICE); 2845 if (!dma_mapping_error(&adapter->pdev->dev, 2846 new_table_pa)) { 2847 new_mode |= VMXNET3_RXM_MCAST; 2848 new_table_pa_valid = true; 2849 rxConf->mfTablePA = cpu_to_le64( 2850 new_table_pa); 2851 } 2852 } 2853 if (!new_table_pa_valid) { 2854 netdev_info(netdev, 2855 "failed to copy mcast list, setting ALL_MULTI\n"); 2856 new_mode |= VMXNET3_RXM_ALL_MULTI; 2857 } 2858 } 2859 2860 if (!(new_mode & VMXNET3_RXM_MCAST)) { 2861 rxConf->mfTableLen = 0; 2862 rxConf->mfTablePA = 0; 2863 } 2864 2865 spin_lock_irqsave(&adapter->cmd_lock, flags); 2866 if (new_mode != rxConf->rxMode) { 2867 rxConf->rxMode = cpu_to_le32(new_mode); 2868 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2869 VMXNET3_CMD_UPDATE_RX_MODE); 2870 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2871 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2872 } 2873 2874 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2875 VMXNET3_CMD_UPDATE_MAC_FILTERS); 2876 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2877 2878 if (new_table_pa_valid) 2879 dma_unmap_single(&adapter->pdev->dev, new_table_pa, 2880 rxConf->mfTableLen, DMA_TO_DEVICE); 2881 kfree(new_table); 2882 } 2883 2884 void 2885 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter) 2886 { 2887 int i; 2888 2889 for (i = 0; i < adapter->num_rx_queues; i++) 2890 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter); 2891 } 2892 2893 2894 /* 2895 * Set up driver_shared based on settings in adapter. 2896 */ 2897 2898 static void 2899 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter) 2900 { 2901 struct Vmxnet3_DriverShared *shared = adapter->shared; 2902 struct Vmxnet3_DSDevRead *devRead = &shared->devRead; 2903 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 2904 struct Vmxnet3_TxQueueConf *tqc; 2905 struct Vmxnet3_RxQueueConf *rqc; 2906 struct Vmxnet3_TxQueueTSConf *tqtsc; 2907 struct Vmxnet3_RxQueueTSConf *rqtsc; 2908 int i; 2909 2910 memset(shared, 0, sizeof(*shared)); 2911 2912 /* driver settings */ 2913 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC); 2914 devRead->misc.driverInfo.version = cpu_to_le32( 2915 VMXNET3_DRIVER_VERSION_NUM); 2916 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ? 2917 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64); 2918 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 2919 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32( 2920 *((u32 *)&devRead->misc.driverInfo.gos)); 2921 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1); 2922 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1); 2923 2924 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa); 2925 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter)); 2926 2927 /* set up feature flags */ 2928 if (adapter->netdev->features & NETIF_F_RXCSUM) 2929 devRead->misc.uptFeatures |= UPT1_F_RXCSUM; 2930 2931 if (adapter->netdev->features & NETIF_F_LRO) { 2932 devRead->misc.uptFeatures |= UPT1_F_LRO; 2933 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS); 2934 } 2935 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2936 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 2937 2938 if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL | 2939 NETIF_F_GSO_UDP_TUNNEL_CSUM)) 2940 devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD; 2941 2942 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2943 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2944 devRead->misc.queueDescLen = cpu_to_le32( 2945 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 2946 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc)); 2947 2948 /* tx queue settings */ 2949 devRead->misc.numTxQueues = adapter->num_tx_queues; 2950 for (i = 0; i < adapter->num_tx_queues; i++) { 2951 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 2952 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL); 2953 tqc = &adapter->tqd_start[i].conf; 2954 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA); 2955 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA); 2956 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA); 2957 tqc->ddPA = cpu_to_le64(~0ULL); 2958 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size); 2959 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size); 2960 tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size); 2961 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size); 2962 tqc->ddLen = cpu_to_le32(0); 2963 tqc->intrIdx = tq->comp_ring.intr_idx; 2964 if (VMXNET3_VERSION_GE_9(adapter)) { 2965 tqtsc = &adapter->tqd_start[i].tsConf; 2966 tqtsc->txTSRingBasePA = cpu_to_le64(tq->ts_ring.basePA); 2967 tqtsc->txTSRingDescSize = cpu_to_le16(tq->tx_ts_desc_size); 2968 } 2969 } 2970 2971 /* rx queue settings */ 2972 devRead->misc.numRxQueues = adapter->num_rx_queues; 2973 for (i = 0; i < adapter->num_rx_queues; i++) { 2974 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2975 rqc = &adapter->rqd_start[i].conf; 2976 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA); 2977 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA); 2978 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA); 2979 rqc->ddPA = cpu_to_le64(~0ULL); 2980 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size); 2981 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size); 2982 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size); 2983 rqc->ddLen = cpu_to_le32(0); 2984 rqc->intrIdx = rq->comp_ring.intr_idx; 2985 if (VMXNET3_VERSION_GE_3(adapter)) { 2986 rqc->rxDataRingBasePA = 2987 cpu_to_le64(rq->data_ring.basePA); 2988 rqc->rxDataRingDescSize = 2989 cpu_to_le16(rq->data_ring.desc_size); 2990 } 2991 if (VMXNET3_VERSION_GE_9(adapter)) { 2992 rqtsc = &adapter->rqd_start[i].tsConf; 2993 rqtsc->rxTSRingBasePA = cpu_to_le64(rq->ts_ring.basePA); 2994 rqtsc->rxTSRingDescSize = cpu_to_le16(rq->rx_ts_desc_size); 2995 } 2996 } 2997 2998 #ifdef VMXNET3_RSS 2999 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf)); 3000 3001 if (adapter->rss) { 3002 struct UPT1_RSSConf *rssConf = adapter->rss_conf; 3003 3004 devRead->misc.uptFeatures |= UPT1_F_RSS; 3005 devRead->misc.numRxQueues = adapter->num_rx_queues; 3006 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 | 3007 UPT1_RSS_HASH_TYPE_IPV4 | 3008 UPT1_RSS_HASH_TYPE_TCP_IPV6 | 3009 UPT1_RSS_HASH_TYPE_IPV6; 3010 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ; 3011 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE; 3012 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE; 3013 netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey)); 3014 3015 for (i = 0; i < rssConf->indTableSize; i++) 3016 rssConf->indTable[i] = ethtool_rxfh_indir_default( 3017 i, adapter->num_rx_queues); 3018 3019 devRead->rssConfDesc.confVer = 1; 3020 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf)); 3021 devRead->rssConfDesc.confPA = 3022 cpu_to_le64(adapter->rss_conf_pa); 3023 } 3024 3025 #endif /* VMXNET3_RSS */ 3026 3027 /* intr settings */ 3028 if (!VMXNET3_VERSION_GE_6(adapter) || 3029 !adapter->queuesExtEnabled) { 3030 devRead->intrConf.autoMask = adapter->intr.mask_mode == 3031 VMXNET3_IMM_AUTO; 3032 devRead->intrConf.numIntrs = adapter->intr.num_intrs; 3033 for (i = 0; i < adapter->intr.num_intrs; i++) 3034 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i]; 3035 3036 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx; 3037 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 3038 } else { 3039 devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode == 3040 VMXNET3_IMM_AUTO; 3041 devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs; 3042 for (i = 0; i < adapter->intr.num_intrs; i++) 3043 devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i]; 3044 3045 devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx; 3046 devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 3047 } 3048 3049 /* rx filter settings */ 3050 devRead->rxFilterConf.rxMode = 0; 3051 vmxnet3_restore_vlan(adapter); 3052 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr); 3053 3054 /* the rest are already zeroed */ 3055 } 3056 3057 static void 3058 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter) 3059 { 3060 struct Vmxnet3_DriverShared *shared = adapter->shared; 3061 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 3062 unsigned long flags; 3063 3064 if (!VMXNET3_VERSION_GE_7(adapter)) 3065 return; 3066 3067 cmdInfo->ringBufSize = adapter->ringBufSize; 3068 spin_lock_irqsave(&adapter->cmd_lock, flags); 3069 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3070 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 3071 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3072 } 3073 3074 static void 3075 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter) 3076 { 3077 struct Vmxnet3_DriverShared *shared = adapter->shared; 3078 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 3079 unsigned long flags; 3080 3081 if (!VMXNET3_VERSION_GE_3(adapter)) 3082 return; 3083 3084 spin_lock_irqsave(&adapter->cmd_lock, flags); 3085 cmdInfo->varConf.confVer = 1; 3086 cmdInfo->varConf.confLen = 3087 cpu_to_le32(sizeof(*adapter->coal_conf)); 3088 cmdInfo->varConf.confPA = cpu_to_le64(adapter->coal_conf_pa); 3089 3090 if (adapter->default_coal_mode) { 3091 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3092 VMXNET3_CMD_GET_COALESCE); 3093 } else { 3094 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3095 VMXNET3_CMD_SET_COALESCE); 3096 } 3097 3098 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3099 } 3100 3101 static void 3102 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter) 3103 { 3104 struct Vmxnet3_DriverShared *shared = adapter->shared; 3105 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 3106 unsigned long flags; 3107 3108 if (!VMXNET3_VERSION_GE_4(adapter)) 3109 return; 3110 3111 spin_lock_irqsave(&adapter->cmd_lock, flags); 3112 3113 if (adapter->default_rss_fields) { 3114 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3115 VMXNET3_CMD_GET_RSS_FIELDS); 3116 adapter->rss_fields = 3117 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3118 } else { 3119 if (VMXNET3_VERSION_GE_7(adapter)) { 3120 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 || 3121 adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) && 3122 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3123 VMXNET3_CAP_UDP_RSS)) { 3124 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS; 3125 } else { 3126 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS); 3127 } 3128 3129 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) && 3130 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3131 VMXNET3_CAP_ESP_RSS_IPV4)) { 3132 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4; 3133 } else { 3134 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4); 3135 } 3136 3137 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) && 3138 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3139 VMXNET3_CAP_ESP_RSS_IPV6)) { 3140 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6; 3141 } else { 3142 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6); 3143 } 3144 3145 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3146 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3147 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3148 } 3149 cmdInfo->setRssFields = adapter->rss_fields; 3150 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3151 VMXNET3_CMD_SET_RSS_FIELDS); 3152 /* Not all requested RSS may get applied, so get and 3153 * cache what was actually applied. 3154 */ 3155 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3156 VMXNET3_CMD_GET_RSS_FIELDS); 3157 adapter->rss_fields = 3158 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3159 } 3160 3161 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3162 } 3163 3164 int 3165 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 3166 { 3167 int err, i; 3168 u32 ret; 3169 unsigned long flags; 3170 3171 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d," 3172 " ring sizes %u %u %u\n", adapter->netdev->name, 3173 adapter->skb_buf_size, adapter->rx_buf_per_pkt, 3174 adapter->tx_queue[0].tx_ring.size, 3175 adapter->rx_queue[0].rx_ring[0].size, 3176 adapter->rx_queue[0].rx_ring[1].size); 3177 3178 vmxnet3_tq_init_all(adapter); 3179 err = vmxnet3_rq_init_all(adapter); 3180 if (err) { 3181 netdev_err(adapter->netdev, 3182 "Failed to init rx queue error %d\n", err); 3183 goto rq_err; 3184 } 3185 3186 err = vmxnet3_request_irqs(adapter); 3187 if (err) { 3188 netdev_err(adapter->netdev, 3189 "Failed to setup irq for error %d\n", err); 3190 goto irq_err; 3191 } 3192 3193 vmxnet3_setup_driver_shared(adapter); 3194 3195 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO( 3196 adapter->shared_pa)); 3197 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI( 3198 adapter->shared_pa)); 3199 spin_lock_irqsave(&adapter->cmd_lock, flags); 3200 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3201 VMXNET3_CMD_ACTIVATE_DEV); 3202 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3203 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3204 3205 if (ret != 0) { 3206 netdev_err(adapter->netdev, 3207 "Failed to activate dev: error %u\n", ret); 3208 err = -EINVAL; 3209 goto activate_err; 3210 } 3211 3212 vmxnet3_init_bufsize(adapter); 3213 vmxnet3_init_coalesce(adapter); 3214 vmxnet3_init_rssfields(adapter); 3215 3216 for (i = 0; i < adapter->num_rx_queues; i++) { 3217 VMXNET3_WRITE_BAR0_REG(adapter, 3218 adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN, 3219 adapter->rx_queue[i].rx_ring[0].next2fill); 3220 VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset + 3221 (i * VMXNET3_REG_ALIGN)), 3222 adapter->rx_queue[i].rx_ring[1].next2fill); 3223 } 3224 3225 /* Apply the rx filter settins last. */ 3226 vmxnet3_set_mc(adapter->netdev); 3227 3228 /* 3229 * Check link state when first activating device. It will start the 3230 * tx queue if the link is up. 3231 */ 3232 vmxnet3_check_link(adapter, true); 3233 netif_tx_wake_all_queues(adapter->netdev); 3234 for (i = 0; i < adapter->num_rx_queues; i++) 3235 napi_enable(&adapter->rx_queue[i].napi); 3236 vmxnet3_enable_all_intrs(adapter); 3237 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3238 return 0; 3239 3240 activate_err: 3241 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0); 3242 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0); 3243 vmxnet3_free_irqs(adapter); 3244 irq_err: 3245 rq_err: 3246 /* free up buffers we allocated */ 3247 vmxnet3_rq_cleanup_all(adapter); 3248 return err; 3249 } 3250 3251 3252 void 3253 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter) 3254 { 3255 unsigned long flags; 3256 spin_lock_irqsave(&adapter->cmd_lock, flags); 3257 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 3258 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3259 } 3260 3261 3262 int 3263 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 3264 { 3265 int i; 3266 unsigned long flags; 3267 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 3268 return 0; 3269 3270 3271 spin_lock_irqsave(&adapter->cmd_lock, flags); 3272 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3273 VMXNET3_CMD_QUIESCE_DEV); 3274 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3275 vmxnet3_disable_all_intrs(adapter); 3276 3277 for (i = 0; i < adapter->num_rx_queues; i++) 3278 napi_disable(&adapter->rx_queue[i].napi); 3279 netif_tx_disable(adapter->netdev); 3280 adapter->link_speed = 0; 3281 netif_carrier_off(adapter->netdev); 3282 3283 vmxnet3_tq_cleanup_all(adapter); 3284 vmxnet3_rq_cleanup_all(adapter); 3285 vmxnet3_free_irqs(adapter); 3286 return 0; 3287 } 3288 3289 3290 static void 3291 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac) 3292 { 3293 u32 tmp; 3294 3295 tmp = *(u32 *)mac; 3296 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp); 3297 3298 tmp = (mac[5] << 8) | mac[4]; 3299 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp); 3300 } 3301 3302 3303 static int 3304 vmxnet3_set_mac_addr(struct net_device *netdev, void *p) 3305 { 3306 struct sockaddr *addr = p; 3307 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3308 3309 dev_addr_set(netdev, addr->sa_data); 3310 vmxnet3_write_mac_addr(adapter, addr->sa_data); 3311 3312 return 0; 3313 } 3314 3315 3316 /* ==================== initialization and cleanup routines ============ */ 3317 3318 static int 3319 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter) 3320 { 3321 int err; 3322 unsigned long mmio_start, mmio_len; 3323 struct pci_dev *pdev = adapter->pdev; 3324 3325 err = pci_enable_device(pdev); 3326 if (err) { 3327 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err); 3328 return err; 3329 } 3330 3331 err = pci_request_selected_regions(pdev, (1 << 2) - 1, 3332 vmxnet3_driver_name); 3333 if (err) { 3334 dev_err(&pdev->dev, 3335 "Failed to request region for adapter: error %d\n", err); 3336 goto err_enable_device; 3337 } 3338 3339 pci_set_master(pdev); 3340 3341 mmio_start = pci_resource_start(pdev, 0); 3342 mmio_len = pci_resource_len(pdev, 0); 3343 adapter->hw_addr0 = ioremap(mmio_start, mmio_len); 3344 if (!adapter->hw_addr0) { 3345 dev_err(&pdev->dev, "Failed to map bar0\n"); 3346 err = -EIO; 3347 goto err_ioremap; 3348 } 3349 3350 mmio_start = pci_resource_start(pdev, 1); 3351 mmio_len = pci_resource_len(pdev, 1); 3352 adapter->hw_addr1 = ioremap(mmio_start, mmio_len); 3353 if (!adapter->hw_addr1) { 3354 dev_err(&pdev->dev, "Failed to map bar1\n"); 3355 err = -EIO; 3356 goto err_bar1; 3357 } 3358 return 0; 3359 3360 err_bar1: 3361 iounmap(adapter->hw_addr0); 3362 err_ioremap: 3363 pci_release_selected_regions(pdev, (1 << 2) - 1); 3364 err_enable_device: 3365 pci_disable_device(pdev); 3366 return err; 3367 } 3368 3369 3370 static void 3371 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter) 3372 { 3373 BUG_ON(!adapter->pdev); 3374 3375 iounmap(adapter->hw_addr0); 3376 iounmap(adapter->hw_addr1); 3377 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1); 3378 pci_disable_device(adapter->pdev); 3379 } 3380 3381 3382 void 3383 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 3384 { 3385 size_t sz, i, ring0_size, ring1_size, comp_size; 3386 /* With version7 ring1 will have only T0 buffers */ 3387 if (!VMXNET3_VERSION_GE_7(adapter)) { 3388 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 3389 VMXNET3_MAX_ETH_HDR_SIZE) { 3390 adapter->skb_buf_size = adapter->netdev->mtu + 3391 VMXNET3_MAX_ETH_HDR_SIZE; 3392 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE) 3393 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE; 3394 3395 adapter->rx_buf_per_pkt = 1; 3396 } else { 3397 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE; 3398 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE + 3399 VMXNET3_MAX_ETH_HDR_SIZE; 3400 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE; 3401 } 3402 } else { 3403 adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE, 3404 VMXNET3_MAX_SKB_BUF_SIZE); 3405 adapter->rx_buf_per_pkt = 1; 3406 adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size); 3407 adapter->ringBufSize.ring1BufSizeType1 = 0; 3408 adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE); 3409 } 3410 3411 /* 3412 * for simplicity, force the ring0 size to be a multiple of 3413 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 3414 */ 3415 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 3416 ring0_size = adapter->rx_queue[0].rx_ring[0].size; 3417 ring0_size = (ring0_size + sz - 1) / sz * sz; 3418 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE / 3419 sz * sz); 3420 ring1_size = adapter->rx_queue[0].rx_ring[1].size; 3421 ring1_size = (ring1_size + sz - 1) / sz * sz; 3422 ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE / 3423 sz * sz); 3424 /* For v7 and later, keep ring size power of 2 for UPT */ 3425 if (VMXNET3_VERSION_GE_7(adapter)) { 3426 ring0_size = rounddown_pow_of_two(ring0_size); 3427 ring1_size = rounddown_pow_of_two(ring1_size); 3428 } 3429 comp_size = ring0_size + ring1_size; 3430 3431 for (i = 0; i < adapter->num_rx_queues; i++) { 3432 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3433 3434 rq->rx_ring[0].size = ring0_size; 3435 rq->rx_ring[1].size = ring1_size; 3436 rq->comp_ring.size = comp_size; 3437 } 3438 } 3439 3440 3441 int 3442 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 3443 u32 rx_ring_size, u32 rx_ring2_size, 3444 u16 txdata_desc_size, u16 rxdata_desc_size) 3445 { 3446 int err = 0, i; 3447 3448 for (i = 0; i < adapter->num_tx_queues; i++) { 3449 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 3450 tq->tx_ring.size = tx_ring_size; 3451 tq->data_ring.size = tx_ring_size; 3452 tq->comp_ring.size = tx_ring_size; 3453 tq->txdata_desc_size = txdata_desc_size; 3454 tq->shared = &adapter->tqd_start[i].ctrl; 3455 tq->stopped = true; 3456 tq->adapter = adapter; 3457 tq->qid = i; 3458 tq->tx_ts_desc_size = adapter->tx_ts_desc_size; 3459 tq->tsPktCount = 1; 3460 err = vmxnet3_tq_create(tq, adapter); 3461 /* 3462 * Too late to change num_tx_queues. We cannot do away with 3463 * lesser number of queues than what we asked for 3464 */ 3465 if (err) 3466 goto queue_err; 3467 } 3468 3469 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size; 3470 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size; 3471 vmxnet3_adjust_rx_ring_size(adapter); 3472 3473 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 3474 for (i = 0; i < adapter->num_rx_queues; i++) { 3475 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3476 /* qid and qid2 for rx queues will be assigned later when num 3477 * of rx queues is finalized after allocating intrs */ 3478 rq->shared = &adapter->rqd_start[i].ctrl; 3479 rq->adapter = adapter; 3480 rq->data_ring.desc_size = rxdata_desc_size; 3481 rq->rx_ts_desc_size = adapter->rx_ts_desc_size; 3482 err = vmxnet3_rq_create(rq, adapter); 3483 if (err) { 3484 if (i == 0) { 3485 netdev_err(adapter->netdev, 3486 "Could not allocate any rx queues. " 3487 "Aborting.\n"); 3488 goto queue_err; 3489 } else { 3490 netdev_info(adapter->netdev, 3491 "Number of rx queues changed " 3492 "to : %d.\n", i); 3493 adapter->num_rx_queues = i; 3494 err = 0; 3495 break; 3496 } 3497 } 3498 } 3499 3500 if (!adapter->rxdataring_enabled) 3501 vmxnet3_rq_destroy_all_rxdataring(adapter); 3502 3503 return err; 3504 queue_err: 3505 vmxnet3_tq_destroy_all(adapter); 3506 return err; 3507 } 3508 3509 static int 3510 vmxnet3_open(struct net_device *netdev) 3511 { 3512 struct vmxnet3_adapter *adapter; 3513 int err, i; 3514 3515 adapter = netdev_priv(netdev); 3516 3517 for (i = 0; i < adapter->num_tx_queues; i++) 3518 spin_lock_init(&adapter->tx_queue[i].tx_lock); 3519 3520 if (VMXNET3_VERSION_GE_3(adapter)) { 3521 unsigned long flags; 3522 u16 txdata_desc_size; 3523 u32 ret; 3524 3525 spin_lock_irqsave(&adapter->cmd_lock, flags); 3526 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3527 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 3528 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3529 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3530 3531 txdata_desc_size = ret & 0xffff; 3532 if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) || 3533 (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) || 3534 (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) { 3535 adapter->txdata_desc_size = 3536 sizeof(struct Vmxnet3_TxDataDesc); 3537 } else { 3538 adapter->txdata_desc_size = txdata_desc_size; 3539 } 3540 if (VMXNET3_VERSION_GE_9(adapter)) 3541 adapter->rxdata_desc_size = (ret >> 16) & 0xffff; 3542 } else { 3543 adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc); 3544 } 3545 3546 if (VMXNET3_VERSION_GE_9(adapter)) { 3547 unsigned long flags; 3548 u16 tx_ts_desc_size = 0; 3549 u16 rx_ts_desc_size = 0; 3550 u32 ret; 3551 3552 spin_lock_irqsave(&adapter->cmd_lock, flags); 3553 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3554 VMXNET3_CMD_GET_TSRING_DESC_SIZE); 3555 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3556 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3557 if (ret > 0) { 3558 tx_ts_desc_size = (ret & 0xff); 3559 rx_ts_desc_size = ((ret >> 16) & 0xff); 3560 } 3561 if (tx_ts_desc_size > VMXNET3_TXTS_DESC_MAX_SIZE || 3562 tx_ts_desc_size & VMXNET3_TXTS_DESC_SIZE_MASK) 3563 tx_ts_desc_size = 0; 3564 if (rx_ts_desc_size > VMXNET3_RXTS_DESC_MAX_SIZE || 3565 rx_ts_desc_size & VMXNET3_RXTS_DESC_SIZE_MASK) 3566 rx_ts_desc_size = 0; 3567 adapter->tx_ts_desc_size = tx_ts_desc_size; 3568 adapter->rx_ts_desc_size = rx_ts_desc_size; 3569 } else { 3570 adapter->tx_ts_desc_size = 0; 3571 adapter->rx_ts_desc_size = 0; 3572 } 3573 3574 err = vmxnet3_create_queues(adapter, 3575 adapter->tx_ring_size, 3576 adapter->rx_ring_size, 3577 adapter->rx_ring2_size, 3578 adapter->txdata_desc_size, 3579 adapter->rxdata_desc_size); 3580 if (err) 3581 goto queue_err; 3582 3583 err = vmxnet3_activate_dev(adapter); 3584 if (err) 3585 goto activate_err; 3586 3587 return 0; 3588 3589 activate_err: 3590 vmxnet3_rq_destroy_all(adapter); 3591 vmxnet3_tq_destroy_all(adapter); 3592 queue_err: 3593 return err; 3594 } 3595 3596 3597 static int 3598 vmxnet3_close(struct net_device *netdev) 3599 { 3600 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3601 3602 /* 3603 * Reset_work may be in the middle of resetting the device, wait for its 3604 * completion. 3605 */ 3606 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3607 usleep_range(1000, 2000); 3608 3609 vmxnet3_quiesce_dev(adapter); 3610 3611 vmxnet3_rq_destroy_all(adapter); 3612 vmxnet3_tq_destroy_all(adapter); 3613 3614 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3615 3616 3617 return 0; 3618 } 3619 3620 3621 void 3622 vmxnet3_force_close(struct vmxnet3_adapter *adapter) 3623 { 3624 int i; 3625 3626 /* 3627 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 3628 * vmxnet3_close() will deadlock. 3629 */ 3630 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 3631 3632 /* we need to enable NAPI, otherwise dev_close will deadlock */ 3633 for (i = 0; i < adapter->num_rx_queues; i++) 3634 napi_enable(&adapter->rx_queue[i].napi); 3635 /* 3636 * Need to clear the quiesce bit to ensure that vmxnet3_close 3637 * can quiesce the device properly 3638 */ 3639 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3640 dev_close(adapter->netdev); 3641 } 3642 3643 3644 static int 3645 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) 3646 { 3647 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3648 int err = 0; 3649 3650 /* 3651 * Reset_work may be in the middle of resetting the device, wait for its 3652 * completion. 3653 */ 3654 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3655 usleep_range(1000, 2000); 3656 3657 if (netif_running(netdev)) { 3658 vmxnet3_quiesce_dev(adapter); 3659 vmxnet3_reset_dev(adapter); 3660 3661 /* we need to re-create the rx queue based on the new mtu */ 3662 vmxnet3_rq_destroy_all(adapter); 3663 WRITE_ONCE(netdev->mtu, new_mtu); 3664 vmxnet3_adjust_rx_ring_size(adapter); 3665 err = vmxnet3_rq_create_all(adapter); 3666 if (err) { 3667 netdev_err(netdev, 3668 "failed to re-create rx queues, " 3669 " error %d. Closing it.\n", err); 3670 goto out; 3671 } 3672 3673 err = vmxnet3_activate_dev(adapter); 3674 if (err) { 3675 netdev_err(netdev, 3676 "failed to re-activate, error %d. " 3677 "Closing it\n", err); 3678 goto out; 3679 } 3680 } else { 3681 WRITE_ONCE(netdev->mtu, new_mtu); 3682 } 3683 3684 out: 3685 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3686 if (err) 3687 vmxnet3_force_close(adapter); 3688 3689 return err; 3690 } 3691 3692 3693 static void 3694 vmxnet3_declare_features(struct vmxnet3_adapter *adapter) 3695 { 3696 struct net_device *netdev = adapter->netdev; 3697 unsigned long flags; 3698 3699 if (VMXNET3_VERSION_GE_9(adapter)) { 3700 spin_lock_irqsave(&adapter->cmd_lock, flags); 3701 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3702 VMXNET3_CMD_GET_DISABLED_OFFLOADS); 3703 adapter->disabledOffloads = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3704 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3705 } 3706 3707 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 3708 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3709 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3710 NETIF_F_LRO | NETIF_F_HIGHDMA; 3711 3712 if (VMXNET3_VERSION_GE_4(adapter)) { 3713 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3714 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3715 3716 netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM | 3717 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3718 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3719 NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL | 3720 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3721 } 3722 3723 if (adapter->disabledOffloads & VMXNET3_OFFLOAD_TSO) { 3724 netdev->hw_features &= ~(NETIF_F_TSO | NETIF_F_TSO6); 3725 netdev->hw_enc_features &= ~(NETIF_F_TSO | NETIF_F_TSO6); 3726 } 3727 3728 if (adapter->disabledOffloads & VMXNET3_OFFLOAD_LRO) { 3729 netdev->hw_features &= ~(NETIF_F_LRO); 3730 netdev->hw_enc_features &= ~(NETIF_F_LRO); 3731 } 3732 3733 if (VMXNET3_VERSION_GE_7(adapter)) { 3734 unsigned long flags; 3735 3736 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3737 VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) { 3738 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD; 3739 } 3740 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3741 VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) { 3742 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD; 3743 } 3744 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3745 VMXNET3_CAP_GENEVE_TSO)) { 3746 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO; 3747 } 3748 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3749 VMXNET3_CAP_VXLAN_TSO)) { 3750 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO; 3751 } 3752 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3753 VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) { 3754 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD; 3755 } 3756 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3757 VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) { 3758 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD; 3759 } 3760 3761 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3762 spin_lock_irqsave(&adapter->cmd_lock, flags); 3763 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3764 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3765 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3766 3767 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) && 3768 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) && 3769 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) && 3770 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) { 3771 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3772 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3773 } 3774 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) && 3775 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) { 3776 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3777 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3778 } 3779 } 3780 3781 netdev->vlan_features = netdev->hw_features & 3782 ~(NETIF_F_HW_VLAN_CTAG_TX | 3783 NETIF_F_HW_VLAN_CTAG_RX); 3784 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 3785 } 3786 3787 3788 static void 3789 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) 3790 { 3791 u32 tmp; 3792 3793 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL); 3794 *(u32 *)mac = tmp; 3795 3796 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH); 3797 mac[4] = tmp & 0xff; 3798 mac[5] = (tmp >> 8) & 0xff; 3799 } 3800 3801 #ifdef CONFIG_PCI_MSI 3802 3803 /* 3804 * Enable MSIx vectors. 3805 * Returns : 3806 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required 3807 * were enabled. 3808 * number of vectors which were enabled otherwise (this number is greater 3809 * than VMXNET3_LINUX_MIN_MSIX_VECT) 3810 */ 3811 3812 static int 3813 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec) 3814 { 3815 int ret = pci_enable_msix_range(adapter->pdev, 3816 adapter->intr.msix_entries, nvec, nvec); 3817 3818 if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) { 3819 dev_err(&adapter->netdev->dev, 3820 "Failed to enable %d MSI-X, trying %d\n", 3821 nvec, VMXNET3_LINUX_MIN_MSIX_VECT); 3822 3823 ret = pci_enable_msix_range(adapter->pdev, 3824 adapter->intr.msix_entries, 3825 VMXNET3_LINUX_MIN_MSIX_VECT, 3826 VMXNET3_LINUX_MIN_MSIX_VECT); 3827 } 3828 3829 if (ret < 0) { 3830 dev_err(&adapter->netdev->dev, 3831 "Failed to enable MSI-X, error: %d\n", ret); 3832 } 3833 3834 return ret; 3835 } 3836 3837 3838 #endif /* CONFIG_PCI_MSI */ 3839 3840 static void 3841 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 3842 { 3843 u32 cfg; 3844 unsigned long flags; 3845 3846 /* intr settings */ 3847 spin_lock_irqsave(&adapter->cmd_lock, flags); 3848 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3849 VMXNET3_CMD_GET_CONF_INTR); 3850 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3851 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3852 adapter->intr.type = cfg & 0x3; 3853 adapter->intr.mask_mode = (cfg >> 2) & 0x3; 3854 3855 if (adapter->intr.type == VMXNET3_IT_AUTO) { 3856 adapter->intr.type = VMXNET3_IT_MSIX; 3857 } 3858 3859 #ifdef CONFIG_PCI_MSI 3860 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3861 int i, nvec, nvec_allocated; 3862 3863 nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ? 3864 1 : adapter->num_tx_queues; 3865 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ? 3866 0 : adapter->num_rx_queues; 3867 nvec += 1; /* for link event */ 3868 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ? 3869 nvec : VMXNET3_LINUX_MIN_MSIX_VECT; 3870 3871 for (i = 0; i < nvec; i++) 3872 adapter->intr.msix_entries[i].entry = i; 3873 3874 nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec); 3875 if (nvec_allocated < 0) 3876 goto msix_err; 3877 3878 /* If we cannot allocate one MSIx vector per queue 3879 * then limit the number of rx queues to 1 3880 */ 3881 if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT && 3882 nvec != VMXNET3_LINUX_MIN_MSIX_VECT) { 3883 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE 3884 || adapter->num_rx_queues != 1) { 3885 adapter->share_intr = VMXNET3_INTR_TXSHARE; 3886 netdev_err(adapter->netdev, 3887 "Number of rx queues : 1\n"); 3888 adapter->num_rx_queues = 1; 3889 } 3890 } 3891 3892 adapter->intr.num_intrs = nvec_allocated; 3893 return; 3894 3895 msix_err: 3896 /* If we cannot allocate MSIx vectors use only one rx queue */ 3897 dev_info(&adapter->pdev->dev, 3898 "Failed to enable MSI-X, error %d. " 3899 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated); 3900 3901 adapter->intr.type = VMXNET3_IT_MSI; 3902 } 3903 3904 if (adapter->intr.type == VMXNET3_IT_MSI) { 3905 if (!pci_enable_msi(adapter->pdev)) { 3906 adapter->num_rx_queues = 1; 3907 adapter->intr.num_intrs = 1; 3908 return; 3909 } 3910 } 3911 #endif /* CONFIG_PCI_MSI */ 3912 3913 adapter->num_rx_queues = 1; 3914 dev_info(&adapter->netdev->dev, 3915 "Using INTx interrupt, #Rx queues: 1.\n"); 3916 adapter->intr.type = VMXNET3_IT_INTX; 3917 3918 /* INT-X related setting */ 3919 adapter->intr.num_intrs = 1; 3920 } 3921 3922 3923 static void 3924 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter) 3925 { 3926 if (adapter->intr.type == VMXNET3_IT_MSIX) 3927 pci_disable_msix(adapter->pdev); 3928 else if (adapter->intr.type == VMXNET3_IT_MSI) 3929 pci_disable_msi(adapter->pdev); 3930 else 3931 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX); 3932 } 3933 3934 3935 static void 3936 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue) 3937 { 3938 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3939 adapter->tx_timeout_count++; 3940 3941 netdev_err(adapter->netdev, "tx hang\n"); 3942 schedule_work(&adapter->work); 3943 } 3944 3945 3946 static void 3947 vmxnet3_reset_work(struct work_struct *data) 3948 { 3949 struct vmxnet3_adapter *adapter; 3950 3951 adapter = container_of(data, struct vmxnet3_adapter, work); 3952 3953 /* if another thread is resetting the device, no need to proceed */ 3954 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3955 return; 3956 3957 /* if the device is closed, we must leave it alone */ 3958 rtnl_lock(); 3959 if (netif_running(adapter->netdev)) { 3960 netdev_notice(adapter->netdev, "resetting\n"); 3961 vmxnet3_quiesce_dev(adapter); 3962 vmxnet3_reset_dev(adapter); 3963 vmxnet3_activate_dev(adapter); 3964 } else { 3965 netdev_info(adapter->netdev, "already closed\n"); 3966 } 3967 rtnl_unlock(); 3968 3969 netif_wake_queue(adapter->netdev); 3970 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3971 } 3972 3973 3974 static int 3975 vmxnet3_probe_device(struct pci_dev *pdev, 3976 const struct pci_device_id *id) 3977 { 3978 static const struct net_device_ops vmxnet3_netdev_ops = { 3979 .ndo_open = vmxnet3_open, 3980 .ndo_stop = vmxnet3_close, 3981 .ndo_start_xmit = vmxnet3_xmit_frame, 3982 .ndo_set_mac_address = vmxnet3_set_mac_addr, 3983 .ndo_change_mtu = vmxnet3_change_mtu, 3984 .ndo_fix_features = vmxnet3_fix_features, 3985 .ndo_set_features = vmxnet3_set_features, 3986 .ndo_features_check = vmxnet3_features_check, 3987 .ndo_get_stats64 = vmxnet3_get_stats64, 3988 .ndo_tx_timeout = vmxnet3_tx_timeout, 3989 .ndo_set_rx_mode = vmxnet3_set_mc, 3990 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid, 3991 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid, 3992 #ifdef CONFIG_NET_POLL_CONTROLLER 3993 .ndo_poll_controller = vmxnet3_netpoll, 3994 #endif 3995 .ndo_bpf = vmxnet3_xdp, 3996 .ndo_xdp_xmit = vmxnet3_xdp_xmit, 3997 }; 3998 int err; 3999 u32 ver; 4000 struct net_device *netdev; 4001 struct vmxnet3_adapter *adapter; 4002 u8 mac[ETH_ALEN]; 4003 int size, i; 4004 int num_tx_queues; 4005 int num_rx_queues; 4006 int queues; 4007 unsigned long flags; 4008 4009 if (!pci_msi_enabled()) 4010 enable_mq = 0; 4011 4012 #ifdef VMXNET3_RSS 4013 if (enable_mq) 4014 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 4015 (int)num_online_cpus()); 4016 else 4017 #endif 4018 num_rx_queues = 1; 4019 4020 if (enable_mq) 4021 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES, 4022 (int)num_online_cpus()); 4023 else 4024 num_tx_queues = 1; 4025 4026 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter), 4027 max(num_tx_queues, num_rx_queues)); 4028 if (!netdev) 4029 return -ENOMEM; 4030 4031 pci_set_drvdata(pdev, netdev); 4032 adapter = netdev_priv(netdev); 4033 adapter->netdev = netdev; 4034 adapter->pdev = pdev; 4035 4036 adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE; 4037 adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE; 4038 adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE; 4039 4040 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4041 if (err) { 4042 dev_err(&pdev->dev, "dma_set_mask failed\n"); 4043 goto err_set_mask; 4044 } 4045 4046 spin_lock_init(&adapter->cmd_lock); 4047 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter, 4048 sizeof(struct vmxnet3_adapter), 4049 DMA_TO_DEVICE); 4050 if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) { 4051 dev_err(&pdev->dev, "Failed to map dma\n"); 4052 err = -EFAULT; 4053 goto err_set_mask; 4054 } 4055 adapter->shared = dma_alloc_coherent( 4056 &adapter->pdev->dev, 4057 sizeof(struct Vmxnet3_DriverShared), 4058 &adapter->shared_pa, GFP_KERNEL); 4059 if (!adapter->shared) { 4060 dev_err(&pdev->dev, "Failed to allocate memory\n"); 4061 err = -ENOMEM; 4062 goto err_alloc_shared; 4063 } 4064 4065 err = vmxnet3_alloc_pci_resources(adapter); 4066 if (err < 0) 4067 goto err_alloc_pci; 4068 4069 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS); 4070 for (i = VMXNET3_REV_9; i >= VMXNET3_REV_1; i--) { 4071 if (ver & (1 << i)) { 4072 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_VRRS, 1 << i); 4073 adapter->version = i + 1; 4074 break; 4075 } 4076 } 4077 if (i < VMXNET3_REV_1) { 4078 dev_err(&pdev->dev, 4079 "Incompatible h/w version (0x%x) for adapter\n", ver); 4080 err = -EBUSY; 4081 goto err_ver; 4082 } 4083 dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version); 4084 4085 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS); 4086 if (ver & 1) { 4087 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1); 4088 } else { 4089 dev_err(&pdev->dev, 4090 "Incompatible upt version (0x%x) for adapter\n", ver); 4091 err = -EBUSY; 4092 goto err_ver; 4093 } 4094 4095 if (VMXNET3_VERSION_GE_7(adapter)) { 4096 adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR); 4097 adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR); 4098 if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 4099 adapter->dev_caps[0] = adapter->devcap_supported[0] & 4100 (1UL << VMXNET3_CAP_LARGE_BAR); 4101 } 4102 if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) && 4103 adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 4104 adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 4105 adapter->dev_caps[0] |= adapter->devcap_supported[0] & 4106 (1UL << VMXNET3_CAP_OOORX_COMP); 4107 } 4108 if (adapter->dev_caps[0]) 4109 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 4110 4111 spin_lock_irqsave(&adapter->cmd_lock, flags); 4112 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 4113 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4114 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4115 } 4116 4117 if (VMXNET3_VERSION_GE_7(adapter) && 4118 adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 4119 adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 4120 adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD; 4121 adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2; 4122 } else { 4123 adapter->tx_prod_offset = VMXNET3_REG_TXPROD; 4124 adapter->rx_prod_offset = VMXNET3_REG_RXPROD; 4125 adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2; 4126 } 4127 4128 if (VMXNET3_VERSION_GE_6(adapter)) { 4129 spin_lock_irqsave(&adapter->cmd_lock, flags); 4130 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4131 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 4132 queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4133 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4134 if (queues > 0) { 4135 adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff)); 4136 adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff)); 4137 } else { 4138 adapter->num_rx_queues = min(num_rx_queues, 4139 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4140 adapter->num_tx_queues = min(num_tx_queues, 4141 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 4142 } 4143 if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES || 4144 adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) { 4145 adapter->queuesExtEnabled = true; 4146 } else { 4147 adapter->queuesExtEnabled = false; 4148 } 4149 } else { 4150 adapter->queuesExtEnabled = false; 4151 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 4152 num_tx_queues = rounddown_pow_of_two(num_tx_queues); 4153 adapter->num_rx_queues = min(num_rx_queues, 4154 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4155 adapter->num_tx_queues = min(num_tx_queues, 4156 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 4157 } 4158 dev_info(&pdev->dev, 4159 "# of Tx queues : %d, # of Rx queues : %d\n", 4160 adapter->num_tx_queues, adapter->num_rx_queues); 4161 4162 adapter->rx_buf_per_pkt = 1; 4163 4164 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 4165 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues; 4166 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size, 4167 &adapter->queue_desc_pa, 4168 GFP_KERNEL); 4169 4170 if (!adapter->tqd_start) { 4171 dev_err(&pdev->dev, "Failed to allocate memory\n"); 4172 err = -ENOMEM; 4173 goto err_ver; 4174 } 4175 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start + 4176 adapter->num_tx_queues); 4177 if (VMXNET3_VERSION_GE_9(adapter)) 4178 adapter->latencyConf = &adapter->tqd_start->tsConf.latencyConf; 4179 4180 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev, 4181 sizeof(struct Vmxnet3_PMConf), 4182 &adapter->pm_conf_pa, 4183 GFP_KERNEL); 4184 if (adapter->pm_conf == NULL) { 4185 err = -ENOMEM; 4186 goto err_alloc_pm; 4187 } 4188 4189 #ifdef VMXNET3_RSS 4190 4191 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev, 4192 sizeof(struct UPT1_RSSConf), 4193 &adapter->rss_conf_pa, 4194 GFP_KERNEL); 4195 if (adapter->rss_conf == NULL) { 4196 err = -ENOMEM; 4197 goto err_alloc_rss; 4198 } 4199 #endif /* VMXNET3_RSS */ 4200 4201 if (VMXNET3_VERSION_GE_3(adapter)) { 4202 adapter->coal_conf = 4203 dma_alloc_coherent(&adapter->pdev->dev, 4204 sizeof(struct Vmxnet3_CoalesceScheme) 4205 , 4206 &adapter->coal_conf_pa, 4207 GFP_KERNEL); 4208 if (!adapter->coal_conf) { 4209 err = -ENOMEM; 4210 goto err_coal_conf; 4211 } 4212 adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED; 4213 adapter->default_coal_mode = true; 4214 } 4215 4216 if (VMXNET3_VERSION_GE_4(adapter)) { 4217 adapter->default_rss_fields = true; 4218 adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT; 4219 } 4220 4221 SET_NETDEV_DEV(netdev, &pdev->dev); 4222 vmxnet3_declare_features(adapter); 4223 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 4224 NETDEV_XDP_ACT_NDO_XMIT; 4225 4226 adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ? 4227 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 4228 4229 if (adapter->num_tx_queues == adapter->num_rx_queues) 4230 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE; 4231 else 4232 adapter->share_intr = VMXNET3_INTR_DONTSHARE; 4233 4234 vmxnet3_alloc_intr_resources(adapter); 4235 4236 #ifdef VMXNET3_RSS 4237 if (adapter->num_rx_queues > 1 && 4238 adapter->intr.type == VMXNET3_IT_MSIX) { 4239 adapter->rss = true; 4240 netdev->hw_features |= NETIF_F_RXHASH; 4241 netdev->features |= NETIF_F_RXHASH; 4242 dev_dbg(&pdev->dev, "RSS is enabled.\n"); 4243 } else { 4244 adapter->rss = false; 4245 } 4246 #endif 4247 4248 vmxnet3_read_mac_addr(adapter, mac); 4249 dev_addr_set(netdev, mac); 4250 4251 netdev->netdev_ops = &vmxnet3_netdev_ops; 4252 vmxnet3_set_ethtool_ops(netdev); 4253 netdev->watchdog_timeo = 5 * HZ; 4254 4255 /* MTU range: 60 - 9190 */ 4256 netdev->min_mtu = VMXNET3_MIN_MTU; 4257 if (VMXNET3_VERSION_GE_6(adapter)) 4258 netdev->max_mtu = VMXNET3_V6_MAX_MTU; 4259 else 4260 netdev->max_mtu = VMXNET3_MAX_MTU; 4261 4262 INIT_WORK(&adapter->work, vmxnet3_reset_work); 4263 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 4264 4265 if (adapter->intr.type == VMXNET3_IT_MSIX) { 4266 int i; 4267 for (i = 0; i < adapter->num_rx_queues; i++) { 4268 netif_napi_add(adapter->netdev, 4269 &adapter->rx_queue[i].napi, 4270 vmxnet3_poll_rx_only); 4271 } 4272 } else { 4273 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi, 4274 vmxnet3_poll); 4275 } 4276 4277 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); 4278 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues); 4279 4280 netif_carrier_off(netdev); 4281 err = register_netdev(netdev); 4282 4283 if (err) { 4284 dev_err(&pdev->dev, "Failed to register adapter\n"); 4285 goto err_register; 4286 } 4287 4288 vmxnet3_check_link(adapter, false); 4289 return 0; 4290 4291 err_register: 4292 if (VMXNET3_VERSION_GE_3(adapter)) { 4293 dma_free_coherent(&adapter->pdev->dev, 4294 sizeof(struct Vmxnet3_CoalesceScheme), 4295 adapter->coal_conf, adapter->coal_conf_pa); 4296 } 4297 vmxnet3_free_intr_resources(adapter); 4298 err_coal_conf: 4299 #ifdef VMXNET3_RSS 4300 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 4301 adapter->rss_conf, adapter->rss_conf_pa); 4302 err_alloc_rss: 4303 #endif 4304 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 4305 adapter->pm_conf, adapter->pm_conf_pa); 4306 err_alloc_pm: 4307 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 4308 adapter->queue_desc_pa); 4309 err_ver: 4310 vmxnet3_free_pci_resources(adapter); 4311 err_alloc_pci: 4312 dma_free_coherent(&adapter->pdev->dev, 4313 sizeof(struct Vmxnet3_DriverShared), 4314 adapter->shared, adapter->shared_pa); 4315 err_alloc_shared: 4316 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4317 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4318 err_set_mask: 4319 free_netdev(netdev); 4320 return err; 4321 } 4322 4323 4324 static void 4325 vmxnet3_remove_device(struct pci_dev *pdev) 4326 { 4327 struct net_device *netdev = pci_get_drvdata(pdev); 4328 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4329 int size = 0; 4330 int num_rx_queues, rx_queues; 4331 unsigned long flags; 4332 4333 #ifdef VMXNET3_RSS 4334 if (enable_mq) 4335 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 4336 (int)num_online_cpus()); 4337 else 4338 #endif 4339 num_rx_queues = 1; 4340 if (!VMXNET3_VERSION_GE_6(adapter)) { 4341 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 4342 } 4343 if (VMXNET3_VERSION_GE_6(adapter)) { 4344 spin_lock_irqsave(&adapter->cmd_lock, flags); 4345 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4346 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 4347 rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4348 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4349 if (rx_queues > 0) 4350 rx_queues = (rx_queues >> 8) & 0xff; 4351 else 4352 rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4353 num_rx_queues = min(num_rx_queues, rx_queues); 4354 } else { 4355 num_rx_queues = min(num_rx_queues, 4356 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4357 } 4358 4359 cancel_work_sync(&adapter->work); 4360 4361 unregister_netdev(netdev); 4362 4363 vmxnet3_free_intr_resources(adapter); 4364 vmxnet3_free_pci_resources(adapter); 4365 if (VMXNET3_VERSION_GE_3(adapter)) { 4366 dma_free_coherent(&adapter->pdev->dev, 4367 sizeof(struct Vmxnet3_CoalesceScheme), 4368 adapter->coal_conf, adapter->coal_conf_pa); 4369 } 4370 #ifdef VMXNET3_RSS 4371 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 4372 adapter->rss_conf, adapter->rss_conf_pa); 4373 #endif 4374 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 4375 adapter->pm_conf, adapter->pm_conf_pa); 4376 4377 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 4378 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues; 4379 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 4380 adapter->queue_desc_pa); 4381 dma_free_coherent(&adapter->pdev->dev, 4382 sizeof(struct Vmxnet3_DriverShared), 4383 adapter->shared, adapter->shared_pa); 4384 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4385 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4386 free_netdev(netdev); 4387 } 4388 4389 static void vmxnet3_shutdown_device(struct pci_dev *pdev) 4390 { 4391 struct net_device *netdev = pci_get_drvdata(pdev); 4392 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4393 unsigned long flags; 4394 4395 /* Reset_work may be in the middle of resetting the device, wait for its 4396 * completion. 4397 */ 4398 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 4399 usleep_range(1000, 2000); 4400 4401 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, 4402 &adapter->state)) { 4403 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4404 return; 4405 } 4406 spin_lock_irqsave(&adapter->cmd_lock, flags); 4407 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4408 VMXNET3_CMD_QUIESCE_DEV); 4409 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4410 vmxnet3_disable_all_intrs(adapter); 4411 4412 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4413 } 4414 4415 4416 #ifdef CONFIG_PM 4417 4418 static int 4419 vmxnet3_suspend(struct device *device) 4420 { 4421 struct pci_dev *pdev = to_pci_dev(device); 4422 struct net_device *netdev = pci_get_drvdata(pdev); 4423 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4424 struct Vmxnet3_PMConf *pmConf; 4425 struct ethhdr *ehdr; 4426 struct arphdr *ahdr; 4427 u8 *arpreq; 4428 struct in_device *in_dev; 4429 struct in_ifaddr *ifa; 4430 unsigned long flags; 4431 int i = 0; 4432 4433 if (!netif_running(netdev)) 4434 return 0; 4435 4436 for (i = 0; i < adapter->num_rx_queues; i++) 4437 napi_disable(&adapter->rx_queue[i].napi); 4438 4439 vmxnet3_disable_all_intrs(adapter); 4440 vmxnet3_free_irqs(adapter); 4441 vmxnet3_free_intr_resources(adapter); 4442 4443 netif_device_detach(netdev); 4444 4445 /* Create wake-up filters. */ 4446 pmConf = adapter->pm_conf; 4447 memset(pmConf, 0, sizeof(*pmConf)); 4448 4449 if (adapter->wol & WAKE_UCAST) { 4450 pmConf->filters[i].patternSize = ETH_ALEN; 4451 pmConf->filters[i].maskSize = 1; 4452 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN); 4453 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */ 4454 4455 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4456 i++; 4457 } 4458 4459 if (adapter->wol & WAKE_ARP) { 4460 rcu_read_lock(); 4461 4462 in_dev = __in_dev_get_rcu(netdev); 4463 if (!in_dev) { 4464 rcu_read_unlock(); 4465 goto skip_arp; 4466 } 4467 4468 ifa = rcu_dereference(in_dev->ifa_list); 4469 if (!ifa) { 4470 rcu_read_unlock(); 4471 goto skip_arp; 4472 } 4473 4474 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/ 4475 sizeof(struct arphdr) + /* ARP header */ 4476 2 * ETH_ALEN + /* 2 Ethernet addresses*/ 4477 2 * sizeof(u32); /*2 IPv4 addresses */ 4478 pmConf->filters[i].maskSize = 4479 (pmConf->filters[i].patternSize - 1) / 8 + 1; 4480 4481 /* ETH_P_ARP in Ethernet header. */ 4482 ehdr = (struct ethhdr *)pmConf->filters[i].pattern; 4483 ehdr->h_proto = htons(ETH_P_ARP); 4484 4485 /* ARPOP_REQUEST in ARP header. */ 4486 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN]; 4487 ahdr->ar_op = htons(ARPOP_REQUEST); 4488 arpreq = (u8 *)(ahdr + 1); 4489 4490 /* The Unicast IPv4 address in 'tip' field. */ 4491 arpreq += 2 * ETH_ALEN + sizeof(u32); 4492 *(__be32 *)arpreq = ifa->ifa_address; 4493 4494 rcu_read_unlock(); 4495 4496 /* The mask for the relevant bits. */ 4497 pmConf->filters[i].mask[0] = 0x00; 4498 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */ 4499 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */ 4500 pmConf->filters[i].mask[3] = 0x00; 4501 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */ 4502 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */ 4503 4504 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4505 i++; 4506 } 4507 4508 skip_arp: 4509 if (adapter->wol & WAKE_MAGIC) 4510 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC; 4511 4512 pmConf->numFilters = i; 4513 4514 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); 4515 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( 4516 *pmConf)); 4517 adapter->shared->devRead.pmConfDesc.confPA = 4518 cpu_to_le64(adapter->pm_conf_pa); 4519 4520 spin_lock_irqsave(&adapter->cmd_lock, flags); 4521 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4522 VMXNET3_CMD_UPDATE_PMCFG); 4523 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4524 4525 pci_save_state(pdev); 4526 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND), 4527 adapter->wol); 4528 pci_disable_device(pdev); 4529 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND)); 4530 4531 return 0; 4532 } 4533 4534 4535 static int 4536 vmxnet3_resume(struct device *device) 4537 { 4538 int err; 4539 unsigned long flags; 4540 struct pci_dev *pdev = to_pci_dev(device); 4541 struct net_device *netdev = pci_get_drvdata(pdev); 4542 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4543 4544 if (!netif_running(netdev)) 4545 return 0; 4546 4547 pci_set_power_state(pdev, PCI_D0); 4548 pci_restore_state(pdev); 4549 err = pci_enable_device_mem(pdev); 4550 if (err != 0) 4551 return err; 4552 4553 pci_enable_wake(pdev, PCI_D0, 0); 4554 4555 vmxnet3_alloc_intr_resources(adapter); 4556 4557 /* During hibernate and suspend, device has to be reinitialized as the 4558 * device state need not be preserved. 4559 */ 4560 4561 /* Need not check adapter state as other reset tasks cannot run during 4562 * device resume. 4563 */ 4564 spin_lock_irqsave(&adapter->cmd_lock, flags); 4565 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4566 VMXNET3_CMD_QUIESCE_DEV); 4567 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4568 vmxnet3_tq_cleanup_all(adapter); 4569 vmxnet3_rq_cleanup_all(adapter); 4570 4571 vmxnet3_reset_dev(adapter); 4572 err = vmxnet3_activate_dev(adapter); 4573 if (err != 0) { 4574 netdev_err(netdev, 4575 "failed to re-activate on resume, error: %d", err); 4576 vmxnet3_force_close(adapter); 4577 return err; 4578 } 4579 netif_device_attach(netdev); 4580 4581 return 0; 4582 } 4583 4584 static const struct dev_pm_ops vmxnet3_pm_ops = { 4585 .suspend = vmxnet3_suspend, 4586 .resume = vmxnet3_resume, 4587 .freeze = vmxnet3_suspend, 4588 .restore = vmxnet3_resume, 4589 }; 4590 #endif 4591 4592 static struct pci_driver vmxnet3_driver = { 4593 .name = vmxnet3_driver_name, 4594 .id_table = vmxnet3_pciid_table, 4595 .probe = vmxnet3_probe_device, 4596 .remove = vmxnet3_remove_device, 4597 .shutdown = vmxnet3_shutdown_device, 4598 #ifdef CONFIG_PM 4599 .driver.pm = &vmxnet3_pm_ops, 4600 #endif 4601 }; 4602 4603 4604 static int __init 4605 vmxnet3_init_module(void) 4606 { 4607 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC, 4608 VMXNET3_DRIVER_VERSION_REPORT); 4609 return pci_register_driver(&vmxnet3_driver); 4610 } 4611 4612 module_init(vmxnet3_init_module); 4613 4614 4615 static void 4616 vmxnet3_exit_module(void) 4617 { 4618 pci_unregister_driver(&vmxnet3_driver); 4619 } 4620 4621 module_exit(vmxnet3_exit_module); 4622 4623 MODULE_AUTHOR("VMware, Inc."); 4624 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC); 4625 MODULE_LICENSE("GPL v2"); 4626 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING); 4627