1 /* 2 * Linux driver for VMware's vmxnet3 ethernet NIC. 3 * 4 * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License as published by the 8 * Free Software Foundation; version 2 of the License and no later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or 13 * NON INFRINGEMENT. See the GNU General Public License for more 14 * details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * The full GNU General Public License is included in this distribution in 21 * the file called "COPYING". 22 * 23 * Maintained by: pv-drivers@vmware.com 24 * 25 */ 26 27 #include <linux/module.h> 28 #include <net/ip6_checksum.h> 29 30 #include "vmxnet3_int.h" 31 #include "vmxnet3_xdp.h" 32 33 char vmxnet3_driver_name[] = "vmxnet3"; 34 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver" 35 36 /* 37 * PCI Device ID Table 38 * Last entry must be all 0s 39 */ 40 static const struct pci_device_id vmxnet3_pciid_table[] = { 41 {PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)}, 42 {0} 43 }; 44 45 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table); 46 47 static int enable_mq = 1; 48 49 static void 50 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac); 51 52 /* 53 * Enable/Disable the given intr 54 */ 55 static void 56 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 57 { 58 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0); 59 } 60 61 62 static void 63 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx) 64 { 65 VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1); 66 } 67 68 69 /* 70 * Enable/Disable all intrs used by the device 71 */ 72 static void 73 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter) 74 { 75 int i; 76 77 for (i = 0; i < adapter->intr.num_intrs; i++) 78 vmxnet3_enable_intr(adapter, i); 79 if (!VMXNET3_VERSION_GE_6(adapter) || 80 !adapter->queuesExtEnabled) { 81 adapter->shared->devRead.intrConf.intrCtrl &= 82 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 83 } else { 84 adapter->shared->devReadExt.intrConfExt.intrCtrl &= 85 cpu_to_le32(~VMXNET3_IC_DISABLE_ALL); 86 } 87 } 88 89 90 static void 91 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter) 92 { 93 int i; 94 95 if (!VMXNET3_VERSION_GE_6(adapter) || 96 !adapter->queuesExtEnabled) { 97 adapter->shared->devRead.intrConf.intrCtrl |= 98 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 99 } else { 100 adapter->shared->devReadExt.intrConfExt.intrCtrl |= 101 cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 102 } 103 for (i = 0; i < adapter->intr.num_intrs; i++) 104 vmxnet3_disable_intr(adapter, i); 105 } 106 107 108 static void 109 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events) 110 { 111 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events); 112 } 113 114 115 static bool 116 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 117 { 118 return tq->stopped; 119 } 120 121 122 static void 123 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 124 { 125 tq->stopped = false; 126 netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue); 127 } 128 129 130 static void 131 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 132 { 133 tq->stopped = false; 134 netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 135 } 136 137 138 static void 139 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter) 140 { 141 tq->stopped = true; 142 tq->num_stop++; 143 netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue)); 144 } 145 146 /* Check if capability is supported by UPT device or 147 * UPT is even requested 148 */ 149 bool 150 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap) 151 { 152 if (cap_supported & (1UL << VMXNET3_DCR_ERROR) || 153 cap_supported & (1UL << cap)) { 154 return true; 155 } 156 157 return false; 158 } 159 160 161 /* 162 * Check the link state. This may start or stop the tx queue. 163 */ 164 static void 165 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue) 166 { 167 u32 ret; 168 int i; 169 unsigned long flags; 170 171 spin_lock_irqsave(&adapter->cmd_lock, flags); 172 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK); 173 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 174 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 175 176 adapter->link_speed = ret >> 16; 177 if (ret & 1) { /* Link is up. */ 178 netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n", 179 adapter->link_speed); 180 netif_carrier_on(adapter->netdev); 181 182 if (affectTxQueue) { 183 for (i = 0; i < adapter->num_tx_queues; i++) 184 vmxnet3_tq_start(&adapter->tx_queue[i], 185 adapter); 186 } 187 } else { 188 netdev_info(adapter->netdev, "NIC Link is Down\n"); 189 netif_carrier_off(adapter->netdev); 190 191 if (affectTxQueue) { 192 for (i = 0; i < adapter->num_tx_queues; i++) 193 vmxnet3_tq_stop(&adapter->tx_queue[i], adapter); 194 } 195 } 196 } 197 198 static void 199 vmxnet3_process_events(struct vmxnet3_adapter *adapter) 200 { 201 int i; 202 unsigned long flags; 203 u32 events = le32_to_cpu(adapter->shared->ecr); 204 if (!events) 205 return; 206 207 vmxnet3_ack_events(adapter, events); 208 209 /* Check if link state has changed */ 210 if (events & VMXNET3_ECR_LINK) 211 vmxnet3_check_link(adapter, true); 212 213 /* Check if there is an error on xmit/recv queues */ 214 if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) { 215 spin_lock_irqsave(&adapter->cmd_lock, flags); 216 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 217 VMXNET3_CMD_GET_QUEUE_STATUS); 218 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 219 220 for (i = 0; i < adapter->num_tx_queues; i++) 221 if (adapter->tqd_start[i].status.stopped) 222 dev_err(&adapter->netdev->dev, 223 "%s: tq[%d] error 0x%x\n", 224 adapter->netdev->name, i, le32_to_cpu( 225 adapter->tqd_start[i].status.error)); 226 for (i = 0; i < adapter->num_rx_queues; i++) 227 if (adapter->rqd_start[i].status.stopped) 228 dev_err(&adapter->netdev->dev, 229 "%s: rq[%d] error 0x%x\n", 230 adapter->netdev->name, i, 231 adapter->rqd_start[i].status.error); 232 233 schedule_work(&adapter->work); 234 } 235 } 236 237 #ifdef __BIG_ENDIAN_BITFIELD 238 /* 239 * The device expects the bitfields in shared structures to be written in 240 * little endian. When CPU is big endian, the following routines are used to 241 * correctly read and write into ABI. 242 * The general technique used here is : double word bitfields are defined in 243 * opposite order for big endian architecture. Then before reading them in 244 * driver the complete double word is translated using le32_to_cpu. Similarly 245 * After the driver writes into bitfields, cpu_to_le32 is used to translate the 246 * double words into required format. 247 * In order to avoid touching bits in shared structure more than once, temporary 248 * descriptors are used. These are passed as srcDesc to following functions. 249 */ 250 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc, 251 struct Vmxnet3_RxDesc *dstDesc) 252 { 253 u32 *src = (u32 *)srcDesc + 2; 254 u32 *dst = (u32 *)dstDesc + 2; 255 dstDesc->addr = le64_to_cpu(srcDesc->addr); 256 *dst = le32_to_cpu(*src); 257 dstDesc->ext1 = le32_to_cpu(srcDesc->ext1); 258 } 259 260 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc, 261 struct Vmxnet3_TxDesc *dstDesc) 262 { 263 int i; 264 u32 *src = (u32 *)(srcDesc + 1); 265 u32 *dst = (u32 *)(dstDesc + 1); 266 267 /* Working backwards so that the gen bit is set at the end. */ 268 for (i = 2; i > 0; i--) { 269 src--; 270 dst--; 271 *dst = cpu_to_le32(*src); 272 } 273 } 274 275 276 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc, 277 struct Vmxnet3_RxCompDesc *dstDesc) 278 { 279 int i = 0; 280 u32 *src = (u32 *)srcDesc; 281 u32 *dst = (u32 *)dstDesc; 282 for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) { 283 *dst = le32_to_cpu(*src); 284 src++; 285 dst++; 286 } 287 } 288 289 290 /* Used to read bitfield values from double words. */ 291 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size) 292 { 293 u32 temp = le32_to_cpu(*bitfield); 294 u32 mask = ((1 << size) - 1) << pos; 295 temp &= mask; 296 temp >>= pos; 297 return temp; 298 } 299 300 301 302 #endif /* __BIG_ENDIAN_BITFIELD */ 303 304 #ifdef __BIG_ENDIAN_BITFIELD 305 306 # define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \ 307 txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \ 308 VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE) 309 # define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \ 310 txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \ 311 VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE) 312 # define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \ 313 VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \ 314 VMXNET3_TCD_GEN_SIZE) 315 # define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \ 316 VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE) 317 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \ 318 (dstrcd) = (tmp); \ 319 vmxnet3_RxCompToCPU((rcd), (tmp)); \ 320 } while (0) 321 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \ 322 (dstrxd) = (tmp); \ 323 vmxnet3_RxDescToCPU((rxd), (tmp)); \ 324 } while (0) 325 326 #else 327 328 # define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen) 329 # define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop) 330 # define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen) 331 # define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx) 332 # define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd) 333 # define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd) 334 335 #endif /* __BIG_ENDIAN_BITFIELD */ 336 337 338 static void 339 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi, 340 struct pci_dev *pdev) 341 { 342 u32 map_type = tbi->map_type; 343 344 if (map_type & VMXNET3_MAP_SINGLE) 345 dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len, 346 DMA_TO_DEVICE); 347 else if (map_type & VMXNET3_MAP_PAGE) 348 dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len, 349 DMA_TO_DEVICE); 350 else 351 BUG_ON(map_type & ~VMXNET3_MAP_XDP); 352 353 tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */ 354 } 355 356 357 static int 358 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq, 359 struct pci_dev *pdev, struct vmxnet3_adapter *adapter, 360 struct xdp_frame_bulk *bq) 361 { 362 struct vmxnet3_tx_buf_info *tbi; 363 int entries = 0; 364 u32 map_type; 365 366 /* no out of order completion */ 367 BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp); 368 BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1); 369 370 tbi = &tq->buf_info[eop_idx]; 371 BUG_ON(!tbi->skb); 372 map_type = tbi->map_type; 373 VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size); 374 375 while (tq->tx_ring.next2comp != eop_idx) { 376 vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp, 377 pdev); 378 379 /* update next2comp w/o tx_lock. Since we are marking more, 380 * instead of less, tx ring entries avail, the worst case is 381 * that the tx routine incorrectly re-queues a pkt due to 382 * insufficient tx ring entries. 383 */ 384 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 385 entries++; 386 } 387 388 if (map_type & VMXNET3_MAP_XDP) 389 xdp_return_frame_bulk(tbi->xdpf, bq); 390 else 391 dev_kfree_skb_any(tbi->skb); 392 393 /* xdpf and skb are in an anonymous union. */ 394 tbi->skb = NULL; 395 396 return entries; 397 } 398 399 400 static int 401 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq, 402 struct vmxnet3_adapter *adapter) 403 { 404 union Vmxnet3_GenericDesc *gdesc; 405 struct xdp_frame_bulk bq; 406 int completed = 0; 407 408 xdp_frame_bulk_init(&bq); 409 rcu_read_lock(); 410 411 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 412 while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) { 413 /* Prevent any &gdesc->tcd field from being (speculatively) 414 * read before (&gdesc->tcd)->gen is read. 415 */ 416 dma_rmb(); 417 418 completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX( 419 &gdesc->tcd), tq, adapter->pdev, 420 adapter, &bq); 421 422 vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring); 423 gdesc = tq->comp_ring.base + tq->comp_ring.next2proc; 424 } 425 xdp_flush_frame_bulk(&bq); 426 rcu_read_unlock(); 427 428 if (completed) { 429 spin_lock(&tq->tx_lock); 430 if (unlikely(vmxnet3_tq_stopped(tq, adapter) && 431 vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) > 432 VMXNET3_WAKE_QUEUE_THRESHOLD(tq) && 433 netif_carrier_ok(adapter->netdev))) { 434 vmxnet3_tq_wake(tq, adapter); 435 } 436 spin_unlock(&tq->tx_lock); 437 } 438 return completed; 439 } 440 441 442 static void 443 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq, 444 struct vmxnet3_adapter *adapter) 445 { 446 struct xdp_frame_bulk bq; 447 u32 map_type; 448 int i; 449 450 xdp_frame_bulk_init(&bq); 451 rcu_read_lock(); 452 453 while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) { 454 struct vmxnet3_tx_buf_info *tbi; 455 456 tbi = tq->buf_info + tq->tx_ring.next2comp; 457 map_type = tbi->map_type; 458 459 vmxnet3_unmap_tx_buf(tbi, adapter->pdev); 460 if (tbi->skb) { 461 if (map_type & VMXNET3_MAP_XDP) 462 xdp_return_frame_bulk(tbi->xdpf, &bq); 463 else 464 dev_kfree_skb_any(tbi->skb); 465 tbi->skb = NULL; 466 } 467 vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring); 468 } 469 470 xdp_flush_frame_bulk(&bq); 471 rcu_read_unlock(); 472 473 /* sanity check, verify all buffers are indeed unmapped */ 474 for (i = 0; i < tq->tx_ring.size; i++) 475 BUG_ON(tq->buf_info[i].map_type != VMXNET3_MAP_NONE); 476 477 tq->tx_ring.gen = VMXNET3_INIT_GEN; 478 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 479 480 tq->comp_ring.gen = VMXNET3_INIT_GEN; 481 tq->comp_ring.next2proc = 0; 482 } 483 484 485 static void 486 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq, 487 struct vmxnet3_adapter *adapter) 488 { 489 if (tq->tx_ring.base) { 490 dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size * 491 sizeof(struct Vmxnet3_TxDesc), 492 tq->tx_ring.base, tq->tx_ring.basePA); 493 tq->tx_ring.base = NULL; 494 } 495 if (tq->data_ring.base) { 496 dma_free_coherent(&adapter->pdev->dev, 497 tq->data_ring.size * tq->txdata_desc_size, 498 tq->data_ring.base, tq->data_ring.basePA); 499 tq->data_ring.base = NULL; 500 } 501 if (tq->comp_ring.base) { 502 dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size * 503 sizeof(struct Vmxnet3_TxCompDesc), 504 tq->comp_ring.base, tq->comp_ring.basePA); 505 tq->comp_ring.base = NULL; 506 } 507 kfree(tq->buf_info); 508 tq->buf_info = NULL; 509 } 510 511 512 /* Destroy all tx queues */ 513 void 514 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter) 515 { 516 int i; 517 518 for (i = 0; i < adapter->num_tx_queues; i++) 519 vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter); 520 } 521 522 523 static void 524 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq, 525 struct vmxnet3_adapter *adapter) 526 { 527 int i; 528 529 /* reset the tx ring contents to 0 and reset the tx ring states */ 530 memset(tq->tx_ring.base, 0, tq->tx_ring.size * 531 sizeof(struct Vmxnet3_TxDesc)); 532 tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0; 533 tq->tx_ring.gen = VMXNET3_INIT_GEN; 534 535 memset(tq->data_ring.base, 0, 536 tq->data_ring.size * tq->txdata_desc_size); 537 538 /* reset the tx comp ring contents to 0 and reset comp ring states */ 539 memset(tq->comp_ring.base, 0, tq->comp_ring.size * 540 sizeof(struct Vmxnet3_TxCompDesc)); 541 tq->comp_ring.next2proc = 0; 542 tq->comp_ring.gen = VMXNET3_INIT_GEN; 543 544 /* reset the bookkeeping data */ 545 memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size); 546 for (i = 0; i < tq->tx_ring.size; i++) 547 tq->buf_info[i].map_type = VMXNET3_MAP_NONE; 548 549 /* stats are not reset */ 550 } 551 552 553 static int 554 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq, 555 struct vmxnet3_adapter *adapter) 556 { 557 BUG_ON(tq->tx_ring.base || tq->data_ring.base || 558 tq->comp_ring.base || tq->buf_info); 559 560 tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 561 tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc), 562 &tq->tx_ring.basePA, GFP_KERNEL); 563 if (!tq->tx_ring.base) { 564 netdev_err(adapter->netdev, "failed to allocate tx ring\n"); 565 goto err; 566 } 567 568 tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 569 tq->data_ring.size * tq->txdata_desc_size, 570 &tq->data_ring.basePA, GFP_KERNEL); 571 if (!tq->data_ring.base) { 572 netdev_err(adapter->netdev, "failed to allocate tx data ring\n"); 573 goto err; 574 } 575 576 tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, 577 tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc), 578 &tq->comp_ring.basePA, GFP_KERNEL); 579 if (!tq->comp_ring.base) { 580 netdev_err(adapter->netdev, "failed to allocate tx comp ring\n"); 581 goto err; 582 } 583 584 tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]), 585 GFP_KERNEL, 586 dev_to_node(&adapter->pdev->dev)); 587 if (!tq->buf_info) 588 goto err; 589 590 return 0; 591 592 err: 593 vmxnet3_tq_destroy(tq, adapter); 594 return -ENOMEM; 595 } 596 597 static void 598 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter) 599 { 600 int i; 601 602 for (i = 0; i < adapter->num_tx_queues; i++) 603 vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter); 604 } 605 606 /* 607 * starting from ring->next2fill, allocate rx buffers for the given ring 608 * of the rx queue and update the rx desc. stop after @num_to_alloc buffers 609 * are allocated or allocation fails 610 */ 611 612 static int 613 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx, 614 int num_to_alloc, struct vmxnet3_adapter *adapter) 615 { 616 int num_allocated = 0; 617 struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx]; 618 struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx]; 619 u32 val; 620 621 while (num_allocated <= num_to_alloc) { 622 struct vmxnet3_rx_buf_info *rbi; 623 union Vmxnet3_GenericDesc *gd; 624 625 rbi = rbi_base + ring->next2fill; 626 gd = ring->base + ring->next2fill; 627 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 628 629 if (rbi->buf_type == VMXNET3_RX_BUF_XDP) { 630 void *data = vmxnet3_pp_get_buff(rq->page_pool, 631 &rbi->dma_addr, 632 GFP_KERNEL); 633 if (!data) { 634 rq->stats.rx_buf_alloc_failure++; 635 break; 636 } 637 rbi->page = virt_to_page(data); 638 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 639 } else if (rbi->buf_type == VMXNET3_RX_BUF_SKB) { 640 if (rbi->skb == NULL) { 641 rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev, 642 rbi->len, 643 GFP_KERNEL); 644 if (unlikely(rbi->skb == NULL)) { 645 rq->stats.rx_buf_alloc_failure++; 646 break; 647 } 648 649 rbi->dma_addr = dma_map_single( 650 &adapter->pdev->dev, 651 rbi->skb->data, rbi->len, 652 DMA_FROM_DEVICE); 653 if (dma_mapping_error(&adapter->pdev->dev, 654 rbi->dma_addr)) { 655 dev_kfree_skb_any(rbi->skb); 656 rbi->skb = NULL; 657 rq->stats.rx_buf_alloc_failure++; 658 break; 659 } 660 } else { 661 /* rx buffer skipped by the device */ 662 } 663 val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT; 664 } else { 665 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE || 666 rbi->len != PAGE_SIZE); 667 668 if (rbi->page == NULL) { 669 rbi->page = alloc_page(GFP_ATOMIC); 670 if (unlikely(rbi->page == NULL)) { 671 rq->stats.rx_buf_alloc_failure++; 672 break; 673 } 674 rbi->dma_addr = dma_map_page( 675 &adapter->pdev->dev, 676 rbi->page, 0, PAGE_SIZE, 677 DMA_FROM_DEVICE); 678 if (dma_mapping_error(&adapter->pdev->dev, 679 rbi->dma_addr)) { 680 put_page(rbi->page); 681 rbi->page = NULL; 682 rq->stats.rx_buf_alloc_failure++; 683 break; 684 } 685 } else { 686 /* rx buffers skipped by the device */ 687 } 688 val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT; 689 } 690 691 gd->rxd.addr = cpu_to_le64(rbi->dma_addr); 692 gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT) 693 | val | rbi->len); 694 695 /* Fill the last buffer but dont mark it ready, or else the 696 * device will think that the queue is full */ 697 if (num_allocated == num_to_alloc) { 698 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 699 break; 700 } 701 702 gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT); 703 num_allocated++; 704 vmxnet3_cmd_ring_adv_next2fill(ring); 705 } 706 707 netdev_dbg(adapter->netdev, 708 "alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n", 709 num_allocated, ring->next2fill, ring->next2comp); 710 711 /* so that the device can distinguish a full ring and an empty ring */ 712 BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp); 713 714 return num_allocated; 715 } 716 717 718 static void 719 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd, 720 struct vmxnet3_rx_buf_info *rbi) 721 { 722 skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags; 723 724 BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS); 725 726 skb_frag_fill_page_desc(frag, rbi->page, 0, rcd->len); 727 skb->data_len += rcd->len; 728 skb->truesize += PAGE_SIZE; 729 skb_shinfo(skb)->nr_frags++; 730 } 731 732 733 static int 734 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx, 735 struct vmxnet3_tx_queue *tq, struct pci_dev *pdev, 736 struct vmxnet3_adapter *adapter) 737 { 738 u32 dw2, len; 739 unsigned long buf_offset; 740 int i; 741 union Vmxnet3_GenericDesc *gdesc; 742 struct vmxnet3_tx_buf_info *tbi = NULL; 743 744 BUG_ON(ctx->copy_size > skb_headlen(skb)); 745 746 /* use the previous gen bit for the SOP desc */ 747 dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT; 748 749 ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill; 750 gdesc = ctx->sop_txd; /* both loops below can be skipped */ 751 752 /* no need to map the buffer if headers are copied */ 753 if (ctx->copy_size) { 754 ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA + 755 tq->tx_ring.next2fill * 756 tq->txdata_desc_size); 757 ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size); 758 ctx->sop_txd->dword[3] = 0; 759 760 tbi = tq->buf_info + tq->tx_ring.next2fill; 761 tbi->map_type = VMXNET3_MAP_NONE; 762 763 netdev_dbg(adapter->netdev, 764 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 765 tq->tx_ring.next2fill, 766 le64_to_cpu(ctx->sop_txd->txd.addr), 767 ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]); 768 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 769 770 /* use the right gen for non-SOP desc */ 771 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 772 } 773 774 /* linear part can use multiple tx desc if it's big */ 775 len = skb_headlen(skb) - ctx->copy_size; 776 buf_offset = ctx->copy_size; 777 while (len) { 778 u32 buf_size; 779 780 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 781 buf_size = len; 782 dw2 |= len; 783 } else { 784 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 785 /* spec says that for TxDesc.len, 0 == 2^14 */ 786 } 787 788 tbi = tq->buf_info + tq->tx_ring.next2fill; 789 tbi->map_type = VMXNET3_MAP_SINGLE; 790 tbi->dma_addr = dma_map_single(&adapter->pdev->dev, 791 skb->data + buf_offset, buf_size, 792 DMA_TO_DEVICE); 793 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 794 return -EFAULT; 795 796 tbi->len = buf_size; 797 798 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 799 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 800 801 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 802 gdesc->dword[2] = cpu_to_le32(dw2); 803 gdesc->dword[3] = 0; 804 805 netdev_dbg(adapter->netdev, 806 "txd[%u]: 0x%Lx 0x%x 0x%x\n", 807 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 808 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 809 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 810 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 811 812 len -= buf_size; 813 buf_offset += buf_size; 814 } 815 816 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 817 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 818 u32 buf_size; 819 820 buf_offset = 0; 821 len = skb_frag_size(frag); 822 while (len) { 823 tbi = tq->buf_info + tq->tx_ring.next2fill; 824 if (len < VMXNET3_MAX_TX_BUF_SIZE) { 825 buf_size = len; 826 dw2 |= len; 827 } else { 828 buf_size = VMXNET3_MAX_TX_BUF_SIZE; 829 /* spec says that for TxDesc.len, 0 == 2^14 */ 830 } 831 tbi->map_type = VMXNET3_MAP_PAGE; 832 tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag, 833 buf_offset, buf_size, 834 DMA_TO_DEVICE); 835 if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr)) 836 return -EFAULT; 837 838 tbi->len = buf_size; 839 840 gdesc = tq->tx_ring.base + tq->tx_ring.next2fill; 841 BUG_ON(gdesc->txd.gen == tq->tx_ring.gen); 842 843 gdesc->txd.addr = cpu_to_le64(tbi->dma_addr); 844 gdesc->dword[2] = cpu_to_le32(dw2); 845 gdesc->dword[3] = 0; 846 847 netdev_dbg(adapter->netdev, 848 "txd[%u]: 0x%llx %u %u\n", 849 tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr), 850 le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]); 851 vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring); 852 dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT; 853 854 len -= buf_size; 855 buf_offset += buf_size; 856 } 857 } 858 859 ctx->eop_txd = gdesc; 860 861 /* set the last buf_info for the pkt */ 862 tbi->skb = skb; 863 tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base; 864 865 return 0; 866 } 867 868 869 /* Init all tx queues */ 870 static void 871 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter) 872 { 873 int i; 874 875 for (i = 0; i < adapter->num_tx_queues; i++) 876 vmxnet3_tq_init(&adapter->tx_queue[i], adapter); 877 } 878 879 880 /* 881 * parse relevant protocol headers: 882 * For a tso pkt, relevant headers are L2/3/4 including options 883 * For a pkt requesting csum offloading, they are L2/3 and may include L4 884 * if it's a TCP/UDP pkt 885 * 886 * Returns: 887 * -1: error happens during parsing 888 * 0: protocol headers parsed, but too big to be copied 889 * 1: protocol headers parsed and copied 890 * 891 * Other effects: 892 * 1. related *ctx fields are updated. 893 * 2. ctx->copy_size is # of bytes copied 894 * 3. the portion to be copied is guaranteed to be in the linear part 895 * 896 */ 897 static int 898 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 899 struct vmxnet3_tx_ctx *ctx, 900 struct vmxnet3_adapter *adapter) 901 { 902 u8 protocol = 0; 903 904 if (ctx->mss) { /* TSO */ 905 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 906 ctx->l4_offset = skb_inner_transport_offset(skb); 907 ctx->l4_hdr_size = inner_tcp_hdrlen(skb); 908 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 909 } else { 910 ctx->l4_offset = skb_transport_offset(skb); 911 ctx->l4_hdr_size = tcp_hdrlen(skb); 912 ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size; 913 } 914 } else { 915 if (skb->ip_summed == CHECKSUM_PARTIAL) { 916 /* For encap packets, skb_checksum_start_offset refers 917 * to inner L4 offset. Thus, below works for encap as 918 * well as non-encap case 919 */ 920 ctx->l4_offset = skb_checksum_start_offset(skb); 921 922 if (VMXNET3_VERSION_GE_4(adapter) && 923 skb->encapsulation) { 924 struct iphdr *iph = inner_ip_hdr(skb); 925 926 if (iph->version == 4) { 927 protocol = iph->protocol; 928 } else { 929 const struct ipv6hdr *ipv6h; 930 931 ipv6h = inner_ipv6_hdr(skb); 932 protocol = ipv6h->nexthdr; 933 } 934 } else { 935 if (ctx->ipv4) { 936 const struct iphdr *iph = ip_hdr(skb); 937 938 protocol = iph->protocol; 939 } else if (ctx->ipv6) { 940 const struct ipv6hdr *ipv6h; 941 942 ipv6h = ipv6_hdr(skb); 943 protocol = ipv6h->nexthdr; 944 } 945 } 946 947 switch (protocol) { 948 case IPPROTO_TCP: 949 ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) : 950 tcp_hdrlen(skb); 951 break; 952 case IPPROTO_UDP: 953 ctx->l4_hdr_size = sizeof(struct udphdr); 954 break; 955 default: 956 ctx->l4_hdr_size = 0; 957 break; 958 } 959 960 ctx->copy_size = min(ctx->l4_offset + 961 ctx->l4_hdr_size, skb->len); 962 } else { 963 ctx->l4_offset = 0; 964 ctx->l4_hdr_size = 0; 965 /* copy as much as allowed */ 966 ctx->copy_size = min_t(unsigned int, 967 tq->txdata_desc_size, 968 skb_headlen(skb)); 969 } 970 971 if (skb->len <= VMXNET3_HDR_COPY_SIZE) 972 ctx->copy_size = skb->len; 973 974 /* make sure headers are accessible directly */ 975 if (unlikely(!pskb_may_pull(skb, ctx->copy_size))) 976 goto err; 977 } 978 979 if (unlikely(ctx->copy_size > tq->txdata_desc_size)) { 980 tq->stats.oversized_hdr++; 981 ctx->copy_size = 0; 982 return 0; 983 } 984 985 return 1; 986 err: 987 return -1; 988 } 989 990 /* 991 * copy relevant protocol headers to the transmit ring: 992 * For a tso pkt, relevant headers are L2/3/4 including options 993 * For a pkt requesting csum offloading, they are L2/3 and may include L4 994 * if it's a TCP/UDP pkt 995 * 996 * 997 * Note that this requires that vmxnet3_parse_hdr be called first to set the 998 * appropriate bits in ctx first 999 */ 1000 static void 1001 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1002 struct vmxnet3_tx_ctx *ctx, 1003 struct vmxnet3_adapter *adapter) 1004 { 1005 struct Vmxnet3_TxDataDesc *tdd; 1006 1007 tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base + 1008 tq->tx_ring.next2fill * 1009 tq->txdata_desc_size); 1010 1011 memcpy(tdd->data, skb->data, ctx->copy_size); 1012 netdev_dbg(adapter->netdev, 1013 "copy %u bytes to dataRing[%u]\n", 1014 ctx->copy_size, tq->tx_ring.next2fill); 1015 } 1016 1017 1018 static void 1019 vmxnet3_prepare_inner_tso(struct sk_buff *skb, 1020 struct vmxnet3_tx_ctx *ctx) 1021 { 1022 struct tcphdr *tcph = inner_tcp_hdr(skb); 1023 struct iphdr *iph = inner_ip_hdr(skb); 1024 1025 if (iph->version == 4) { 1026 iph->check = 0; 1027 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1028 IPPROTO_TCP, 0); 1029 } else { 1030 struct ipv6hdr *iph = inner_ipv6_hdr(skb); 1031 1032 tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0, 1033 IPPROTO_TCP, 0); 1034 } 1035 } 1036 1037 static void 1038 vmxnet3_prepare_tso(struct sk_buff *skb, 1039 struct vmxnet3_tx_ctx *ctx) 1040 { 1041 struct tcphdr *tcph = tcp_hdr(skb); 1042 1043 if (ctx->ipv4) { 1044 struct iphdr *iph = ip_hdr(skb); 1045 1046 iph->check = 0; 1047 tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0, 1048 IPPROTO_TCP, 0); 1049 } else if (ctx->ipv6) { 1050 tcp_v6_gso_csum_prep(skb); 1051 } 1052 } 1053 1054 static int txd_estimate(const struct sk_buff *skb) 1055 { 1056 int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1057 int i; 1058 1059 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1060 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1061 1062 count += VMXNET3_TXD_NEEDED(skb_frag_size(frag)); 1063 } 1064 return count; 1065 } 1066 1067 /* 1068 * Transmits a pkt thru a given tq 1069 * Returns: 1070 * NETDEV_TX_OK: descriptors are setup successfully 1071 * NETDEV_TX_OK: error occurred, the pkt is dropped 1072 * NETDEV_TX_BUSY: tx ring is full, queue is stopped 1073 * 1074 * Side-effects: 1075 * 1. tx ring may be changed 1076 * 2. tq stats may be updated accordingly 1077 * 3. shared->txNumDeferred may be updated 1078 */ 1079 1080 static int 1081 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq, 1082 struct vmxnet3_adapter *adapter, struct net_device *netdev) 1083 { 1084 int ret; 1085 u32 count; 1086 int num_pkts; 1087 int tx_num_deferred; 1088 unsigned long flags; 1089 struct vmxnet3_tx_ctx ctx; 1090 union Vmxnet3_GenericDesc *gdesc; 1091 #ifdef __BIG_ENDIAN_BITFIELD 1092 /* Use temporary descriptor to avoid touching bits multiple times */ 1093 union Vmxnet3_GenericDesc tempTxDesc; 1094 #endif 1095 1096 count = txd_estimate(skb); 1097 1098 ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP)); 1099 ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6)); 1100 1101 ctx.mss = skb_shinfo(skb)->gso_size; 1102 if (ctx.mss) { 1103 if (skb_header_cloned(skb)) { 1104 if (unlikely(pskb_expand_head(skb, 0, 0, 1105 GFP_ATOMIC) != 0)) { 1106 tq->stats.drop_tso++; 1107 goto drop_pkt; 1108 } 1109 tq->stats.copy_skb_header++; 1110 } 1111 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1112 /* tso pkts must not use more than 1113 * VMXNET3_MAX_TSO_TXD_PER_PKT entries 1114 */ 1115 if (skb_linearize(skb) != 0) { 1116 tq->stats.drop_too_many_frags++; 1117 goto drop_pkt; 1118 } 1119 tq->stats.linearized++; 1120 1121 /* recalculate the # of descriptors to use */ 1122 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1123 if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) { 1124 tq->stats.drop_too_many_frags++; 1125 goto drop_pkt; 1126 } 1127 } 1128 if (skb->encapsulation) { 1129 vmxnet3_prepare_inner_tso(skb, &ctx); 1130 } else { 1131 vmxnet3_prepare_tso(skb, &ctx); 1132 } 1133 } else { 1134 if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) { 1135 1136 /* non-tso pkts must not use more than 1137 * VMXNET3_MAX_TXD_PER_PKT entries 1138 */ 1139 if (skb_linearize(skb) != 0) { 1140 tq->stats.drop_too_many_frags++; 1141 goto drop_pkt; 1142 } 1143 tq->stats.linearized++; 1144 1145 /* recalculate the # of descriptors to use */ 1146 count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1; 1147 } 1148 } 1149 1150 ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter); 1151 if (ret >= 0) { 1152 BUG_ON(ret <= 0 && ctx.copy_size != 0); 1153 /* hdrs parsed, check against other limits */ 1154 if (ctx.mss) { 1155 if (unlikely(ctx.l4_offset + ctx.l4_hdr_size > 1156 VMXNET3_MAX_TX_BUF_SIZE)) { 1157 tq->stats.drop_oversized_hdr++; 1158 goto drop_pkt; 1159 } 1160 } else { 1161 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1162 if (unlikely(ctx.l4_offset + 1163 skb->csum_offset > 1164 VMXNET3_MAX_CSUM_OFFSET)) { 1165 tq->stats.drop_oversized_hdr++; 1166 goto drop_pkt; 1167 } 1168 } 1169 } 1170 } else { 1171 tq->stats.drop_hdr_inspect_err++; 1172 goto drop_pkt; 1173 } 1174 1175 spin_lock_irqsave(&tq->tx_lock, flags); 1176 1177 if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) { 1178 tq->stats.tx_ring_full++; 1179 netdev_dbg(adapter->netdev, 1180 "tx queue stopped on %s, next2comp %u" 1181 " next2fill %u\n", adapter->netdev->name, 1182 tq->tx_ring.next2comp, tq->tx_ring.next2fill); 1183 1184 vmxnet3_tq_stop(tq, adapter); 1185 spin_unlock_irqrestore(&tq->tx_lock, flags); 1186 return NETDEV_TX_BUSY; 1187 } 1188 1189 1190 vmxnet3_copy_hdr(skb, tq, &ctx, adapter); 1191 1192 /* fill tx descs related to addr & len */ 1193 if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter)) 1194 goto unlock_drop_pkt; 1195 1196 /* setup the EOP desc */ 1197 ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP); 1198 1199 /* setup the SOP desc */ 1200 #ifdef __BIG_ENDIAN_BITFIELD 1201 gdesc = &tempTxDesc; 1202 gdesc->dword[2] = ctx.sop_txd->dword[2]; 1203 gdesc->dword[3] = ctx.sop_txd->dword[3]; 1204 #else 1205 gdesc = ctx.sop_txd; 1206 #endif 1207 tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred); 1208 if (ctx.mss) { 1209 if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) { 1210 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1211 if (VMXNET3_VERSION_GE_7(adapter)) { 1212 gdesc->txd.om = VMXNET3_OM_TSO; 1213 gdesc->txd.ext1 = 1; 1214 } else { 1215 gdesc->txd.om = VMXNET3_OM_ENCAP; 1216 } 1217 gdesc->txd.msscof = ctx.mss; 1218 1219 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) 1220 gdesc->txd.oco = 1; 1221 } else { 1222 gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size; 1223 gdesc->txd.om = VMXNET3_OM_TSO; 1224 gdesc->txd.msscof = ctx.mss; 1225 } 1226 num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss; 1227 } else { 1228 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1229 if (VMXNET3_VERSION_GE_4(adapter) && 1230 skb->encapsulation) { 1231 gdesc->txd.hlen = ctx.l4_offset + 1232 ctx.l4_hdr_size; 1233 if (VMXNET3_VERSION_GE_7(adapter)) { 1234 gdesc->txd.om = VMXNET3_OM_CSUM; 1235 gdesc->txd.msscof = ctx.l4_offset + 1236 skb->csum_offset; 1237 gdesc->txd.ext1 = 1; 1238 } else { 1239 gdesc->txd.om = VMXNET3_OM_ENCAP; 1240 gdesc->txd.msscof = 0; /* Reserved */ 1241 } 1242 } else { 1243 gdesc->txd.hlen = ctx.l4_offset; 1244 gdesc->txd.om = VMXNET3_OM_CSUM; 1245 gdesc->txd.msscof = ctx.l4_offset + 1246 skb->csum_offset; 1247 } 1248 } else { 1249 gdesc->txd.om = 0; 1250 gdesc->txd.msscof = 0; 1251 } 1252 num_pkts = 1; 1253 } 1254 le32_add_cpu(&tq->shared->txNumDeferred, num_pkts); 1255 tx_num_deferred += num_pkts; 1256 1257 if (skb_vlan_tag_present(skb)) { 1258 gdesc->txd.ti = 1; 1259 gdesc->txd.tci = skb_vlan_tag_get(skb); 1260 } 1261 1262 /* Ensure that the write to (&gdesc->txd)->gen will be observed after 1263 * all other writes to &gdesc->txd. 1264 */ 1265 dma_wmb(); 1266 1267 /* finally flips the GEN bit of the SOP desc. */ 1268 gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^ 1269 VMXNET3_TXD_GEN); 1270 #ifdef __BIG_ENDIAN_BITFIELD 1271 /* Finished updating in bitfields of Tx Desc, so write them in original 1272 * place. 1273 */ 1274 vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc, 1275 (struct Vmxnet3_TxDesc *)ctx.sop_txd); 1276 gdesc = ctx.sop_txd; 1277 #endif 1278 netdev_dbg(adapter->netdev, 1279 "txd[%u]: SOP 0x%Lx 0x%x 0x%x\n", 1280 (u32)(ctx.sop_txd - 1281 tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr), 1282 le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3])); 1283 1284 spin_unlock_irqrestore(&tq->tx_lock, flags); 1285 1286 if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) { 1287 tq->shared->txNumDeferred = 0; 1288 VMXNET3_WRITE_BAR0_REG(adapter, 1289 adapter->tx_prod_offset + tq->qid * 8, 1290 tq->tx_ring.next2fill); 1291 } 1292 1293 return NETDEV_TX_OK; 1294 1295 unlock_drop_pkt: 1296 spin_unlock_irqrestore(&tq->tx_lock, flags); 1297 drop_pkt: 1298 tq->stats.drop_total++; 1299 dev_kfree_skb_any(skb); 1300 return NETDEV_TX_OK; 1301 } 1302 1303 static int 1304 vmxnet3_create_pp(struct vmxnet3_adapter *adapter, 1305 struct vmxnet3_rx_queue *rq, int size) 1306 { 1307 bool xdp_prog = vmxnet3_xdp_enabled(adapter); 1308 const struct page_pool_params pp_params = { 1309 .order = 0, 1310 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, 1311 .pool_size = size, 1312 .nid = NUMA_NO_NODE, 1313 .dev = &adapter->pdev->dev, 1314 .offset = VMXNET3_XDP_RX_OFFSET, 1315 .max_len = VMXNET3_XDP_MAX_FRSIZE, 1316 .dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE, 1317 }; 1318 struct page_pool *pp; 1319 int err; 1320 1321 pp = page_pool_create(&pp_params); 1322 if (IS_ERR(pp)) 1323 return PTR_ERR(pp); 1324 1325 err = xdp_rxq_info_reg(&rq->xdp_rxq, adapter->netdev, rq->qid, 1326 rq->napi.napi_id); 1327 if (err < 0) 1328 goto err_free_pp; 1329 1330 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_PAGE_POOL, pp); 1331 if (err) 1332 goto err_unregister_rxq; 1333 1334 rq->page_pool = pp; 1335 1336 return 0; 1337 1338 err_unregister_rxq: 1339 xdp_rxq_info_unreg(&rq->xdp_rxq); 1340 err_free_pp: 1341 page_pool_destroy(pp); 1342 1343 return err; 1344 } 1345 1346 void * 1347 vmxnet3_pp_get_buff(struct page_pool *pp, dma_addr_t *dma_addr, 1348 gfp_t gfp_mask) 1349 { 1350 struct page *page; 1351 1352 page = page_pool_alloc_pages(pp, gfp_mask | __GFP_NOWARN); 1353 if (unlikely(!page)) 1354 return NULL; 1355 1356 *dma_addr = page_pool_get_dma_addr(page) + pp->p.offset; 1357 1358 return page_address(page); 1359 } 1360 1361 static netdev_tx_t 1362 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 1363 { 1364 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 1365 1366 BUG_ON(skb->queue_mapping > adapter->num_tx_queues); 1367 return vmxnet3_tq_xmit(skb, 1368 &adapter->tx_queue[skb->queue_mapping], 1369 adapter, netdev); 1370 } 1371 1372 1373 static void 1374 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter, 1375 struct sk_buff *skb, 1376 union Vmxnet3_GenericDesc *gdesc) 1377 { 1378 if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) { 1379 if (gdesc->rcd.v4 && 1380 (le32_to_cpu(gdesc->dword[3]) & 1381 VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) { 1382 skb->ip_summed = CHECKSUM_UNNECESSARY; 1383 if ((le32_to_cpu(gdesc->dword[0]) & 1384 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1385 skb->csum_level = 1; 1386 } 1387 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1388 !(le32_to_cpu(gdesc->dword[0]) & 1389 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1390 WARN_ON_ONCE(gdesc->rcd.frg && 1391 !(le32_to_cpu(gdesc->dword[0]) & 1392 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1393 } else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) & 1394 (1 << VMXNET3_RCD_TUC_SHIFT))) { 1395 skb->ip_summed = CHECKSUM_UNNECESSARY; 1396 if ((le32_to_cpu(gdesc->dword[0]) & 1397 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) { 1398 skb->csum_level = 1; 1399 } 1400 WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) && 1401 !(le32_to_cpu(gdesc->dword[0]) & 1402 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1403 WARN_ON_ONCE(gdesc->rcd.frg && 1404 !(le32_to_cpu(gdesc->dword[0]) & 1405 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))); 1406 } else { 1407 if (gdesc->rcd.csum) { 1408 skb->csum = htons(gdesc->rcd.csum); 1409 skb->ip_summed = CHECKSUM_PARTIAL; 1410 } else { 1411 skb_checksum_none_assert(skb); 1412 } 1413 } 1414 } else { 1415 skb_checksum_none_assert(skb); 1416 } 1417 } 1418 1419 1420 static void 1421 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd, 1422 struct vmxnet3_rx_ctx *ctx, struct vmxnet3_adapter *adapter) 1423 { 1424 rq->stats.drop_err++; 1425 if (!rcd->fcs) 1426 rq->stats.drop_fcs++; 1427 1428 rq->stats.drop_total++; 1429 1430 /* 1431 * We do not unmap and chain the rx buffer to the skb. 1432 * We basically pretend this buffer is not used and will be recycled 1433 * by vmxnet3_rq_alloc_rx_buf() 1434 */ 1435 1436 /* 1437 * ctx->skb may be NULL if this is the first and the only one 1438 * desc for the pkt 1439 */ 1440 if (ctx->skb) 1441 dev_kfree_skb_irq(ctx->skb); 1442 1443 ctx->skb = NULL; 1444 } 1445 1446 1447 static u32 1448 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb, 1449 union Vmxnet3_GenericDesc *gdesc) 1450 { 1451 u32 hlen, maplen; 1452 union { 1453 void *ptr; 1454 struct ethhdr *eth; 1455 struct vlan_ethhdr *veth; 1456 struct iphdr *ipv4; 1457 struct ipv6hdr *ipv6; 1458 struct tcphdr *tcp; 1459 } hdr; 1460 BUG_ON(gdesc->rcd.tcp == 0); 1461 1462 maplen = skb_headlen(skb); 1463 if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen)) 1464 return 0; 1465 1466 if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || 1467 skb->protocol == cpu_to_be16(ETH_P_8021AD)) 1468 hlen = sizeof(struct vlan_ethhdr); 1469 else 1470 hlen = sizeof(struct ethhdr); 1471 1472 hdr.eth = eth_hdr(skb); 1473 if (gdesc->rcd.v4) { 1474 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) && 1475 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP)); 1476 hdr.ptr += hlen; 1477 BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP); 1478 hlen = hdr.ipv4->ihl << 2; 1479 hdr.ptr += hdr.ipv4->ihl << 2; 1480 } else if (gdesc->rcd.v6) { 1481 BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) && 1482 hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6)); 1483 hdr.ptr += hlen; 1484 /* Use an estimated value, since we also need to handle 1485 * TSO case. 1486 */ 1487 if (hdr.ipv6->nexthdr != IPPROTO_TCP) 1488 return sizeof(struct ipv6hdr) + sizeof(struct tcphdr); 1489 hlen = sizeof(struct ipv6hdr); 1490 hdr.ptr += sizeof(struct ipv6hdr); 1491 } else { 1492 /* Non-IP pkt, dont estimate header length */ 1493 return 0; 1494 } 1495 1496 if (hlen + sizeof(struct tcphdr) > maplen) 1497 return 0; 1498 1499 return (hlen + (hdr.tcp->doff << 2)); 1500 } 1501 1502 static int 1503 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq, 1504 struct vmxnet3_adapter *adapter, int quota) 1505 { 1506 u32 rxprod_reg[2] = { 1507 adapter->rx_prod_offset, adapter->rx_prod2_offset 1508 }; 1509 u32 num_pkts = 0; 1510 bool skip_page_frags = false; 1511 bool encap_lro = false; 1512 struct Vmxnet3_RxCompDesc *rcd; 1513 struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx; 1514 u16 segCnt = 0, mss = 0; 1515 int comp_offset, fill_offset; 1516 #ifdef __BIG_ENDIAN_BITFIELD 1517 struct Vmxnet3_RxDesc rxCmdDesc; 1518 struct Vmxnet3_RxCompDesc rxComp; 1519 #endif 1520 bool need_flush = false; 1521 1522 vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, 1523 &rxComp); 1524 while (rcd->gen == rq->comp_ring.gen) { 1525 struct vmxnet3_rx_buf_info *rbi; 1526 struct sk_buff *skb, *new_skb = NULL; 1527 struct page *new_page = NULL; 1528 dma_addr_t new_dma_addr; 1529 int num_to_alloc; 1530 struct Vmxnet3_RxDesc *rxd; 1531 u32 idx, ring_idx; 1532 struct vmxnet3_cmd_ring *ring = NULL; 1533 if (num_pkts >= quota) { 1534 /* we may stop even before we see the EOP desc of 1535 * the current pkt 1536 */ 1537 break; 1538 } 1539 1540 /* Prevent any rcd field from being (speculatively) read before 1541 * rcd->gen is read. 1542 */ 1543 dma_rmb(); 1544 1545 BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 && 1546 rcd->rqID != rq->dataRingQid); 1547 idx = rcd->rxdIdx; 1548 ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID); 1549 ring = rq->rx_ring + ring_idx; 1550 vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd, 1551 &rxCmdDesc); 1552 rbi = rq->buf_info[ring_idx] + idx; 1553 1554 BUG_ON(rxd->addr != rbi->dma_addr || 1555 rxd->len != rbi->len); 1556 1557 if (unlikely(rcd->eop && rcd->err)) { 1558 vmxnet3_rx_error(rq, rcd, ctx, adapter); 1559 goto rcd_done; 1560 } 1561 1562 if (rcd->sop && rcd->eop && vmxnet3_xdp_enabled(adapter)) { 1563 struct sk_buff *skb_xdp_pass; 1564 int act; 1565 1566 if (VMXNET3_RX_DATA_RING(adapter, rcd->rqID)) { 1567 ctx->skb = NULL; 1568 goto skip_xdp; /* Handle it later. */ 1569 } 1570 1571 if (rbi->buf_type != VMXNET3_RX_BUF_XDP) 1572 goto rcd_done; 1573 1574 act = vmxnet3_process_xdp(adapter, rq, rcd, rbi, rxd, 1575 &skb_xdp_pass); 1576 if (act == XDP_PASS) { 1577 ctx->skb = skb_xdp_pass; 1578 goto sop_done; 1579 } 1580 ctx->skb = NULL; 1581 need_flush |= act == XDP_REDIRECT; 1582 1583 goto rcd_done; 1584 } 1585 skip_xdp: 1586 1587 if (rcd->sop) { /* first buf of the pkt */ 1588 bool rxDataRingUsed; 1589 u16 len; 1590 1591 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD || 1592 (rcd->rqID != rq->qid && 1593 rcd->rqID != rq->dataRingQid)); 1594 1595 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB && 1596 rbi->buf_type != VMXNET3_RX_BUF_XDP); 1597 BUG_ON(ctx->skb != NULL || rbi->skb == NULL); 1598 1599 if (unlikely(rcd->len == 0)) { 1600 /* Pretend the rx buffer is skipped. */ 1601 BUG_ON(!(rcd->sop && rcd->eop)); 1602 netdev_dbg(adapter->netdev, 1603 "rxRing[%u][%u] 0 length\n", 1604 ring_idx, idx); 1605 goto rcd_done; 1606 } 1607 1608 skip_page_frags = false; 1609 ctx->skb = rbi->skb; 1610 1611 rxDataRingUsed = 1612 VMXNET3_RX_DATA_RING(adapter, rcd->rqID); 1613 len = rxDataRingUsed ? rcd->len : rbi->len; 1614 1615 if (rxDataRingUsed && vmxnet3_xdp_enabled(adapter)) { 1616 struct sk_buff *skb_xdp_pass; 1617 size_t sz; 1618 int act; 1619 1620 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1621 act = vmxnet3_process_xdp_small(adapter, rq, 1622 &rq->data_ring.base[sz], 1623 rcd->len, 1624 &skb_xdp_pass); 1625 if (act == XDP_PASS) { 1626 ctx->skb = skb_xdp_pass; 1627 goto sop_done; 1628 } 1629 need_flush |= act == XDP_REDIRECT; 1630 1631 goto rcd_done; 1632 } 1633 new_skb = netdev_alloc_skb_ip_align(adapter->netdev, 1634 len); 1635 if (new_skb == NULL) { 1636 /* Skb allocation failed, do not handover this 1637 * skb to stack. Reuse it. Drop the existing pkt 1638 */ 1639 rq->stats.rx_buf_alloc_failure++; 1640 ctx->skb = NULL; 1641 rq->stats.drop_total++; 1642 skip_page_frags = true; 1643 goto rcd_done; 1644 } 1645 1646 if (rxDataRingUsed && adapter->rxdataring_enabled) { 1647 size_t sz; 1648 1649 BUG_ON(rcd->len > rq->data_ring.desc_size); 1650 1651 ctx->skb = new_skb; 1652 sz = rcd->rxdIdx * rq->data_ring.desc_size; 1653 memcpy(new_skb->data, 1654 &rq->data_ring.base[sz], rcd->len); 1655 } else { 1656 ctx->skb = rbi->skb; 1657 1658 new_dma_addr = 1659 dma_map_single(&adapter->pdev->dev, 1660 new_skb->data, rbi->len, 1661 DMA_FROM_DEVICE); 1662 if (dma_mapping_error(&adapter->pdev->dev, 1663 new_dma_addr)) { 1664 dev_kfree_skb(new_skb); 1665 /* Skb allocation failed, do not 1666 * handover this skb to stack. Reuse 1667 * it. Drop the existing pkt. 1668 */ 1669 rq->stats.rx_buf_alloc_failure++; 1670 ctx->skb = NULL; 1671 rq->stats.drop_total++; 1672 skip_page_frags = true; 1673 goto rcd_done; 1674 } 1675 1676 dma_unmap_single(&adapter->pdev->dev, 1677 rbi->dma_addr, 1678 rbi->len, 1679 DMA_FROM_DEVICE); 1680 1681 /* Immediate refill */ 1682 rbi->skb = new_skb; 1683 rbi->dma_addr = new_dma_addr; 1684 rxd->addr = cpu_to_le64(rbi->dma_addr); 1685 rxd->len = rbi->len; 1686 } 1687 1688 skb_record_rx_queue(ctx->skb, rq->qid); 1689 skb_put(ctx->skb, rcd->len); 1690 1691 if (VMXNET3_VERSION_GE_2(adapter) && 1692 rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) { 1693 struct Vmxnet3_RxCompDescExt *rcdlro; 1694 union Vmxnet3_GenericDesc *gdesc; 1695 1696 rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd; 1697 gdesc = (union Vmxnet3_GenericDesc *)rcd; 1698 1699 segCnt = rcdlro->segCnt; 1700 WARN_ON_ONCE(segCnt == 0); 1701 mss = rcdlro->mss; 1702 if (unlikely(segCnt <= 1)) 1703 segCnt = 0; 1704 encap_lro = (le32_to_cpu(gdesc->dword[0]) & 1705 (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)); 1706 } else { 1707 segCnt = 0; 1708 } 1709 } else { 1710 BUG_ON(ctx->skb == NULL && !skip_page_frags); 1711 1712 /* non SOP buffer must be type 1 in most cases */ 1713 BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE); 1714 BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY); 1715 1716 /* If an sop buffer was dropped, skip all 1717 * following non-sop fragments. They will be reused. 1718 */ 1719 if (skip_page_frags) 1720 goto rcd_done; 1721 1722 if (rcd->len) { 1723 new_page = alloc_page(GFP_ATOMIC); 1724 /* Replacement page frag could not be allocated. 1725 * Reuse this page. Drop the pkt and free the 1726 * skb which contained this page as a frag. Skip 1727 * processing all the following non-sop frags. 1728 */ 1729 if (unlikely(!new_page)) { 1730 rq->stats.rx_buf_alloc_failure++; 1731 dev_kfree_skb(ctx->skb); 1732 ctx->skb = NULL; 1733 skip_page_frags = true; 1734 goto rcd_done; 1735 } 1736 new_dma_addr = dma_map_page(&adapter->pdev->dev, 1737 new_page, 1738 0, PAGE_SIZE, 1739 DMA_FROM_DEVICE); 1740 if (dma_mapping_error(&adapter->pdev->dev, 1741 new_dma_addr)) { 1742 put_page(new_page); 1743 rq->stats.rx_buf_alloc_failure++; 1744 dev_kfree_skb(ctx->skb); 1745 ctx->skb = NULL; 1746 skip_page_frags = true; 1747 goto rcd_done; 1748 } 1749 1750 dma_unmap_page(&adapter->pdev->dev, 1751 rbi->dma_addr, rbi->len, 1752 DMA_FROM_DEVICE); 1753 1754 vmxnet3_append_frag(ctx->skb, rcd, rbi); 1755 1756 /* Immediate refill */ 1757 rbi->page = new_page; 1758 rbi->dma_addr = new_dma_addr; 1759 rxd->addr = cpu_to_le64(rbi->dma_addr); 1760 rxd->len = rbi->len; 1761 } 1762 } 1763 1764 1765 sop_done: 1766 skb = ctx->skb; 1767 if (rcd->eop) { 1768 u32 mtu = adapter->netdev->mtu; 1769 skb->len += skb->data_len; 1770 1771 #ifdef VMXNET3_RSS 1772 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE && 1773 (adapter->netdev->features & NETIF_F_RXHASH)) { 1774 enum pkt_hash_types hash_type; 1775 1776 switch (rcd->rssType) { 1777 case VMXNET3_RCD_RSS_TYPE_IPV4: 1778 case VMXNET3_RCD_RSS_TYPE_IPV6: 1779 hash_type = PKT_HASH_TYPE_L3; 1780 break; 1781 case VMXNET3_RCD_RSS_TYPE_TCPIPV4: 1782 case VMXNET3_RCD_RSS_TYPE_TCPIPV6: 1783 case VMXNET3_RCD_RSS_TYPE_UDPIPV4: 1784 case VMXNET3_RCD_RSS_TYPE_UDPIPV6: 1785 hash_type = PKT_HASH_TYPE_L4; 1786 break; 1787 default: 1788 hash_type = PKT_HASH_TYPE_L3; 1789 break; 1790 } 1791 skb_set_hash(skb, 1792 le32_to_cpu(rcd->rssHash), 1793 hash_type); 1794 } 1795 #endif 1796 vmxnet3_rx_csum(adapter, skb, 1797 (union Vmxnet3_GenericDesc *)rcd); 1798 skb->protocol = eth_type_trans(skb, adapter->netdev); 1799 if ((!rcd->tcp && !encap_lro) || 1800 !(adapter->netdev->features & NETIF_F_LRO)) 1801 goto not_lro; 1802 1803 if (segCnt != 0 && mss != 0) { 1804 skb_shinfo(skb)->gso_type = rcd->v4 ? 1805 SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1806 skb_shinfo(skb)->gso_size = mss; 1807 skb_shinfo(skb)->gso_segs = segCnt; 1808 } else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) { 1809 u32 hlen; 1810 1811 hlen = vmxnet3_get_hdr_len(adapter, skb, 1812 (union Vmxnet3_GenericDesc *)rcd); 1813 if (hlen == 0) 1814 goto not_lro; 1815 1816 skb_shinfo(skb)->gso_type = 1817 rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6; 1818 if (segCnt != 0) { 1819 skb_shinfo(skb)->gso_segs = segCnt; 1820 skb_shinfo(skb)->gso_size = 1821 DIV_ROUND_UP(skb->len - 1822 hlen, segCnt); 1823 } else { 1824 skb_shinfo(skb)->gso_size = mtu - hlen; 1825 } 1826 } 1827 not_lro: 1828 if (unlikely(rcd->ts)) 1829 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci); 1830 1831 /* Use GRO callback if UPT is enabled */ 1832 if ((adapter->netdev->features & NETIF_F_LRO) && 1833 !rq->shared->updateRxProd) 1834 netif_receive_skb(skb); 1835 else 1836 napi_gro_receive(&rq->napi, skb); 1837 1838 ctx->skb = NULL; 1839 encap_lro = false; 1840 num_pkts++; 1841 } 1842 1843 rcd_done: 1844 /* device may have skipped some rx descs */ 1845 ring = rq->rx_ring + ring_idx; 1846 rbi->comp_state = VMXNET3_RXD_COMP_DONE; 1847 1848 comp_offset = vmxnet3_cmd_ring_desc_avail(ring); 1849 fill_offset = (idx > ring->next2fill ? 0 : ring->size) + 1850 idx - ring->next2fill - 1; 1851 if (!ring->isOutOfOrder || fill_offset >= comp_offset) 1852 ring->next2comp = idx; 1853 num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring); 1854 1855 /* Ensure that the writes to rxd->gen bits will be observed 1856 * after all other writes to rxd objects. 1857 */ 1858 dma_wmb(); 1859 1860 while (num_to_alloc) { 1861 rbi = rq->buf_info[ring_idx] + ring->next2fill; 1862 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP))) 1863 goto refill_buf; 1864 if (ring_idx == 0) { 1865 /* ring0 Type1 buffers can get skipped; re-fill them */ 1866 if (rbi->buf_type != VMXNET3_RX_BUF_SKB) 1867 goto refill_buf; 1868 } 1869 if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) { 1870 refill_buf: 1871 vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd, 1872 &rxCmdDesc); 1873 WARN_ON(!rxd->addr); 1874 1875 /* Recv desc is ready to be used by the device */ 1876 rxd->gen = ring->gen; 1877 vmxnet3_cmd_ring_adv_next2fill(ring); 1878 rbi->comp_state = VMXNET3_RXD_COMP_PENDING; 1879 num_to_alloc--; 1880 } else { 1881 /* rx completion hasn't occurred */ 1882 ring->isOutOfOrder = 1; 1883 break; 1884 } 1885 } 1886 1887 if (num_to_alloc == 0) { 1888 ring->isOutOfOrder = 0; 1889 } 1890 1891 /* if needed, update the register */ 1892 if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) { 1893 VMXNET3_WRITE_BAR0_REG(adapter, 1894 rxprod_reg[ring_idx] + rq->qid * 8, 1895 ring->next2fill); 1896 } 1897 1898 vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring); 1899 vmxnet3_getRxComp(rcd, 1900 &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp); 1901 } 1902 if (need_flush) 1903 xdp_do_flush(); 1904 1905 return num_pkts; 1906 } 1907 1908 1909 static void 1910 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq, 1911 struct vmxnet3_adapter *adapter) 1912 { 1913 u32 i, ring_idx; 1914 struct Vmxnet3_RxDesc *rxd; 1915 1916 /* ring has already been cleaned up */ 1917 if (!rq->rx_ring[0].base) 1918 return; 1919 1920 for (ring_idx = 0; ring_idx < 2; ring_idx++) { 1921 for (i = 0; i < rq->rx_ring[ring_idx].size; i++) { 1922 struct vmxnet3_rx_buf_info *rbi; 1923 #ifdef __BIG_ENDIAN_BITFIELD 1924 struct Vmxnet3_RxDesc rxDesc; 1925 #endif 1926 1927 rbi = &rq->buf_info[ring_idx][i]; 1928 vmxnet3_getRxDesc(rxd, 1929 &rq->rx_ring[ring_idx].base[i].rxd, &rxDesc); 1930 1931 if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 1932 rbi->page && rbi->buf_type == VMXNET3_RX_BUF_XDP) { 1933 page_pool_recycle_direct(rq->page_pool, 1934 rbi->page); 1935 rbi->page = NULL; 1936 } else if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD && 1937 rbi->skb) { 1938 dma_unmap_single(&adapter->pdev->dev, rxd->addr, 1939 rxd->len, DMA_FROM_DEVICE); 1940 dev_kfree_skb(rbi->skb); 1941 rbi->skb = NULL; 1942 } else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY && 1943 rbi->page) { 1944 dma_unmap_page(&adapter->pdev->dev, rxd->addr, 1945 rxd->len, DMA_FROM_DEVICE); 1946 put_page(rbi->page); 1947 rbi->page = NULL; 1948 } 1949 } 1950 1951 rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN; 1952 rq->rx_ring[ring_idx].next2fill = 1953 rq->rx_ring[ring_idx].next2comp = 0; 1954 } 1955 1956 rq->comp_ring.gen = VMXNET3_INIT_GEN; 1957 rq->comp_ring.next2proc = 0; 1958 } 1959 1960 1961 static void 1962 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter) 1963 { 1964 int i; 1965 1966 for (i = 0; i < adapter->num_rx_queues; i++) 1967 vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter); 1968 rcu_assign_pointer(adapter->xdp_bpf_prog, NULL); 1969 } 1970 1971 1972 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq, 1973 struct vmxnet3_adapter *adapter) 1974 { 1975 int i; 1976 int j; 1977 1978 /* all rx buffers must have already been freed */ 1979 for (i = 0; i < 2; i++) { 1980 if (rq->buf_info[i]) { 1981 for (j = 0; j < rq->rx_ring[i].size; j++) 1982 BUG_ON(rq->buf_info[i][j].page != NULL); 1983 } 1984 } 1985 1986 1987 for (i = 0; i < 2; i++) { 1988 if (rq->rx_ring[i].base) { 1989 dma_free_coherent(&adapter->pdev->dev, 1990 rq->rx_ring[i].size 1991 * sizeof(struct Vmxnet3_RxDesc), 1992 rq->rx_ring[i].base, 1993 rq->rx_ring[i].basePA); 1994 rq->rx_ring[i].base = NULL; 1995 } 1996 } 1997 1998 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) 1999 xdp_rxq_info_unreg(&rq->xdp_rxq); 2000 page_pool_destroy(rq->page_pool); 2001 rq->page_pool = NULL; 2002 2003 if (rq->data_ring.base) { 2004 dma_free_coherent(&adapter->pdev->dev, 2005 rq->rx_ring[0].size * rq->data_ring.desc_size, 2006 rq->data_ring.base, rq->data_ring.basePA); 2007 rq->data_ring.base = NULL; 2008 } 2009 2010 if (rq->comp_ring.base) { 2011 dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size 2012 * sizeof(struct Vmxnet3_RxCompDesc), 2013 rq->comp_ring.base, rq->comp_ring.basePA); 2014 rq->comp_ring.base = NULL; 2015 } 2016 2017 kfree(rq->buf_info[0]); 2018 rq->buf_info[0] = NULL; 2019 rq->buf_info[1] = NULL; 2020 } 2021 2022 static void 2023 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter) 2024 { 2025 int i; 2026 2027 for (i = 0; i < adapter->num_rx_queues; i++) { 2028 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2029 2030 if (rq->data_ring.base) { 2031 dma_free_coherent(&adapter->pdev->dev, 2032 (rq->rx_ring[0].size * 2033 rq->data_ring.desc_size), 2034 rq->data_ring.base, 2035 rq->data_ring.basePA); 2036 rq->data_ring.base = NULL; 2037 rq->data_ring.desc_size = 0; 2038 } 2039 } 2040 } 2041 2042 static int 2043 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq, 2044 struct vmxnet3_adapter *adapter) 2045 { 2046 int i, err; 2047 2048 /* initialize buf_info */ 2049 for (i = 0; i < rq->rx_ring[0].size; i++) { 2050 2051 /* 1st buf for a pkt is skbuff or xdp page */ 2052 if (i % adapter->rx_buf_per_pkt == 0) { 2053 rq->buf_info[0][i].buf_type = vmxnet3_xdp_enabled(adapter) ? 2054 VMXNET3_RX_BUF_XDP : 2055 VMXNET3_RX_BUF_SKB; 2056 rq->buf_info[0][i].len = adapter->skb_buf_size; 2057 } else { /* subsequent bufs for a pkt is frag */ 2058 rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE; 2059 rq->buf_info[0][i].len = PAGE_SIZE; 2060 } 2061 } 2062 for (i = 0; i < rq->rx_ring[1].size; i++) { 2063 rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE; 2064 rq->buf_info[1][i].len = PAGE_SIZE; 2065 } 2066 2067 /* reset internal state and allocate buffers for both rings */ 2068 for (i = 0; i < 2; i++) { 2069 rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0; 2070 2071 memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size * 2072 sizeof(struct Vmxnet3_RxDesc)); 2073 rq->rx_ring[i].gen = VMXNET3_INIT_GEN; 2074 rq->rx_ring[i].isOutOfOrder = 0; 2075 } 2076 2077 err = vmxnet3_create_pp(adapter, rq, 2078 rq->rx_ring[0].size + rq->rx_ring[1].size); 2079 if (err) 2080 return err; 2081 2082 if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1, 2083 adapter) == 0) { 2084 xdp_rxq_info_unreg(&rq->xdp_rxq); 2085 page_pool_destroy(rq->page_pool); 2086 rq->page_pool = NULL; 2087 2088 /* at least has 1 rx buffer for the 1st ring */ 2089 return -ENOMEM; 2090 } 2091 vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter); 2092 2093 /* reset the comp ring */ 2094 rq->comp_ring.next2proc = 0; 2095 memset(rq->comp_ring.base, 0, rq->comp_ring.size * 2096 sizeof(struct Vmxnet3_RxCompDesc)); 2097 rq->comp_ring.gen = VMXNET3_INIT_GEN; 2098 2099 /* reset rxctx */ 2100 rq->rx_ctx.skb = NULL; 2101 2102 /* stats are not reset */ 2103 return 0; 2104 } 2105 2106 2107 static int 2108 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter) 2109 { 2110 int i, err = 0; 2111 2112 for (i = 0; i < adapter->num_rx_queues; i++) { 2113 err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter); 2114 if (unlikely(err)) { 2115 dev_err(&adapter->netdev->dev, "%s: failed to " 2116 "initialize rx queue%i\n", 2117 adapter->netdev->name, i); 2118 break; 2119 } 2120 } 2121 return err; 2122 2123 } 2124 2125 2126 static int 2127 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter) 2128 { 2129 int i; 2130 size_t sz; 2131 struct vmxnet3_rx_buf_info *bi; 2132 2133 for (i = 0; i < 2; i++) { 2134 2135 sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc); 2136 rq->rx_ring[i].base = dma_alloc_coherent( 2137 &adapter->pdev->dev, sz, 2138 &rq->rx_ring[i].basePA, 2139 GFP_KERNEL); 2140 if (!rq->rx_ring[i].base) { 2141 netdev_err(adapter->netdev, 2142 "failed to allocate rx ring %d\n", i); 2143 goto err; 2144 } 2145 } 2146 2147 if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) { 2148 sz = rq->rx_ring[0].size * rq->data_ring.desc_size; 2149 rq->data_ring.base = 2150 dma_alloc_coherent(&adapter->pdev->dev, sz, 2151 &rq->data_ring.basePA, 2152 GFP_KERNEL); 2153 if (!rq->data_ring.base) { 2154 netdev_err(adapter->netdev, 2155 "rx data ring will be disabled\n"); 2156 adapter->rxdataring_enabled = false; 2157 } 2158 } else { 2159 rq->data_ring.base = NULL; 2160 rq->data_ring.desc_size = 0; 2161 } 2162 2163 sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc); 2164 rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz, 2165 &rq->comp_ring.basePA, 2166 GFP_KERNEL); 2167 if (!rq->comp_ring.base) { 2168 netdev_err(adapter->netdev, "failed to allocate rx comp ring\n"); 2169 goto err; 2170 } 2171 2172 bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size, 2173 sizeof(rq->buf_info[0][0]), GFP_KERNEL, 2174 dev_to_node(&adapter->pdev->dev)); 2175 if (!bi) 2176 goto err; 2177 2178 rq->buf_info[0] = bi; 2179 rq->buf_info[1] = bi + rq->rx_ring[0].size; 2180 2181 return 0; 2182 2183 err: 2184 vmxnet3_rq_destroy(rq, adapter); 2185 return -ENOMEM; 2186 } 2187 2188 2189 int 2190 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter) 2191 { 2192 int i, err = 0; 2193 2194 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 2195 2196 for (i = 0; i < adapter->num_rx_queues; i++) { 2197 err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter); 2198 if (unlikely(err)) { 2199 dev_err(&adapter->netdev->dev, 2200 "%s: failed to create rx queue%i\n", 2201 adapter->netdev->name, i); 2202 goto err_out; 2203 } 2204 } 2205 2206 if (!adapter->rxdataring_enabled) 2207 vmxnet3_rq_destroy_all_rxdataring(adapter); 2208 2209 return err; 2210 err_out: 2211 vmxnet3_rq_destroy_all(adapter); 2212 return err; 2213 2214 } 2215 2216 /* Multiple queue aware polling function for tx and rx */ 2217 2218 static int 2219 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget) 2220 { 2221 int rcd_done = 0, i; 2222 if (unlikely(adapter->shared->ecr)) 2223 vmxnet3_process_events(adapter); 2224 for (i = 0; i < adapter->num_tx_queues; i++) 2225 vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter); 2226 2227 for (i = 0; i < adapter->num_rx_queues; i++) 2228 rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i], 2229 adapter, budget); 2230 return rcd_done; 2231 } 2232 2233 2234 static int 2235 vmxnet3_poll(struct napi_struct *napi, int budget) 2236 { 2237 struct vmxnet3_rx_queue *rx_queue = container_of(napi, 2238 struct vmxnet3_rx_queue, napi); 2239 int rxd_done; 2240 2241 rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget); 2242 2243 if (rxd_done < budget) { 2244 napi_complete_done(napi, rxd_done); 2245 vmxnet3_enable_all_intrs(rx_queue->adapter); 2246 } 2247 return rxd_done; 2248 } 2249 2250 /* 2251 * NAPI polling function for MSI-X mode with multiple Rx queues 2252 * Returns the # of the NAPI credit consumed (# of rx descriptors processed) 2253 */ 2254 2255 static int 2256 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget) 2257 { 2258 struct vmxnet3_rx_queue *rq = container_of(napi, 2259 struct vmxnet3_rx_queue, napi); 2260 struct vmxnet3_adapter *adapter = rq->adapter; 2261 int rxd_done; 2262 2263 /* When sharing interrupt with corresponding tx queue, process 2264 * tx completions in that queue as well 2265 */ 2266 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) { 2267 struct vmxnet3_tx_queue *tq = 2268 &adapter->tx_queue[rq - adapter->rx_queue]; 2269 vmxnet3_tq_tx_complete(tq, adapter); 2270 } 2271 2272 rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget); 2273 2274 if (rxd_done < budget) { 2275 napi_complete_done(napi, rxd_done); 2276 vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx); 2277 } 2278 return rxd_done; 2279 } 2280 2281 2282 #ifdef CONFIG_PCI_MSI 2283 2284 /* 2285 * Handle completion interrupts on tx queues 2286 * Returns whether or not the intr is handled 2287 */ 2288 2289 static irqreturn_t 2290 vmxnet3_msix_tx(int irq, void *data) 2291 { 2292 struct vmxnet3_tx_queue *tq = data; 2293 struct vmxnet3_adapter *adapter = tq->adapter; 2294 2295 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2296 vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx); 2297 2298 /* Handle the case where only one irq is allocate for all tx queues */ 2299 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2300 int i; 2301 for (i = 0; i < adapter->num_tx_queues; i++) { 2302 struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i]; 2303 vmxnet3_tq_tx_complete(txq, adapter); 2304 } 2305 } else { 2306 vmxnet3_tq_tx_complete(tq, adapter); 2307 } 2308 vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx); 2309 2310 return IRQ_HANDLED; 2311 } 2312 2313 2314 /* 2315 * Handle completion interrupts on rx queues. Returns whether or not the 2316 * intr is handled 2317 */ 2318 2319 static irqreturn_t 2320 vmxnet3_msix_rx(int irq, void *data) 2321 { 2322 struct vmxnet3_rx_queue *rq = data; 2323 struct vmxnet3_adapter *adapter = rq->adapter; 2324 2325 /* disable intr if needed */ 2326 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2327 vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx); 2328 napi_schedule(&rq->napi); 2329 2330 return IRQ_HANDLED; 2331 } 2332 2333 /* 2334 *---------------------------------------------------------------------------- 2335 * 2336 * vmxnet3_msix_event -- 2337 * 2338 * vmxnet3 msix event intr handler 2339 * 2340 * Result: 2341 * whether or not the intr is handled 2342 * 2343 *---------------------------------------------------------------------------- 2344 */ 2345 2346 static irqreturn_t 2347 vmxnet3_msix_event(int irq, void *data) 2348 { 2349 struct net_device *dev = data; 2350 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2351 2352 /* disable intr if needed */ 2353 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2354 vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx); 2355 2356 if (adapter->shared->ecr) 2357 vmxnet3_process_events(adapter); 2358 2359 vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx); 2360 2361 return IRQ_HANDLED; 2362 } 2363 2364 #endif /* CONFIG_PCI_MSI */ 2365 2366 2367 /* Interrupt handler for vmxnet3 */ 2368 static irqreturn_t 2369 vmxnet3_intr(int irq, void *dev_id) 2370 { 2371 struct net_device *dev = dev_id; 2372 struct vmxnet3_adapter *adapter = netdev_priv(dev); 2373 2374 if (adapter->intr.type == VMXNET3_IT_INTX) { 2375 u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR); 2376 if (unlikely(icr == 0)) 2377 /* not ours */ 2378 return IRQ_NONE; 2379 } 2380 2381 2382 /* disable intr if needed */ 2383 if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE) 2384 vmxnet3_disable_all_intrs(adapter); 2385 2386 napi_schedule(&adapter->rx_queue[0].napi); 2387 2388 return IRQ_HANDLED; 2389 } 2390 2391 #ifdef CONFIG_NET_POLL_CONTROLLER 2392 2393 /* netpoll callback. */ 2394 static void 2395 vmxnet3_netpoll(struct net_device *netdev) 2396 { 2397 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2398 2399 switch (adapter->intr.type) { 2400 #ifdef CONFIG_PCI_MSI 2401 case VMXNET3_IT_MSIX: { 2402 int i; 2403 for (i = 0; i < adapter->num_rx_queues; i++) 2404 vmxnet3_msix_rx(0, &adapter->rx_queue[i]); 2405 break; 2406 } 2407 #endif 2408 case VMXNET3_IT_MSI: 2409 default: 2410 vmxnet3_intr(0, adapter->netdev); 2411 break; 2412 } 2413 2414 } 2415 #endif /* CONFIG_NET_POLL_CONTROLLER */ 2416 2417 static int 2418 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter) 2419 { 2420 struct vmxnet3_intr *intr = &adapter->intr; 2421 int err = 0, i; 2422 int vector = 0; 2423 2424 #ifdef CONFIG_PCI_MSI 2425 if (adapter->intr.type == VMXNET3_IT_MSIX) { 2426 for (i = 0; i < adapter->num_tx_queues; i++) { 2427 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2428 sprintf(adapter->tx_queue[i].name, "%s-tx-%d", 2429 adapter->netdev->name, vector); 2430 err = request_irq( 2431 intr->msix_entries[vector].vector, 2432 vmxnet3_msix_tx, 0, 2433 adapter->tx_queue[i].name, 2434 &adapter->tx_queue[i]); 2435 } else { 2436 sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d", 2437 adapter->netdev->name, vector); 2438 } 2439 if (err) { 2440 dev_err(&adapter->netdev->dev, 2441 "Failed to request irq for MSIX, %s, " 2442 "error %d\n", 2443 adapter->tx_queue[i].name, err); 2444 return err; 2445 } 2446 2447 /* Handle the case where only 1 MSIx was allocated for 2448 * all tx queues */ 2449 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) { 2450 for (; i < adapter->num_tx_queues; i++) 2451 adapter->tx_queue[i].comp_ring.intr_idx 2452 = vector; 2453 vector++; 2454 break; 2455 } else { 2456 adapter->tx_queue[i].comp_ring.intr_idx 2457 = vector++; 2458 } 2459 } 2460 if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) 2461 vector = 0; 2462 2463 for (i = 0; i < adapter->num_rx_queues; i++) { 2464 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) 2465 sprintf(adapter->rx_queue[i].name, "%s-rx-%d", 2466 adapter->netdev->name, vector); 2467 else 2468 sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d", 2469 adapter->netdev->name, vector); 2470 err = request_irq(intr->msix_entries[vector].vector, 2471 vmxnet3_msix_rx, 0, 2472 adapter->rx_queue[i].name, 2473 &(adapter->rx_queue[i])); 2474 if (err) { 2475 netdev_err(adapter->netdev, 2476 "Failed to request irq for MSIX, " 2477 "%s, error %d\n", 2478 adapter->rx_queue[i].name, err); 2479 return err; 2480 } 2481 2482 adapter->rx_queue[i].comp_ring.intr_idx = vector++; 2483 } 2484 2485 sprintf(intr->event_msi_vector_name, "%s-event-%d", 2486 adapter->netdev->name, vector); 2487 err = request_irq(intr->msix_entries[vector].vector, 2488 vmxnet3_msix_event, 0, 2489 intr->event_msi_vector_name, adapter->netdev); 2490 intr->event_intr_idx = vector; 2491 2492 } else if (intr->type == VMXNET3_IT_MSI) { 2493 adapter->num_rx_queues = 1; 2494 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0, 2495 adapter->netdev->name, adapter->netdev); 2496 } else { 2497 #endif 2498 adapter->num_rx_queues = 1; 2499 err = request_irq(adapter->pdev->irq, vmxnet3_intr, 2500 IRQF_SHARED, adapter->netdev->name, 2501 adapter->netdev); 2502 #ifdef CONFIG_PCI_MSI 2503 } 2504 #endif 2505 intr->num_intrs = vector + 1; 2506 if (err) { 2507 netdev_err(adapter->netdev, 2508 "Failed to request irq (intr type:%d), error %d\n", 2509 intr->type, err); 2510 } else { 2511 /* Number of rx queues will not change after this */ 2512 for (i = 0; i < adapter->num_rx_queues; i++) { 2513 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2514 rq->qid = i; 2515 rq->qid2 = i + adapter->num_rx_queues; 2516 rq->dataRingQid = i + 2 * adapter->num_rx_queues; 2517 } 2518 2519 /* init our intr settings */ 2520 for (i = 0; i < intr->num_intrs; i++) 2521 intr->mod_levels[i] = UPT1_IML_ADAPTIVE; 2522 if (adapter->intr.type != VMXNET3_IT_MSIX) { 2523 adapter->intr.event_intr_idx = 0; 2524 for (i = 0; i < adapter->num_tx_queues; i++) 2525 adapter->tx_queue[i].comp_ring.intr_idx = 0; 2526 adapter->rx_queue[0].comp_ring.intr_idx = 0; 2527 } 2528 2529 netdev_info(adapter->netdev, 2530 "intr type %u, mode %u, %u vectors allocated\n", 2531 intr->type, intr->mask_mode, intr->num_intrs); 2532 } 2533 2534 return err; 2535 } 2536 2537 2538 static void 2539 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter) 2540 { 2541 struct vmxnet3_intr *intr = &adapter->intr; 2542 BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0); 2543 2544 switch (intr->type) { 2545 #ifdef CONFIG_PCI_MSI 2546 case VMXNET3_IT_MSIX: 2547 { 2548 int i, vector = 0; 2549 2550 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) { 2551 for (i = 0; i < adapter->num_tx_queues; i++) { 2552 free_irq(intr->msix_entries[vector++].vector, 2553 &(adapter->tx_queue[i])); 2554 if (adapter->share_intr == VMXNET3_INTR_TXSHARE) 2555 break; 2556 } 2557 } 2558 2559 for (i = 0; i < adapter->num_rx_queues; i++) { 2560 free_irq(intr->msix_entries[vector++].vector, 2561 &(adapter->rx_queue[i])); 2562 } 2563 2564 free_irq(intr->msix_entries[vector].vector, 2565 adapter->netdev); 2566 BUG_ON(vector >= intr->num_intrs); 2567 break; 2568 } 2569 #endif 2570 case VMXNET3_IT_MSI: 2571 free_irq(adapter->pdev->irq, adapter->netdev); 2572 break; 2573 case VMXNET3_IT_INTX: 2574 free_irq(adapter->pdev->irq, adapter->netdev); 2575 break; 2576 default: 2577 BUG(); 2578 } 2579 } 2580 2581 2582 static void 2583 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter) 2584 { 2585 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2586 u16 vid; 2587 2588 /* allow untagged pkts */ 2589 VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0); 2590 2591 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID) 2592 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2593 } 2594 2595 2596 static int 2597 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 2598 { 2599 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2600 2601 if (!(netdev->flags & IFF_PROMISC)) { 2602 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2603 unsigned long flags; 2604 2605 VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid); 2606 spin_lock_irqsave(&adapter->cmd_lock, flags); 2607 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2608 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2609 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2610 } 2611 2612 set_bit(vid, adapter->active_vlans); 2613 2614 return 0; 2615 } 2616 2617 2618 static int 2619 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 2620 { 2621 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2622 2623 if (!(netdev->flags & IFF_PROMISC)) { 2624 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2625 unsigned long flags; 2626 2627 VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid); 2628 spin_lock_irqsave(&adapter->cmd_lock, flags); 2629 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2630 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2631 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2632 } 2633 2634 clear_bit(vid, adapter->active_vlans); 2635 2636 return 0; 2637 } 2638 2639 2640 static u8 * 2641 vmxnet3_copy_mc(struct net_device *netdev) 2642 { 2643 u8 *buf = NULL; 2644 u32 sz = netdev_mc_count(netdev) * ETH_ALEN; 2645 2646 /* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */ 2647 if (sz <= 0xffff) { 2648 /* We may be called with BH disabled */ 2649 buf = kmalloc(sz, GFP_ATOMIC); 2650 if (buf) { 2651 struct netdev_hw_addr *ha; 2652 int i = 0; 2653 2654 netdev_for_each_mc_addr(ha, netdev) 2655 memcpy(buf + i++ * ETH_ALEN, ha->addr, 2656 ETH_ALEN); 2657 } 2658 } 2659 return buf; 2660 } 2661 2662 2663 static void 2664 vmxnet3_set_mc(struct net_device *netdev) 2665 { 2666 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 2667 unsigned long flags; 2668 struct Vmxnet3_RxFilterConf *rxConf = 2669 &adapter->shared->devRead.rxFilterConf; 2670 u8 *new_table = NULL; 2671 dma_addr_t new_table_pa = 0; 2672 bool new_table_pa_valid = false; 2673 u32 new_mode = VMXNET3_RXM_UCAST; 2674 2675 if (netdev->flags & IFF_PROMISC) { 2676 u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable; 2677 memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable)); 2678 2679 new_mode |= VMXNET3_RXM_PROMISC; 2680 } else { 2681 vmxnet3_restore_vlan(adapter); 2682 } 2683 2684 if (netdev->flags & IFF_BROADCAST) 2685 new_mode |= VMXNET3_RXM_BCAST; 2686 2687 if (netdev->flags & IFF_ALLMULTI) 2688 new_mode |= VMXNET3_RXM_ALL_MULTI; 2689 else 2690 if (!netdev_mc_empty(netdev)) { 2691 new_table = vmxnet3_copy_mc(netdev); 2692 if (new_table) { 2693 size_t sz = netdev_mc_count(netdev) * ETH_ALEN; 2694 2695 rxConf->mfTableLen = cpu_to_le16(sz); 2696 new_table_pa = dma_map_single( 2697 &adapter->pdev->dev, 2698 new_table, 2699 sz, 2700 DMA_TO_DEVICE); 2701 if (!dma_mapping_error(&adapter->pdev->dev, 2702 new_table_pa)) { 2703 new_mode |= VMXNET3_RXM_MCAST; 2704 new_table_pa_valid = true; 2705 rxConf->mfTablePA = cpu_to_le64( 2706 new_table_pa); 2707 } 2708 } 2709 if (!new_table_pa_valid) { 2710 netdev_info(netdev, 2711 "failed to copy mcast list, setting ALL_MULTI\n"); 2712 new_mode |= VMXNET3_RXM_ALL_MULTI; 2713 } 2714 } 2715 2716 if (!(new_mode & VMXNET3_RXM_MCAST)) { 2717 rxConf->mfTableLen = 0; 2718 rxConf->mfTablePA = 0; 2719 } 2720 2721 spin_lock_irqsave(&adapter->cmd_lock, flags); 2722 if (new_mode != rxConf->rxMode) { 2723 rxConf->rxMode = cpu_to_le32(new_mode); 2724 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2725 VMXNET3_CMD_UPDATE_RX_MODE); 2726 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2727 VMXNET3_CMD_UPDATE_VLAN_FILTERS); 2728 } 2729 2730 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2731 VMXNET3_CMD_UPDATE_MAC_FILTERS); 2732 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2733 2734 if (new_table_pa_valid) 2735 dma_unmap_single(&adapter->pdev->dev, new_table_pa, 2736 rxConf->mfTableLen, DMA_TO_DEVICE); 2737 kfree(new_table); 2738 } 2739 2740 void 2741 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter) 2742 { 2743 int i; 2744 2745 for (i = 0; i < adapter->num_rx_queues; i++) 2746 vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter); 2747 } 2748 2749 2750 /* 2751 * Set up driver_shared based on settings in adapter. 2752 */ 2753 2754 static void 2755 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter) 2756 { 2757 struct Vmxnet3_DriverShared *shared = adapter->shared; 2758 struct Vmxnet3_DSDevRead *devRead = &shared->devRead; 2759 struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt; 2760 struct Vmxnet3_TxQueueConf *tqc; 2761 struct Vmxnet3_RxQueueConf *rqc; 2762 int i; 2763 2764 memset(shared, 0, sizeof(*shared)); 2765 2766 /* driver settings */ 2767 shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC); 2768 devRead->misc.driverInfo.version = cpu_to_le32( 2769 VMXNET3_DRIVER_VERSION_NUM); 2770 devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ? 2771 VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64); 2772 devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX; 2773 *((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32( 2774 *((u32 *)&devRead->misc.driverInfo.gos)); 2775 devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1); 2776 devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1); 2777 2778 devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa); 2779 devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter)); 2780 2781 /* set up feature flags */ 2782 if (adapter->netdev->features & NETIF_F_RXCSUM) 2783 devRead->misc.uptFeatures |= UPT1_F_RXCSUM; 2784 2785 if (adapter->netdev->features & NETIF_F_LRO) { 2786 devRead->misc.uptFeatures |= UPT1_F_LRO; 2787 devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS); 2788 } 2789 if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2790 devRead->misc.uptFeatures |= UPT1_F_RXVLAN; 2791 2792 if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL | 2793 NETIF_F_GSO_UDP_TUNNEL_CSUM)) 2794 devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD; 2795 2796 devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu); 2797 devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa); 2798 devRead->misc.queueDescLen = cpu_to_le32( 2799 adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) + 2800 adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc)); 2801 2802 /* tx queue settings */ 2803 devRead->misc.numTxQueues = adapter->num_tx_queues; 2804 for (i = 0; i < adapter->num_tx_queues; i++) { 2805 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 2806 BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL); 2807 tqc = &adapter->tqd_start[i].conf; 2808 tqc->txRingBasePA = cpu_to_le64(tq->tx_ring.basePA); 2809 tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA); 2810 tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA); 2811 tqc->ddPA = cpu_to_le64(~0ULL); 2812 tqc->txRingSize = cpu_to_le32(tq->tx_ring.size); 2813 tqc->dataRingSize = cpu_to_le32(tq->data_ring.size); 2814 tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size); 2815 tqc->compRingSize = cpu_to_le32(tq->comp_ring.size); 2816 tqc->ddLen = cpu_to_le32(0); 2817 tqc->intrIdx = tq->comp_ring.intr_idx; 2818 } 2819 2820 /* rx queue settings */ 2821 devRead->misc.numRxQueues = adapter->num_rx_queues; 2822 for (i = 0; i < adapter->num_rx_queues; i++) { 2823 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 2824 rqc = &adapter->rqd_start[i].conf; 2825 rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA); 2826 rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA); 2827 rqc->compRingBasePA = cpu_to_le64(rq->comp_ring.basePA); 2828 rqc->ddPA = cpu_to_le64(~0ULL); 2829 rqc->rxRingSize[0] = cpu_to_le32(rq->rx_ring[0].size); 2830 rqc->rxRingSize[1] = cpu_to_le32(rq->rx_ring[1].size); 2831 rqc->compRingSize = cpu_to_le32(rq->comp_ring.size); 2832 rqc->ddLen = cpu_to_le32(0); 2833 rqc->intrIdx = rq->comp_ring.intr_idx; 2834 if (VMXNET3_VERSION_GE_3(adapter)) { 2835 rqc->rxDataRingBasePA = 2836 cpu_to_le64(rq->data_ring.basePA); 2837 rqc->rxDataRingDescSize = 2838 cpu_to_le16(rq->data_ring.desc_size); 2839 } 2840 } 2841 2842 #ifdef VMXNET3_RSS 2843 memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf)); 2844 2845 if (adapter->rss) { 2846 struct UPT1_RSSConf *rssConf = adapter->rss_conf; 2847 2848 devRead->misc.uptFeatures |= UPT1_F_RSS; 2849 devRead->misc.numRxQueues = adapter->num_rx_queues; 2850 rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 | 2851 UPT1_RSS_HASH_TYPE_IPV4 | 2852 UPT1_RSS_HASH_TYPE_TCP_IPV6 | 2853 UPT1_RSS_HASH_TYPE_IPV6; 2854 rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ; 2855 rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE; 2856 rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE; 2857 netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey)); 2858 2859 for (i = 0; i < rssConf->indTableSize; i++) 2860 rssConf->indTable[i] = ethtool_rxfh_indir_default( 2861 i, adapter->num_rx_queues); 2862 2863 devRead->rssConfDesc.confVer = 1; 2864 devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf)); 2865 devRead->rssConfDesc.confPA = 2866 cpu_to_le64(adapter->rss_conf_pa); 2867 } 2868 2869 #endif /* VMXNET3_RSS */ 2870 2871 /* intr settings */ 2872 if (!VMXNET3_VERSION_GE_6(adapter) || 2873 !adapter->queuesExtEnabled) { 2874 devRead->intrConf.autoMask = adapter->intr.mask_mode == 2875 VMXNET3_IMM_AUTO; 2876 devRead->intrConf.numIntrs = adapter->intr.num_intrs; 2877 for (i = 0; i < adapter->intr.num_intrs; i++) 2878 devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i]; 2879 2880 devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx; 2881 devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2882 } else { 2883 devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode == 2884 VMXNET3_IMM_AUTO; 2885 devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs; 2886 for (i = 0; i < adapter->intr.num_intrs; i++) 2887 devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i]; 2888 2889 devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx; 2890 devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL); 2891 } 2892 2893 /* rx filter settings */ 2894 devRead->rxFilterConf.rxMode = 0; 2895 vmxnet3_restore_vlan(adapter); 2896 vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr); 2897 2898 /* the rest are already zeroed */ 2899 } 2900 2901 static void 2902 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter) 2903 { 2904 struct Vmxnet3_DriverShared *shared = adapter->shared; 2905 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2906 unsigned long flags; 2907 2908 if (!VMXNET3_VERSION_GE_7(adapter)) 2909 return; 2910 2911 cmdInfo->ringBufSize = adapter->ringBufSize; 2912 spin_lock_irqsave(&adapter->cmd_lock, flags); 2913 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2914 VMXNET3_CMD_SET_RING_BUFFER_SIZE); 2915 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2916 } 2917 2918 static void 2919 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter) 2920 { 2921 struct Vmxnet3_DriverShared *shared = adapter->shared; 2922 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2923 unsigned long flags; 2924 2925 if (!VMXNET3_VERSION_GE_3(adapter)) 2926 return; 2927 2928 spin_lock_irqsave(&adapter->cmd_lock, flags); 2929 cmdInfo->varConf.confVer = 1; 2930 cmdInfo->varConf.confLen = 2931 cpu_to_le32(sizeof(*adapter->coal_conf)); 2932 cmdInfo->varConf.confPA = cpu_to_le64(adapter->coal_conf_pa); 2933 2934 if (adapter->default_coal_mode) { 2935 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2936 VMXNET3_CMD_GET_COALESCE); 2937 } else { 2938 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2939 VMXNET3_CMD_SET_COALESCE); 2940 } 2941 2942 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 2943 } 2944 2945 static void 2946 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter) 2947 { 2948 struct Vmxnet3_DriverShared *shared = adapter->shared; 2949 union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo; 2950 unsigned long flags; 2951 2952 if (!VMXNET3_VERSION_GE_4(adapter)) 2953 return; 2954 2955 spin_lock_irqsave(&adapter->cmd_lock, flags); 2956 2957 if (adapter->default_rss_fields) { 2958 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2959 VMXNET3_CMD_GET_RSS_FIELDS); 2960 adapter->rss_fields = 2961 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2962 } else { 2963 if (VMXNET3_VERSION_GE_7(adapter)) { 2964 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 || 2965 adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) && 2966 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2967 VMXNET3_CAP_UDP_RSS)) { 2968 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS; 2969 } else { 2970 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS); 2971 } 2972 2973 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) && 2974 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2975 VMXNET3_CAP_ESP_RSS_IPV4)) { 2976 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4; 2977 } else { 2978 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4); 2979 } 2980 2981 if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) && 2982 vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 2983 VMXNET3_CAP_ESP_RSS_IPV6)) { 2984 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6; 2985 } else { 2986 adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6); 2987 } 2988 2989 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 2990 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 2991 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 2992 } 2993 cmdInfo->setRssFields = adapter->rss_fields; 2994 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 2995 VMXNET3_CMD_SET_RSS_FIELDS); 2996 /* Not all requested RSS may get applied, so get and 2997 * cache what was actually applied. 2998 */ 2999 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3000 VMXNET3_CMD_GET_RSS_FIELDS); 3001 adapter->rss_fields = 3002 VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3003 } 3004 3005 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3006 } 3007 3008 int 3009 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter) 3010 { 3011 int err, i; 3012 u32 ret; 3013 unsigned long flags; 3014 3015 netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d," 3016 " ring sizes %u %u %u\n", adapter->netdev->name, 3017 adapter->skb_buf_size, adapter->rx_buf_per_pkt, 3018 adapter->tx_queue[0].tx_ring.size, 3019 adapter->rx_queue[0].rx_ring[0].size, 3020 adapter->rx_queue[0].rx_ring[1].size); 3021 3022 vmxnet3_tq_init_all(adapter); 3023 err = vmxnet3_rq_init_all(adapter); 3024 if (err) { 3025 netdev_err(adapter->netdev, 3026 "Failed to init rx queue error %d\n", err); 3027 goto rq_err; 3028 } 3029 3030 err = vmxnet3_request_irqs(adapter); 3031 if (err) { 3032 netdev_err(adapter->netdev, 3033 "Failed to setup irq for error %d\n", err); 3034 goto irq_err; 3035 } 3036 3037 vmxnet3_setup_driver_shared(adapter); 3038 3039 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO( 3040 adapter->shared_pa)); 3041 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI( 3042 adapter->shared_pa)); 3043 spin_lock_irqsave(&adapter->cmd_lock, flags); 3044 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3045 VMXNET3_CMD_ACTIVATE_DEV); 3046 ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3047 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3048 3049 if (ret != 0) { 3050 netdev_err(adapter->netdev, 3051 "Failed to activate dev: error %u\n", ret); 3052 err = -EINVAL; 3053 goto activate_err; 3054 } 3055 3056 vmxnet3_init_bufsize(adapter); 3057 vmxnet3_init_coalesce(adapter); 3058 vmxnet3_init_rssfields(adapter); 3059 3060 for (i = 0; i < adapter->num_rx_queues; i++) { 3061 VMXNET3_WRITE_BAR0_REG(adapter, 3062 adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN, 3063 adapter->rx_queue[i].rx_ring[0].next2fill); 3064 VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset + 3065 (i * VMXNET3_REG_ALIGN)), 3066 adapter->rx_queue[i].rx_ring[1].next2fill); 3067 } 3068 3069 /* Apply the rx filter settins last. */ 3070 vmxnet3_set_mc(adapter->netdev); 3071 3072 /* 3073 * Check link state when first activating device. It will start the 3074 * tx queue if the link is up. 3075 */ 3076 vmxnet3_check_link(adapter, true); 3077 netif_tx_wake_all_queues(adapter->netdev); 3078 for (i = 0; i < adapter->num_rx_queues; i++) 3079 napi_enable(&adapter->rx_queue[i].napi); 3080 vmxnet3_enable_all_intrs(adapter); 3081 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3082 return 0; 3083 3084 activate_err: 3085 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0); 3086 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0); 3087 vmxnet3_free_irqs(adapter); 3088 irq_err: 3089 rq_err: 3090 /* free up buffers we allocated */ 3091 vmxnet3_rq_cleanup_all(adapter); 3092 return err; 3093 } 3094 3095 3096 void 3097 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter) 3098 { 3099 unsigned long flags; 3100 spin_lock_irqsave(&adapter->cmd_lock, flags); 3101 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV); 3102 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3103 } 3104 3105 3106 int 3107 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter) 3108 { 3109 int i; 3110 unsigned long flags; 3111 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)) 3112 return 0; 3113 3114 3115 spin_lock_irqsave(&adapter->cmd_lock, flags); 3116 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3117 VMXNET3_CMD_QUIESCE_DEV); 3118 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3119 vmxnet3_disable_all_intrs(adapter); 3120 3121 for (i = 0; i < adapter->num_rx_queues; i++) 3122 napi_disable(&adapter->rx_queue[i].napi); 3123 netif_tx_disable(adapter->netdev); 3124 adapter->link_speed = 0; 3125 netif_carrier_off(adapter->netdev); 3126 3127 vmxnet3_tq_cleanup_all(adapter); 3128 vmxnet3_rq_cleanup_all(adapter); 3129 vmxnet3_free_irqs(adapter); 3130 return 0; 3131 } 3132 3133 3134 static void 3135 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac) 3136 { 3137 u32 tmp; 3138 3139 tmp = *(u32 *)mac; 3140 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp); 3141 3142 tmp = (mac[5] << 8) | mac[4]; 3143 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp); 3144 } 3145 3146 3147 static int 3148 vmxnet3_set_mac_addr(struct net_device *netdev, void *p) 3149 { 3150 struct sockaddr *addr = p; 3151 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3152 3153 dev_addr_set(netdev, addr->sa_data); 3154 vmxnet3_write_mac_addr(adapter, addr->sa_data); 3155 3156 return 0; 3157 } 3158 3159 3160 /* ==================== initialization and cleanup routines ============ */ 3161 3162 static int 3163 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter) 3164 { 3165 int err; 3166 unsigned long mmio_start, mmio_len; 3167 struct pci_dev *pdev = adapter->pdev; 3168 3169 err = pci_enable_device(pdev); 3170 if (err) { 3171 dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err); 3172 return err; 3173 } 3174 3175 err = pci_request_selected_regions(pdev, (1 << 2) - 1, 3176 vmxnet3_driver_name); 3177 if (err) { 3178 dev_err(&pdev->dev, 3179 "Failed to request region for adapter: error %d\n", err); 3180 goto err_enable_device; 3181 } 3182 3183 pci_set_master(pdev); 3184 3185 mmio_start = pci_resource_start(pdev, 0); 3186 mmio_len = pci_resource_len(pdev, 0); 3187 adapter->hw_addr0 = ioremap(mmio_start, mmio_len); 3188 if (!adapter->hw_addr0) { 3189 dev_err(&pdev->dev, "Failed to map bar0\n"); 3190 err = -EIO; 3191 goto err_ioremap; 3192 } 3193 3194 mmio_start = pci_resource_start(pdev, 1); 3195 mmio_len = pci_resource_len(pdev, 1); 3196 adapter->hw_addr1 = ioremap(mmio_start, mmio_len); 3197 if (!adapter->hw_addr1) { 3198 dev_err(&pdev->dev, "Failed to map bar1\n"); 3199 err = -EIO; 3200 goto err_bar1; 3201 } 3202 return 0; 3203 3204 err_bar1: 3205 iounmap(adapter->hw_addr0); 3206 err_ioremap: 3207 pci_release_selected_regions(pdev, (1 << 2) - 1); 3208 err_enable_device: 3209 pci_disable_device(pdev); 3210 return err; 3211 } 3212 3213 3214 static void 3215 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter) 3216 { 3217 BUG_ON(!adapter->pdev); 3218 3219 iounmap(adapter->hw_addr0); 3220 iounmap(adapter->hw_addr1); 3221 pci_release_selected_regions(adapter->pdev, (1 << 2) - 1); 3222 pci_disable_device(adapter->pdev); 3223 } 3224 3225 3226 void 3227 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter) 3228 { 3229 size_t sz, i, ring0_size, ring1_size, comp_size; 3230 /* With version7 ring1 will have only T0 buffers */ 3231 if (!VMXNET3_VERSION_GE_7(adapter)) { 3232 if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE - 3233 VMXNET3_MAX_ETH_HDR_SIZE) { 3234 adapter->skb_buf_size = adapter->netdev->mtu + 3235 VMXNET3_MAX_ETH_HDR_SIZE; 3236 if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE) 3237 adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE; 3238 3239 adapter->rx_buf_per_pkt = 1; 3240 } else { 3241 adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE; 3242 sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE + 3243 VMXNET3_MAX_ETH_HDR_SIZE; 3244 adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE; 3245 } 3246 } else { 3247 adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE, 3248 VMXNET3_MAX_SKB_BUF_SIZE); 3249 adapter->rx_buf_per_pkt = 1; 3250 adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size); 3251 adapter->ringBufSize.ring1BufSizeType1 = 0; 3252 adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE); 3253 } 3254 3255 /* 3256 * for simplicity, force the ring0 size to be a multiple of 3257 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN 3258 */ 3259 sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN; 3260 ring0_size = adapter->rx_queue[0].rx_ring[0].size; 3261 ring0_size = (ring0_size + sz - 1) / sz * sz; 3262 ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE / 3263 sz * sz); 3264 ring1_size = adapter->rx_queue[0].rx_ring[1].size; 3265 ring1_size = (ring1_size + sz - 1) / sz * sz; 3266 ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE / 3267 sz * sz); 3268 /* For v7 and later, keep ring size power of 2 for UPT */ 3269 if (VMXNET3_VERSION_GE_7(adapter)) { 3270 ring0_size = rounddown_pow_of_two(ring0_size); 3271 ring1_size = rounddown_pow_of_two(ring1_size); 3272 } 3273 comp_size = ring0_size + ring1_size; 3274 3275 for (i = 0; i < adapter->num_rx_queues; i++) { 3276 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3277 3278 rq->rx_ring[0].size = ring0_size; 3279 rq->rx_ring[1].size = ring1_size; 3280 rq->comp_ring.size = comp_size; 3281 } 3282 } 3283 3284 3285 int 3286 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size, 3287 u32 rx_ring_size, u32 rx_ring2_size, 3288 u16 txdata_desc_size, u16 rxdata_desc_size) 3289 { 3290 int err = 0, i; 3291 3292 for (i = 0; i < adapter->num_tx_queues; i++) { 3293 struct vmxnet3_tx_queue *tq = &adapter->tx_queue[i]; 3294 tq->tx_ring.size = tx_ring_size; 3295 tq->data_ring.size = tx_ring_size; 3296 tq->comp_ring.size = tx_ring_size; 3297 tq->txdata_desc_size = txdata_desc_size; 3298 tq->shared = &adapter->tqd_start[i].ctrl; 3299 tq->stopped = true; 3300 tq->adapter = adapter; 3301 tq->qid = i; 3302 err = vmxnet3_tq_create(tq, adapter); 3303 /* 3304 * Too late to change num_tx_queues. We cannot do away with 3305 * lesser number of queues than what we asked for 3306 */ 3307 if (err) 3308 goto queue_err; 3309 } 3310 3311 adapter->rx_queue[0].rx_ring[0].size = rx_ring_size; 3312 adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size; 3313 vmxnet3_adjust_rx_ring_size(adapter); 3314 3315 adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter); 3316 for (i = 0; i < adapter->num_rx_queues; i++) { 3317 struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i]; 3318 /* qid and qid2 for rx queues will be assigned later when num 3319 * of rx queues is finalized after allocating intrs */ 3320 rq->shared = &adapter->rqd_start[i].ctrl; 3321 rq->adapter = adapter; 3322 rq->data_ring.desc_size = rxdata_desc_size; 3323 err = vmxnet3_rq_create(rq, adapter); 3324 if (err) { 3325 if (i == 0) { 3326 netdev_err(adapter->netdev, 3327 "Could not allocate any rx queues. " 3328 "Aborting.\n"); 3329 goto queue_err; 3330 } else { 3331 netdev_info(adapter->netdev, 3332 "Number of rx queues changed " 3333 "to : %d.\n", i); 3334 adapter->num_rx_queues = i; 3335 err = 0; 3336 break; 3337 } 3338 } 3339 } 3340 3341 if (!adapter->rxdataring_enabled) 3342 vmxnet3_rq_destroy_all_rxdataring(adapter); 3343 3344 return err; 3345 queue_err: 3346 vmxnet3_tq_destroy_all(adapter); 3347 return err; 3348 } 3349 3350 static int 3351 vmxnet3_open(struct net_device *netdev) 3352 { 3353 struct vmxnet3_adapter *adapter; 3354 int err, i; 3355 3356 adapter = netdev_priv(netdev); 3357 3358 for (i = 0; i < adapter->num_tx_queues; i++) 3359 spin_lock_init(&adapter->tx_queue[i].tx_lock); 3360 3361 if (VMXNET3_VERSION_GE_3(adapter)) { 3362 unsigned long flags; 3363 u16 txdata_desc_size; 3364 3365 spin_lock_irqsave(&adapter->cmd_lock, flags); 3366 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3367 VMXNET3_CMD_GET_TXDATA_DESC_SIZE); 3368 txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter, 3369 VMXNET3_REG_CMD); 3370 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3371 3372 if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) || 3373 (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) || 3374 (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) { 3375 adapter->txdata_desc_size = 3376 sizeof(struct Vmxnet3_TxDataDesc); 3377 } else { 3378 adapter->txdata_desc_size = txdata_desc_size; 3379 } 3380 } else { 3381 adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc); 3382 } 3383 3384 err = vmxnet3_create_queues(adapter, 3385 adapter->tx_ring_size, 3386 adapter->rx_ring_size, 3387 adapter->rx_ring2_size, 3388 adapter->txdata_desc_size, 3389 adapter->rxdata_desc_size); 3390 if (err) 3391 goto queue_err; 3392 3393 err = vmxnet3_activate_dev(adapter); 3394 if (err) 3395 goto activate_err; 3396 3397 return 0; 3398 3399 activate_err: 3400 vmxnet3_rq_destroy_all(adapter); 3401 vmxnet3_tq_destroy_all(adapter); 3402 queue_err: 3403 return err; 3404 } 3405 3406 3407 static int 3408 vmxnet3_close(struct net_device *netdev) 3409 { 3410 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3411 3412 /* 3413 * Reset_work may be in the middle of resetting the device, wait for its 3414 * completion. 3415 */ 3416 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3417 usleep_range(1000, 2000); 3418 3419 vmxnet3_quiesce_dev(adapter); 3420 3421 vmxnet3_rq_destroy_all(adapter); 3422 vmxnet3_tq_destroy_all(adapter); 3423 3424 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3425 3426 3427 return 0; 3428 } 3429 3430 3431 void 3432 vmxnet3_force_close(struct vmxnet3_adapter *adapter) 3433 { 3434 int i; 3435 3436 /* 3437 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise 3438 * vmxnet3_close() will deadlock. 3439 */ 3440 BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)); 3441 3442 /* we need to enable NAPI, otherwise dev_close will deadlock */ 3443 for (i = 0; i < adapter->num_rx_queues; i++) 3444 napi_enable(&adapter->rx_queue[i].napi); 3445 /* 3446 * Need to clear the quiesce bit to ensure that vmxnet3_close 3447 * can quiesce the device properly 3448 */ 3449 clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 3450 dev_close(adapter->netdev); 3451 } 3452 3453 3454 static int 3455 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu) 3456 { 3457 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3458 int err = 0; 3459 3460 netdev->mtu = new_mtu; 3461 3462 /* 3463 * Reset_work may be in the middle of resetting the device, wait for its 3464 * completion. 3465 */ 3466 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3467 usleep_range(1000, 2000); 3468 3469 if (netif_running(netdev)) { 3470 vmxnet3_quiesce_dev(adapter); 3471 vmxnet3_reset_dev(adapter); 3472 3473 /* we need to re-create the rx queue based on the new mtu */ 3474 vmxnet3_rq_destroy_all(adapter); 3475 vmxnet3_adjust_rx_ring_size(adapter); 3476 err = vmxnet3_rq_create_all(adapter); 3477 if (err) { 3478 netdev_err(netdev, 3479 "failed to re-create rx queues, " 3480 " error %d. Closing it.\n", err); 3481 goto out; 3482 } 3483 3484 err = vmxnet3_activate_dev(adapter); 3485 if (err) { 3486 netdev_err(netdev, 3487 "failed to re-activate, error %d. " 3488 "Closing it\n", err); 3489 goto out; 3490 } 3491 } 3492 3493 out: 3494 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3495 if (err) 3496 vmxnet3_force_close(adapter); 3497 3498 return err; 3499 } 3500 3501 3502 static void 3503 vmxnet3_declare_features(struct vmxnet3_adapter *adapter) 3504 { 3505 struct net_device *netdev = adapter->netdev; 3506 3507 netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM | 3508 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3509 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3510 NETIF_F_LRO | NETIF_F_HIGHDMA; 3511 3512 if (VMXNET3_VERSION_GE_4(adapter)) { 3513 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3514 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3515 3516 netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM | 3517 NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX | 3518 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 | 3519 NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL | 3520 NETIF_F_GSO_UDP_TUNNEL_CSUM; 3521 } 3522 3523 if (VMXNET3_VERSION_GE_7(adapter)) { 3524 unsigned long flags; 3525 3526 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3527 VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) { 3528 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD; 3529 } 3530 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3531 VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) { 3532 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD; 3533 } 3534 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3535 VMXNET3_CAP_GENEVE_TSO)) { 3536 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO; 3537 } 3538 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3539 VMXNET3_CAP_VXLAN_TSO)) { 3540 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO; 3541 } 3542 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3543 VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) { 3544 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD; 3545 } 3546 if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0], 3547 VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) { 3548 adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD; 3549 } 3550 3551 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3552 spin_lock_irqsave(&adapter->cmd_lock, flags); 3553 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3554 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3555 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3556 3557 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) && 3558 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) && 3559 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) && 3560 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) { 3561 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3562 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL; 3563 } 3564 if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) && 3565 !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) { 3566 netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3567 netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM; 3568 } 3569 } 3570 3571 netdev->vlan_features = netdev->hw_features & 3572 ~(NETIF_F_HW_VLAN_CTAG_TX | 3573 NETIF_F_HW_VLAN_CTAG_RX); 3574 netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 3575 } 3576 3577 3578 static void 3579 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac) 3580 { 3581 u32 tmp; 3582 3583 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL); 3584 *(u32 *)mac = tmp; 3585 3586 tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH); 3587 mac[4] = tmp & 0xff; 3588 mac[5] = (tmp >> 8) & 0xff; 3589 } 3590 3591 #ifdef CONFIG_PCI_MSI 3592 3593 /* 3594 * Enable MSIx vectors. 3595 * Returns : 3596 * VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required 3597 * were enabled. 3598 * number of vectors which were enabled otherwise (this number is greater 3599 * than VMXNET3_LINUX_MIN_MSIX_VECT) 3600 */ 3601 3602 static int 3603 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec) 3604 { 3605 int ret = pci_enable_msix_range(adapter->pdev, 3606 adapter->intr.msix_entries, nvec, nvec); 3607 3608 if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) { 3609 dev_err(&adapter->netdev->dev, 3610 "Failed to enable %d MSI-X, trying %d\n", 3611 nvec, VMXNET3_LINUX_MIN_MSIX_VECT); 3612 3613 ret = pci_enable_msix_range(adapter->pdev, 3614 adapter->intr.msix_entries, 3615 VMXNET3_LINUX_MIN_MSIX_VECT, 3616 VMXNET3_LINUX_MIN_MSIX_VECT); 3617 } 3618 3619 if (ret < 0) { 3620 dev_err(&adapter->netdev->dev, 3621 "Failed to enable MSI-X, error: %d\n", ret); 3622 } 3623 3624 return ret; 3625 } 3626 3627 3628 #endif /* CONFIG_PCI_MSI */ 3629 3630 static void 3631 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter) 3632 { 3633 u32 cfg; 3634 unsigned long flags; 3635 3636 /* intr settings */ 3637 spin_lock_irqsave(&adapter->cmd_lock, flags); 3638 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3639 VMXNET3_CMD_GET_CONF_INTR); 3640 cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3641 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3642 adapter->intr.type = cfg & 0x3; 3643 adapter->intr.mask_mode = (cfg >> 2) & 0x3; 3644 3645 if (adapter->intr.type == VMXNET3_IT_AUTO) { 3646 adapter->intr.type = VMXNET3_IT_MSIX; 3647 } 3648 3649 #ifdef CONFIG_PCI_MSI 3650 if (adapter->intr.type == VMXNET3_IT_MSIX) { 3651 int i, nvec, nvec_allocated; 3652 3653 nvec = adapter->share_intr == VMXNET3_INTR_TXSHARE ? 3654 1 : adapter->num_tx_queues; 3655 nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ? 3656 0 : adapter->num_rx_queues; 3657 nvec += 1; /* for link event */ 3658 nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ? 3659 nvec : VMXNET3_LINUX_MIN_MSIX_VECT; 3660 3661 for (i = 0; i < nvec; i++) 3662 adapter->intr.msix_entries[i].entry = i; 3663 3664 nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec); 3665 if (nvec_allocated < 0) 3666 goto msix_err; 3667 3668 /* If we cannot allocate one MSIx vector per queue 3669 * then limit the number of rx queues to 1 3670 */ 3671 if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT && 3672 nvec != VMXNET3_LINUX_MIN_MSIX_VECT) { 3673 if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE 3674 || adapter->num_rx_queues != 1) { 3675 adapter->share_intr = VMXNET3_INTR_TXSHARE; 3676 netdev_err(adapter->netdev, 3677 "Number of rx queues : 1\n"); 3678 adapter->num_rx_queues = 1; 3679 } 3680 } 3681 3682 adapter->intr.num_intrs = nvec_allocated; 3683 return; 3684 3685 msix_err: 3686 /* If we cannot allocate MSIx vectors use only one rx queue */ 3687 dev_info(&adapter->pdev->dev, 3688 "Failed to enable MSI-X, error %d. " 3689 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated); 3690 3691 adapter->intr.type = VMXNET3_IT_MSI; 3692 } 3693 3694 if (adapter->intr.type == VMXNET3_IT_MSI) { 3695 if (!pci_enable_msi(adapter->pdev)) { 3696 adapter->num_rx_queues = 1; 3697 adapter->intr.num_intrs = 1; 3698 return; 3699 } 3700 } 3701 #endif /* CONFIG_PCI_MSI */ 3702 3703 adapter->num_rx_queues = 1; 3704 dev_info(&adapter->netdev->dev, 3705 "Using INTx interrupt, #Rx queues: 1.\n"); 3706 adapter->intr.type = VMXNET3_IT_INTX; 3707 3708 /* INT-X related setting */ 3709 adapter->intr.num_intrs = 1; 3710 } 3711 3712 3713 static void 3714 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter) 3715 { 3716 if (adapter->intr.type == VMXNET3_IT_MSIX) 3717 pci_disable_msix(adapter->pdev); 3718 else if (adapter->intr.type == VMXNET3_IT_MSI) 3719 pci_disable_msi(adapter->pdev); 3720 else 3721 BUG_ON(adapter->intr.type != VMXNET3_IT_INTX); 3722 } 3723 3724 3725 static void 3726 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue) 3727 { 3728 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 3729 adapter->tx_timeout_count++; 3730 3731 netdev_err(adapter->netdev, "tx hang\n"); 3732 schedule_work(&adapter->work); 3733 } 3734 3735 3736 static void 3737 vmxnet3_reset_work(struct work_struct *data) 3738 { 3739 struct vmxnet3_adapter *adapter; 3740 3741 adapter = container_of(data, struct vmxnet3_adapter, work); 3742 3743 /* if another thread is resetting the device, no need to proceed */ 3744 if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 3745 return; 3746 3747 /* if the device is closed, we must leave it alone */ 3748 rtnl_lock(); 3749 if (netif_running(adapter->netdev)) { 3750 netdev_notice(adapter->netdev, "resetting\n"); 3751 vmxnet3_quiesce_dev(adapter); 3752 vmxnet3_reset_dev(adapter); 3753 vmxnet3_activate_dev(adapter); 3754 } else { 3755 netdev_info(adapter->netdev, "already closed\n"); 3756 } 3757 rtnl_unlock(); 3758 3759 netif_wake_queue(adapter->netdev); 3760 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 3761 } 3762 3763 3764 static int 3765 vmxnet3_probe_device(struct pci_dev *pdev, 3766 const struct pci_device_id *id) 3767 { 3768 static const struct net_device_ops vmxnet3_netdev_ops = { 3769 .ndo_open = vmxnet3_open, 3770 .ndo_stop = vmxnet3_close, 3771 .ndo_start_xmit = vmxnet3_xmit_frame, 3772 .ndo_set_mac_address = vmxnet3_set_mac_addr, 3773 .ndo_change_mtu = vmxnet3_change_mtu, 3774 .ndo_fix_features = vmxnet3_fix_features, 3775 .ndo_set_features = vmxnet3_set_features, 3776 .ndo_features_check = vmxnet3_features_check, 3777 .ndo_get_stats64 = vmxnet3_get_stats64, 3778 .ndo_tx_timeout = vmxnet3_tx_timeout, 3779 .ndo_set_rx_mode = vmxnet3_set_mc, 3780 .ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid, 3781 .ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid, 3782 #ifdef CONFIG_NET_POLL_CONTROLLER 3783 .ndo_poll_controller = vmxnet3_netpoll, 3784 #endif 3785 .ndo_bpf = vmxnet3_xdp, 3786 .ndo_xdp_xmit = vmxnet3_xdp_xmit, 3787 }; 3788 int err; 3789 u32 ver; 3790 struct net_device *netdev; 3791 struct vmxnet3_adapter *adapter; 3792 u8 mac[ETH_ALEN]; 3793 int size; 3794 int num_tx_queues; 3795 int num_rx_queues; 3796 int queues; 3797 unsigned long flags; 3798 3799 if (!pci_msi_enabled()) 3800 enable_mq = 0; 3801 3802 #ifdef VMXNET3_RSS 3803 if (enable_mq) 3804 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 3805 (int)num_online_cpus()); 3806 else 3807 #endif 3808 num_rx_queues = 1; 3809 3810 if (enable_mq) 3811 num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES, 3812 (int)num_online_cpus()); 3813 else 3814 num_tx_queues = 1; 3815 3816 netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter), 3817 max(num_tx_queues, num_rx_queues)); 3818 if (!netdev) 3819 return -ENOMEM; 3820 3821 pci_set_drvdata(pdev, netdev); 3822 adapter = netdev_priv(netdev); 3823 adapter->netdev = netdev; 3824 adapter->pdev = pdev; 3825 3826 adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE; 3827 adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE; 3828 adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE; 3829 3830 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 3831 if (err) { 3832 dev_err(&pdev->dev, "dma_set_mask failed\n"); 3833 goto err_set_mask; 3834 } 3835 3836 spin_lock_init(&adapter->cmd_lock); 3837 adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter, 3838 sizeof(struct vmxnet3_adapter), 3839 DMA_TO_DEVICE); 3840 if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) { 3841 dev_err(&pdev->dev, "Failed to map dma\n"); 3842 err = -EFAULT; 3843 goto err_set_mask; 3844 } 3845 adapter->shared = dma_alloc_coherent( 3846 &adapter->pdev->dev, 3847 sizeof(struct Vmxnet3_DriverShared), 3848 &adapter->shared_pa, GFP_KERNEL); 3849 if (!adapter->shared) { 3850 dev_err(&pdev->dev, "Failed to allocate memory\n"); 3851 err = -ENOMEM; 3852 goto err_alloc_shared; 3853 } 3854 3855 err = vmxnet3_alloc_pci_resources(adapter); 3856 if (err < 0) 3857 goto err_alloc_pci; 3858 3859 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS); 3860 if (ver & (1 << VMXNET3_REV_7)) { 3861 VMXNET3_WRITE_BAR1_REG(adapter, 3862 VMXNET3_REG_VRRS, 3863 1 << VMXNET3_REV_7); 3864 adapter->version = VMXNET3_REV_7 + 1; 3865 } else if (ver & (1 << VMXNET3_REV_6)) { 3866 VMXNET3_WRITE_BAR1_REG(adapter, 3867 VMXNET3_REG_VRRS, 3868 1 << VMXNET3_REV_6); 3869 adapter->version = VMXNET3_REV_6 + 1; 3870 } else if (ver & (1 << VMXNET3_REV_5)) { 3871 VMXNET3_WRITE_BAR1_REG(adapter, 3872 VMXNET3_REG_VRRS, 3873 1 << VMXNET3_REV_5); 3874 adapter->version = VMXNET3_REV_5 + 1; 3875 } else if (ver & (1 << VMXNET3_REV_4)) { 3876 VMXNET3_WRITE_BAR1_REG(adapter, 3877 VMXNET3_REG_VRRS, 3878 1 << VMXNET3_REV_4); 3879 adapter->version = VMXNET3_REV_4 + 1; 3880 } else if (ver & (1 << VMXNET3_REV_3)) { 3881 VMXNET3_WRITE_BAR1_REG(adapter, 3882 VMXNET3_REG_VRRS, 3883 1 << VMXNET3_REV_3); 3884 adapter->version = VMXNET3_REV_3 + 1; 3885 } else if (ver & (1 << VMXNET3_REV_2)) { 3886 VMXNET3_WRITE_BAR1_REG(adapter, 3887 VMXNET3_REG_VRRS, 3888 1 << VMXNET3_REV_2); 3889 adapter->version = VMXNET3_REV_2 + 1; 3890 } else if (ver & (1 << VMXNET3_REV_1)) { 3891 VMXNET3_WRITE_BAR1_REG(adapter, 3892 VMXNET3_REG_VRRS, 3893 1 << VMXNET3_REV_1); 3894 adapter->version = VMXNET3_REV_1 + 1; 3895 } else { 3896 dev_err(&pdev->dev, 3897 "Incompatible h/w version (0x%x) for adapter\n", ver); 3898 err = -EBUSY; 3899 goto err_ver; 3900 } 3901 dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version); 3902 3903 ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS); 3904 if (ver & 1) { 3905 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1); 3906 } else { 3907 dev_err(&pdev->dev, 3908 "Incompatible upt version (0x%x) for adapter\n", ver); 3909 err = -EBUSY; 3910 goto err_ver; 3911 } 3912 3913 if (VMXNET3_VERSION_GE_7(adapter)) { 3914 adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR); 3915 adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR); 3916 if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 3917 adapter->dev_caps[0] = adapter->devcap_supported[0] & 3918 (1UL << VMXNET3_CAP_LARGE_BAR); 3919 } 3920 if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) && 3921 adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) && 3922 adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) { 3923 adapter->dev_caps[0] |= adapter->devcap_supported[0] & 3924 (1UL << VMXNET3_CAP_OOORX_COMP); 3925 } 3926 if (adapter->dev_caps[0]) 3927 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]); 3928 3929 spin_lock_irqsave(&adapter->cmd_lock, flags); 3930 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG); 3931 adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3932 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3933 } 3934 3935 if (VMXNET3_VERSION_GE_7(adapter) && 3936 adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) { 3937 adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD; 3938 adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD; 3939 adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2; 3940 } else { 3941 adapter->tx_prod_offset = VMXNET3_REG_TXPROD; 3942 adapter->rx_prod_offset = VMXNET3_REG_RXPROD; 3943 adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2; 3944 } 3945 3946 if (VMXNET3_VERSION_GE_6(adapter)) { 3947 spin_lock_irqsave(&adapter->cmd_lock, flags); 3948 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 3949 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 3950 queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 3951 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 3952 if (queues > 0) { 3953 adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff)); 3954 adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff)); 3955 } else { 3956 adapter->num_rx_queues = min(num_rx_queues, 3957 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3958 adapter->num_tx_queues = min(num_tx_queues, 3959 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 3960 } 3961 if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES || 3962 adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) { 3963 adapter->queuesExtEnabled = true; 3964 } else { 3965 adapter->queuesExtEnabled = false; 3966 } 3967 } else { 3968 adapter->queuesExtEnabled = false; 3969 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 3970 num_tx_queues = rounddown_pow_of_two(num_tx_queues); 3971 adapter->num_rx_queues = min(num_rx_queues, 3972 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 3973 adapter->num_tx_queues = min(num_tx_queues, 3974 VMXNET3_DEVICE_DEFAULT_TX_QUEUES); 3975 } 3976 dev_info(&pdev->dev, 3977 "# of Tx queues : %d, # of Rx queues : %d\n", 3978 adapter->num_tx_queues, adapter->num_rx_queues); 3979 3980 adapter->rx_buf_per_pkt = 1; 3981 3982 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 3983 size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues; 3984 adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size, 3985 &adapter->queue_desc_pa, 3986 GFP_KERNEL); 3987 3988 if (!adapter->tqd_start) { 3989 dev_err(&pdev->dev, "Failed to allocate memory\n"); 3990 err = -ENOMEM; 3991 goto err_ver; 3992 } 3993 adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start + 3994 adapter->num_tx_queues); 3995 3996 adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev, 3997 sizeof(struct Vmxnet3_PMConf), 3998 &adapter->pm_conf_pa, 3999 GFP_KERNEL); 4000 if (adapter->pm_conf == NULL) { 4001 err = -ENOMEM; 4002 goto err_alloc_pm; 4003 } 4004 4005 #ifdef VMXNET3_RSS 4006 4007 adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev, 4008 sizeof(struct UPT1_RSSConf), 4009 &adapter->rss_conf_pa, 4010 GFP_KERNEL); 4011 if (adapter->rss_conf == NULL) { 4012 err = -ENOMEM; 4013 goto err_alloc_rss; 4014 } 4015 #endif /* VMXNET3_RSS */ 4016 4017 if (VMXNET3_VERSION_GE_3(adapter)) { 4018 adapter->coal_conf = 4019 dma_alloc_coherent(&adapter->pdev->dev, 4020 sizeof(struct Vmxnet3_CoalesceScheme) 4021 , 4022 &adapter->coal_conf_pa, 4023 GFP_KERNEL); 4024 if (!adapter->coal_conf) { 4025 err = -ENOMEM; 4026 goto err_coal_conf; 4027 } 4028 adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED; 4029 adapter->default_coal_mode = true; 4030 } 4031 4032 if (VMXNET3_VERSION_GE_4(adapter)) { 4033 adapter->default_rss_fields = true; 4034 adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT; 4035 } 4036 4037 SET_NETDEV_DEV(netdev, &pdev->dev); 4038 vmxnet3_declare_features(adapter); 4039 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 4040 NETDEV_XDP_ACT_NDO_XMIT; 4041 4042 adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ? 4043 VMXNET3_DEF_RXDATA_DESC_SIZE : 0; 4044 4045 if (adapter->num_tx_queues == adapter->num_rx_queues) 4046 adapter->share_intr = VMXNET3_INTR_BUDDYSHARE; 4047 else 4048 adapter->share_intr = VMXNET3_INTR_DONTSHARE; 4049 4050 vmxnet3_alloc_intr_resources(adapter); 4051 4052 #ifdef VMXNET3_RSS 4053 if (adapter->num_rx_queues > 1 && 4054 adapter->intr.type == VMXNET3_IT_MSIX) { 4055 adapter->rss = true; 4056 netdev->hw_features |= NETIF_F_RXHASH; 4057 netdev->features |= NETIF_F_RXHASH; 4058 dev_dbg(&pdev->dev, "RSS is enabled.\n"); 4059 } else { 4060 adapter->rss = false; 4061 } 4062 #endif 4063 4064 vmxnet3_read_mac_addr(adapter, mac); 4065 dev_addr_set(netdev, mac); 4066 4067 netdev->netdev_ops = &vmxnet3_netdev_ops; 4068 vmxnet3_set_ethtool_ops(netdev); 4069 netdev->watchdog_timeo = 5 * HZ; 4070 4071 /* MTU range: 60 - 9190 */ 4072 netdev->min_mtu = VMXNET3_MIN_MTU; 4073 if (VMXNET3_VERSION_GE_6(adapter)) 4074 netdev->max_mtu = VMXNET3_V6_MAX_MTU; 4075 else 4076 netdev->max_mtu = VMXNET3_MAX_MTU; 4077 4078 INIT_WORK(&adapter->work, vmxnet3_reset_work); 4079 set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state); 4080 4081 if (adapter->intr.type == VMXNET3_IT_MSIX) { 4082 int i; 4083 for (i = 0; i < adapter->num_rx_queues; i++) { 4084 netif_napi_add(adapter->netdev, 4085 &adapter->rx_queue[i].napi, 4086 vmxnet3_poll_rx_only); 4087 } 4088 } else { 4089 netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi, 4090 vmxnet3_poll); 4091 } 4092 4093 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues); 4094 netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues); 4095 4096 netif_carrier_off(netdev); 4097 err = register_netdev(netdev); 4098 4099 if (err) { 4100 dev_err(&pdev->dev, "Failed to register adapter\n"); 4101 goto err_register; 4102 } 4103 4104 vmxnet3_check_link(adapter, false); 4105 return 0; 4106 4107 err_register: 4108 if (VMXNET3_VERSION_GE_3(adapter)) { 4109 dma_free_coherent(&adapter->pdev->dev, 4110 sizeof(struct Vmxnet3_CoalesceScheme), 4111 adapter->coal_conf, adapter->coal_conf_pa); 4112 } 4113 vmxnet3_free_intr_resources(adapter); 4114 err_coal_conf: 4115 #ifdef VMXNET3_RSS 4116 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 4117 adapter->rss_conf, adapter->rss_conf_pa); 4118 err_alloc_rss: 4119 #endif 4120 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 4121 adapter->pm_conf, adapter->pm_conf_pa); 4122 err_alloc_pm: 4123 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 4124 adapter->queue_desc_pa); 4125 err_ver: 4126 vmxnet3_free_pci_resources(adapter); 4127 err_alloc_pci: 4128 dma_free_coherent(&adapter->pdev->dev, 4129 sizeof(struct Vmxnet3_DriverShared), 4130 adapter->shared, adapter->shared_pa); 4131 err_alloc_shared: 4132 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4133 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4134 err_set_mask: 4135 free_netdev(netdev); 4136 return err; 4137 } 4138 4139 4140 static void 4141 vmxnet3_remove_device(struct pci_dev *pdev) 4142 { 4143 struct net_device *netdev = pci_get_drvdata(pdev); 4144 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4145 int size = 0; 4146 int num_rx_queues, rx_queues; 4147 unsigned long flags; 4148 4149 #ifdef VMXNET3_RSS 4150 if (enable_mq) 4151 num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES, 4152 (int)num_online_cpus()); 4153 else 4154 #endif 4155 num_rx_queues = 1; 4156 if (!VMXNET3_VERSION_GE_6(adapter)) { 4157 num_rx_queues = rounddown_pow_of_two(num_rx_queues); 4158 } 4159 if (VMXNET3_VERSION_GE_6(adapter)) { 4160 spin_lock_irqsave(&adapter->cmd_lock, flags); 4161 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4162 VMXNET3_CMD_GET_MAX_QUEUES_CONF); 4163 rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD); 4164 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4165 if (rx_queues > 0) 4166 rx_queues = (rx_queues >> 8) & 0xff; 4167 else 4168 rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4169 num_rx_queues = min(num_rx_queues, rx_queues); 4170 } else { 4171 num_rx_queues = min(num_rx_queues, 4172 VMXNET3_DEVICE_DEFAULT_RX_QUEUES); 4173 } 4174 4175 cancel_work_sync(&adapter->work); 4176 4177 unregister_netdev(netdev); 4178 4179 vmxnet3_free_intr_resources(adapter); 4180 vmxnet3_free_pci_resources(adapter); 4181 if (VMXNET3_VERSION_GE_3(adapter)) { 4182 dma_free_coherent(&adapter->pdev->dev, 4183 sizeof(struct Vmxnet3_CoalesceScheme), 4184 adapter->coal_conf, adapter->coal_conf_pa); 4185 } 4186 #ifdef VMXNET3_RSS 4187 dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf), 4188 adapter->rss_conf, adapter->rss_conf_pa); 4189 #endif 4190 dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf), 4191 adapter->pm_conf, adapter->pm_conf_pa); 4192 4193 size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues; 4194 size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues; 4195 dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start, 4196 adapter->queue_desc_pa); 4197 dma_free_coherent(&adapter->pdev->dev, 4198 sizeof(struct Vmxnet3_DriverShared), 4199 adapter->shared, adapter->shared_pa); 4200 dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa, 4201 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE); 4202 free_netdev(netdev); 4203 } 4204 4205 static void vmxnet3_shutdown_device(struct pci_dev *pdev) 4206 { 4207 struct net_device *netdev = pci_get_drvdata(pdev); 4208 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4209 unsigned long flags; 4210 4211 /* Reset_work may be in the middle of resetting the device, wait for its 4212 * completion. 4213 */ 4214 while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)) 4215 usleep_range(1000, 2000); 4216 4217 if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, 4218 &adapter->state)) { 4219 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4220 return; 4221 } 4222 spin_lock_irqsave(&adapter->cmd_lock, flags); 4223 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4224 VMXNET3_CMD_QUIESCE_DEV); 4225 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4226 vmxnet3_disable_all_intrs(adapter); 4227 4228 clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state); 4229 } 4230 4231 4232 #ifdef CONFIG_PM 4233 4234 static int 4235 vmxnet3_suspend(struct device *device) 4236 { 4237 struct pci_dev *pdev = to_pci_dev(device); 4238 struct net_device *netdev = pci_get_drvdata(pdev); 4239 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4240 struct Vmxnet3_PMConf *pmConf; 4241 struct ethhdr *ehdr; 4242 struct arphdr *ahdr; 4243 u8 *arpreq; 4244 struct in_device *in_dev; 4245 struct in_ifaddr *ifa; 4246 unsigned long flags; 4247 int i = 0; 4248 4249 if (!netif_running(netdev)) 4250 return 0; 4251 4252 for (i = 0; i < adapter->num_rx_queues; i++) 4253 napi_disable(&adapter->rx_queue[i].napi); 4254 4255 vmxnet3_disable_all_intrs(adapter); 4256 vmxnet3_free_irqs(adapter); 4257 vmxnet3_free_intr_resources(adapter); 4258 4259 netif_device_detach(netdev); 4260 4261 /* Create wake-up filters. */ 4262 pmConf = adapter->pm_conf; 4263 memset(pmConf, 0, sizeof(*pmConf)); 4264 4265 if (adapter->wol & WAKE_UCAST) { 4266 pmConf->filters[i].patternSize = ETH_ALEN; 4267 pmConf->filters[i].maskSize = 1; 4268 memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN); 4269 pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */ 4270 4271 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4272 i++; 4273 } 4274 4275 if (adapter->wol & WAKE_ARP) { 4276 rcu_read_lock(); 4277 4278 in_dev = __in_dev_get_rcu(netdev); 4279 if (!in_dev) { 4280 rcu_read_unlock(); 4281 goto skip_arp; 4282 } 4283 4284 ifa = rcu_dereference(in_dev->ifa_list); 4285 if (!ifa) { 4286 rcu_read_unlock(); 4287 goto skip_arp; 4288 } 4289 4290 pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/ 4291 sizeof(struct arphdr) + /* ARP header */ 4292 2 * ETH_ALEN + /* 2 Ethernet addresses*/ 4293 2 * sizeof(u32); /*2 IPv4 addresses */ 4294 pmConf->filters[i].maskSize = 4295 (pmConf->filters[i].patternSize - 1) / 8 + 1; 4296 4297 /* ETH_P_ARP in Ethernet header. */ 4298 ehdr = (struct ethhdr *)pmConf->filters[i].pattern; 4299 ehdr->h_proto = htons(ETH_P_ARP); 4300 4301 /* ARPOP_REQUEST in ARP header. */ 4302 ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN]; 4303 ahdr->ar_op = htons(ARPOP_REQUEST); 4304 arpreq = (u8 *)(ahdr + 1); 4305 4306 /* The Unicast IPv4 address in 'tip' field. */ 4307 arpreq += 2 * ETH_ALEN + sizeof(u32); 4308 *(__be32 *)arpreq = ifa->ifa_address; 4309 4310 rcu_read_unlock(); 4311 4312 /* The mask for the relevant bits. */ 4313 pmConf->filters[i].mask[0] = 0x00; 4314 pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */ 4315 pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */ 4316 pmConf->filters[i].mask[3] = 0x00; 4317 pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */ 4318 pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */ 4319 4320 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER; 4321 i++; 4322 } 4323 4324 skip_arp: 4325 if (adapter->wol & WAKE_MAGIC) 4326 pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC; 4327 4328 pmConf->numFilters = i; 4329 4330 adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1); 4331 adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof( 4332 *pmConf)); 4333 adapter->shared->devRead.pmConfDesc.confPA = 4334 cpu_to_le64(adapter->pm_conf_pa); 4335 4336 spin_lock_irqsave(&adapter->cmd_lock, flags); 4337 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4338 VMXNET3_CMD_UPDATE_PMCFG); 4339 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4340 4341 pci_save_state(pdev); 4342 pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND), 4343 adapter->wol); 4344 pci_disable_device(pdev); 4345 pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND)); 4346 4347 return 0; 4348 } 4349 4350 4351 static int 4352 vmxnet3_resume(struct device *device) 4353 { 4354 int err; 4355 unsigned long flags; 4356 struct pci_dev *pdev = to_pci_dev(device); 4357 struct net_device *netdev = pci_get_drvdata(pdev); 4358 struct vmxnet3_adapter *adapter = netdev_priv(netdev); 4359 4360 if (!netif_running(netdev)) 4361 return 0; 4362 4363 pci_set_power_state(pdev, PCI_D0); 4364 pci_restore_state(pdev); 4365 err = pci_enable_device_mem(pdev); 4366 if (err != 0) 4367 return err; 4368 4369 pci_enable_wake(pdev, PCI_D0, 0); 4370 4371 vmxnet3_alloc_intr_resources(adapter); 4372 4373 /* During hibernate and suspend, device has to be reinitialized as the 4374 * device state need not be preserved. 4375 */ 4376 4377 /* Need not check adapter state as other reset tasks cannot run during 4378 * device resume. 4379 */ 4380 spin_lock_irqsave(&adapter->cmd_lock, flags); 4381 VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, 4382 VMXNET3_CMD_QUIESCE_DEV); 4383 spin_unlock_irqrestore(&adapter->cmd_lock, flags); 4384 vmxnet3_tq_cleanup_all(adapter); 4385 vmxnet3_rq_cleanup_all(adapter); 4386 4387 vmxnet3_reset_dev(adapter); 4388 err = vmxnet3_activate_dev(adapter); 4389 if (err != 0) { 4390 netdev_err(netdev, 4391 "failed to re-activate on resume, error: %d", err); 4392 vmxnet3_force_close(adapter); 4393 return err; 4394 } 4395 netif_device_attach(netdev); 4396 4397 return 0; 4398 } 4399 4400 static const struct dev_pm_ops vmxnet3_pm_ops = { 4401 .suspend = vmxnet3_suspend, 4402 .resume = vmxnet3_resume, 4403 .freeze = vmxnet3_suspend, 4404 .restore = vmxnet3_resume, 4405 }; 4406 #endif 4407 4408 static struct pci_driver vmxnet3_driver = { 4409 .name = vmxnet3_driver_name, 4410 .id_table = vmxnet3_pciid_table, 4411 .probe = vmxnet3_probe_device, 4412 .remove = vmxnet3_remove_device, 4413 .shutdown = vmxnet3_shutdown_device, 4414 #ifdef CONFIG_PM 4415 .driver.pm = &vmxnet3_pm_ops, 4416 #endif 4417 }; 4418 4419 4420 static int __init 4421 vmxnet3_init_module(void) 4422 { 4423 pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC, 4424 VMXNET3_DRIVER_VERSION_REPORT); 4425 return pci_register_driver(&vmxnet3_driver); 4426 } 4427 4428 module_init(vmxnet3_init_module); 4429 4430 4431 static void 4432 vmxnet3_exit_module(void) 4433 { 4434 pci_unregister_driver(&vmxnet3_driver); 4435 } 4436 4437 module_exit(vmxnet3_exit_module); 4438 4439 MODULE_AUTHOR("VMware, Inc."); 4440 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC); 4441 MODULE_LICENSE("GPL v2"); 4442 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING); 4443