1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include <linux/net/intel/libie/rx.h> 5 #include <net/netdev_lock.h> 6 7 #include "iavf.h" 8 #include "iavf_ptp.h" 9 #include "iavf_prototype.h" 10 /* All iavf tracepoints are defined by the include below, which must 11 * be included exactly once across the whole kernel with 12 * CREATE_TRACE_POINTS defined 13 */ 14 #define CREATE_TRACE_POINTS 15 #include "iavf_trace.h" 16 17 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter); 18 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter); 19 static int iavf_close(struct net_device *netdev); 20 static void iavf_init_get_resources(struct iavf_adapter *adapter); 21 static int iavf_check_reset_complete(struct iavf_hw *hw); 22 23 char iavf_driver_name[] = "iavf"; 24 static const char iavf_driver_string[] = 25 "Intel(R) Ethernet Adaptive Virtual Function Network Driver"; 26 27 static const char iavf_copyright[] = 28 "Copyright (c) 2013 - 2018 Intel Corporation."; 29 30 /* iavf_pci_tbl - PCI Device ID Table 31 * 32 * Wildcard entries (PCI_ANY_ID) should come last 33 * Last entry must be all 0s 34 * 35 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 36 * Class, Class Mask, private data (not used) } 37 */ 38 static const struct pci_device_id iavf_pci_tbl[] = { 39 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0}, 40 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0}, 41 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0}, 42 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0}, 43 /* required last entry */ 44 {0, } 45 }; 46 47 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl); 48 49 MODULE_ALIAS("i40evf"); 50 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver"); 51 MODULE_IMPORT_NS("LIBETH"); 52 MODULE_IMPORT_NS("LIBIE"); 53 MODULE_LICENSE("GPL v2"); 54 55 static const struct net_device_ops iavf_netdev_ops; 56 57 int iavf_status_to_errno(enum iavf_status status) 58 { 59 switch (status) { 60 case IAVF_SUCCESS: 61 return 0; 62 case IAVF_ERR_PARAM: 63 case IAVF_ERR_MAC_TYPE: 64 case IAVF_ERR_INVALID_MAC_ADDR: 65 case IAVF_ERR_INVALID_LINK_SETTINGS: 66 case IAVF_ERR_INVALID_PD_ID: 67 case IAVF_ERR_INVALID_QP_ID: 68 case IAVF_ERR_INVALID_CQ_ID: 69 case IAVF_ERR_INVALID_CEQ_ID: 70 case IAVF_ERR_INVALID_AEQ_ID: 71 case IAVF_ERR_INVALID_SIZE: 72 case IAVF_ERR_INVALID_ARP_INDEX: 73 case IAVF_ERR_INVALID_FPM_FUNC_ID: 74 case IAVF_ERR_QP_INVALID_MSG_SIZE: 75 case IAVF_ERR_INVALID_FRAG_COUNT: 76 case IAVF_ERR_INVALID_ALIGNMENT: 77 case IAVF_ERR_INVALID_PUSH_PAGE_INDEX: 78 case IAVF_ERR_INVALID_IMM_DATA_SIZE: 79 case IAVF_ERR_INVALID_VF_ID: 80 case IAVF_ERR_INVALID_HMCFN_ID: 81 case IAVF_ERR_INVALID_PBLE_INDEX: 82 case IAVF_ERR_INVALID_SD_INDEX: 83 case IAVF_ERR_INVALID_PAGE_DESC_INDEX: 84 case IAVF_ERR_INVALID_SD_TYPE: 85 case IAVF_ERR_INVALID_HMC_OBJ_INDEX: 86 case IAVF_ERR_INVALID_HMC_OBJ_COUNT: 87 case IAVF_ERR_INVALID_SRQ_ARM_LIMIT: 88 return -EINVAL; 89 case IAVF_ERR_NVM: 90 case IAVF_ERR_NVM_CHECKSUM: 91 case IAVF_ERR_PHY: 92 case IAVF_ERR_CONFIG: 93 case IAVF_ERR_UNKNOWN_PHY: 94 case IAVF_ERR_LINK_SETUP: 95 case IAVF_ERR_ADAPTER_STOPPED: 96 case IAVF_ERR_PRIMARY_REQUESTS_PENDING: 97 case IAVF_ERR_AUTONEG_NOT_COMPLETE: 98 case IAVF_ERR_RESET_FAILED: 99 case IAVF_ERR_BAD_PTR: 100 case IAVF_ERR_SWFW_SYNC: 101 case IAVF_ERR_QP_TOOMANY_WRS_POSTED: 102 case IAVF_ERR_QUEUE_EMPTY: 103 case IAVF_ERR_FLUSHED_QUEUE: 104 case IAVF_ERR_OPCODE_MISMATCH: 105 case IAVF_ERR_CQP_COMPL_ERROR: 106 case IAVF_ERR_BACKING_PAGE_ERROR: 107 case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE: 108 case IAVF_ERR_MEMCPY_FAILED: 109 case IAVF_ERR_SRQ_ENABLED: 110 case IAVF_ERR_ADMIN_QUEUE_ERROR: 111 case IAVF_ERR_ADMIN_QUEUE_FULL: 112 case IAVF_ERR_BAD_RDMA_CQE: 113 case IAVF_ERR_NVM_BLANK_MODE: 114 case IAVF_ERR_PE_DOORBELL_NOT_ENABLED: 115 case IAVF_ERR_DIAG_TEST_FAILED: 116 case IAVF_ERR_FIRMWARE_API_VERSION: 117 case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR: 118 return -EIO; 119 case IAVF_ERR_DEVICE_NOT_SUPPORTED: 120 return -ENODEV; 121 case IAVF_ERR_NO_AVAILABLE_VSI: 122 case IAVF_ERR_RING_FULL: 123 return -ENOSPC; 124 case IAVF_ERR_NO_MEMORY: 125 return -ENOMEM; 126 case IAVF_ERR_TIMEOUT: 127 case IAVF_ERR_ADMIN_QUEUE_TIMEOUT: 128 return -ETIMEDOUT; 129 case IAVF_ERR_NOT_IMPLEMENTED: 130 case IAVF_NOT_SUPPORTED: 131 return -EOPNOTSUPP; 132 case IAVF_ERR_ADMIN_QUEUE_NO_WORK: 133 return -EALREADY; 134 case IAVF_ERR_NOT_READY: 135 return -EBUSY; 136 case IAVF_ERR_BUF_TOO_SHORT: 137 return -EMSGSIZE; 138 } 139 140 return -EIO; 141 } 142 143 int virtchnl_status_to_errno(enum virtchnl_status_code v_status) 144 { 145 switch (v_status) { 146 case VIRTCHNL_STATUS_SUCCESS: 147 return 0; 148 case VIRTCHNL_STATUS_ERR_PARAM: 149 case VIRTCHNL_STATUS_ERR_INVALID_VF_ID: 150 return -EINVAL; 151 case VIRTCHNL_STATUS_ERR_NO_MEMORY: 152 return -ENOMEM; 153 case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH: 154 case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR: 155 case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR: 156 return -EIO; 157 case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED: 158 return -EOPNOTSUPP; 159 } 160 161 return -EIO; 162 } 163 164 /** 165 * iavf_pdev_to_adapter - go from pci_dev to adapter 166 * @pdev: pci_dev pointer 167 */ 168 static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev) 169 { 170 return netdev_priv(pci_get_drvdata(pdev)); 171 } 172 173 /** 174 * iavf_is_reset_in_progress - Check if a reset is in progress 175 * @adapter: board private structure 176 */ 177 static bool iavf_is_reset_in_progress(struct iavf_adapter *adapter) 178 { 179 if (adapter->state == __IAVF_RESETTING || 180 adapter->flags & (IAVF_FLAG_RESET_PENDING | 181 IAVF_FLAG_RESET_NEEDED)) 182 return true; 183 184 return false; 185 } 186 187 /** 188 * iavf_wait_for_reset - Wait for reset to finish. 189 * @adapter: board private structure 190 * 191 * Returns 0 if reset finished successfully, negative on timeout or interrupt. 192 */ 193 int iavf_wait_for_reset(struct iavf_adapter *adapter) 194 { 195 int ret = wait_event_interruptible_timeout(adapter->reset_waitqueue, 196 !iavf_is_reset_in_progress(adapter), 197 msecs_to_jiffies(5000)); 198 199 /* If ret < 0 then it means wait was interrupted. 200 * If ret == 0 then it means we got a timeout while waiting 201 * for reset to finish. 202 * If ret > 0 it means reset has finished. 203 */ 204 if (ret > 0) 205 return 0; 206 else if (ret < 0) 207 return -EINTR; 208 else 209 return -EBUSY; 210 } 211 212 /** 213 * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code 214 * @hw: pointer to the HW structure 215 * @mem: ptr to mem struct to fill out 216 * @size: size of memory requested 217 * @alignment: what to align the allocation to 218 **/ 219 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw, 220 struct iavf_dma_mem *mem, 221 u64 size, u32 alignment) 222 { 223 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 224 225 if (!mem) 226 return IAVF_ERR_PARAM; 227 228 mem->size = ALIGN(size, alignment); 229 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size, 230 (dma_addr_t *)&mem->pa, GFP_KERNEL); 231 if (mem->va) 232 return 0; 233 else 234 return IAVF_ERR_NO_MEMORY; 235 } 236 237 /** 238 * iavf_free_dma_mem - wrapper for DMA memory freeing 239 * @hw: pointer to the HW structure 240 * @mem: ptr to mem struct to free 241 **/ 242 enum iavf_status iavf_free_dma_mem(struct iavf_hw *hw, struct iavf_dma_mem *mem) 243 { 244 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 245 246 if (!mem || !mem->va) 247 return IAVF_ERR_PARAM; 248 dma_free_coherent(&adapter->pdev->dev, mem->size, 249 mem->va, (dma_addr_t)mem->pa); 250 return 0; 251 } 252 253 /** 254 * iavf_allocate_virt_mem - virt memory alloc wrapper 255 * @hw: pointer to the HW structure 256 * @mem: ptr to mem struct to fill out 257 * @size: size of memory requested 258 **/ 259 enum iavf_status iavf_allocate_virt_mem(struct iavf_hw *hw, 260 struct iavf_virt_mem *mem, u32 size) 261 { 262 if (!mem) 263 return IAVF_ERR_PARAM; 264 265 mem->size = size; 266 mem->va = kzalloc(size, GFP_KERNEL); 267 268 if (mem->va) 269 return 0; 270 else 271 return IAVF_ERR_NO_MEMORY; 272 } 273 274 /** 275 * iavf_free_virt_mem - virt memory free wrapper 276 * @hw: pointer to the HW structure 277 * @mem: ptr to mem struct to free 278 **/ 279 void iavf_free_virt_mem(struct iavf_hw *hw, struct iavf_virt_mem *mem) 280 { 281 kfree(mem->va); 282 } 283 284 /** 285 * iavf_schedule_reset - Set the flags and schedule a reset event 286 * @adapter: board private structure 287 * @flags: IAVF_FLAG_RESET_PENDING or IAVF_FLAG_RESET_NEEDED 288 **/ 289 void iavf_schedule_reset(struct iavf_adapter *adapter, u64 flags) 290 { 291 if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section) && 292 !(adapter->flags & 293 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) { 294 adapter->flags |= flags; 295 queue_work(adapter->wq, &adapter->reset_task); 296 } 297 } 298 299 /** 300 * iavf_schedule_aq_request - Set the flags and schedule aq request 301 * @adapter: board private structure 302 * @flags: requested aq flags 303 **/ 304 void iavf_schedule_aq_request(struct iavf_adapter *adapter, u64 flags) 305 { 306 adapter->aq_required |= flags; 307 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0); 308 } 309 310 /** 311 * iavf_tx_timeout - Respond to a Tx Hang 312 * @netdev: network interface device structure 313 * @txqueue: queue number that is timing out 314 **/ 315 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue) 316 { 317 struct iavf_adapter *adapter = netdev_priv(netdev); 318 319 adapter->tx_timeout_count++; 320 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); 321 } 322 323 /** 324 * iavf_misc_irq_disable - Mask off interrupt generation on the NIC 325 * @adapter: board private structure 326 **/ 327 static void iavf_misc_irq_disable(struct iavf_adapter *adapter) 328 { 329 struct iavf_hw *hw = &adapter->hw; 330 331 if (!adapter->msix_entries) 332 return; 333 334 wr32(hw, IAVF_VFINT_DYN_CTL01, 0); 335 336 iavf_flush(hw); 337 338 synchronize_irq(adapter->msix_entries[0].vector); 339 } 340 341 /** 342 * iavf_misc_irq_enable - Enable default interrupt generation settings 343 * @adapter: board private structure 344 **/ 345 static void iavf_misc_irq_enable(struct iavf_adapter *adapter) 346 { 347 struct iavf_hw *hw = &adapter->hw; 348 349 wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK | 350 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 351 wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 352 353 iavf_flush(hw); 354 } 355 356 /** 357 * iavf_irq_disable - Mask off interrupt generation on the NIC 358 * @adapter: board private structure 359 **/ 360 static void iavf_irq_disable(struct iavf_adapter *adapter) 361 { 362 int i; 363 struct iavf_hw *hw = &adapter->hw; 364 365 if (!adapter->msix_entries) 366 return; 367 368 for (i = 1; i < adapter->num_msix_vectors; i++) { 369 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0); 370 synchronize_irq(adapter->msix_entries[i].vector); 371 } 372 iavf_flush(hw); 373 } 374 375 /** 376 * iavf_irq_enable_queues - Enable interrupt for all queues 377 * @adapter: board private structure 378 **/ 379 static void iavf_irq_enable_queues(struct iavf_adapter *adapter) 380 { 381 struct iavf_hw *hw = &adapter->hw; 382 int i; 383 384 for (i = 1; i < adapter->num_msix_vectors; i++) { 385 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 386 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 387 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 388 } 389 } 390 391 /** 392 * iavf_irq_enable - Enable default interrupt generation settings 393 * @adapter: board private structure 394 * @flush: boolean value whether to run rd32() 395 **/ 396 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush) 397 { 398 struct iavf_hw *hw = &adapter->hw; 399 400 iavf_misc_irq_enable(adapter); 401 iavf_irq_enable_queues(adapter); 402 403 if (flush) 404 iavf_flush(hw); 405 } 406 407 /** 408 * iavf_msix_aq - Interrupt handler for vector 0 409 * @irq: interrupt number 410 * @data: pointer to netdev 411 **/ 412 static irqreturn_t iavf_msix_aq(int irq, void *data) 413 { 414 struct net_device *netdev = data; 415 struct iavf_adapter *adapter = netdev_priv(netdev); 416 struct iavf_hw *hw = &adapter->hw; 417 418 /* handle non-queue interrupts, these reads clear the registers */ 419 rd32(hw, IAVF_VFINT_ICR01); 420 rd32(hw, IAVF_VFINT_ICR0_ENA1); 421 422 if (adapter->state != __IAVF_REMOVE) 423 /* schedule work on the private workqueue */ 424 queue_work(adapter->wq, &adapter->adminq_task); 425 426 return IRQ_HANDLED; 427 } 428 429 /** 430 * iavf_msix_clean_rings - MSIX mode Interrupt Handler 431 * @irq: interrupt number 432 * @data: pointer to a q_vector 433 **/ 434 static irqreturn_t iavf_msix_clean_rings(int irq, void *data) 435 { 436 struct iavf_q_vector *q_vector = data; 437 438 if (!q_vector->tx.ring && !q_vector->rx.ring) 439 return IRQ_HANDLED; 440 441 napi_schedule_irqoff(&q_vector->napi); 442 443 return IRQ_HANDLED; 444 } 445 446 /** 447 * iavf_map_vector_to_rxq - associate irqs with rx queues 448 * @adapter: board private structure 449 * @v_idx: interrupt number 450 * @r_idx: queue number 451 **/ 452 static void 453 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx) 454 { 455 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 456 struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx]; 457 struct iavf_hw *hw = &adapter->hw; 458 459 rx_ring->q_vector = q_vector; 460 rx_ring->next = q_vector->rx.ring; 461 rx_ring->vsi = &adapter->vsi; 462 q_vector->rx.ring = rx_ring; 463 q_vector->rx.count++; 464 q_vector->rx.next_update = jiffies + 1; 465 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting); 466 q_vector->ring_mask |= BIT(r_idx); 467 wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx), 468 q_vector->rx.current_itr >> 1); 469 q_vector->rx.current_itr = q_vector->rx.target_itr; 470 } 471 472 /** 473 * iavf_map_vector_to_txq - associate irqs with tx queues 474 * @adapter: board private structure 475 * @v_idx: interrupt number 476 * @t_idx: queue number 477 **/ 478 static void 479 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx) 480 { 481 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 482 struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx]; 483 struct iavf_hw *hw = &adapter->hw; 484 485 tx_ring->q_vector = q_vector; 486 tx_ring->next = q_vector->tx.ring; 487 tx_ring->vsi = &adapter->vsi; 488 q_vector->tx.ring = tx_ring; 489 q_vector->tx.count++; 490 q_vector->tx.next_update = jiffies + 1; 491 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting); 492 q_vector->num_ringpairs++; 493 wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx), 494 q_vector->tx.target_itr >> 1); 495 q_vector->tx.current_itr = q_vector->tx.target_itr; 496 } 497 498 /** 499 * iavf_map_rings_to_vectors - Maps descriptor rings to vectors 500 * @adapter: board private structure to initialize 501 * 502 * This function maps descriptor rings to the queue-specific vectors 503 * we were allotted through the MSI-X enabling code. Ideally, we'd have 504 * one vector per ring/queue, but on a constrained vector budget, we 505 * group the rings as "efficiently" as possible. You would add new 506 * mapping configurations in here. 507 **/ 508 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter) 509 { 510 int rings_remaining = adapter->num_active_queues; 511 int ridx = 0, vidx = 0; 512 int q_vectors; 513 514 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 515 516 for (; ridx < rings_remaining; ridx++) { 517 iavf_map_vector_to_rxq(adapter, vidx, ridx); 518 iavf_map_vector_to_txq(adapter, vidx, ridx); 519 520 /* In the case where we have more queues than vectors, continue 521 * round-robin on vectors until all queues are mapped. 522 */ 523 if (++vidx >= q_vectors) 524 vidx = 0; 525 } 526 527 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 528 } 529 530 /** 531 * iavf_request_traffic_irqs - Initialize MSI-X interrupts 532 * @adapter: board private structure 533 * @basename: device basename 534 * 535 * Allocates MSI-X vectors for tx and rx handling, and requests 536 * interrupts from the kernel. 537 **/ 538 static int 539 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename) 540 { 541 unsigned int vector, q_vectors; 542 unsigned int rx_int_idx = 0, tx_int_idx = 0; 543 int irq_num, err; 544 545 iavf_irq_disable(adapter); 546 /* Decrement for Other and TCP Timer vectors */ 547 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 548 549 for (vector = 0; vector < q_vectors; vector++) { 550 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector]; 551 552 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 553 554 if (q_vector->tx.ring && q_vector->rx.ring) { 555 snprintf(q_vector->name, sizeof(q_vector->name), 556 "iavf-%s-TxRx-%u", basename, rx_int_idx++); 557 tx_int_idx++; 558 } else if (q_vector->rx.ring) { 559 snprintf(q_vector->name, sizeof(q_vector->name), 560 "iavf-%s-rx-%u", basename, rx_int_idx++); 561 } else if (q_vector->tx.ring) { 562 snprintf(q_vector->name, sizeof(q_vector->name), 563 "iavf-%s-tx-%u", basename, tx_int_idx++); 564 } else { 565 /* skip this unused q_vector */ 566 continue; 567 } 568 err = request_irq(irq_num, 569 iavf_msix_clean_rings, 570 0, 571 q_vector->name, 572 q_vector); 573 if (err) { 574 dev_info(&adapter->pdev->dev, 575 "Request_irq failed, error: %d\n", err); 576 goto free_queue_irqs; 577 } 578 } 579 580 return 0; 581 582 free_queue_irqs: 583 while (vector) { 584 vector--; 585 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 586 free_irq(irq_num, &adapter->q_vectors[vector]); 587 } 588 return err; 589 } 590 591 /** 592 * iavf_request_misc_irq - Initialize MSI-X interrupts 593 * @adapter: board private structure 594 * 595 * Allocates MSI-X vector 0 and requests interrupts from the kernel. This 596 * vector is only for the admin queue, and stays active even when the netdev 597 * is closed. 598 **/ 599 static int iavf_request_misc_irq(struct iavf_adapter *adapter) 600 { 601 struct net_device *netdev = adapter->netdev; 602 int err; 603 604 snprintf(adapter->misc_vector_name, 605 sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx", 606 dev_name(&adapter->pdev->dev)); 607 err = request_irq(adapter->msix_entries[0].vector, 608 &iavf_msix_aq, 0, 609 adapter->misc_vector_name, netdev); 610 if (err) { 611 dev_err(&adapter->pdev->dev, 612 "request_irq for %s failed: %d\n", 613 adapter->misc_vector_name, err); 614 free_irq(adapter->msix_entries[0].vector, netdev); 615 } 616 return err; 617 } 618 619 /** 620 * iavf_free_traffic_irqs - Free MSI-X interrupts 621 * @adapter: board private structure 622 * 623 * Frees all MSI-X vectors other than 0. 624 **/ 625 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter) 626 { 627 struct iavf_q_vector *q_vector; 628 int vector, irq_num, q_vectors; 629 630 if (!adapter->msix_entries) 631 return; 632 633 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 634 635 for (vector = 0; vector < q_vectors; vector++) { 636 q_vector = &adapter->q_vectors[vector]; 637 netif_napi_set_irq_locked(&q_vector->napi, -1); 638 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 639 free_irq(irq_num, q_vector); 640 } 641 } 642 643 /** 644 * iavf_free_misc_irq - Free MSI-X miscellaneous vector 645 * @adapter: board private structure 646 * 647 * Frees MSI-X vector 0. 648 **/ 649 static void iavf_free_misc_irq(struct iavf_adapter *adapter) 650 { 651 struct net_device *netdev = adapter->netdev; 652 653 if (!adapter->msix_entries) 654 return; 655 656 free_irq(adapter->msix_entries[0].vector, netdev); 657 } 658 659 /** 660 * iavf_configure_tx - Configure Transmit Unit after Reset 661 * @adapter: board private structure 662 * 663 * Configure the Tx unit of the MAC after a reset. 664 **/ 665 static void iavf_configure_tx(struct iavf_adapter *adapter) 666 { 667 struct iavf_hw *hw = &adapter->hw; 668 int i; 669 670 for (i = 0; i < adapter->num_active_queues; i++) 671 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i); 672 } 673 674 /** 675 * iavf_select_rx_desc_format - Select Rx descriptor format 676 * @adapter: adapter private structure 677 * 678 * Select what Rx descriptor format based on availability and enabled 679 * features. 680 * 681 * Return: the desired RXDID to select for a given Rx queue, as defined by 682 * enum virtchnl_rxdid_format. 683 */ 684 static u8 iavf_select_rx_desc_format(const struct iavf_adapter *adapter) 685 { 686 u64 rxdids = adapter->supp_rxdids; 687 688 /* If we did not negotiate VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC, we must 689 * stick with the default value of the legacy 32 byte format. 690 */ 691 if (!IAVF_RXDID_ALLOWED(adapter)) 692 return VIRTCHNL_RXDID_1_32B_BASE; 693 694 /* Rx timestamping requires the use of flexible NIC descriptors */ 695 if (iavf_ptp_cap_supported(adapter, VIRTCHNL_1588_PTP_CAP_RX_TSTAMP)) { 696 if (rxdids & BIT(VIRTCHNL_RXDID_2_FLEX_SQ_NIC)) 697 return VIRTCHNL_RXDID_2_FLEX_SQ_NIC; 698 699 pci_warn(adapter->pdev, 700 "Unable to negotiate flexible descriptor format\n"); 701 } 702 703 /* Warn if the PF does not list support for the default legacy 704 * descriptor format. This shouldn't happen, as this is the format 705 * used if VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC is not supported. It is 706 * likely caused by a bug in the PF implementation failing to indicate 707 * support for the format. 708 */ 709 if (!(rxdids & VIRTCHNL_RXDID_1_32B_BASE_M)) 710 netdev_warn(adapter->netdev, "PF does not list support for default Rx descriptor format\n"); 711 712 return VIRTCHNL_RXDID_1_32B_BASE; 713 } 714 715 /** 716 * iavf_configure_rx - Configure Receive Unit after Reset 717 * @adapter: board private structure 718 * 719 * Configure the Rx unit of the MAC after a reset. 720 **/ 721 static void iavf_configure_rx(struct iavf_adapter *adapter) 722 { 723 struct iavf_hw *hw = &adapter->hw; 724 725 adapter->rxdid = iavf_select_rx_desc_format(adapter); 726 727 for (u32 i = 0; i < adapter->num_active_queues; i++) { 728 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i); 729 adapter->rx_rings[i].rxdid = adapter->rxdid; 730 } 731 } 732 733 /** 734 * iavf_find_vlan - Search filter list for specific vlan filter 735 * @adapter: board private structure 736 * @vlan: vlan tag 737 * 738 * Returns ptr to the filter object or NULL. Must be called while holding the 739 * mac_vlan_list_lock. 740 **/ 741 static struct 742 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, 743 struct iavf_vlan vlan) 744 { 745 struct iavf_vlan_filter *f; 746 747 list_for_each_entry(f, &adapter->vlan_filter_list, list) { 748 if (f->vlan.vid == vlan.vid && 749 f->vlan.tpid == vlan.tpid) 750 return f; 751 } 752 753 return NULL; 754 } 755 756 /** 757 * iavf_add_vlan - Add a vlan filter to the list 758 * @adapter: board private structure 759 * @vlan: VLAN tag 760 * 761 * Returns ptr to the filter object or NULL when no memory available. 762 **/ 763 static struct 764 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, 765 struct iavf_vlan vlan) 766 { 767 struct iavf_vlan_filter *f = NULL; 768 769 spin_lock_bh(&adapter->mac_vlan_list_lock); 770 771 f = iavf_find_vlan(adapter, vlan); 772 if (!f) { 773 f = kzalloc(sizeof(*f), GFP_ATOMIC); 774 if (!f) 775 goto clearout; 776 777 f->vlan = vlan; 778 779 list_add_tail(&f->list, &adapter->vlan_filter_list); 780 f->state = IAVF_VLAN_ADD; 781 adapter->num_vlan_filters++; 782 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER); 783 } else if (f->state == IAVF_VLAN_REMOVE) { 784 /* IAVF_VLAN_REMOVE means that VLAN wasn't yet removed. 785 * We can safely only change the state here. 786 */ 787 f->state = IAVF_VLAN_ACTIVE; 788 } 789 790 clearout: 791 spin_unlock_bh(&adapter->mac_vlan_list_lock); 792 return f; 793 } 794 795 /** 796 * iavf_del_vlan - Remove a vlan filter from the list 797 * @adapter: board private structure 798 * @vlan: VLAN tag 799 **/ 800 static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan) 801 { 802 struct iavf_vlan_filter *f; 803 804 spin_lock_bh(&adapter->mac_vlan_list_lock); 805 806 f = iavf_find_vlan(adapter, vlan); 807 if (f) { 808 /* IAVF_ADD_VLAN means that VLAN wasn't even added yet. 809 * Remove it from the list. 810 */ 811 if (f->state == IAVF_VLAN_ADD) { 812 list_del(&f->list); 813 kfree(f); 814 adapter->num_vlan_filters--; 815 } else { 816 f->state = IAVF_VLAN_REMOVE; 817 iavf_schedule_aq_request(adapter, 818 IAVF_FLAG_AQ_DEL_VLAN_FILTER); 819 } 820 } 821 822 spin_unlock_bh(&adapter->mac_vlan_list_lock); 823 } 824 825 /** 826 * iavf_restore_filters 827 * @adapter: board private structure 828 * 829 * Restore existing non MAC filters when VF netdev comes back up 830 **/ 831 static void iavf_restore_filters(struct iavf_adapter *adapter) 832 { 833 struct iavf_vlan_filter *f; 834 835 /* re-add all VLAN filters */ 836 spin_lock_bh(&adapter->mac_vlan_list_lock); 837 838 list_for_each_entry(f, &adapter->vlan_filter_list, list) { 839 if (f->state == IAVF_VLAN_INACTIVE) 840 f->state = IAVF_VLAN_ADD; 841 } 842 843 spin_unlock_bh(&adapter->mac_vlan_list_lock); 844 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER; 845 } 846 847 /** 848 * iavf_get_num_vlans_added - get number of VLANs added 849 * @adapter: board private structure 850 */ 851 u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter) 852 { 853 return adapter->num_vlan_filters; 854 } 855 856 /** 857 * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF 858 * @adapter: board private structure 859 * 860 * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN, 861 * do not impose a limit as that maintains current behavior and for 862 * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF. 863 **/ 864 static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter) 865 { 866 /* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has 867 * never been a limit on the VF driver side 868 */ 869 if (VLAN_ALLOWED(adapter)) 870 return VLAN_N_VID; 871 else if (VLAN_V2_ALLOWED(adapter)) 872 return adapter->vlan_v2_caps.filtering.max_filters; 873 874 return 0; 875 } 876 877 /** 878 * iavf_max_vlans_added - check if maximum VLANs allowed already exist 879 * @adapter: board private structure 880 **/ 881 static bool iavf_max_vlans_added(struct iavf_adapter *adapter) 882 { 883 if (iavf_get_num_vlans_added(adapter) < 884 iavf_get_max_vlans_allowed(adapter)) 885 return false; 886 887 return true; 888 } 889 890 /** 891 * iavf_vlan_rx_add_vid - Add a VLAN filter to a device 892 * @netdev: network device struct 893 * @proto: unused protocol data 894 * @vid: VLAN tag 895 **/ 896 static int iavf_vlan_rx_add_vid(struct net_device *netdev, 897 __always_unused __be16 proto, u16 vid) 898 { 899 struct iavf_adapter *adapter = netdev_priv(netdev); 900 901 /* Do not track VLAN 0 filter, always added by the PF on VF init */ 902 if (!vid) 903 return 0; 904 905 if (!VLAN_FILTERING_ALLOWED(adapter)) 906 return -EIO; 907 908 if (iavf_max_vlans_added(adapter)) { 909 netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n", 910 iavf_get_max_vlans_allowed(adapter)); 911 return -EIO; 912 } 913 914 if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)))) 915 return -ENOMEM; 916 917 return 0; 918 } 919 920 /** 921 * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device 922 * @netdev: network device struct 923 * @proto: unused protocol data 924 * @vid: VLAN tag 925 **/ 926 static int iavf_vlan_rx_kill_vid(struct net_device *netdev, 927 __always_unused __be16 proto, u16 vid) 928 { 929 struct iavf_adapter *adapter = netdev_priv(netdev); 930 931 /* We do not track VLAN 0 filter */ 932 if (!vid) 933 return 0; 934 935 iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))); 936 return 0; 937 } 938 939 /** 940 * iavf_find_filter - Search filter list for specific mac filter 941 * @adapter: board private structure 942 * @macaddr: the MAC address 943 * 944 * Returns ptr to the filter object or NULL. Must be called while holding the 945 * mac_vlan_list_lock. 946 **/ 947 static struct 948 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter, 949 const u8 *macaddr) 950 { 951 struct iavf_mac_filter *f; 952 953 if (!macaddr) 954 return NULL; 955 956 list_for_each_entry(f, &adapter->mac_filter_list, list) { 957 if (ether_addr_equal(macaddr, f->macaddr)) 958 return f; 959 } 960 return NULL; 961 } 962 963 /** 964 * iavf_add_filter - Add a mac filter to the filter list 965 * @adapter: board private structure 966 * @macaddr: the MAC address 967 * 968 * Returns ptr to the filter object or NULL when no memory available. 969 **/ 970 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, 971 const u8 *macaddr) 972 { 973 struct iavf_mac_filter *f; 974 975 if (!macaddr) 976 return NULL; 977 978 f = iavf_find_filter(adapter, macaddr); 979 if (!f) { 980 f = kzalloc(sizeof(*f), GFP_ATOMIC); 981 if (!f) 982 return f; 983 984 ether_addr_copy(f->macaddr, macaddr); 985 986 list_add_tail(&f->list, &adapter->mac_filter_list); 987 f->add = true; 988 f->add_handled = false; 989 f->is_new_mac = true; 990 f->is_primary = ether_addr_equal(macaddr, adapter->hw.mac.addr); 991 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 992 } else { 993 f->remove = false; 994 } 995 996 return f; 997 } 998 999 /** 1000 * iavf_replace_primary_mac - Replace current primary address 1001 * @adapter: board private structure 1002 * @new_mac: new MAC address to be applied 1003 * 1004 * Replace current dev_addr and send request to PF for removal of previous 1005 * primary MAC address filter and addition of new primary MAC filter. 1006 * Return 0 for success, -ENOMEM for failure. 1007 * 1008 * Do not call this with mac_vlan_list_lock! 1009 **/ 1010 static int iavf_replace_primary_mac(struct iavf_adapter *adapter, 1011 const u8 *new_mac) 1012 { 1013 struct iavf_hw *hw = &adapter->hw; 1014 struct iavf_mac_filter *new_f; 1015 struct iavf_mac_filter *old_f; 1016 1017 spin_lock_bh(&adapter->mac_vlan_list_lock); 1018 1019 new_f = iavf_add_filter(adapter, new_mac); 1020 if (!new_f) { 1021 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1022 return -ENOMEM; 1023 } 1024 1025 old_f = iavf_find_filter(adapter, hw->mac.addr); 1026 if (old_f) { 1027 old_f->is_primary = false; 1028 old_f->remove = true; 1029 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 1030 } 1031 /* Always send the request to add if changing primary MAC, 1032 * even if filter is already present on the list 1033 */ 1034 new_f->is_primary = true; 1035 new_f->add = true; 1036 ether_addr_copy(hw->mac.addr, new_mac); 1037 1038 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1039 1040 /* schedule the watchdog task to immediately process the request */ 1041 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_MAC_FILTER); 1042 return 0; 1043 } 1044 1045 /** 1046 * iavf_is_mac_set_handled - wait for a response to set MAC from PF 1047 * @netdev: network interface device structure 1048 * @macaddr: MAC address to set 1049 * 1050 * Returns true on success, false on failure 1051 */ 1052 static bool iavf_is_mac_set_handled(struct net_device *netdev, 1053 const u8 *macaddr) 1054 { 1055 struct iavf_adapter *adapter = netdev_priv(netdev); 1056 struct iavf_mac_filter *f; 1057 bool ret = false; 1058 1059 spin_lock_bh(&adapter->mac_vlan_list_lock); 1060 1061 f = iavf_find_filter(adapter, macaddr); 1062 1063 if (!f || (!f->add && f->add_handled)) 1064 ret = true; 1065 1066 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1067 1068 return ret; 1069 } 1070 1071 /** 1072 * iavf_set_mac - NDO callback to set port MAC address 1073 * @netdev: network interface device structure 1074 * @p: pointer to an address structure 1075 * 1076 * Returns 0 on success, negative on failure 1077 */ 1078 static int iavf_set_mac(struct net_device *netdev, void *p) 1079 { 1080 struct iavf_adapter *adapter = netdev_priv(netdev); 1081 struct sockaddr *addr = p; 1082 int ret; 1083 1084 if (!is_valid_ether_addr(addr->sa_data)) 1085 return -EADDRNOTAVAIL; 1086 1087 ret = iavf_replace_primary_mac(adapter, addr->sa_data); 1088 1089 if (ret) 1090 return ret; 1091 1092 ret = wait_event_interruptible_timeout(adapter->vc_waitqueue, 1093 iavf_is_mac_set_handled(netdev, addr->sa_data), 1094 msecs_to_jiffies(2500)); 1095 1096 /* If ret < 0 then it means wait was interrupted. 1097 * If ret == 0 then it means we got a timeout. 1098 * else it means we got response for set MAC from PF, 1099 * check if netdev MAC was updated to requested MAC, 1100 * if yes then set MAC succeeded otherwise it failed return -EACCES 1101 */ 1102 if (ret < 0) 1103 return ret; 1104 1105 if (!ret) 1106 return -EAGAIN; 1107 1108 if (!ether_addr_equal(netdev->dev_addr, addr->sa_data)) 1109 return -EACCES; 1110 1111 return 0; 1112 } 1113 1114 /** 1115 * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address 1116 * @netdev: the netdevice 1117 * @addr: address to add 1118 * 1119 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1120 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1121 */ 1122 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr) 1123 { 1124 struct iavf_adapter *adapter = netdev_priv(netdev); 1125 1126 if (iavf_add_filter(adapter, addr)) 1127 return 0; 1128 else 1129 return -ENOMEM; 1130 } 1131 1132 /** 1133 * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1134 * @netdev: the netdevice 1135 * @addr: address to add 1136 * 1137 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1138 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1139 */ 1140 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr) 1141 { 1142 struct iavf_adapter *adapter = netdev_priv(netdev); 1143 struct iavf_mac_filter *f; 1144 1145 /* Under some circumstances, we might receive a request to delete 1146 * our own device address from our uc list. Because we store the 1147 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1148 * such requests and not delete our device address from this list. 1149 */ 1150 if (ether_addr_equal(addr, netdev->dev_addr)) 1151 return 0; 1152 1153 f = iavf_find_filter(adapter, addr); 1154 if (f) { 1155 f->remove = true; 1156 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 1157 } 1158 return 0; 1159 } 1160 1161 /** 1162 * iavf_promiscuous_mode_changed - check if promiscuous mode bits changed 1163 * @adapter: device specific adapter 1164 */ 1165 bool iavf_promiscuous_mode_changed(struct iavf_adapter *adapter) 1166 { 1167 return (adapter->current_netdev_promisc_flags ^ adapter->netdev->flags) & 1168 (IFF_PROMISC | IFF_ALLMULTI); 1169 } 1170 1171 /** 1172 * iavf_set_rx_mode - NDO callback to set the netdev filters 1173 * @netdev: network interface device structure 1174 **/ 1175 static void iavf_set_rx_mode(struct net_device *netdev) 1176 { 1177 struct iavf_adapter *adapter = netdev_priv(netdev); 1178 1179 spin_lock_bh(&adapter->mac_vlan_list_lock); 1180 __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 1181 __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 1182 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1183 1184 spin_lock_bh(&adapter->current_netdev_promisc_flags_lock); 1185 if (iavf_promiscuous_mode_changed(adapter)) 1186 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE; 1187 spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock); 1188 } 1189 1190 /** 1191 * iavf_napi_enable_all - enable NAPI on all queue vectors 1192 * @adapter: board private structure 1193 **/ 1194 static void iavf_napi_enable_all(struct iavf_adapter *adapter) 1195 { 1196 int q_idx; 1197 struct iavf_q_vector *q_vector; 1198 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1199 1200 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1201 struct napi_struct *napi; 1202 1203 q_vector = &adapter->q_vectors[q_idx]; 1204 napi = &q_vector->napi; 1205 napi_enable_locked(napi); 1206 } 1207 } 1208 1209 /** 1210 * iavf_napi_disable_all - disable NAPI on all queue vectors 1211 * @adapter: board private structure 1212 **/ 1213 static void iavf_napi_disable_all(struct iavf_adapter *adapter) 1214 { 1215 int q_idx; 1216 struct iavf_q_vector *q_vector; 1217 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1218 1219 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1220 q_vector = &adapter->q_vectors[q_idx]; 1221 napi_disable_locked(&q_vector->napi); 1222 } 1223 } 1224 1225 /** 1226 * iavf_configure - set up transmit and receive data structures 1227 * @adapter: board private structure 1228 **/ 1229 static void iavf_configure(struct iavf_adapter *adapter) 1230 { 1231 struct net_device *netdev = adapter->netdev; 1232 int i; 1233 1234 iavf_set_rx_mode(netdev); 1235 1236 iavf_configure_tx(adapter); 1237 iavf_configure_rx(adapter); 1238 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES; 1239 1240 for (i = 0; i < adapter->num_active_queues; i++) { 1241 struct iavf_ring *ring = &adapter->rx_rings[i]; 1242 1243 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring)); 1244 } 1245 } 1246 1247 /** 1248 * iavf_up_complete - Finish the last steps of bringing up a connection 1249 * @adapter: board private structure 1250 */ 1251 static void iavf_up_complete(struct iavf_adapter *adapter) 1252 { 1253 netdev_assert_locked(adapter->netdev); 1254 1255 iavf_change_state(adapter, __IAVF_RUNNING); 1256 clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1257 1258 iavf_napi_enable_all(adapter); 1259 1260 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ENABLE_QUEUES); 1261 } 1262 1263 /** 1264 * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF 1265 * yet and mark other to be removed. 1266 * @adapter: board private structure 1267 **/ 1268 static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter) 1269 { 1270 struct iavf_vlan_filter *vlf, *vlftmp; 1271 struct iavf_mac_filter *f, *ftmp; 1272 1273 spin_lock_bh(&adapter->mac_vlan_list_lock); 1274 /* clear the sync flag on all filters */ 1275 __dev_uc_unsync(adapter->netdev, NULL); 1276 __dev_mc_unsync(adapter->netdev, NULL); 1277 1278 /* remove all MAC filters */ 1279 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, 1280 list) { 1281 if (f->add) { 1282 list_del(&f->list); 1283 kfree(f); 1284 } else { 1285 f->remove = true; 1286 } 1287 } 1288 1289 /* disable all VLAN filters */ 1290 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list, 1291 list) 1292 vlf->state = IAVF_VLAN_DISABLE; 1293 1294 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1295 } 1296 1297 /** 1298 * iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and 1299 * mark other to be removed. 1300 * @adapter: board private structure 1301 **/ 1302 static void iavf_clear_cloud_filters(struct iavf_adapter *adapter) 1303 { 1304 struct iavf_cloud_filter *cf, *cftmp; 1305 1306 /* remove all cloud filters */ 1307 spin_lock_bh(&adapter->cloud_filter_list_lock); 1308 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, 1309 list) { 1310 if (cf->add) { 1311 list_del(&cf->list); 1312 kfree(cf); 1313 adapter->num_cloud_filters--; 1314 } else { 1315 cf->del = true; 1316 } 1317 } 1318 spin_unlock_bh(&adapter->cloud_filter_list_lock); 1319 } 1320 1321 /** 1322 * iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark 1323 * other to be removed. 1324 * @adapter: board private structure 1325 **/ 1326 static void iavf_clear_fdir_filters(struct iavf_adapter *adapter) 1327 { 1328 struct iavf_fdir_fltr *fdir; 1329 1330 /* remove all Flow Director filters */ 1331 spin_lock_bh(&adapter->fdir_fltr_lock); 1332 list_for_each_entry(fdir, &adapter->fdir_list_head, list) { 1333 if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) { 1334 /* Cancel a request, keep filter as inactive */ 1335 fdir->state = IAVF_FDIR_FLTR_INACTIVE; 1336 } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING || 1337 fdir->state == IAVF_FDIR_FLTR_ACTIVE) { 1338 /* Disable filters which are active or have a pending 1339 * request to PF to be added 1340 */ 1341 fdir->state = IAVF_FDIR_FLTR_DIS_REQUEST; 1342 } 1343 } 1344 spin_unlock_bh(&adapter->fdir_fltr_lock); 1345 } 1346 1347 /** 1348 * iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark 1349 * other to be removed. 1350 * @adapter: board private structure 1351 **/ 1352 static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter) 1353 { 1354 struct iavf_adv_rss *rss, *rsstmp; 1355 1356 /* remove all advance RSS configuration */ 1357 spin_lock_bh(&adapter->adv_rss_lock); 1358 list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head, 1359 list) { 1360 if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) { 1361 list_del(&rss->list); 1362 kfree(rss); 1363 } else { 1364 rss->state = IAVF_ADV_RSS_DEL_REQUEST; 1365 } 1366 } 1367 spin_unlock_bh(&adapter->adv_rss_lock); 1368 } 1369 1370 /** 1371 * iavf_down - Shutdown the connection processing 1372 * @adapter: board private structure 1373 */ 1374 void iavf_down(struct iavf_adapter *adapter) 1375 { 1376 struct net_device *netdev = adapter->netdev; 1377 1378 netdev_assert_locked(netdev); 1379 1380 if (adapter->state <= __IAVF_DOWN_PENDING) 1381 return; 1382 1383 netif_carrier_off(netdev); 1384 netif_tx_disable(netdev); 1385 adapter->link_up = false; 1386 iavf_napi_disable_all(adapter); 1387 iavf_irq_disable(adapter); 1388 1389 iavf_clear_mac_vlan_filters(adapter); 1390 iavf_clear_cloud_filters(adapter); 1391 iavf_clear_fdir_filters(adapter); 1392 iavf_clear_adv_rss_conf(adapter); 1393 1394 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 1395 return; 1396 1397 if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) { 1398 /* cancel any current operation */ 1399 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1400 /* Schedule operations to close down the HW. Don't wait 1401 * here for this to complete. The watchdog is still running 1402 * and it will take care of this. 1403 */ 1404 if (!list_empty(&adapter->mac_filter_list)) 1405 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 1406 if (!list_empty(&adapter->vlan_filter_list)) 1407 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; 1408 if (!list_empty(&adapter->cloud_filter_list)) 1409 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 1410 if (!list_empty(&adapter->fdir_list_head)) 1411 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER; 1412 if (!list_empty(&adapter->adv_rss_list_head)) 1413 adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG; 1414 } 1415 1416 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DISABLE_QUEUES); 1417 } 1418 1419 /** 1420 * iavf_acquire_msix_vectors - Setup the MSIX capability 1421 * @adapter: board private structure 1422 * @vectors: number of vectors to request 1423 * 1424 * Work with the OS to set up the MSIX vectors needed. 1425 * 1426 * Returns 0 on success, negative on failure 1427 **/ 1428 static int 1429 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors) 1430 { 1431 int err, vector_threshold; 1432 1433 /* We'll want at least 3 (vector_threshold): 1434 * 0) Other (Admin Queue and link, mostly) 1435 * 1) TxQ[0] Cleanup 1436 * 2) RxQ[0] Cleanup 1437 */ 1438 vector_threshold = MIN_MSIX_COUNT; 1439 1440 /* The more we get, the more we will assign to Tx/Rx Cleanup 1441 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1442 * Right now, we simply care about how many we'll get; we'll 1443 * set them up later while requesting irq's. 1444 */ 1445 err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 1446 vector_threshold, vectors); 1447 if (err < 0) { 1448 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n"); 1449 kfree(adapter->msix_entries); 1450 adapter->msix_entries = NULL; 1451 return err; 1452 } 1453 1454 /* Adjust for only the vectors we'll use, which is minimum 1455 * of max_msix_q_vectors + NONQ_VECS, or the number of 1456 * vectors we were allocated. 1457 */ 1458 adapter->num_msix_vectors = err; 1459 return 0; 1460 } 1461 1462 /** 1463 * iavf_free_queues - Free memory for all rings 1464 * @adapter: board private structure to initialize 1465 * 1466 * Free all of the memory associated with queue pairs. 1467 **/ 1468 static void iavf_free_queues(struct iavf_adapter *adapter) 1469 { 1470 if (!adapter->vsi_res) 1471 return; 1472 adapter->num_active_queues = 0; 1473 kfree(adapter->tx_rings); 1474 adapter->tx_rings = NULL; 1475 kfree(adapter->rx_rings); 1476 adapter->rx_rings = NULL; 1477 } 1478 1479 /** 1480 * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload 1481 * @adapter: board private structure 1482 * 1483 * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or 1484 * stripped in certain descriptor fields. Instead of checking the offload 1485 * capability bits in the hot path, cache the location the ring specific 1486 * flags. 1487 */ 1488 void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter) 1489 { 1490 int i; 1491 1492 for (i = 0; i < adapter->num_active_queues; i++) { 1493 struct iavf_ring *tx_ring = &adapter->tx_rings[i]; 1494 struct iavf_ring *rx_ring = &adapter->rx_rings[i]; 1495 1496 /* prevent multiple L2TAG bits being set after VFR */ 1497 tx_ring->flags &= 1498 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 | 1499 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2); 1500 rx_ring->flags &= 1501 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 | 1502 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2); 1503 1504 if (VLAN_ALLOWED(adapter)) { 1505 tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1506 rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1507 } else if (VLAN_V2_ALLOWED(adapter)) { 1508 struct virtchnl_vlan_supported_caps *stripping_support; 1509 struct virtchnl_vlan_supported_caps *insertion_support; 1510 1511 stripping_support = 1512 &adapter->vlan_v2_caps.offloads.stripping_support; 1513 insertion_support = 1514 &adapter->vlan_v2_caps.offloads.insertion_support; 1515 1516 if (stripping_support->outer) { 1517 if (stripping_support->outer & 1518 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1519 rx_ring->flags |= 1520 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1521 else if (stripping_support->outer & 1522 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2) 1523 rx_ring->flags |= 1524 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2; 1525 } else if (stripping_support->inner) { 1526 if (stripping_support->inner & 1527 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1528 rx_ring->flags |= 1529 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1530 else if (stripping_support->inner & 1531 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2) 1532 rx_ring->flags |= 1533 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2; 1534 } 1535 1536 if (insertion_support->outer) { 1537 if (insertion_support->outer & 1538 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1539 tx_ring->flags |= 1540 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1541 else if (insertion_support->outer & 1542 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2) 1543 tx_ring->flags |= 1544 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2; 1545 } else if (insertion_support->inner) { 1546 if (insertion_support->inner & 1547 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1548 tx_ring->flags |= 1549 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1550 else if (insertion_support->inner & 1551 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2) 1552 tx_ring->flags |= 1553 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2; 1554 } 1555 } 1556 } 1557 } 1558 1559 /** 1560 * iavf_alloc_queues - Allocate memory for all rings 1561 * @adapter: board private structure to initialize 1562 * 1563 * We allocate one ring per queue at run-time since we don't know the 1564 * number of queues at compile-time. The polling_netdev array is 1565 * intended for Multiqueue, but should work fine with a single queue. 1566 **/ 1567 static int iavf_alloc_queues(struct iavf_adapter *adapter) 1568 { 1569 int i, num_active_queues; 1570 1571 /* If we're in reset reallocating queues we don't actually know yet for 1572 * certain the PF gave us the number of queues we asked for but we'll 1573 * assume it did. Once basic reset is finished we'll confirm once we 1574 * start negotiating config with PF. 1575 */ 1576 if (adapter->num_req_queues) 1577 num_active_queues = adapter->num_req_queues; 1578 else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1579 adapter->num_tc) 1580 num_active_queues = adapter->ch_config.total_qps; 1581 else 1582 num_active_queues = min_t(int, 1583 adapter->vsi_res->num_queue_pairs, 1584 (int)(num_online_cpus())); 1585 1586 1587 adapter->tx_rings = kcalloc(num_active_queues, 1588 sizeof(struct iavf_ring), GFP_KERNEL); 1589 if (!adapter->tx_rings) 1590 goto err_out; 1591 adapter->rx_rings = kcalloc(num_active_queues, 1592 sizeof(struct iavf_ring), GFP_KERNEL); 1593 if (!adapter->rx_rings) 1594 goto err_out; 1595 1596 for (i = 0; i < num_active_queues; i++) { 1597 struct iavf_ring *tx_ring; 1598 struct iavf_ring *rx_ring; 1599 1600 tx_ring = &adapter->tx_rings[i]; 1601 1602 tx_ring->queue_index = i; 1603 tx_ring->netdev = adapter->netdev; 1604 tx_ring->dev = &adapter->pdev->dev; 1605 tx_ring->count = adapter->tx_desc_count; 1606 tx_ring->itr_setting = IAVF_ITR_TX_DEF; 1607 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE) 1608 tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR; 1609 1610 rx_ring = &adapter->rx_rings[i]; 1611 rx_ring->queue_index = i; 1612 rx_ring->netdev = adapter->netdev; 1613 rx_ring->count = adapter->rx_desc_count; 1614 rx_ring->itr_setting = IAVF_ITR_RX_DEF; 1615 } 1616 1617 adapter->num_active_queues = num_active_queues; 1618 1619 iavf_set_queue_vlan_tag_loc(adapter); 1620 1621 return 0; 1622 1623 err_out: 1624 iavf_free_queues(adapter); 1625 return -ENOMEM; 1626 } 1627 1628 /** 1629 * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported 1630 * @adapter: board private structure to initialize 1631 * 1632 * Attempt to configure the interrupts using the best available 1633 * capabilities of the hardware and the kernel. 1634 **/ 1635 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter) 1636 { 1637 int vector, v_budget; 1638 int pairs = 0; 1639 int err = 0; 1640 1641 if (!adapter->vsi_res) { 1642 err = -EIO; 1643 goto out; 1644 } 1645 pairs = adapter->num_active_queues; 1646 1647 /* It's easy to be greedy for MSI-X vectors, but it really doesn't do 1648 * us much good if we have more vectors than CPUs. However, we already 1649 * limit the total number of queues by the number of CPUs so we do not 1650 * need any further limiting here. 1651 */ 1652 v_budget = min_t(int, pairs + NONQ_VECS, 1653 (int)adapter->vf_res->max_vectors); 1654 1655 adapter->msix_entries = kcalloc(v_budget, 1656 sizeof(struct msix_entry), GFP_KERNEL); 1657 if (!adapter->msix_entries) { 1658 err = -ENOMEM; 1659 goto out; 1660 } 1661 1662 for (vector = 0; vector < v_budget; vector++) 1663 adapter->msix_entries[vector].entry = vector; 1664 1665 err = iavf_acquire_msix_vectors(adapter, v_budget); 1666 if (!err) 1667 iavf_schedule_finish_config(adapter); 1668 1669 out: 1670 return err; 1671 } 1672 1673 /** 1674 * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands 1675 * @adapter: board private structure 1676 * 1677 * Return 0 on success, negative on failure 1678 **/ 1679 static int iavf_config_rss_aq(struct iavf_adapter *adapter) 1680 { 1681 struct iavf_aqc_get_set_rss_key_data *rss_key = 1682 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key; 1683 struct iavf_hw *hw = &adapter->hw; 1684 enum iavf_status status; 1685 1686 if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { 1687 /* bail because we already have a command pending */ 1688 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n", 1689 adapter->current_op); 1690 return -EBUSY; 1691 } 1692 1693 status = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key); 1694 if (status) { 1695 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n", 1696 iavf_stat_str(hw, status), 1697 iavf_aq_str(hw, hw->aq.asq_last_status)); 1698 return iavf_status_to_errno(status); 1699 1700 } 1701 1702 status = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false, 1703 adapter->rss_lut, adapter->rss_lut_size); 1704 if (status) { 1705 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n", 1706 iavf_stat_str(hw, status), 1707 iavf_aq_str(hw, hw->aq.asq_last_status)); 1708 return iavf_status_to_errno(status); 1709 } 1710 1711 return 0; 1712 1713 } 1714 1715 /** 1716 * iavf_config_rss_reg - Configure RSS keys and lut by writing registers 1717 * @adapter: board private structure 1718 * 1719 * Returns 0 on success, negative on failure 1720 **/ 1721 static int iavf_config_rss_reg(struct iavf_adapter *adapter) 1722 { 1723 struct iavf_hw *hw = &adapter->hw; 1724 u32 *dw; 1725 u16 i; 1726 1727 dw = (u32 *)adapter->rss_key; 1728 for (i = 0; i <= adapter->rss_key_size / 4; i++) 1729 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]); 1730 1731 dw = (u32 *)adapter->rss_lut; 1732 for (i = 0; i <= adapter->rss_lut_size / 4; i++) 1733 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]); 1734 1735 iavf_flush(hw); 1736 1737 return 0; 1738 } 1739 1740 /** 1741 * iavf_config_rss - Configure RSS keys and lut 1742 * @adapter: board private structure 1743 * 1744 * Returns 0 on success, negative on failure 1745 **/ 1746 int iavf_config_rss(struct iavf_adapter *adapter) 1747 { 1748 1749 if (RSS_PF(adapter)) { 1750 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT | 1751 IAVF_FLAG_AQ_SET_RSS_KEY; 1752 return 0; 1753 } else if (RSS_AQ(adapter)) { 1754 return iavf_config_rss_aq(adapter); 1755 } else { 1756 return iavf_config_rss_reg(adapter); 1757 } 1758 } 1759 1760 /** 1761 * iavf_fill_rss_lut - Fill the lut with default values 1762 * @adapter: board private structure 1763 **/ 1764 static void iavf_fill_rss_lut(struct iavf_adapter *adapter) 1765 { 1766 u16 i; 1767 1768 for (i = 0; i < adapter->rss_lut_size; i++) 1769 adapter->rss_lut[i] = i % adapter->num_active_queues; 1770 } 1771 1772 /** 1773 * iavf_init_rss - Prepare for RSS 1774 * @adapter: board private structure 1775 * 1776 * Return 0 on success, negative on failure 1777 **/ 1778 static int iavf_init_rss(struct iavf_adapter *adapter) 1779 { 1780 struct iavf_hw *hw = &adapter->hw; 1781 1782 if (!RSS_PF(adapter)) { 1783 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */ 1784 if (adapter->vf_res->vf_cap_flags & 1785 VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2) 1786 adapter->rss_hashcfg = 1787 IAVF_DEFAULT_RSS_HASHCFG_EXPANDED; 1788 else 1789 adapter->rss_hashcfg = IAVF_DEFAULT_RSS_HASHCFG; 1790 1791 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->rss_hashcfg); 1792 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->rss_hashcfg >> 32)); 1793 } 1794 1795 iavf_fill_rss_lut(adapter); 1796 netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size); 1797 1798 return iavf_config_rss(adapter); 1799 } 1800 1801 /** 1802 * iavf_alloc_q_vectors - Allocate memory for interrupt vectors 1803 * @adapter: board private structure to initialize 1804 * 1805 * We allocate one q_vector per queue interrupt. If allocation fails we 1806 * return -ENOMEM. 1807 **/ 1808 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter) 1809 { 1810 int q_idx = 0, num_q_vectors, irq_num; 1811 struct iavf_q_vector *q_vector; 1812 1813 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1814 adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector), 1815 GFP_KERNEL); 1816 if (!adapter->q_vectors) 1817 return -ENOMEM; 1818 1819 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1820 irq_num = adapter->msix_entries[q_idx + NONQ_VECS].vector; 1821 q_vector = &adapter->q_vectors[q_idx]; 1822 q_vector->adapter = adapter; 1823 q_vector->vsi = &adapter->vsi; 1824 q_vector->v_idx = q_idx; 1825 q_vector->reg_idx = q_idx; 1826 netif_napi_add_config_locked(adapter->netdev, &q_vector->napi, 1827 iavf_napi_poll, q_idx); 1828 netif_napi_set_irq_locked(&q_vector->napi, irq_num); 1829 } 1830 1831 return 0; 1832 } 1833 1834 /** 1835 * iavf_free_q_vectors - Free memory allocated for interrupt vectors 1836 * @adapter: board private structure to initialize 1837 * 1838 * This function frees the memory allocated to the q_vectors. In addition if 1839 * NAPI is enabled it will delete any references to the NAPI struct prior 1840 * to freeing the q_vector. 1841 **/ 1842 static void iavf_free_q_vectors(struct iavf_adapter *adapter) 1843 { 1844 int q_idx, num_q_vectors; 1845 1846 if (!adapter->q_vectors) 1847 return; 1848 1849 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1850 1851 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1852 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx]; 1853 1854 netif_napi_del_locked(&q_vector->napi); 1855 } 1856 kfree(adapter->q_vectors); 1857 adapter->q_vectors = NULL; 1858 } 1859 1860 /** 1861 * iavf_reset_interrupt_capability - Reset MSIX setup 1862 * @adapter: board private structure 1863 * 1864 **/ 1865 static void iavf_reset_interrupt_capability(struct iavf_adapter *adapter) 1866 { 1867 if (!adapter->msix_entries) 1868 return; 1869 1870 pci_disable_msix(adapter->pdev); 1871 kfree(adapter->msix_entries); 1872 adapter->msix_entries = NULL; 1873 } 1874 1875 /** 1876 * iavf_init_interrupt_scheme - Determine if MSIX is supported and init 1877 * @adapter: board private structure to initialize 1878 * 1879 **/ 1880 static int iavf_init_interrupt_scheme(struct iavf_adapter *adapter) 1881 { 1882 int err; 1883 1884 err = iavf_alloc_queues(adapter); 1885 if (err) { 1886 dev_err(&adapter->pdev->dev, 1887 "Unable to allocate memory for queues\n"); 1888 goto err_alloc_queues; 1889 } 1890 1891 err = iavf_set_interrupt_capability(adapter); 1892 if (err) { 1893 dev_err(&adapter->pdev->dev, 1894 "Unable to setup interrupt capabilities\n"); 1895 goto err_set_interrupt; 1896 } 1897 1898 err = iavf_alloc_q_vectors(adapter); 1899 if (err) { 1900 dev_err(&adapter->pdev->dev, 1901 "Unable to allocate memory for queue vectors\n"); 1902 goto err_alloc_q_vectors; 1903 } 1904 1905 /* If we've made it so far while ADq flag being ON, then we haven't 1906 * bailed out anywhere in middle. And ADq isn't just enabled but actual 1907 * resources have been allocated in the reset path. 1908 * Now we can truly claim that ADq is enabled. 1909 */ 1910 if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1911 adapter->num_tc) 1912 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created", 1913 adapter->num_tc); 1914 1915 dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u", 1916 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled", 1917 adapter->num_active_queues); 1918 1919 return 0; 1920 err_alloc_q_vectors: 1921 iavf_reset_interrupt_capability(adapter); 1922 err_set_interrupt: 1923 iavf_free_queues(adapter); 1924 err_alloc_queues: 1925 return err; 1926 } 1927 1928 /** 1929 * iavf_free_interrupt_scheme - Undo what iavf_init_interrupt_scheme does 1930 * @adapter: board private structure 1931 **/ 1932 static void iavf_free_interrupt_scheme(struct iavf_adapter *adapter) 1933 { 1934 iavf_free_q_vectors(adapter); 1935 iavf_reset_interrupt_capability(adapter); 1936 iavf_free_queues(adapter); 1937 } 1938 1939 /** 1940 * iavf_free_rss - Free memory used by RSS structs 1941 * @adapter: board private structure 1942 **/ 1943 static void iavf_free_rss(struct iavf_adapter *adapter) 1944 { 1945 kfree(adapter->rss_key); 1946 adapter->rss_key = NULL; 1947 1948 kfree(adapter->rss_lut); 1949 adapter->rss_lut = NULL; 1950 } 1951 1952 /** 1953 * iavf_reinit_interrupt_scheme - Reallocate queues and vectors 1954 * @adapter: board private structure 1955 * @running: true if adapter->state == __IAVF_RUNNING 1956 * 1957 * Returns 0 on success, negative on failure 1958 **/ 1959 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter, bool running) 1960 { 1961 struct net_device *netdev = adapter->netdev; 1962 int err; 1963 1964 if (running) 1965 iavf_free_traffic_irqs(adapter); 1966 iavf_free_misc_irq(adapter); 1967 iavf_free_interrupt_scheme(adapter); 1968 1969 err = iavf_init_interrupt_scheme(adapter); 1970 if (err) 1971 goto err; 1972 1973 netif_tx_stop_all_queues(netdev); 1974 1975 err = iavf_request_misc_irq(adapter); 1976 if (err) 1977 goto err; 1978 1979 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1980 1981 iavf_map_rings_to_vectors(adapter); 1982 err: 1983 return err; 1984 } 1985 1986 /** 1987 * iavf_finish_config - do all netdev work that needs RTNL 1988 * @work: our work_struct 1989 * 1990 * Do work that needs RTNL. 1991 */ 1992 static void iavf_finish_config(struct work_struct *work) 1993 { 1994 struct iavf_adapter *adapter; 1995 bool netdev_released = false; 1996 int pairs, err; 1997 1998 adapter = container_of(work, struct iavf_adapter, finish_config); 1999 2000 /* Always take RTNL first to prevent circular lock dependency; 2001 * the dev->lock (== netdev lock) is needed to update the queue number. 2002 */ 2003 rtnl_lock(); 2004 netdev_lock(adapter->netdev); 2005 2006 if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES) && 2007 adapter->netdev->reg_state == NETREG_REGISTERED && 2008 !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) { 2009 netdev_update_features(adapter->netdev); 2010 adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES; 2011 } 2012 2013 switch (adapter->state) { 2014 case __IAVF_DOWN: 2015 /* Set the real number of queues when reset occurs while 2016 * state == __IAVF_DOWN 2017 */ 2018 pairs = adapter->num_active_queues; 2019 netif_set_real_num_rx_queues(adapter->netdev, pairs); 2020 netif_set_real_num_tx_queues(adapter->netdev, pairs); 2021 2022 if (adapter->netdev->reg_state != NETREG_REGISTERED) { 2023 netdev_unlock(adapter->netdev); 2024 netdev_released = true; 2025 err = register_netdevice(adapter->netdev); 2026 if (err) { 2027 dev_err(&adapter->pdev->dev, "Unable to register netdev (%d)\n", 2028 err); 2029 2030 /* go back and try again.*/ 2031 netdev_lock(adapter->netdev); 2032 iavf_free_rss(adapter); 2033 iavf_free_misc_irq(adapter); 2034 iavf_reset_interrupt_capability(adapter); 2035 iavf_change_state(adapter, 2036 __IAVF_INIT_CONFIG_ADAPTER); 2037 netdev_unlock(adapter->netdev); 2038 goto out; 2039 } 2040 } 2041 break; 2042 case __IAVF_RUNNING: 2043 pairs = adapter->num_active_queues; 2044 netif_set_real_num_rx_queues(adapter->netdev, pairs); 2045 netif_set_real_num_tx_queues(adapter->netdev, pairs); 2046 break; 2047 2048 default: 2049 break; 2050 } 2051 2052 out: 2053 if (!netdev_released) 2054 netdev_unlock(adapter->netdev); 2055 rtnl_unlock(); 2056 } 2057 2058 /** 2059 * iavf_schedule_finish_config - Set the flags and schedule a reset event 2060 * @adapter: board private structure 2061 **/ 2062 void iavf_schedule_finish_config(struct iavf_adapter *adapter) 2063 { 2064 if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) 2065 queue_work(adapter->wq, &adapter->finish_config); 2066 } 2067 2068 /** 2069 * iavf_process_aq_command - process aq_required flags 2070 * and sends aq command 2071 * @adapter: pointer to iavf adapter structure 2072 * 2073 * Returns 0 on success 2074 * Returns error code if no command was sent 2075 * or error code if the command failed. 2076 **/ 2077 static int iavf_process_aq_command(struct iavf_adapter *adapter) 2078 { 2079 if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG) 2080 return iavf_send_vf_config_msg(adapter); 2081 if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS) 2082 return iavf_send_vf_offload_vlan_v2_msg(adapter); 2083 if (adapter->aq_required & IAVF_FLAG_AQ_GET_SUPPORTED_RXDIDS) 2084 return iavf_send_vf_supported_rxdids_msg(adapter); 2085 if (adapter->aq_required & IAVF_FLAG_AQ_GET_PTP_CAPS) 2086 return iavf_send_vf_ptp_caps_msg(adapter); 2087 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) { 2088 iavf_disable_queues(adapter); 2089 return 0; 2090 } 2091 2092 if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) { 2093 iavf_map_queues(adapter); 2094 return 0; 2095 } 2096 2097 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) { 2098 iavf_add_ether_addrs(adapter); 2099 return 0; 2100 } 2101 2102 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) { 2103 iavf_add_vlans(adapter); 2104 return 0; 2105 } 2106 2107 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) { 2108 iavf_del_ether_addrs(adapter); 2109 return 0; 2110 } 2111 2112 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) { 2113 iavf_del_vlans(adapter); 2114 return 0; 2115 } 2116 2117 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) { 2118 iavf_enable_vlan_stripping(adapter); 2119 return 0; 2120 } 2121 2122 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) { 2123 iavf_disable_vlan_stripping(adapter); 2124 return 0; 2125 } 2126 2127 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES_BW) { 2128 iavf_cfg_queues_bw(adapter); 2129 return 0; 2130 } 2131 2132 if (adapter->aq_required & IAVF_FLAG_AQ_GET_QOS_CAPS) { 2133 iavf_get_qos_caps(adapter); 2134 return 0; 2135 } 2136 2137 if (adapter->aq_required & IAVF_FLAG_AQ_CFG_QUEUES_QUANTA_SIZE) { 2138 iavf_cfg_queues_quanta_size(adapter); 2139 return 0; 2140 } 2141 2142 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) { 2143 iavf_configure_queues(adapter); 2144 return 0; 2145 } 2146 2147 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) { 2148 iavf_enable_queues(adapter); 2149 return 0; 2150 } 2151 2152 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) { 2153 /* This message goes straight to the firmware, not the 2154 * PF, so we don't have to set current_op as we will 2155 * not get a response through the ARQ. 2156 */ 2157 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS; 2158 return 0; 2159 } 2160 if (adapter->aq_required & IAVF_FLAG_AQ_GET_RSS_HASHCFG) { 2161 iavf_get_rss_hashcfg(adapter); 2162 return 0; 2163 } 2164 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_HASHCFG) { 2165 iavf_set_rss_hashcfg(adapter); 2166 return 0; 2167 } 2168 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) { 2169 iavf_set_rss_key(adapter); 2170 return 0; 2171 } 2172 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) { 2173 iavf_set_rss_lut(adapter); 2174 return 0; 2175 } 2176 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_HFUNC) { 2177 iavf_set_rss_hfunc(adapter); 2178 return 0; 2179 } 2180 2181 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE) { 2182 iavf_set_promiscuous(adapter); 2183 return 0; 2184 } 2185 2186 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) { 2187 iavf_enable_channels(adapter); 2188 return 0; 2189 } 2190 2191 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) { 2192 iavf_disable_channels(adapter); 2193 return 0; 2194 } 2195 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) { 2196 iavf_add_cloud_filter(adapter); 2197 return 0; 2198 } 2199 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) { 2200 iavf_del_cloud_filter(adapter); 2201 return 0; 2202 } 2203 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) { 2204 iavf_add_fdir_filter(adapter); 2205 return IAVF_SUCCESS; 2206 } 2207 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) { 2208 iavf_del_fdir_filter(adapter); 2209 return IAVF_SUCCESS; 2210 } 2211 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) { 2212 iavf_add_adv_rss_cfg(adapter); 2213 return 0; 2214 } 2215 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) { 2216 iavf_del_adv_rss_cfg(adapter); 2217 return 0; 2218 } 2219 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) { 2220 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q); 2221 return 0; 2222 } 2223 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) { 2224 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD); 2225 return 0; 2226 } 2227 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) { 2228 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q); 2229 return 0; 2230 } 2231 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) { 2232 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD); 2233 return 0; 2234 } 2235 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) { 2236 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q); 2237 return 0; 2238 } 2239 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) { 2240 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD); 2241 return 0; 2242 } 2243 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) { 2244 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q); 2245 return 0; 2246 } 2247 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) { 2248 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD); 2249 return 0; 2250 } 2251 if (adapter->aq_required & IAVF_FLAG_AQ_SEND_PTP_CMD) { 2252 iavf_virtchnl_send_ptp_cmd(adapter); 2253 return IAVF_SUCCESS; 2254 } 2255 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) { 2256 iavf_request_stats(adapter); 2257 return 0; 2258 } 2259 2260 return -EAGAIN; 2261 } 2262 2263 /** 2264 * iavf_set_vlan_offload_features - set VLAN offload configuration 2265 * @adapter: board private structure 2266 * @prev_features: previous features used for comparison 2267 * @features: updated features used for configuration 2268 * 2269 * Set the aq_required bit(s) based on the requested features passed in to 2270 * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule 2271 * the watchdog if any changes are requested to expedite the request via 2272 * virtchnl. 2273 **/ 2274 static void 2275 iavf_set_vlan_offload_features(struct iavf_adapter *adapter, 2276 netdev_features_t prev_features, 2277 netdev_features_t features) 2278 { 2279 bool enable_stripping = true, enable_insertion = true; 2280 u16 vlan_ethertype = 0; 2281 u64 aq_required = 0; 2282 2283 /* keep cases separate because one ethertype for offloads can be 2284 * disabled at the same time as another is disabled, so check for an 2285 * enabled ethertype first, then check for disabled. Default to 2286 * ETH_P_8021Q so an ethertype is specified if disabling insertion and 2287 * stripping. 2288 */ 2289 if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) 2290 vlan_ethertype = ETH_P_8021AD; 2291 else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) 2292 vlan_ethertype = ETH_P_8021Q; 2293 else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) 2294 vlan_ethertype = ETH_P_8021AD; 2295 else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) 2296 vlan_ethertype = ETH_P_8021Q; 2297 else 2298 vlan_ethertype = ETH_P_8021Q; 2299 2300 if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX))) 2301 enable_stripping = false; 2302 if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX))) 2303 enable_insertion = false; 2304 2305 if (VLAN_ALLOWED(adapter)) { 2306 /* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN 2307 * stripping via virtchnl. VLAN insertion can be toggled on the 2308 * netdev, but it doesn't require a virtchnl message 2309 */ 2310 if (enable_stripping) 2311 aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING; 2312 else 2313 aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING; 2314 2315 } else if (VLAN_V2_ALLOWED(adapter)) { 2316 switch (vlan_ethertype) { 2317 case ETH_P_8021Q: 2318 if (enable_stripping) 2319 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING; 2320 else 2321 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING; 2322 2323 if (enable_insertion) 2324 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION; 2325 else 2326 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION; 2327 break; 2328 case ETH_P_8021AD: 2329 if (enable_stripping) 2330 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING; 2331 else 2332 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING; 2333 2334 if (enable_insertion) 2335 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION; 2336 else 2337 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION; 2338 break; 2339 } 2340 } 2341 2342 if (aq_required) 2343 iavf_schedule_aq_request(adapter, aq_required); 2344 } 2345 2346 /** 2347 * iavf_startup - first step of driver startup 2348 * @adapter: board private structure 2349 * 2350 * Function process __IAVF_STARTUP driver state. 2351 * When success the state is changed to __IAVF_INIT_VERSION_CHECK 2352 * when fails the state is changed to __IAVF_INIT_FAILED 2353 **/ 2354 static void iavf_startup(struct iavf_adapter *adapter) 2355 { 2356 struct pci_dev *pdev = adapter->pdev; 2357 struct iavf_hw *hw = &adapter->hw; 2358 enum iavf_status status; 2359 int ret; 2360 2361 WARN_ON(adapter->state != __IAVF_STARTUP); 2362 2363 /* driver loaded, probe complete */ 2364 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 2365 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 2366 2367 ret = iavf_check_reset_complete(hw); 2368 if (ret) { 2369 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n", 2370 ret); 2371 goto err; 2372 } 2373 hw->aq.num_arq_entries = IAVF_AQ_LEN; 2374 hw->aq.num_asq_entries = IAVF_AQ_LEN; 2375 hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 2376 hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 2377 2378 status = iavf_init_adminq(hw); 2379 if (status) { 2380 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n", 2381 status); 2382 goto err; 2383 } 2384 ret = iavf_send_api_ver(adapter); 2385 if (ret) { 2386 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret); 2387 iavf_shutdown_adminq(hw); 2388 goto err; 2389 } 2390 iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK); 2391 return; 2392 err: 2393 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2394 } 2395 2396 /** 2397 * iavf_init_version_check - second step of driver startup 2398 * @adapter: board private structure 2399 * 2400 * Function process __IAVF_INIT_VERSION_CHECK driver state. 2401 * When success the state is changed to __IAVF_INIT_GET_RESOURCES 2402 * when fails the state is changed to __IAVF_INIT_FAILED 2403 **/ 2404 static void iavf_init_version_check(struct iavf_adapter *adapter) 2405 { 2406 struct pci_dev *pdev = adapter->pdev; 2407 struct iavf_hw *hw = &adapter->hw; 2408 int err = -EAGAIN; 2409 2410 WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK); 2411 2412 if (!iavf_asq_done(hw)) { 2413 dev_err(&pdev->dev, "Admin queue command never completed\n"); 2414 iavf_shutdown_adminq(hw); 2415 iavf_change_state(adapter, __IAVF_STARTUP); 2416 goto err; 2417 } 2418 2419 /* aq msg sent, awaiting reply */ 2420 err = iavf_verify_api_ver(adapter); 2421 if (err) { 2422 if (err == -EALREADY) 2423 err = iavf_send_api_ver(adapter); 2424 else 2425 dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n", 2426 adapter->pf_version.major, 2427 adapter->pf_version.minor, 2428 VIRTCHNL_VERSION_MAJOR, 2429 VIRTCHNL_VERSION_MINOR); 2430 goto err; 2431 } 2432 err = iavf_send_vf_config_msg(adapter); 2433 if (err) { 2434 dev_err(&pdev->dev, "Unable to send config request (%d)\n", 2435 err); 2436 goto err; 2437 } 2438 iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES); 2439 return; 2440 err: 2441 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2442 } 2443 2444 /** 2445 * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES 2446 * @adapter: board private structure 2447 */ 2448 int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter) 2449 { 2450 int i, num_req_queues = adapter->num_req_queues; 2451 struct iavf_vsi *vsi = &adapter->vsi; 2452 2453 for (i = 0; i < adapter->vf_res->num_vsis; i++) { 2454 if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 2455 adapter->vsi_res = &adapter->vf_res->vsi_res[i]; 2456 } 2457 if (!adapter->vsi_res) { 2458 dev_err(&adapter->pdev->dev, "No LAN VSI found\n"); 2459 return -ENODEV; 2460 } 2461 2462 if (num_req_queues && 2463 num_req_queues > adapter->vsi_res->num_queue_pairs) { 2464 /* Problem. The PF gave us fewer queues than what we had 2465 * negotiated in our request. Need a reset to see if we can't 2466 * get back to a working state. 2467 */ 2468 dev_err(&adapter->pdev->dev, 2469 "Requested %d queues, but PF only gave us %d.\n", 2470 num_req_queues, 2471 adapter->vsi_res->num_queue_pairs); 2472 adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED; 2473 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs; 2474 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); 2475 2476 return -EAGAIN; 2477 } 2478 adapter->num_req_queues = 0; 2479 adapter->vsi.id = adapter->vsi_res->vsi_id; 2480 2481 adapter->vsi.back = adapter; 2482 adapter->vsi.base_vector = 1; 2483 vsi->netdev = adapter->netdev; 2484 vsi->qs_handle = adapter->vsi_res->qset_handle; 2485 if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2486 adapter->rss_key_size = adapter->vf_res->rss_key_size; 2487 adapter->rss_lut_size = adapter->vf_res->rss_lut_size; 2488 } else { 2489 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE; 2490 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE; 2491 } 2492 2493 return 0; 2494 } 2495 2496 /** 2497 * iavf_init_get_resources - third step of driver startup 2498 * @adapter: board private structure 2499 * 2500 * Function process __IAVF_INIT_GET_RESOURCES driver state and 2501 * finishes driver initialization procedure. 2502 * When success the state is changed to __IAVF_DOWN 2503 * when fails the state is changed to __IAVF_INIT_FAILED 2504 **/ 2505 static void iavf_init_get_resources(struct iavf_adapter *adapter) 2506 { 2507 struct pci_dev *pdev = adapter->pdev; 2508 struct iavf_hw *hw = &adapter->hw; 2509 int err; 2510 2511 WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES); 2512 /* aq msg sent, awaiting reply */ 2513 if (!adapter->vf_res) { 2514 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE, 2515 GFP_KERNEL); 2516 if (!adapter->vf_res) { 2517 err = -ENOMEM; 2518 goto err; 2519 } 2520 } 2521 err = iavf_get_vf_config(adapter); 2522 if (err == -EALREADY) { 2523 err = iavf_send_vf_config_msg(adapter); 2524 goto err; 2525 } else if (err == -EINVAL) { 2526 /* We only get -EINVAL if the device is in a very bad 2527 * state or if we've been disabled for previous bad 2528 * behavior. Either way, we're done now. 2529 */ 2530 iavf_shutdown_adminq(hw); 2531 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n"); 2532 return; 2533 } 2534 if (err) { 2535 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err); 2536 goto err_alloc; 2537 } 2538 2539 err = iavf_parse_vf_resource_msg(adapter); 2540 if (err) { 2541 dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n", 2542 err); 2543 goto err_alloc; 2544 } 2545 /* Some features require additional messages to negotiate extended 2546 * capabilities. These are processed in sequence by the 2547 * __IAVF_INIT_EXTENDED_CAPS driver state. 2548 */ 2549 adapter->extended_caps = IAVF_EXTENDED_CAPS; 2550 2551 iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS); 2552 return; 2553 2554 err_alloc: 2555 kfree(adapter->vf_res); 2556 adapter->vf_res = NULL; 2557 err: 2558 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2559 } 2560 2561 /** 2562 * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps 2563 * @adapter: board private structure 2564 * 2565 * Function processes send of the extended VLAN V2 capability message to the 2566 * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent, 2567 * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2. 2568 */ 2569 static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter) 2570 { 2571 int ret; 2572 2573 WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2)); 2574 2575 ret = iavf_send_vf_offload_vlan_v2_msg(adapter); 2576 if (ret && ret == -EOPNOTSUPP) { 2577 /* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case, 2578 * we did not send the capability exchange message and do not 2579 * expect a response. 2580 */ 2581 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2; 2582 } 2583 2584 /* We sent the message, so move on to the next step */ 2585 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2; 2586 } 2587 2588 /** 2589 * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps 2590 * @adapter: board private structure 2591 * 2592 * Function processes receipt of the extended VLAN V2 capability message from 2593 * the PF. 2594 **/ 2595 static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter) 2596 { 2597 int ret; 2598 2599 WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2)); 2600 2601 memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps)); 2602 2603 ret = iavf_get_vf_vlan_v2_caps(adapter); 2604 if (ret) 2605 goto err; 2606 2607 /* We've processed receipt of the VLAN V2 caps message */ 2608 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2; 2609 return; 2610 err: 2611 /* We didn't receive a reply. Make sure we try sending again when 2612 * __IAVF_INIT_FAILED attempts to recover. 2613 */ 2614 adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2; 2615 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2616 } 2617 2618 /** 2619 * iavf_init_send_supported_rxdids - part of querying for supported RXDID 2620 * formats 2621 * @adapter: board private structure 2622 * 2623 * Function processes send of the request for supported RXDIDs to the PF. 2624 * Must clear IAVF_EXTENDED_CAP_RECV_RXDID if the message is not sent, e.g. 2625 * due to the PF not negotiating VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC. 2626 */ 2627 static void iavf_init_send_supported_rxdids(struct iavf_adapter *adapter) 2628 { 2629 int ret; 2630 2631 ret = iavf_send_vf_supported_rxdids_msg(adapter); 2632 if (ret == -EOPNOTSUPP) { 2633 /* PF does not support VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC. In this 2634 * case, we did not send the capability exchange message and 2635 * do not expect a response. 2636 */ 2637 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_RXDID; 2638 } 2639 2640 /* We sent the message, so move on to the next step */ 2641 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_RXDID; 2642 } 2643 2644 /** 2645 * iavf_init_recv_supported_rxdids - part of querying for supported RXDID 2646 * formats 2647 * @adapter: board private structure 2648 * 2649 * Function processes receipt of the supported RXDIDs message from the PF. 2650 **/ 2651 static void iavf_init_recv_supported_rxdids(struct iavf_adapter *adapter) 2652 { 2653 int ret; 2654 2655 memset(&adapter->supp_rxdids, 0, sizeof(adapter->supp_rxdids)); 2656 2657 ret = iavf_get_vf_supported_rxdids(adapter); 2658 if (ret) 2659 goto err; 2660 2661 /* We've processed the PF response to the 2662 * VIRTCHNL_OP_GET_SUPPORTED_RXDIDS message we sent previously. 2663 */ 2664 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_RXDID; 2665 return; 2666 2667 err: 2668 /* We didn't receive a reply. Make sure we try sending again when 2669 * __IAVF_INIT_FAILED attempts to recover. 2670 */ 2671 adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_RXDID; 2672 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2673 } 2674 2675 /** 2676 * iavf_init_send_ptp_caps - part of querying for extended PTP capabilities 2677 * @adapter: board private structure 2678 * 2679 * Function processes send of the request for 1588 PTP capabilities to the PF. 2680 * Must clear IAVF_EXTENDED_CAP_SEND_PTP if the message is not sent, e.g. 2681 * due to the PF not negotiating VIRTCHNL_VF_PTP_CAP 2682 */ 2683 static void iavf_init_send_ptp_caps(struct iavf_adapter *adapter) 2684 { 2685 if (iavf_send_vf_ptp_caps_msg(adapter) == -EOPNOTSUPP) { 2686 /* PF does not support VIRTCHNL_VF_PTP_CAP. In this case, we 2687 * did not send the capability exchange message and do not 2688 * expect a response. 2689 */ 2690 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_PTP; 2691 } 2692 2693 /* We sent the message, so move on to the next step */ 2694 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_PTP; 2695 } 2696 2697 /** 2698 * iavf_init_recv_ptp_caps - part of querying for supported PTP capabilities 2699 * @adapter: board private structure 2700 * 2701 * Function processes receipt of the PTP capabilities supported on this VF. 2702 **/ 2703 static void iavf_init_recv_ptp_caps(struct iavf_adapter *adapter) 2704 { 2705 memset(&adapter->ptp.hw_caps, 0, sizeof(adapter->ptp.hw_caps)); 2706 2707 if (iavf_get_vf_ptp_caps(adapter)) 2708 goto err; 2709 2710 /* We've processed the PF response to the VIRTCHNL_OP_1588_PTP_GET_CAPS 2711 * message we sent previously. 2712 */ 2713 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_PTP; 2714 return; 2715 2716 err: 2717 /* We didn't receive a reply. Make sure we try sending again when 2718 * __IAVF_INIT_FAILED attempts to recover. 2719 */ 2720 adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_PTP; 2721 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2722 } 2723 2724 /** 2725 * iavf_init_process_extended_caps - Part of driver startup 2726 * @adapter: board private structure 2727 * 2728 * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state 2729 * handles negotiating capabilities for features which require an additional 2730 * message. 2731 * 2732 * Once all extended capabilities exchanges are finished, the driver will 2733 * transition into __IAVF_INIT_CONFIG_ADAPTER. 2734 */ 2735 static void iavf_init_process_extended_caps(struct iavf_adapter *adapter) 2736 { 2737 WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS); 2738 2739 /* Process capability exchange for VLAN V2 */ 2740 if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) { 2741 iavf_init_send_offload_vlan_v2_caps(adapter); 2742 return; 2743 } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) { 2744 iavf_init_recv_offload_vlan_v2_caps(adapter); 2745 return; 2746 } 2747 2748 /* Process capability exchange for RXDID formats */ 2749 if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_RXDID) { 2750 iavf_init_send_supported_rxdids(adapter); 2751 return; 2752 } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_RXDID) { 2753 iavf_init_recv_supported_rxdids(adapter); 2754 return; 2755 } 2756 2757 /* Process capability exchange for PTP features */ 2758 if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_PTP) { 2759 iavf_init_send_ptp_caps(adapter); 2760 return; 2761 } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_PTP) { 2762 iavf_init_recv_ptp_caps(adapter); 2763 return; 2764 } 2765 2766 /* When we reach here, no further extended capabilities exchanges are 2767 * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER 2768 */ 2769 iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER); 2770 } 2771 2772 /** 2773 * iavf_init_config_adapter - last part of driver startup 2774 * @adapter: board private structure 2775 * 2776 * After all the supported capabilities are negotiated, then the 2777 * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization. 2778 */ 2779 static void iavf_init_config_adapter(struct iavf_adapter *adapter) 2780 { 2781 struct net_device *netdev = adapter->netdev; 2782 struct pci_dev *pdev = adapter->pdev; 2783 int err; 2784 2785 WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER); 2786 2787 if (iavf_process_config(adapter)) 2788 goto err; 2789 2790 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2791 2792 adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED; 2793 2794 netdev->netdev_ops = &iavf_netdev_ops; 2795 iavf_set_ethtool_ops(netdev); 2796 netdev->watchdog_timeo = 5 * HZ; 2797 2798 netdev->min_mtu = ETH_MIN_MTU; 2799 netdev->max_mtu = LIBIE_MAX_MTU; 2800 2801 if (!is_valid_ether_addr(adapter->hw.mac.addr)) { 2802 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n", 2803 adapter->hw.mac.addr); 2804 eth_hw_addr_random(netdev); 2805 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); 2806 } else { 2807 eth_hw_addr_set(netdev, adapter->hw.mac.addr); 2808 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); 2809 } 2810 2811 adapter->tx_desc_count = IAVF_DEFAULT_TXD; 2812 adapter->rx_desc_count = IAVF_DEFAULT_RXD; 2813 err = iavf_init_interrupt_scheme(adapter); 2814 if (err) 2815 goto err_sw_init; 2816 iavf_map_rings_to_vectors(adapter); 2817 if (adapter->vf_res->vf_cap_flags & 2818 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) 2819 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE; 2820 2821 err = iavf_request_misc_irq(adapter); 2822 if (err) 2823 goto err_sw_init; 2824 2825 netif_carrier_off(netdev); 2826 adapter->link_up = false; 2827 netif_tx_stop_all_queues(netdev); 2828 2829 dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr); 2830 if (netdev->features & NETIF_F_GRO) 2831 dev_info(&pdev->dev, "GRO is enabled\n"); 2832 2833 iavf_change_state(adapter, __IAVF_DOWN); 2834 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 2835 2836 iavf_misc_irq_enable(adapter); 2837 wake_up(&adapter->down_waitqueue); 2838 2839 adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL); 2840 adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL); 2841 if (!adapter->rss_key || !adapter->rss_lut) { 2842 err = -ENOMEM; 2843 goto err_mem; 2844 } 2845 if (RSS_AQ(adapter)) 2846 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 2847 else 2848 iavf_init_rss(adapter); 2849 2850 if (VLAN_V2_ALLOWED(adapter)) 2851 /* request initial VLAN offload settings */ 2852 iavf_set_vlan_offload_features(adapter, 0, netdev->features); 2853 2854 if (QOS_ALLOWED(adapter)) 2855 adapter->aq_required |= IAVF_FLAG_AQ_GET_QOS_CAPS; 2856 2857 /* Setup initial PTP configuration */ 2858 iavf_ptp_init(adapter); 2859 2860 iavf_schedule_finish_config(adapter); 2861 return; 2862 2863 err_mem: 2864 iavf_free_rss(adapter); 2865 iavf_free_misc_irq(adapter); 2866 err_sw_init: 2867 iavf_reset_interrupt_capability(adapter); 2868 err: 2869 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2870 } 2871 2872 static const int IAVF_NO_RESCHED = -1; 2873 2874 /* return: msec delay for requeueing itself */ 2875 static int iavf_watchdog_step(struct iavf_adapter *adapter) 2876 { 2877 struct iavf_hw *hw = &adapter->hw; 2878 u32 reg_val; 2879 2880 netdev_assert_locked(adapter->netdev); 2881 2882 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 2883 iavf_change_state(adapter, __IAVF_COMM_FAILED); 2884 2885 switch (adapter->state) { 2886 case __IAVF_STARTUP: 2887 iavf_startup(adapter); 2888 return 30; 2889 case __IAVF_INIT_VERSION_CHECK: 2890 iavf_init_version_check(adapter); 2891 return 30; 2892 case __IAVF_INIT_GET_RESOURCES: 2893 iavf_init_get_resources(adapter); 2894 return 1; 2895 case __IAVF_INIT_EXTENDED_CAPS: 2896 iavf_init_process_extended_caps(adapter); 2897 return 1; 2898 case __IAVF_INIT_CONFIG_ADAPTER: 2899 iavf_init_config_adapter(adapter); 2900 return 1; 2901 case __IAVF_INIT_FAILED: 2902 if (test_bit(__IAVF_IN_REMOVE_TASK, 2903 &adapter->crit_section)) { 2904 /* Do not update the state and do not reschedule 2905 * watchdog task, iavf_remove should handle this state 2906 * as it can loop forever 2907 */ 2908 return IAVF_NO_RESCHED; 2909 } 2910 if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) { 2911 dev_err(&adapter->pdev->dev, 2912 "Failed to communicate with PF; waiting before retry\n"); 2913 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 2914 iavf_shutdown_adminq(hw); 2915 return 5000; 2916 } 2917 /* Try again from failed step*/ 2918 iavf_change_state(adapter, adapter->last_state); 2919 return 1000; 2920 case __IAVF_COMM_FAILED: 2921 if (test_bit(__IAVF_IN_REMOVE_TASK, 2922 &adapter->crit_section)) { 2923 /* Set state to __IAVF_INIT_FAILED and perform remove 2924 * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task 2925 * doesn't bring the state back to __IAVF_COMM_FAILED. 2926 */ 2927 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2928 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 2929 return IAVF_NO_RESCHED; 2930 } 2931 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 2932 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 2933 if (reg_val == VIRTCHNL_VFR_VFACTIVE || 2934 reg_val == VIRTCHNL_VFR_COMPLETED) { 2935 /* A chance for redemption! */ 2936 dev_err(&adapter->pdev->dev, 2937 "Hardware came out of reset. Attempting reinit.\n"); 2938 /* When init task contacts the PF and 2939 * gets everything set up again, it'll restart the 2940 * watchdog for us. Down, boy. Sit. Stay. Woof. 2941 */ 2942 iavf_change_state(adapter, __IAVF_STARTUP); 2943 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 2944 } 2945 adapter->aq_required = 0; 2946 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2947 return 10; 2948 case __IAVF_RESETTING: 2949 return 2000; 2950 case __IAVF_DOWN: 2951 case __IAVF_DOWN_PENDING: 2952 case __IAVF_TESTING: 2953 case __IAVF_RUNNING: 2954 if (adapter->current_op) { 2955 if (!iavf_asq_done(hw)) { 2956 dev_dbg(&adapter->pdev->dev, 2957 "Admin queue timeout\n"); 2958 iavf_send_api_ver(adapter); 2959 } 2960 } else { 2961 int ret = iavf_process_aq_command(adapter); 2962 2963 /* An error will be returned if no commands were 2964 * processed; use this opportunity to update stats 2965 * if the error isn't -ENOTSUPP 2966 */ 2967 if (ret && ret != -EOPNOTSUPP && 2968 adapter->state == __IAVF_RUNNING) 2969 iavf_request_stats(adapter); 2970 } 2971 if (adapter->state == __IAVF_RUNNING) 2972 iavf_detect_recover_hung(&adapter->vsi); 2973 break; 2974 case __IAVF_REMOVE: 2975 default: 2976 return IAVF_NO_RESCHED; 2977 } 2978 2979 /* check for hw reset */ 2980 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK; 2981 if (!reg_val) { 2982 adapter->aq_required = 0; 2983 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2984 dev_err(&adapter->pdev->dev, "Hardware reset detected\n"); 2985 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_PENDING); 2986 } 2987 2988 return adapter->aq_required ? 20 : 2000; 2989 } 2990 2991 static void iavf_watchdog_task(struct work_struct *work) 2992 { 2993 struct iavf_adapter *adapter = container_of(work, 2994 struct iavf_adapter, 2995 watchdog_task.work); 2996 struct net_device *netdev = adapter->netdev; 2997 int msec_delay; 2998 2999 netdev_lock(netdev); 3000 msec_delay = iavf_watchdog_step(adapter); 3001 /* note that we schedule a different task */ 3002 if (adapter->state >= __IAVF_DOWN) 3003 queue_work(adapter->wq, &adapter->adminq_task); 3004 3005 if (msec_delay != IAVF_NO_RESCHED) 3006 queue_delayed_work(adapter->wq, &adapter->watchdog_task, 3007 msecs_to_jiffies(msec_delay)); 3008 netdev_unlock(netdev); 3009 } 3010 3011 /** 3012 * iavf_disable_vf - disable VF 3013 * @adapter: board private structure 3014 * 3015 * Set communication failed flag and free all resources. 3016 */ 3017 static void iavf_disable_vf(struct iavf_adapter *adapter) 3018 { 3019 struct iavf_mac_filter *f, *ftmp; 3020 struct iavf_vlan_filter *fv, *fvtmp; 3021 struct iavf_cloud_filter *cf, *cftmp; 3022 3023 netdev_assert_locked(adapter->netdev); 3024 3025 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 3026 3027 /* We don't use netif_running() because it may be true prior to 3028 * ndo_open() returning, so we can't assume it means all our open 3029 * tasks have finished, since we're not holding the rtnl_lock here. 3030 */ 3031 if (adapter->state == __IAVF_RUNNING) { 3032 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 3033 netif_carrier_off(adapter->netdev); 3034 netif_tx_disable(adapter->netdev); 3035 adapter->link_up = false; 3036 iavf_napi_disable_all(adapter); 3037 iavf_irq_disable(adapter); 3038 iavf_free_traffic_irqs(adapter); 3039 iavf_free_all_tx_resources(adapter); 3040 iavf_free_all_rx_resources(adapter); 3041 } 3042 3043 spin_lock_bh(&adapter->mac_vlan_list_lock); 3044 3045 /* Delete all of the filters */ 3046 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 3047 list_del(&f->list); 3048 kfree(f); 3049 } 3050 3051 list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) { 3052 list_del(&fv->list); 3053 kfree(fv); 3054 } 3055 adapter->num_vlan_filters = 0; 3056 3057 spin_unlock_bh(&adapter->mac_vlan_list_lock); 3058 3059 spin_lock_bh(&adapter->cloud_filter_list_lock); 3060 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 3061 list_del(&cf->list); 3062 kfree(cf); 3063 adapter->num_cloud_filters--; 3064 } 3065 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3066 3067 iavf_free_misc_irq(adapter); 3068 iavf_free_interrupt_scheme(adapter); 3069 memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE); 3070 iavf_shutdown_adminq(&adapter->hw); 3071 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 3072 iavf_change_state(adapter, __IAVF_DOWN); 3073 wake_up(&adapter->down_waitqueue); 3074 dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n"); 3075 } 3076 3077 /** 3078 * iavf_reconfig_qs_bw - Call-back task to handle hardware reset 3079 * @adapter: board private structure 3080 * 3081 * After a reset, the shaper parameters of queues need to be replayed again. 3082 * Since the net_shaper object inside TX rings persists across reset, 3083 * set the update flag for all queues so that the virtchnl message is triggered 3084 * for all queues. 3085 **/ 3086 static void iavf_reconfig_qs_bw(struct iavf_adapter *adapter) 3087 { 3088 int i, num = 0; 3089 3090 for (i = 0; i < adapter->num_active_queues; i++) 3091 if (adapter->tx_rings[i].q_shaper.bw_min || 3092 adapter->tx_rings[i].q_shaper.bw_max) { 3093 adapter->tx_rings[i].q_shaper_update = true; 3094 num++; 3095 } 3096 3097 if (num) 3098 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES_BW; 3099 } 3100 3101 /** 3102 * iavf_reset_task - Call-back task to handle hardware reset 3103 * @work: pointer to work_struct 3104 * 3105 * During reset we need to shut down and reinitialize the admin queue 3106 * before we can use it to communicate with the PF again. We also clear 3107 * and reinit the rings because that context is lost as well. 3108 **/ 3109 static void iavf_reset_task(struct work_struct *work) 3110 { 3111 struct iavf_adapter *adapter = container_of(work, 3112 struct iavf_adapter, 3113 reset_task); 3114 struct virtchnl_vf_resource *vfres = adapter->vf_res; 3115 struct net_device *netdev = adapter->netdev; 3116 struct iavf_hw *hw = &adapter->hw; 3117 struct iavf_mac_filter *f, *ftmp; 3118 struct iavf_cloud_filter *cf; 3119 enum iavf_status status; 3120 u32 reg_val; 3121 int i = 0, err; 3122 bool running; 3123 3124 netdev_lock(netdev); 3125 3126 iavf_misc_irq_disable(adapter); 3127 if (adapter->flags & IAVF_FLAG_RESET_NEEDED) { 3128 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED; 3129 /* Restart the AQ here. If we have been reset but didn't 3130 * detect it, or if the PF had to reinit, our AQ will be hosed. 3131 */ 3132 iavf_shutdown_adminq(hw); 3133 iavf_init_adminq(hw); 3134 iavf_request_reset(adapter); 3135 } 3136 adapter->flags |= IAVF_FLAG_RESET_PENDING; 3137 3138 /* poll until we see the reset actually happen */ 3139 for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) { 3140 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & 3141 IAVF_VF_ARQLEN1_ARQENABLE_MASK; 3142 if (!reg_val) 3143 break; 3144 usleep_range(5000, 10000); 3145 } 3146 if (i == IAVF_RESET_WAIT_DETECTED_COUNT) { 3147 dev_info(&adapter->pdev->dev, "Never saw reset\n"); 3148 goto continue_reset; /* act like the reset happened */ 3149 } 3150 3151 /* wait until the reset is complete and the PF is responding to us */ 3152 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { 3153 /* sleep first to make sure a minimum wait time is met */ 3154 msleep(IAVF_RESET_WAIT_MS); 3155 3156 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 3157 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 3158 if (reg_val == VIRTCHNL_VFR_VFACTIVE) 3159 break; 3160 } 3161 3162 pci_set_master(adapter->pdev); 3163 pci_restore_msi_state(adapter->pdev); 3164 3165 if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) { 3166 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n", 3167 reg_val); 3168 iavf_disable_vf(adapter); 3169 netdev_unlock(netdev); 3170 return; /* Do not attempt to reinit. It's dead, Jim. */ 3171 } 3172 3173 continue_reset: 3174 /* If we are still early in the state machine, just restart. */ 3175 if (adapter->state <= __IAVF_INIT_FAILED) { 3176 iavf_shutdown_adminq(hw); 3177 iavf_change_state(adapter, __IAVF_STARTUP); 3178 iavf_startup(adapter); 3179 queue_delayed_work(adapter->wq, &adapter->watchdog_task, 3180 msecs_to_jiffies(30)); 3181 netdev_unlock(netdev); 3182 return; 3183 } 3184 3185 /* We don't use netif_running() because it may be true prior to 3186 * ndo_open() returning, so we can't assume it means all our open 3187 * tasks have finished, since we're not holding the rtnl_lock here. 3188 */ 3189 running = adapter->state == __IAVF_RUNNING; 3190 3191 if (running) { 3192 netif_carrier_off(netdev); 3193 netif_tx_stop_all_queues(netdev); 3194 adapter->link_up = false; 3195 iavf_napi_disable_all(adapter); 3196 } 3197 iavf_irq_disable(adapter); 3198 3199 iavf_change_state(adapter, __IAVF_RESETTING); 3200 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 3201 3202 /* free the Tx/Rx rings and descriptors, might be better to just 3203 * re-use them sometime in the future 3204 */ 3205 iavf_free_all_rx_resources(adapter); 3206 iavf_free_all_tx_resources(adapter); 3207 3208 adapter->flags |= IAVF_FLAG_QUEUES_DISABLED; 3209 /* kill and reinit the admin queue */ 3210 iavf_shutdown_adminq(hw); 3211 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 3212 status = iavf_init_adminq(hw); 3213 if (status) { 3214 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n", 3215 status); 3216 goto reset_err; 3217 } 3218 adapter->aq_required = 0; 3219 3220 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) || 3221 (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) { 3222 err = iavf_reinit_interrupt_scheme(adapter, running); 3223 if (err) 3224 goto reset_err; 3225 } 3226 3227 if (RSS_AQ(adapter)) { 3228 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 3229 } else { 3230 err = iavf_init_rss(adapter); 3231 if (err) 3232 goto reset_err; 3233 } 3234 3235 adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG; 3236 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 3237 3238 /* Certain capabilities require an extended negotiation process using 3239 * extra messages that must be processed after getting the VF 3240 * configuration. The related checks such as VLAN_V2_ALLOWED() are not 3241 * reliable here, since the configuration has not yet been negotiated. 3242 * 3243 * Always set these flags, since them related VIRTCHNL messages won't 3244 * be sent until after VIRTCHNL_OP_GET_VF_RESOURCES. 3245 */ 3246 adapter->aq_required |= IAVF_FLAG_AQ_EXTENDED_CAPS; 3247 3248 spin_lock_bh(&adapter->mac_vlan_list_lock); 3249 3250 /* Delete filter for the current MAC address, it could have 3251 * been changed by the PF via administratively set MAC. 3252 * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES. 3253 */ 3254 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 3255 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) { 3256 list_del(&f->list); 3257 kfree(f); 3258 } 3259 } 3260 /* re-add all MAC filters */ 3261 list_for_each_entry(f, &adapter->mac_filter_list, list) { 3262 f->add = true; 3263 } 3264 spin_unlock_bh(&adapter->mac_vlan_list_lock); 3265 3266 /* check if TCs are running and re-add all cloud filters */ 3267 spin_lock_bh(&adapter->cloud_filter_list_lock); 3268 if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 3269 adapter->num_tc) { 3270 list_for_each_entry(cf, &adapter->cloud_filter_list, list) { 3271 cf->add = true; 3272 } 3273 } 3274 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3275 3276 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 3277 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 3278 iavf_misc_irq_enable(adapter); 3279 3280 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 2); 3281 3282 /* We were running when the reset started, so we need to restore some 3283 * state here. 3284 */ 3285 if (running) { 3286 /* allocate transmit descriptors */ 3287 err = iavf_setup_all_tx_resources(adapter); 3288 if (err) 3289 goto reset_err; 3290 3291 /* allocate receive descriptors */ 3292 err = iavf_setup_all_rx_resources(adapter); 3293 if (err) 3294 goto reset_err; 3295 3296 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) || 3297 (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) { 3298 err = iavf_request_traffic_irqs(adapter, netdev->name); 3299 if (err) 3300 goto reset_err; 3301 3302 adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED; 3303 } 3304 3305 iavf_configure(adapter); 3306 3307 /* iavf_up_complete() will switch device back 3308 * to __IAVF_RUNNING 3309 */ 3310 iavf_up_complete(adapter); 3311 3312 iavf_irq_enable(adapter, true); 3313 3314 iavf_reconfig_qs_bw(adapter); 3315 } else { 3316 iavf_change_state(adapter, __IAVF_DOWN); 3317 wake_up(&adapter->down_waitqueue); 3318 } 3319 3320 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 3321 3322 wake_up(&adapter->reset_waitqueue); 3323 netdev_unlock(netdev); 3324 3325 return; 3326 reset_err: 3327 if (running) { 3328 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 3329 iavf_free_traffic_irqs(adapter); 3330 } 3331 iavf_disable_vf(adapter); 3332 3333 netdev_unlock(netdev); 3334 dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n"); 3335 } 3336 3337 /** 3338 * iavf_adminq_task - worker thread to clean the admin queue 3339 * @work: pointer to work_struct containing our data 3340 **/ 3341 static void iavf_adminq_task(struct work_struct *work) 3342 { 3343 struct iavf_adapter *adapter = 3344 container_of(work, struct iavf_adapter, adminq_task); 3345 struct net_device *netdev = adapter->netdev; 3346 struct iavf_hw *hw = &adapter->hw; 3347 struct iavf_arq_event_info event; 3348 enum virtchnl_ops v_op; 3349 enum iavf_status ret, v_ret; 3350 u32 val, oldval; 3351 u16 pending; 3352 3353 netdev_lock(netdev); 3354 3355 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 3356 goto unlock; 3357 3358 event.buf_len = IAVF_MAX_AQ_BUF_SIZE; 3359 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 3360 if (!event.msg_buf) 3361 goto unlock; 3362 3363 do { 3364 ret = iavf_clean_arq_element(hw, &event, &pending); 3365 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high); 3366 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low); 3367 3368 if (ret || !v_op) 3369 break; /* No event to process or error cleaning ARQ */ 3370 3371 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf, 3372 event.msg_len); 3373 if (pending != 0) 3374 memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE); 3375 } while (pending); 3376 3377 if (iavf_is_reset_in_progress(adapter)) 3378 goto freedom; 3379 3380 /* check for error indications */ 3381 val = rd32(hw, IAVF_VF_ARQLEN1); 3382 if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */ 3383 goto freedom; 3384 oldval = val; 3385 if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) { 3386 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n"); 3387 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK; 3388 } 3389 if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) { 3390 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n"); 3391 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK; 3392 } 3393 if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) { 3394 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n"); 3395 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK; 3396 } 3397 if (oldval != val) 3398 wr32(hw, IAVF_VF_ARQLEN1, val); 3399 3400 val = rd32(hw, IAVF_VF_ATQLEN1); 3401 oldval = val; 3402 if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) { 3403 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n"); 3404 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK; 3405 } 3406 if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) { 3407 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n"); 3408 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK; 3409 } 3410 if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) { 3411 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n"); 3412 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK; 3413 } 3414 if (oldval != val) 3415 wr32(hw, IAVF_VF_ATQLEN1, val); 3416 3417 freedom: 3418 kfree(event.msg_buf); 3419 unlock: 3420 netdev_unlock(netdev); 3421 /* re-enable Admin queue interrupt cause */ 3422 iavf_misc_irq_enable(adapter); 3423 } 3424 3425 /** 3426 * iavf_free_all_tx_resources - Free Tx Resources for All Queues 3427 * @adapter: board private structure 3428 * 3429 * Free all transmit software resources 3430 **/ 3431 void iavf_free_all_tx_resources(struct iavf_adapter *adapter) 3432 { 3433 int i; 3434 3435 if (!adapter->tx_rings) 3436 return; 3437 3438 for (i = 0; i < adapter->num_active_queues; i++) 3439 if (adapter->tx_rings[i].desc) 3440 iavf_free_tx_resources(&adapter->tx_rings[i]); 3441 } 3442 3443 /** 3444 * iavf_setup_all_tx_resources - allocate all queues Tx resources 3445 * @adapter: board private structure 3446 * 3447 * If this function returns with an error, then it's possible one or 3448 * more of the rings is populated (while the rest are not). It is the 3449 * callers duty to clean those orphaned rings. 3450 * 3451 * Return 0 on success, negative on failure 3452 **/ 3453 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter) 3454 { 3455 int i, err = 0; 3456 3457 for (i = 0; i < adapter->num_active_queues; i++) { 3458 adapter->tx_rings[i].count = adapter->tx_desc_count; 3459 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]); 3460 if (!err) 3461 continue; 3462 dev_err(&adapter->pdev->dev, 3463 "Allocation for Tx Queue %u failed\n", i); 3464 break; 3465 } 3466 3467 return err; 3468 } 3469 3470 /** 3471 * iavf_setup_all_rx_resources - allocate all queues Rx resources 3472 * @adapter: board private structure 3473 * 3474 * If this function returns with an error, then it's possible one or 3475 * more of the rings is populated (while the rest are not). It is the 3476 * callers duty to clean those orphaned rings. 3477 * 3478 * Return 0 on success, negative on failure 3479 **/ 3480 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter) 3481 { 3482 int i, err = 0; 3483 3484 for (i = 0; i < adapter->num_active_queues; i++) { 3485 adapter->rx_rings[i].count = adapter->rx_desc_count; 3486 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]); 3487 if (!err) 3488 continue; 3489 dev_err(&adapter->pdev->dev, 3490 "Allocation for Rx Queue %u failed\n", i); 3491 break; 3492 } 3493 return err; 3494 } 3495 3496 /** 3497 * iavf_free_all_rx_resources - Free Rx Resources for All Queues 3498 * @adapter: board private structure 3499 * 3500 * Free all receive software resources 3501 **/ 3502 void iavf_free_all_rx_resources(struct iavf_adapter *adapter) 3503 { 3504 int i; 3505 3506 if (!adapter->rx_rings) 3507 return; 3508 3509 for (i = 0; i < adapter->num_active_queues; i++) 3510 if (adapter->rx_rings[i].desc) 3511 iavf_free_rx_resources(&adapter->rx_rings[i]); 3512 } 3513 3514 /** 3515 * iavf_validate_tx_bandwidth - validate the max Tx bandwidth 3516 * @adapter: board private structure 3517 * @max_tx_rate: max Tx bw for a tc 3518 **/ 3519 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter, 3520 u64 max_tx_rate) 3521 { 3522 int speed = 0, ret = 0; 3523 3524 if (ADV_LINK_SUPPORT(adapter)) { 3525 if (adapter->link_speed_mbps < U32_MAX) { 3526 speed = adapter->link_speed_mbps; 3527 goto validate_bw; 3528 } else { 3529 dev_err(&adapter->pdev->dev, "Unknown link speed\n"); 3530 return -EINVAL; 3531 } 3532 } 3533 3534 switch (adapter->link_speed) { 3535 case VIRTCHNL_LINK_SPEED_40GB: 3536 speed = SPEED_40000; 3537 break; 3538 case VIRTCHNL_LINK_SPEED_25GB: 3539 speed = SPEED_25000; 3540 break; 3541 case VIRTCHNL_LINK_SPEED_20GB: 3542 speed = SPEED_20000; 3543 break; 3544 case VIRTCHNL_LINK_SPEED_10GB: 3545 speed = SPEED_10000; 3546 break; 3547 case VIRTCHNL_LINK_SPEED_5GB: 3548 speed = SPEED_5000; 3549 break; 3550 case VIRTCHNL_LINK_SPEED_2_5GB: 3551 speed = SPEED_2500; 3552 break; 3553 case VIRTCHNL_LINK_SPEED_1GB: 3554 speed = SPEED_1000; 3555 break; 3556 case VIRTCHNL_LINK_SPEED_100MB: 3557 speed = SPEED_100; 3558 break; 3559 default: 3560 break; 3561 } 3562 3563 validate_bw: 3564 if (max_tx_rate > speed) { 3565 dev_err(&adapter->pdev->dev, 3566 "Invalid tx rate specified\n"); 3567 ret = -EINVAL; 3568 } 3569 3570 return ret; 3571 } 3572 3573 /** 3574 * iavf_validate_ch_config - validate queue mapping info 3575 * @adapter: board private structure 3576 * @mqprio_qopt: queue parameters 3577 * 3578 * This function validates if the config provided by the user to 3579 * configure queue channels is valid or not. Returns 0 on a valid 3580 * config. 3581 **/ 3582 static int iavf_validate_ch_config(struct iavf_adapter *adapter, 3583 struct tc_mqprio_qopt_offload *mqprio_qopt) 3584 { 3585 u64 total_max_rate = 0; 3586 u32 tx_rate_rem = 0; 3587 int i, num_qps = 0; 3588 u64 tx_rate = 0; 3589 int ret = 0; 3590 3591 if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS || 3592 mqprio_qopt->qopt.num_tc < 1) 3593 return -EINVAL; 3594 3595 for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) { 3596 if (!mqprio_qopt->qopt.count[i] || 3597 mqprio_qopt->qopt.offset[i] != num_qps) 3598 return -EINVAL; 3599 if (mqprio_qopt->min_rate[i]) { 3600 dev_err(&adapter->pdev->dev, 3601 "Invalid min tx rate (greater than 0) specified for TC%d\n", 3602 i); 3603 return -EINVAL; 3604 } 3605 3606 /* convert to Mbps */ 3607 tx_rate = div_u64(mqprio_qopt->max_rate[i], 3608 IAVF_MBPS_DIVISOR); 3609 3610 if (mqprio_qopt->max_rate[i] && 3611 tx_rate < IAVF_MBPS_QUANTA) { 3612 dev_err(&adapter->pdev->dev, 3613 "Invalid max tx rate for TC%d, minimum %dMbps\n", 3614 i, IAVF_MBPS_QUANTA); 3615 return -EINVAL; 3616 } 3617 3618 (void)div_u64_rem(tx_rate, IAVF_MBPS_QUANTA, &tx_rate_rem); 3619 3620 if (tx_rate_rem != 0) { 3621 dev_err(&adapter->pdev->dev, 3622 "Invalid max tx rate for TC%d, not divisible by %d\n", 3623 i, IAVF_MBPS_QUANTA); 3624 return -EINVAL; 3625 } 3626 3627 total_max_rate += tx_rate; 3628 num_qps += mqprio_qopt->qopt.count[i]; 3629 } 3630 if (num_qps > adapter->num_active_queues) { 3631 dev_err(&adapter->pdev->dev, 3632 "Cannot support requested number of queues\n"); 3633 return -EINVAL; 3634 } 3635 3636 ret = iavf_validate_tx_bandwidth(adapter, total_max_rate); 3637 return ret; 3638 } 3639 3640 /** 3641 * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes 3642 * @adapter: board private structure 3643 **/ 3644 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter) 3645 { 3646 struct iavf_cloud_filter *cf, *cftmp; 3647 3648 spin_lock_bh(&adapter->cloud_filter_list_lock); 3649 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, 3650 list) { 3651 list_del(&cf->list); 3652 kfree(cf); 3653 adapter->num_cloud_filters--; 3654 } 3655 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3656 } 3657 3658 /** 3659 * iavf_is_tc_config_same - Compare the mqprio TC config with the 3660 * TC config already configured on this adapter. 3661 * @adapter: board private structure 3662 * @mqprio_qopt: TC config received from kernel. 3663 * 3664 * This function compares the TC config received from the kernel 3665 * with the config already configured on the adapter. 3666 * 3667 * Return: True if configuration is same, false otherwise. 3668 **/ 3669 static bool iavf_is_tc_config_same(struct iavf_adapter *adapter, 3670 struct tc_mqprio_qopt *mqprio_qopt) 3671 { 3672 struct virtchnl_channel_info *ch = &adapter->ch_config.ch_info[0]; 3673 int i; 3674 3675 if (adapter->num_tc != mqprio_qopt->num_tc) 3676 return false; 3677 3678 for (i = 0; i < adapter->num_tc; i++) { 3679 if (ch[i].count != mqprio_qopt->count[i] || 3680 ch[i].offset != mqprio_qopt->offset[i]) 3681 return false; 3682 } 3683 return true; 3684 } 3685 3686 /** 3687 * __iavf_setup_tc - configure multiple traffic classes 3688 * @netdev: network interface device structure 3689 * @type_data: tc offload data 3690 * 3691 * This function processes the config information provided by the 3692 * user to configure traffic classes/queue channels and packages the 3693 * information to request the PF to setup traffic classes. 3694 * 3695 * Returns 0 on success. 3696 **/ 3697 static int __iavf_setup_tc(struct net_device *netdev, void *type_data) 3698 { 3699 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 3700 struct iavf_adapter *adapter = netdev_priv(netdev); 3701 struct virtchnl_vf_resource *vfres = adapter->vf_res; 3702 u8 num_tc = 0, total_qps = 0; 3703 int ret = 0, netdev_tc = 0; 3704 u64 max_tx_rate; 3705 u16 mode; 3706 int i; 3707 3708 num_tc = mqprio_qopt->qopt.num_tc; 3709 mode = mqprio_qopt->mode; 3710 3711 /* delete queue_channel */ 3712 if (!mqprio_qopt->qopt.hw) { 3713 if (adapter->ch_config.state == __IAVF_TC_RUNNING) { 3714 /* reset the tc configuration */ 3715 netdev_reset_tc(netdev); 3716 adapter->num_tc = 0; 3717 netif_tx_stop_all_queues(netdev); 3718 netif_tx_disable(netdev); 3719 iavf_del_all_cloud_filters(adapter); 3720 adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS; 3721 total_qps = adapter->orig_num_active_queues; 3722 goto exit; 3723 } else { 3724 return -EINVAL; 3725 } 3726 } 3727 3728 /* add queue channel */ 3729 if (mode == TC_MQPRIO_MODE_CHANNEL) { 3730 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) { 3731 dev_err(&adapter->pdev->dev, "ADq not supported\n"); 3732 return -EOPNOTSUPP; 3733 } 3734 if (adapter->ch_config.state != __IAVF_TC_INVALID) { 3735 dev_err(&adapter->pdev->dev, "TC configuration already exists\n"); 3736 return -EINVAL; 3737 } 3738 3739 ret = iavf_validate_ch_config(adapter, mqprio_qopt); 3740 if (ret) 3741 return ret; 3742 /* Return if same TC config is requested */ 3743 if (iavf_is_tc_config_same(adapter, &mqprio_qopt->qopt)) 3744 return 0; 3745 adapter->num_tc = num_tc; 3746 3747 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 3748 if (i < num_tc) { 3749 adapter->ch_config.ch_info[i].count = 3750 mqprio_qopt->qopt.count[i]; 3751 adapter->ch_config.ch_info[i].offset = 3752 mqprio_qopt->qopt.offset[i]; 3753 total_qps += mqprio_qopt->qopt.count[i]; 3754 max_tx_rate = mqprio_qopt->max_rate[i]; 3755 /* convert to Mbps */ 3756 max_tx_rate = div_u64(max_tx_rate, 3757 IAVF_MBPS_DIVISOR); 3758 adapter->ch_config.ch_info[i].max_tx_rate = 3759 max_tx_rate; 3760 } else { 3761 adapter->ch_config.ch_info[i].count = 1; 3762 adapter->ch_config.ch_info[i].offset = 0; 3763 } 3764 } 3765 3766 /* Take snapshot of original config such as "num_active_queues" 3767 * It is used later when delete ADQ flow is exercised, so that 3768 * once delete ADQ flow completes, VF shall go back to its 3769 * original queue configuration 3770 */ 3771 3772 adapter->orig_num_active_queues = adapter->num_active_queues; 3773 3774 /* Store queue info based on TC so that VF gets configured 3775 * with correct number of queues when VF completes ADQ config 3776 * flow 3777 */ 3778 adapter->ch_config.total_qps = total_qps; 3779 3780 netif_tx_stop_all_queues(netdev); 3781 netif_tx_disable(netdev); 3782 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS; 3783 netdev_reset_tc(netdev); 3784 /* Report the tc mapping up the stack */ 3785 netdev_set_num_tc(adapter->netdev, num_tc); 3786 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 3787 u16 qcount = mqprio_qopt->qopt.count[i]; 3788 u16 qoffset = mqprio_qopt->qopt.offset[i]; 3789 3790 if (i < num_tc) 3791 netdev_set_tc_queue(netdev, netdev_tc++, qcount, 3792 qoffset); 3793 } 3794 } 3795 exit: 3796 if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) 3797 return 0; 3798 3799 netif_set_real_num_rx_queues(netdev, total_qps); 3800 netif_set_real_num_tx_queues(netdev, total_qps); 3801 3802 return ret; 3803 } 3804 3805 /** 3806 * iavf_parse_cls_flower - Parse tc flower filters provided by kernel 3807 * @adapter: board private structure 3808 * @f: pointer to struct flow_cls_offload 3809 * @filter: pointer to cloud filter structure 3810 */ 3811 static int iavf_parse_cls_flower(struct iavf_adapter *adapter, 3812 struct flow_cls_offload *f, 3813 struct iavf_cloud_filter *filter) 3814 { 3815 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 3816 struct flow_dissector *dissector = rule->match.dissector; 3817 u16 n_proto_mask = 0; 3818 u16 n_proto_key = 0; 3819 u8 field_flags = 0; 3820 u16 addr_type = 0; 3821 u16 n_proto = 0; 3822 int i = 0; 3823 struct virtchnl_filter *vf = &filter->f; 3824 3825 if (dissector->used_keys & 3826 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) | 3827 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) | 3828 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 3829 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) | 3830 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 3831 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 3832 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) | 3833 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 3834 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%llx\n", 3835 dissector->used_keys); 3836 return -EOPNOTSUPP; 3837 } 3838 3839 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 3840 struct flow_match_enc_keyid match; 3841 3842 flow_rule_match_enc_keyid(rule, &match); 3843 if (match.mask->keyid != 0) 3844 field_flags |= IAVF_CLOUD_FIELD_TEN_ID; 3845 } 3846 3847 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 3848 struct flow_match_basic match; 3849 3850 flow_rule_match_basic(rule, &match); 3851 n_proto_key = ntohs(match.key->n_proto); 3852 n_proto_mask = ntohs(match.mask->n_proto); 3853 3854 if (n_proto_key == ETH_P_ALL) { 3855 n_proto_key = 0; 3856 n_proto_mask = 0; 3857 } 3858 n_proto = n_proto_key & n_proto_mask; 3859 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6) 3860 return -EINVAL; 3861 if (n_proto == ETH_P_IPV6) { 3862 /* specify flow type as TCP IPv6 */ 3863 vf->flow_type = VIRTCHNL_TCP_V6_FLOW; 3864 } 3865 3866 if (match.key->ip_proto != IPPROTO_TCP) { 3867 dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n"); 3868 return -EINVAL; 3869 } 3870 } 3871 3872 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 3873 struct flow_match_eth_addrs match; 3874 3875 flow_rule_match_eth_addrs(rule, &match); 3876 3877 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 3878 if (!is_zero_ether_addr(match.mask->dst)) { 3879 if (is_broadcast_ether_addr(match.mask->dst)) { 3880 field_flags |= IAVF_CLOUD_FIELD_OMAC; 3881 } else { 3882 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n", 3883 match.mask->dst); 3884 return -EINVAL; 3885 } 3886 } 3887 3888 if (!is_zero_ether_addr(match.mask->src)) { 3889 if (is_broadcast_ether_addr(match.mask->src)) { 3890 field_flags |= IAVF_CLOUD_FIELD_IMAC; 3891 } else { 3892 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n", 3893 match.mask->src); 3894 return -EINVAL; 3895 } 3896 } 3897 3898 if (!is_zero_ether_addr(match.key->dst)) 3899 if (is_valid_ether_addr(match.key->dst) || 3900 is_multicast_ether_addr(match.key->dst)) { 3901 /* set the mask if a valid dst_mac address */ 3902 for (i = 0; i < ETH_ALEN; i++) 3903 vf->mask.tcp_spec.dst_mac[i] |= 0xff; 3904 ether_addr_copy(vf->data.tcp_spec.dst_mac, 3905 match.key->dst); 3906 } 3907 3908 if (!is_zero_ether_addr(match.key->src)) 3909 if (is_valid_ether_addr(match.key->src) || 3910 is_multicast_ether_addr(match.key->src)) { 3911 /* set the mask if a valid dst_mac address */ 3912 for (i = 0; i < ETH_ALEN; i++) 3913 vf->mask.tcp_spec.src_mac[i] |= 0xff; 3914 ether_addr_copy(vf->data.tcp_spec.src_mac, 3915 match.key->src); 3916 } 3917 } 3918 3919 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 3920 struct flow_match_vlan match; 3921 3922 flow_rule_match_vlan(rule, &match); 3923 if (match.mask->vlan_id) { 3924 if (match.mask->vlan_id == VLAN_VID_MASK) { 3925 field_flags |= IAVF_CLOUD_FIELD_IVLAN; 3926 } else { 3927 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n", 3928 match.mask->vlan_id); 3929 return -EINVAL; 3930 } 3931 } 3932 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff); 3933 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id); 3934 } 3935 3936 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 3937 struct flow_match_control match; 3938 3939 flow_rule_match_control(rule, &match); 3940 addr_type = match.key->addr_type; 3941 3942 if (flow_rule_has_control_flags(match.mask->flags, 3943 f->common.extack)) 3944 return -EOPNOTSUPP; 3945 } 3946 3947 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 3948 struct flow_match_ipv4_addrs match; 3949 3950 flow_rule_match_ipv4_addrs(rule, &match); 3951 if (match.mask->dst) { 3952 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 3953 field_flags |= IAVF_CLOUD_FIELD_IIP; 3954 } else { 3955 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n", 3956 be32_to_cpu(match.mask->dst)); 3957 return -EINVAL; 3958 } 3959 } 3960 3961 if (match.mask->src) { 3962 if (match.mask->src == cpu_to_be32(0xffffffff)) { 3963 field_flags |= IAVF_CLOUD_FIELD_IIP; 3964 } else { 3965 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n", 3966 be32_to_cpu(match.mask->src)); 3967 return -EINVAL; 3968 } 3969 } 3970 3971 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) { 3972 dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n"); 3973 return -EINVAL; 3974 } 3975 if (match.key->dst) { 3976 vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff); 3977 vf->data.tcp_spec.dst_ip[0] = match.key->dst; 3978 } 3979 if (match.key->src) { 3980 vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff); 3981 vf->data.tcp_spec.src_ip[0] = match.key->src; 3982 } 3983 } 3984 3985 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 3986 struct flow_match_ipv6_addrs match; 3987 3988 flow_rule_match_ipv6_addrs(rule, &match); 3989 3990 /* validate mask, make sure it is not IPV6_ADDR_ANY */ 3991 if (ipv6_addr_any(&match.mask->dst)) { 3992 dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n", 3993 IPV6_ADDR_ANY); 3994 return -EINVAL; 3995 } 3996 3997 /* src and dest IPv6 address should not be LOOPBACK 3998 * (0:0:0:0:0:0:0:1) which can be represented as ::1 3999 */ 4000 if (ipv6_addr_loopback(&match.key->dst) || 4001 ipv6_addr_loopback(&match.key->src)) { 4002 dev_err(&adapter->pdev->dev, 4003 "ipv6 addr should not be loopback\n"); 4004 return -EINVAL; 4005 } 4006 if (!ipv6_addr_any(&match.mask->dst) || 4007 !ipv6_addr_any(&match.mask->src)) 4008 field_flags |= IAVF_CLOUD_FIELD_IIP; 4009 4010 for (i = 0; i < 4; i++) 4011 vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff); 4012 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32, 4013 sizeof(vf->data.tcp_spec.dst_ip)); 4014 for (i = 0; i < 4; i++) 4015 vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff); 4016 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32, 4017 sizeof(vf->data.tcp_spec.src_ip)); 4018 } 4019 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 4020 struct flow_match_ports match; 4021 4022 flow_rule_match_ports(rule, &match); 4023 if (match.mask->src) { 4024 if (match.mask->src == cpu_to_be16(0xffff)) { 4025 field_flags |= IAVF_CLOUD_FIELD_IIP; 4026 } else { 4027 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n", 4028 be16_to_cpu(match.mask->src)); 4029 return -EINVAL; 4030 } 4031 } 4032 4033 if (match.mask->dst) { 4034 if (match.mask->dst == cpu_to_be16(0xffff)) { 4035 field_flags |= IAVF_CLOUD_FIELD_IIP; 4036 } else { 4037 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n", 4038 be16_to_cpu(match.mask->dst)); 4039 return -EINVAL; 4040 } 4041 } 4042 if (match.key->dst) { 4043 vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff); 4044 vf->data.tcp_spec.dst_port = match.key->dst; 4045 } 4046 4047 if (match.key->src) { 4048 vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff); 4049 vf->data.tcp_spec.src_port = match.key->src; 4050 } 4051 } 4052 vf->field_flags = field_flags; 4053 4054 return 0; 4055 } 4056 4057 /** 4058 * iavf_handle_tclass - Forward to a traffic class on the device 4059 * @adapter: board private structure 4060 * @tc: traffic class index on the device 4061 * @filter: pointer to cloud filter structure 4062 */ 4063 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc, 4064 struct iavf_cloud_filter *filter) 4065 { 4066 if (tc == 0) 4067 return 0; 4068 if (tc < adapter->num_tc) { 4069 if (!filter->f.data.tcp_spec.dst_port) { 4070 dev_err(&adapter->pdev->dev, 4071 "Specify destination port to redirect to traffic class other than TC0\n"); 4072 return -EINVAL; 4073 } 4074 } 4075 /* redirect to a traffic class on the same device */ 4076 filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT; 4077 filter->f.action_meta = tc; 4078 return 0; 4079 } 4080 4081 /** 4082 * iavf_find_cf - Find the cloud filter in the list 4083 * @adapter: Board private structure 4084 * @cookie: filter specific cookie 4085 * 4086 * Returns ptr to the filter object or NULL. Must be called while holding the 4087 * cloud_filter_list_lock. 4088 */ 4089 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter, 4090 unsigned long *cookie) 4091 { 4092 struct iavf_cloud_filter *filter = NULL; 4093 4094 if (!cookie) 4095 return NULL; 4096 4097 list_for_each_entry(filter, &adapter->cloud_filter_list, list) { 4098 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 4099 return filter; 4100 } 4101 return NULL; 4102 } 4103 4104 /** 4105 * iavf_configure_clsflower - Add tc flower filters 4106 * @adapter: board private structure 4107 * @cls_flower: Pointer to struct flow_cls_offload 4108 */ 4109 static int iavf_configure_clsflower(struct iavf_adapter *adapter, 4110 struct flow_cls_offload *cls_flower) 4111 { 4112 int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid); 4113 struct iavf_cloud_filter *filter; 4114 int err; 4115 4116 if (tc < 0) { 4117 dev_err(&adapter->pdev->dev, "Invalid traffic class\n"); 4118 return -EINVAL; 4119 } 4120 4121 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 4122 if (!filter) 4123 return -ENOMEM; 4124 filter->cookie = cls_flower->cookie; 4125 4126 netdev_lock(adapter->netdev); 4127 4128 /* bail out here if filter already exists */ 4129 spin_lock_bh(&adapter->cloud_filter_list_lock); 4130 if (iavf_find_cf(adapter, &cls_flower->cookie)) { 4131 dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n"); 4132 err = -EEXIST; 4133 goto spin_unlock; 4134 } 4135 spin_unlock_bh(&adapter->cloud_filter_list_lock); 4136 4137 /* set the mask to all zeroes to begin with */ 4138 memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec)); 4139 /* start out with flow type and eth type IPv4 to begin with */ 4140 filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW; 4141 err = iavf_parse_cls_flower(adapter, cls_flower, filter); 4142 if (err) 4143 goto err; 4144 4145 err = iavf_handle_tclass(adapter, tc, filter); 4146 if (err) 4147 goto err; 4148 4149 /* add filter to the list */ 4150 spin_lock_bh(&adapter->cloud_filter_list_lock); 4151 list_add_tail(&filter->list, &adapter->cloud_filter_list); 4152 adapter->num_cloud_filters++; 4153 filter->add = true; 4154 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 4155 spin_unlock: 4156 spin_unlock_bh(&adapter->cloud_filter_list_lock); 4157 err: 4158 if (err) 4159 kfree(filter); 4160 4161 netdev_unlock(adapter->netdev); 4162 return err; 4163 } 4164 4165 /** 4166 * iavf_delete_clsflower - Remove tc flower filters 4167 * @adapter: board private structure 4168 * @cls_flower: Pointer to struct flow_cls_offload 4169 */ 4170 static int iavf_delete_clsflower(struct iavf_adapter *adapter, 4171 struct flow_cls_offload *cls_flower) 4172 { 4173 struct iavf_cloud_filter *filter = NULL; 4174 int err = 0; 4175 4176 spin_lock_bh(&adapter->cloud_filter_list_lock); 4177 filter = iavf_find_cf(adapter, &cls_flower->cookie); 4178 if (filter) { 4179 filter->del = true; 4180 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 4181 } else { 4182 err = -EINVAL; 4183 } 4184 spin_unlock_bh(&adapter->cloud_filter_list_lock); 4185 4186 return err; 4187 } 4188 4189 /** 4190 * iavf_setup_tc_cls_flower - flower classifier offloads 4191 * @adapter: pointer to iavf adapter structure 4192 * @cls_flower: pointer to flow_cls_offload struct with flow info 4193 */ 4194 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, 4195 struct flow_cls_offload *cls_flower) 4196 { 4197 switch (cls_flower->command) { 4198 case FLOW_CLS_REPLACE: 4199 return iavf_configure_clsflower(adapter, cls_flower); 4200 case FLOW_CLS_DESTROY: 4201 return iavf_delete_clsflower(adapter, cls_flower); 4202 case FLOW_CLS_STATS: 4203 return -EOPNOTSUPP; 4204 default: 4205 return -EOPNOTSUPP; 4206 } 4207 } 4208 4209 /** 4210 * iavf_add_cls_u32 - Add U32 classifier offloads 4211 * @adapter: pointer to iavf adapter structure 4212 * @cls_u32: pointer to tc_cls_u32_offload struct with flow info 4213 * 4214 * Return: 0 on success or negative errno on failure. 4215 */ 4216 static int iavf_add_cls_u32(struct iavf_adapter *adapter, 4217 struct tc_cls_u32_offload *cls_u32) 4218 { 4219 struct netlink_ext_ack *extack = cls_u32->common.extack; 4220 struct virtchnl_fdir_rule *rule_cfg; 4221 struct virtchnl_filter_action *vact; 4222 struct virtchnl_proto_hdrs *hdrs; 4223 struct ethhdr *spec_h, *mask_h; 4224 const struct tc_action *act; 4225 struct iavf_fdir_fltr *fltr; 4226 struct tcf_exts *exts; 4227 unsigned int q_index; 4228 int i, status = 0; 4229 int off_base = 0; 4230 4231 if (cls_u32->knode.link_handle) { 4232 NL_SET_ERR_MSG_MOD(extack, "Linking not supported"); 4233 return -EOPNOTSUPP; 4234 } 4235 4236 fltr = kzalloc(sizeof(*fltr), GFP_KERNEL); 4237 if (!fltr) 4238 return -ENOMEM; 4239 4240 rule_cfg = &fltr->vc_add_msg.rule_cfg; 4241 hdrs = &rule_cfg->proto_hdrs; 4242 hdrs->count = 0; 4243 4244 /* The parser lib at the PF expects the packet starting with MAC hdr */ 4245 switch (ntohs(cls_u32->common.protocol)) { 4246 case ETH_P_802_3: 4247 break; 4248 case ETH_P_IP: 4249 spec_h = (struct ethhdr *)hdrs->raw.spec; 4250 mask_h = (struct ethhdr *)hdrs->raw.mask; 4251 spec_h->h_proto = htons(ETH_P_IP); 4252 mask_h->h_proto = htons(0xFFFF); 4253 off_base += ETH_HLEN; 4254 break; 4255 default: 4256 NL_SET_ERR_MSG_MOD(extack, "Only 802_3 and ip filter protocols are supported"); 4257 status = -EOPNOTSUPP; 4258 goto free_alloc; 4259 } 4260 4261 for (i = 0; i < cls_u32->knode.sel->nkeys; i++) { 4262 __be32 val, mask; 4263 int off; 4264 4265 off = off_base + cls_u32->knode.sel->keys[i].off; 4266 val = cls_u32->knode.sel->keys[i].val; 4267 mask = cls_u32->knode.sel->keys[i].mask; 4268 4269 if (off >= sizeof(hdrs->raw.spec)) { 4270 NL_SET_ERR_MSG_MOD(extack, "Input exceeds maximum allowed."); 4271 status = -EINVAL; 4272 goto free_alloc; 4273 } 4274 4275 memcpy(&hdrs->raw.spec[off], &val, sizeof(val)); 4276 memcpy(&hdrs->raw.mask[off], &mask, sizeof(mask)); 4277 hdrs->raw.pkt_len = off + sizeof(val); 4278 } 4279 4280 /* Only one action is allowed */ 4281 rule_cfg->action_set.count = 1; 4282 vact = &rule_cfg->action_set.actions[0]; 4283 exts = cls_u32->knode.exts; 4284 4285 tcf_exts_for_each_action(i, act, exts) { 4286 /* FDIR queue */ 4287 if (is_tcf_skbedit_rx_queue_mapping(act)) { 4288 q_index = tcf_skbedit_rx_queue_mapping(act); 4289 if (q_index >= adapter->num_active_queues) { 4290 status = -EINVAL; 4291 goto free_alloc; 4292 } 4293 4294 vact->type = VIRTCHNL_ACTION_QUEUE; 4295 vact->act_conf.queue.index = q_index; 4296 break; 4297 } 4298 4299 /* Drop */ 4300 if (is_tcf_gact_shot(act)) { 4301 vact->type = VIRTCHNL_ACTION_DROP; 4302 break; 4303 } 4304 4305 /* Unsupported action */ 4306 NL_SET_ERR_MSG_MOD(extack, "Unsupported action."); 4307 status = -EOPNOTSUPP; 4308 goto free_alloc; 4309 } 4310 4311 fltr->vc_add_msg.vsi_id = adapter->vsi.id; 4312 fltr->cls_u32_handle = cls_u32->knode.handle; 4313 return iavf_fdir_add_fltr(adapter, fltr); 4314 4315 free_alloc: 4316 kfree(fltr); 4317 return status; 4318 } 4319 4320 /** 4321 * iavf_del_cls_u32 - Delete U32 classifier offloads 4322 * @adapter: pointer to iavf adapter structure 4323 * @cls_u32: pointer to tc_cls_u32_offload struct with flow info 4324 * 4325 * Return: 0 on success or negative errno on failure. 4326 */ 4327 static int iavf_del_cls_u32(struct iavf_adapter *adapter, 4328 struct tc_cls_u32_offload *cls_u32) 4329 { 4330 return iavf_fdir_del_fltr(adapter, true, cls_u32->knode.handle); 4331 } 4332 4333 /** 4334 * iavf_setup_tc_cls_u32 - U32 filter offloads 4335 * @adapter: pointer to iavf adapter structure 4336 * @cls_u32: pointer to tc_cls_u32_offload struct with flow info 4337 * 4338 * Return: 0 on success or negative errno on failure. 4339 */ 4340 static int iavf_setup_tc_cls_u32(struct iavf_adapter *adapter, 4341 struct tc_cls_u32_offload *cls_u32) 4342 { 4343 if (!TC_U32_SUPPORT(adapter) || !FDIR_FLTR_SUPPORT(adapter)) 4344 return -EOPNOTSUPP; 4345 4346 switch (cls_u32->command) { 4347 case TC_CLSU32_NEW_KNODE: 4348 case TC_CLSU32_REPLACE_KNODE: 4349 return iavf_add_cls_u32(adapter, cls_u32); 4350 case TC_CLSU32_DELETE_KNODE: 4351 return iavf_del_cls_u32(adapter, cls_u32); 4352 default: 4353 return -EOPNOTSUPP; 4354 } 4355 } 4356 4357 /** 4358 * iavf_setup_tc_block_cb - block callback for tc 4359 * @type: type of offload 4360 * @type_data: offload data 4361 * @cb_priv: 4362 * 4363 * This function is the block callback for traffic classes 4364 **/ 4365 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 4366 void *cb_priv) 4367 { 4368 struct iavf_adapter *adapter = cb_priv; 4369 4370 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data)) 4371 return -EOPNOTSUPP; 4372 4373 switch (type) { 4374 case TC_SETUP_CLSFLOWER: 4375 return iavf_setup_tc_cls_flower(cb_priv, type_data); 4376 case TC_SETUP_CLSU32: 4377 return iavf_setup_tc_cls_u32(cb_priv, type_data); 4378 default: 4379 return -EOPNOTSUPP; 4380 } 4381 } 4382 4383 static LIST_HEAD(iavf_block_cb_list); 4384 4385 /** 4386 * iavf_setup_tc - configure multiple traffic classes 4387 * @netdev: network interface device structure 4388 * @type: type of offload 4389 * @type_data: tc offload data 4390 * 4391 * This function is the callback to ndo_setup_tc in the 4392 * netdev_ops. 4393 * 4394 * Returns 0 on success 4395 **/ 4396 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type, 4397 void *type_data) 4398 { 4399 struct iavf_adapter *adapter = netdev_priv(netdev); 4400 4401 switch (type) { 4402 case TC_SETUP_QDISC_MQPRIO: 4403 return __iavf_setup_tc(netdev, type_data); 4404 case TC_SETUP_BLOCK: 4405 return flow_block_cb_setup_simple(type_data, 4406 &iavf_block_cb_list, 4407 iavf_setup_tc_block_cb, 4408 adapter, adapter, true); 4409 default: 4410 return -EOPNOTSUPP; 4411 } 4412 } 4413 4414 /** 4415 * iavf_restore_fdir_filters 4416 * @adapter: board private structure 4417 * 4418 * Restore existing FDIR filters when VF netdev comes back up. 4419 **/ 4420 static void iavf_restore_fdir_filters(struct iavf_adapter *adapter) 4421 { 4422 struct iavf_fdir_fltr *f; 4423 4424 spin_lock_bh(&adapter->fdir_fltr_lock); 4425 list_for_each_entry(f, &adapter->fdir_list_head, list) { 4426 if (f->state == IAVF_FDIR_FLTR_DIS_REQUEST) { 4427 /* Cancel a request, keep filter as active */ 4428 f->state = IAVF_FDIR_FLTR_ACTIVE; 4429 } else if (f->state == IAVF_FDIR_FLTR_DIS_PENDING || 4430 f->state == IAVF_FDIR_FLTR_INACTIVE) { 4431 /* Add filters which are inactive or have a pending 4432 * request to PF to be deleted 4433 */ 4434 f->state = IAVF_FDIR_FLTR_ADD_REQUEST; 4435 adapter->aq_required |= IAVF_FLAG_AQ_ADD_FDIR_FILTER; 4436 } 4437 } 4438 spin_unlock_bh(&adapter->fdir_fltr_lock); 4439 } 4440 4441 /** 4442 * iavf_open - Called when a network interface is made active 4443 * @netdev: network interface device structure 4444 * 4445 * Returns 0 on success, negative value on failure 4446 * 4447 * The open entry point is called when a network interface is made 4448 * active by the system (IFF_UP). At this point all resources needed 4449 * for transmit and receive operations are allocated, the interrupt 4450 * handler is registered with the OS, the watchdog is started, 4451 * and the stack is notified that the interface is ready. 4452 **/ 4453 static int iavf_open(struct net_device *netdev) 4454 { 4455 struct iavf_adapter *adapter = netdev_priv(netdev); 4456 int err; 4457 4458 netdev_assert_locked(netdev); 4459 4460 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) { 4461 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n"); 4462 return -EIO; 4463 } 4464 4465 if (adapter->state != __IAVF_DOWN) 4466 return -EBUSY; 4467 4468 if (adapter->state == __IAVF_RUNNING && 4469 !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) { 4470 dev_dbg(&adapter->pdev->dev, "VF is already open.\n"); 4471 return 0; 4472 } 4473 4474 /* allocate transmit descriptors */ 4475 err = iavf_setup_all_tx_resources(adapter); 4476 if (err) 4477 goto err_setup_tx; 4478 4479 /* allocate receive descriptors */ 4480 err = iavf_setup_all_rx_resources(adapter); 4481 if (err) 4482 goto err_setup_rx; 4483 4484 /* clear any pending interrupts, may auto mask */ 4485 err = iavf_request_traffic_irqs(adapter, netdev->name); 4486 if (err) 4487 goto err_req_irq; 4488 4489 spin_lock_bh(&adapter->mac_vlan_list_lock); 4490 iavf_add_filter(adapter, adapter->hw.mac.addr); 4491 spin_unlock_bh(&adapter->mac_vlan_list_lock); 4492 4493 /* Restore filters that were removed with IFF_DOWN */ 4494 iavf_restore_filters(adapter); 4495 iavf_restore_fdir_filters(adapter); 4496 4497 iavf_configure(adapter); 4498 4499 iavf_up_complete(adapter); 4500 4501 iavf_irq_enable(adapter, true); 4502 4503 return 0; 4504 4505 err_req_irq: 4506 iavf_down(adapter); 4507 iavf_free_traffic_irqs(adapter); 4508 err_setup_rx: 4509 iavf_free_all_rx_resources(adapter); 4510 err_setup_tx: 4511 iavf_free_all_tx_resources(adapter); 4512 4513 return err; 4514 } 4515 4516 /** 4517 * iavf_close - Disables a network interface 4518 * @netdev: network interface device structure 4519 * 4520 * Returns 0, this is not allowed to fail 4521 * 4522 * The close entry point is called when an interface is de-activated 4523 * by the OS. The hardware is still under the drivers control, but 4524 * needs to be disabled. All IRQs except vector 0 (reserved for admin queue) 4525 * are freed, along with all transmit and receive resources. 4526 **/ 4527 static int iavf_close(struct net_device *netdev) 4528 { 4529 struct iavf_adapter *adapter = netdev_priv(netdev); 4530 u64 aq_to_restore; 4531 int status; 4532 4533 netdev_assert_locked(netdev); 4534 4535 if (adapter->state <= __IAVF_DOWN_PENDING) 4536 return 0; 4537 4538 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 4539 /* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before 4540 * IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl 4541 * deadlock with adminq_task() until iavf_close timeouts. We must send 4542 * IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make 4543 * disable queues possible for vf. Give only necessary flags to 4544 * iavf_down and save other to set them right before iavf_close() 4545 * returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and 4546 * iavf will be in DOWN state. 4547 */ 4548 aq_to_restore = adapter->aq_required; 4549 adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG; 4550 4551 /* Remove flags which we do not want to send after close or we want to 4552 * send before disable queues. 4553 */ 4554 aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG | 4555 IAVF_FLAG_AQ_ENABLE_QUEUES | 4556 IAVF_FLAG_AQ_CONFIGURE_QUEUES | 4557 IAVF_FLAG_AQ_ADD_VLAN_FILTER | 4558 IAVF_FLAG_AQ_ADD_MAC_FILTER | 4559 IAVF_FLAG_AQ_ADD_CLOUD_FILTER | 4560 IAVF_FLAG_AQ_ADD_FDIR_FILTER | 4561 IAVF_FLAG_AQ_ADD_ADV_RSS_CFG); 4562 4563 iavf_down(adapter); 4564 iavf_change_state(adapter, __IAVF_DOWN_PENDING); 4565 iavf_free_traffic_irqs(adapter); 4566 4567 netdev_unlock(netdev); 4568 4569 /* We explicitly don't free resources here because the hardware is 4570 * still active and can DMA into memory. Resources are cleared in 4571 * iavf_virtchnl_completion() after we get confirmation from the PF 4572 * driver that the rings have been stopped. 4573 * 4574 * Also, we wait for state to transition to __IAVF_DOWN before 4575 * returning. State change occurs in iavf_virtchnl_completion() after 4576 * VF resources are released (which occurs after PF driver processes and 4577 * responds to admin queue commands). 4578 */ 4579 4580 status = wait_event_timeout(adapter->down_waitqueue, 4581 adapter->state == __IAVF_DOWN, 4582 msecs_to_jiffies(500)); 4583 if (!status) 4584 netdev_warn(netdev, "Device resources not yet released\n"); 4585 netdev_lock(netdev); 4586 4587 adapter->aq_required |= aq_to_restore; 4588 4589 return 0; 4590 } 4591 4592 /** 4593 * iavf_change_mtu - Change the Maximum Transfer Unit 4594 * @netdev: network interface device structure 4595 * @new_mtu: new value for maximum frame size 4596 * 4597 * Returns 0 on success, negative on failure 4598 **/ 4599 static int iavf_change_mtu(struct net_device *netdev, int new_mtu) 4600 { 4601 struct iavf_adapter *adapter = netdev_priv(netdev); 4602 int ret = 0; 4603 4604 netdev_dbg(netdev, "changing MTU from %d to %d\n", 4605 netdev->mtu, new_mtu); 4606 WRITE_ONCE(netdev->mtu, new_mtu); 4607 4608 if (netif_running(netdev)) { 4609 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); 4610 ret = iavf_wait_for_reset(adapter); 4611 if (ret < 0) 4612 netdev_warn(netdev, "MTU change interrupted waiting for reset"); 4613 else if (ret) 4614 netdev_warn(netdev, "MTU change timed out waiting for reset"); 4615 } 4616 4617 return ret; 4618 } 4619 4620 /** 4621 * iavf_disable_fdir - disable Flow Director and clear existing filters 4622 * @adapter: board private structure 4623 **/ 4624 static void iavf_disable_fdir(struct iavf_adapter *adapter) 4625 { 4626 struct iavf_fdir_fltr *fdir, *fdirtmp; 4627 bool del_filters = false; 4628 4629 adapter->flags &= ~IAVF_FLAG_FDIR_ENABLED; 4630 4631 /* remove all Flow Director filters */ 4632 spin_lock_bh(&adapter->fdir_fltr_lock); 4633 list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, 4634 list) { 4635 if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST || 4636 fdir->state == IAVF_FDIR_FLTR_INACTIVE) { 4637 /* Delete filters not registered in PF */ 4638 list_del(&fdir->list); 4639 iavf_dec_fdir_active_fltr(adapter, fdir); 4640 kfree(fdir); 4641 } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING || 4642 fdir->state == IAVF_FDIR_FLTR_DIS_REQUEST || 4643 fdir->state == IAVF_FDIR_FLTR_ACTIVE) { 4644 /* Filters registered in PF, schedule their deletion */ 4645 fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST; 4646 del_filters = true; 4647 } else if (fdir->state == IAVF_FDIR_FLTR_DIS_PENDING) { 4648 /* Request to delete filter already sent to PF, change 4649 * state to DEL_PENDING to delete filter after PF's 4650 * response, not set as INACTIVE 4651 */ 4652 fdir->state = IAVF_FDIR_FLTR_DEL_PENDING; 4653 } 4654 } 4655 spin_unlock_bh(&adapter->fdir_fltr_lock); 4656 4657 if (del_filters) { 4658 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER; 4659 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0); 4660 } 4661 } 4662 4663 #define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ 4664 NETIF_F_HW_VLAN_CTAG_TX | \ 4665 NETIF_F_HW_VLAN_STAG_RX | \ 4666 NETIF_F_HW_VLAN_STAG_TX) 4667 4668 /** 4669 * iavf_set_features - set the netdev feature flags 4670 * @netdev: ptr to the netdev being adjusted 4671 * @features: the feature set that the stack is suggesting 4672 * Note: expects to be called while under rtnl_lock() 4673 **/ 4674 static int iavf_set_features(struct net_device *netdev, 4675 netdev_features_t features) 4676 { 4677 struct iavf_adapter *adapter = netdev_priv(netdev); 4678 4679 /* trigger update on any VLAN feature change */ 4680 if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^ 4681 (features & NETIF_VLAN_OFFLOAD_FEATURES)) 4682 iavf_set_vlan_offload_features(adapter, netdev->features, 4683 features); 4684 if (CRC_OFFLOAD_ALLOWED(adapter) && 4685 ((netdev->features & NETIF_F_RXFCS) ^ (features & NETIF_F_RXFCS))) 4686 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED); 4687 4688 if ((netdev->features & NETIF_F_NTUPLE) ^ (features & NETIF_F_NTUPLE)) { 4689 if (features & NETIF_F_NTUPLE) 4690 adapter->flags |= IAVF_FLAG_FDIR_ENABLED; 4691 else 4692 iavf_disable_fdir(adapter); 4693 } 4694 4695 return 0; 4696 } 4697 4698 /** 4699 * iavf_features_check - Validate encapsulated packet conforms to limits 4700 * @skb: skb buff 4701 * @dev: This physical port's netdev 4702 * @features: Offload features that the stack believes apply 4703 **/ 4704 static netdev_features_t iavf_features_check(struct sk_buff *skb, 4705 struct net_device *dev, 4706 netdev_features_t features) 4707 { 4708 size_t len; 4709 4710 /* No point in doing any of this if neither checksum nor GSO are 4711 * being requested for this frame. We can rule out both by just 4712 * checking for CHECKSUM_PARTIAL 4713 */ 4714 if (skb->ip_summed != CHECKSUM_PARTIAL) 4715 return features; 4716 4717 /* We cannot support GSO if the MSS is going to be less than 4718 * 64 bytes. If it is then we need to drop support for GSO. 4719 */ 4720 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 4721 features &= ~NETIF_F_GSO_MASK; 4722 4723 /* MACLEN can support at most 63 words */ 4724 len = skb_network_offset(skb); 4725 if (len & ~(63 * 2)) 4726 goto out_err; 4727 4728 /* IPLEN and EIPLEN can support at most 127 dwords */ 4729 len = skb_network_header_len(skb); 4730 if (len & ~(127 * 4)) 4731 goto out_err; 4732 4733 if (skb->encapsulation) { 4734 /* L4TUNLEN can support 127 words */ 4735 len = skb_inner_network_header(skb) - skb_transport_header(skb); 4736 if (len & ~(127 * 2)) 4737 goto out_err; 4738 4739 /* IPLEN can support at most 127 dwords */ 4740 len = skb_inner_transport_header(skb) - 4741 skb_inner_network_header(skb); 4742 if (len & ~(127 * 4)) 4743 goto out_err; 4744 } 4745 4746 /* No need to validate L4LEN as TCP is the only protocol with a 4747 * flexible value and we support all possible values supported 4748 * by TCP, which is at most 15 dwords 4749 */ 4750 4751 return features; 4752 out_err: 4753 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 4754 } 4755 4756 /** 4757 * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off 4758 * @adapter: board private structure 4759 * 4760 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2 4761 * were negotiated determine the VLAN features that can be toggled on and off. 4762 **/ 4763 static netdev_features_t 4764 iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter) 4765 { 4766 netdev_features_t hw_features = 0; 4767 4768 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags) 4769 return hw_features; 4770 4771 /* Enable VLAN features if supported */ 4772 if (VLAN_ALLOWED(adapter)) { 4773 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX | 4774 NETIF_F_HW_VLAN_CTAG_RX); 4775 } else if (VLAN_V2_ALLOWED(adapter)) { 4776 struct virtchnl_vlan_caps *vlan_v2_caps = 4777 &adapter->vlan_v2_caps; 4778 struct virtchnl_vlan_supported_caps *stripping_support = 4779 &vlan_v2_caps->offloads.stripping_support; 4780 struct virtchnl_vlan_supported_caps *insertion_support = 4781 &vlan_v2_caps->offloads.insertion_support; 4782 4783 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED && 4784 stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) { 4785 if (stripping_support->outer & 4786 VIRTCHNL_VLAN_ETHERTYPE_8100) 4787 hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 4788 if (stripping_support->outer & 4789 VIRTCHNL_VLAN_ETHERTYPE_88A8) 4790 hw_features |= NETIF_F_HW_VLAN_STAG_RX; 4791 } else if (stripping_support->inner != 4792 VIRTCHNL_VLAN_UNSUPPORTED && 4793 stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) { 4794 if (stripping_support->inner & 4795 VIRTCHNL_VLAN_ETHERTYPE_8100) 4796 hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 4797 } 4798 4799 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED && 4800 insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) { 4801 if (insertion_support->outer & 4802 VIRTCHNL_VLAN_ETHERTYPE_8100) 4803 hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 4804 if (insertion_support->outer & 4805 VIRTCHNL_VLAN_ETHERTYPE_88A8) 4806 hw_features |= NETIF_F_HW_VLAN_STAG_TX; 4807 } else if (insertion_support->inner && 4808 insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) { 4809 if (insertion_support->inner & 4810 VIRTCHNL_VLAN_ETHERTYPE_8100) 4811 hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 4812 } 4813 } 4814 4815 if (CRC_OFFLOAD_ALLOWED(adapter)) 4816 hw_features |= NETIF_F_RXFCS; 4817 4818 return hw_features; 4819 } 4820 4821 /** 4822 * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures 4823 * @adapter: board private structure 4824 * 4825 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2 4826 * were negotiated determine the VLAN features that are enabled by default. 4827 **/ 4828 static netdev_features_t 4829 iavf_get_netdev_vlan_features(struct iavf_adapter *adapter) 4830 { 4831 netdev_features_t features = 0; 4832 4833 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags) 4834 return features; 4835 4836 if (VLAN_ALLOWED(adapter)) { 4837 features |= NETIF_F_HW_VLAN_CTAG_FILTER | 4838 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX; 4839 } else if (VLAN_V2_ALLOWED(adapter)) { 4840 struct virtchnl_vlan_caps *vlan_v2_caps = 4841 &adapter->vlan_v2_caps; 4842 struct virtchnl_vlan_supported_caps *filtering_support = 4843 &vlan_v2_caps->filtering.filtering_support; 4844 struct virtchnl_vlan_supported_caps *stripping_support = 4845 &vlan_v2_caps->offloads.stripping_support; 4846 struct virtchnl_vlan_supported_caps *insertion_support = 4847 &vlan_v2_caps->offloads.insertion_support; 4848 u32 ethertype_init; 4849 4850 /* give priority to outer stripping and don't support both outer 4851 * and inner stripping 4852 */ 4853 ethertype_init = vlan_v2_caps->offloads.ethertype_init; 4854 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { 4855 if (stripping_support->outer & 4856 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4857 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4858 features |= NETIF_F_HW_VLAN_CTAG_RX; 4859 else if (stripping_support->outer & 4860 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4861 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4862 features |= NETIF_F_HW_VLAN_STAG_RX; 4863 } else if (stripping_support->inner != 4864 VIRTCHNL_VLAN_UNSUPPORTED) { 4865 if (stripping_support->inner & 4866 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4867 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4868 features |= NETIF_F_HW_VLAN_CTAG_RX; 4869 } 4870 4871 /* give priority to outer insertion and don't support both outer 4872 * and inner insertion 4873 */ 4874 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { 4875 if (insertion_support->outer & 4876 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4877 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4878 features |= NETIF_F_HW_VLAN_CTAG_TX; 4879 else if (insertion_support->outer & 4880 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4881 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4882 features |= NETIF_F_HW_VLAN_STAG_TX; 4883 } else if (insertion_support->inner != 4884 VIRTCHNL_VLAN_UNSUPPORTED) { 4885 if (insertion_support->inner & 4886 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4887 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4888 features |= NETIF_F_HW_VLAN_CTAG_TX; 4889 } 4890 4891 /* give priority to outer filtering and don't bother if both 4892 * outer and inner filtering are enabled 4893 */ 4894 ethertype_init = vlan_v2_caps->filtering.ethertype_init; 4895 if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { 4896 if (filtering_support->outer & 4897 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4898 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4899 features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4900 if (filtering_support->outer & 4901 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4902 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4903 features |= NETIF_F_HW_VLAN_STAG_FILTER; 4904 } else if (filtering_support->inner != 4905 VIRTCHNL_VLAN_UNSUPPORTED) { 4906 if (filtering_support->inner & 4907 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4908 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4909 features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4910 if (filtering_support->inner & 4911 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4912 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4913 features |= NETIF_F_HW_VLAN_STAG_FILTER; 4914 } 4915 } 4916 4917 return features; 4918 } 4919 4920 #define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \ 4921 (!(((requested) & (feature_bit)) && \ 4922 !((allowed) & (feature_bit)))) 4923 4924 /** 4925 * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support 4926 * @adapter: board private structure 4927 * @requested_features: stack requested NETDEV features 4928 **/ 4929 static netdev_features_t 4930 iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter, 4931 netdev_features_t requested_features) 4932 { 4933 netdev_features_t allowed_features; 4934 4935 allowed_features = iavf_get_netdev_vlan_hw_features(adapter) | 4936 iavf_get_netdev_vlan_features(adapter); 4937 4938 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4939 allowed_features, 4940 NETIF_F_HW_VLAN_CTAG_TX)) 4941 requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX; 4942 4943 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4944 allowed_features, 4945 NETIF_F_HW_VLAN_CTAG_RX)) 4946 requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX; 4947 4948 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4949 allowed_features, 4950 NETIF_F_HW_VLAN_STAG_TX)) 4951 requested_features &= ~NETIF_F_HW_VLAN_STAG_TX; 4952 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4953 allowed_features, 4954 NETIF_F_HW_VLAN_STAG_RX)) 4955 requested_features &= ~NETIF_F_HW_VLAN_STAG_RX; 4956 4957 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4958 allowed_features, 4959 NETIF_F_HW_VLAN_CTAG_FILTER)) 4960 requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4961 4962 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4963 allowed_features, 4964 NETIF_F_HW_VLAN_STAG_FILTER)) 4965 requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER; 4966 4967 if ((requested_features & 4968 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) && 4969 (requested_features & 4970 (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) && 4971 adapter->vlan_v2_caps.offloads.ethertype_match == 4972 VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) { 4973 netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n"); 4974 requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX | 4975 NETIF_F_HW_VLAN_STAG_TX); 4976 } 4977 4978 return requested_features; 4979 } 4980 4981 /** 4982 * iavf_fix_strip_features - fix NETDEV CRC and VLAN strip features 4983 * @adapter: board private structure 4984 * @requested_features: stack requested NETDEV features 4985 * 4986 * Returns fixed-up features bits 4987 **/ 4988 static netdev_features_t 4989 iavf_fix_strip_features(struct iavf_adapter *adapter, 4990 netdev_features_t requested_features) 4991 { 4992 struct net_device *netdev = adapter->netdev; 4993 bool crc_offload_req, is_vlan_strip; 4994 netdev_features_t vlan_strip; 4995 int num_non_zero_vlan; 4996 4997 crc_offload_req = CRC_OFFLOAD_ALLOWED(adapter) && 4998 (requested_features & NETIF_F_RXFCS); 4999 num_non_zero_vlan = iavf_get_num_vlans_added(adapter); 5000 vlan_strip = (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX); 5001 is_vlan_strip = requested_features & vlan_strip; 5002 5003 if (!crc_offload_req) 5004 return requested_features; 5005 5006 if (!num_non_zero_vlan && (netdev->features & vlan_strip) && 5007 !(netdev->features & NETIF_F_RXFCS) && is_vlan_strip) { 5008 requested_features &= ~vlan_strip; 5009 netdev_info(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n"); 5010 return requested_features; 5011 } 5012 5013 if ((netdev->features & NETIF_F_RXFCS) && is_vlan_strip) { 5014 requested_features &= ~vlan_strip; 5015 if (!(netdev->features & vlan_strip)) 5016 netdev_info(netdev, "To enable VLAN stripping, first need to enable FCS/CRC stripping"); 5017 5018 return requested_features; 5019 } 5020 5021 if (num_non_zero_vlan && is_vlan_strip && 5022 !(netdev->features & NETIF_F_RXFCS)) { 5023 requested_features &= ~NETIF_F_RXFCS; 5024 netdev_info(netdev, "To disable FCS/CRC stripping, first need to disable VLAN stripping"); 5025 } 5026 5027 return requested_features; 5028 } 5029 5030 /** 5031 * iavf_fix_features - fix up the netdev feature bits 5032 * @netdev: our net device 5033 * @features: desired feature bits 5034 * 5035 * Returns fixed-up features bits 5036 **/ 5037 static netdev_features_t iavf_fix_features(struct net_device *netdev, 5038 netdev_features_t features) 5039 { 5040 struct iavf_adapter *adapter = netdev_priv(netdev); 5041 5042 features = iavf_fix_netdev_vlan_features(adapter, features); 5043 5044 if (!FDIR_FLTR_SUPPORT(adapter)) 5045 features &= ~NETIF_F_NTUPLE; 5046 5047 return iavf_fix_strip_features(adapter, features); 5048 } 5049 5050 static int iavf_hwstamp_get(struct net_device *netdev, 5051 struct kernel_hwtstamp_config *config) 5052 { 5053 struct iavf_adapter *adapter = netdev_priv(netdev); 5054 5055 *config = adapter->ptp.hwtstamp_config; 5056 5057 return 0; 5058 } 5059 5060 static int iavf_hwstamp_set(struct net_device *netdev, 5061 struct kernel_hwtstamp_config *config, 5062 struct netlink_ext_ack *extack) 5063 { 5064 struct iavf_adapter *adapter = netdev_priv(netdev); 5065 5066 return iavf_ptp_set_ts_config(adapter, config, extack); 5067 } 5068 5069 static int 5070 iavf_verify_shaper(struct net_shaper_binding *binding, 5071 const struct net_shaper *shaper, 5072 struct netlink_ext_ack *extack) 5073 { 5074 struct iavf_adapter *adapter = netdev_priv(binding->netdev); 5075 u64 vf_max; 5076 5077 if (shaper->handle.scope == NET_SHAPER_SCOPE_QUEUE) { 5078 vf_max = adapter->qos_caps->cap[0].shaper.peak; 5079 if (vf_max && shaper->bw_max > vf_max) { 5080 NL_SET_ERR_MSG_FMT(extack, "Max rate (%llu) of queue %d can't exceed max TX rate of VF (%llu kbps)", 5081 shaper->bw_max, shaper->handle.id, 5082 vf_max); 5083 return -EINVAL; 5084 } 5085 } 5086 return 0; 5087 } 5088 5089 static int 5090 iavf_shaper_set(struct net_shaper_binding *binding, 5091 const struct net_shaper *shaper, 5092 struct netlink_ext_ack *extack) 5093 { 5094 struct iavf_adapter *adapter = netdev_priv(binding->netdev); 5095 const struct net_shaper_handle *handle = &shaper->handle; 5096 struct iavf_ring *tx_ring; 5097 int ret; 5098 5099 netdev_assert_locked(adapter->netdev); 5100 5101 if (handle->id >= adapter->num_active_queues) 5102 return 0; 5103 5104 ret = iavf_verify_shaper(binding, shaper, extack); 5105 if (ret) 5106 return ret; 5107 5108 tx_ring = &adapter->tx_rings[handle->id]; 5109 5110 tx_ring->q_shaper.bw_min = div_u64(shaper->bw_min, 1000); 5111 tx_ring->q_shaper.bw_max = div_u64(shaper->bw_max, 1000); 5112 tx_ring->q_shaper_update = true; 5113 5114 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES_BW; 5115 5116 return 0; 5117 } 5118 5119 static int iavf_shaper_del(struct net_shaper_binding *binding, 5120 const struct net_shaper_handle *handle, 5121 struct netlink_ext_ack *extack) 5122 { 5123 struct iavf_adapter *adapter = netdev_priv(binding->netdev); 5124 struct iavf_ring *tx_ring; 5125 5126 netdev_assert_locked(adapter->netdev); 5127 5128 if (handle->id >= adapter->num_active_queues) 5129 return 0; 5130 5131 tx_ring = &adapter->tx_rings[handle->id]; 5132 tx_ring->q_shaper.bw_min = 0; 5133 tx_ring->q_shaper.bw_max = 0; 5134 tx_ring->q_shaper_update = true; 5135 5136 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES_BW; 5137 5138 return 0; 5139 } 5140 5141 static void iavf_shaper_cap(struct net_shaper_binding *binding, 5142 enum net_shaper_scope scope, 5143 unsigned long *flags) 5144 { 5145 if (scope != NET_SHAPER_SCOPE_QUEUE) 5146 return; 5147 5148 *flags = BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MIN) | 5149 BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MAX) | 5150 BIT(NET_SHAPER_A_CAPS_SUPPORT_METRIC_BPS); 5151 } 5152 5153 static const struct net_shaper_ops iavf_shaper_ops = { 5154 .set = iavf_shaper_set, 5155 .delete = iavf_shaper_del, 5156 .capabilities = iavf_shaper_cap, 5157 }; 5158 5159 static const struct net_device_ops iavf_netdev_ops = { 5160 .ndo_open = iavf_open, 5161 .ndo_stop = iavf_close, 5162 .ndo_start_xmit = iavf_xmit_frame, 5163 .ndo_set_rx_mode = iavf_set_rx_mode, 5164 .ndo_validate_addr = eth_validate_addr, 5165 .ndo_set_mac_address = iavf_set_mac, 5166 .ndo_change_mtu = iavf_change_mtu, 5167 .ndo_tx_timeout = iavf_tx_timeout, 5168 .ndo_vlan_rx_add_vid = iavf_vlan_rx_add_vid, 5169 .ndo_vlan_rx_kill_vid = iavf_vlan_rx_kill_vid, 5170 .ndo_features_check = iavf_features_check, 5171 .ndo_fix_features = iavf_fix_features, 5172 .ndo_set_features = iavf_set_features, 5173 .ndo_setup_tc = iavf_setup_tc, 5174 .net_shaper_ops = &iavf_shaper_ops, 5175 .ndo_hwtstamp_get = iavf_hwstamp_get, 5176 .ndo_hwtstamp_set = iavf_hwstamp_set, 5177 }; 5178 5179 /** 5180 * iavf_check_reset_complete - check that VF reset is complete 5181 * @hw: pointer to hw struct 5182 * 5183 * Returns 0 if device is ready to use, or -EBUSY if it's in reset. 5184 **/ 5185 static int iavf_check_reset_complete(struct iavf_hw *hw) 5186 { 5187 u32 rstat; 5188 int i; 5189 5190 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { 5191 rstat = rd32(hw, IAVF_VFGEN_RSTAT) & 5192 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 5193 if ((rstat == VIRTCHNL_VFR_VFACTIVE) || 5194 (rstat == VIRTCHNL_VFR_COMPLETED)) 5195 return 0; 5196 msleep(IAVF_RESET_WAIT_MS); 5197 } 5198 return -EBUSY; 5199 } 5200 5201 /** 5202 * iavf_process_config - Process the config information we got from the PF 5203 * @adapter: board private structure 5204 * 5205 * Verify that we have a valid config struct, and set up our netdev features 5206 * and our VSI struct. 5207 **/ 5208 int iavf_process_config(struct iavf_adapter *adapter) 5209 { 5210 struct virtchnl_vf_resource *vfres = adapter->vf_res; 5211 netdev_features_t hw_vlan_features, vlan_features; 5212 struct net_device *netdev = adapter->netdev; 5213 netdev_features_t hw_enc_features; 5214 netdev_features_t hw_features; 5215 5216 hw_enc_features = NETIF_F_SG | 5217 NETIF_F_IP_CSUM | 5218 NETIF_F_IPV6_CSUM | 5219 NETIF_F_HIGHDMA | 5220 NETIF_F_SOFT_FEATURES | 5221 NETIF_F_TSO | 5222 NETIF_F_TSO_ECN | 5223 NETIF_F_TSO6 | 5224 NETIF_F_SCTP_CRC | 5225 NETIF_F_RXHASH | 5226 NETIF_F_RXCSUM | 5227 0; 5228 5229 /* advertise to stack only if offloads for encapsulated packets is 5230 * supported 5231 */ 5232 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) { 5233 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 5234 NETIF_F_GSO_GRE | 5235 NETIF_F_GSO_GRE_CSUM | 5236 NETIF_F_GSO_IPXIP4 | 5237 NETIF_F_GSO_IPXIP6 | 5238 NETIF_F_GSO_UDP_TUNNEL_CSUM | 5239 NETIF_F_GSO_PARTIAL | 5240 0; 5241 5242 if (!(vfres->vf_cap_flags & 5243 VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM)) 5244 netdev->gso_partial_features |= 5245 NETIF_F_GSO_UDP_TUNNEL_CSUM; 5246 5247 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 5248 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 5249 netdev->hw_enc_features |= hw_enc_features; 5250 } 5251 /* record features VLANs can make use of */ 5252 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 5253 5254 /* Write features and hw_features separately to avoid polluting 5255 * with, or dropping, features that are set when we registered. 5256 */ 5257 hw_features = hw_enc_features; 5258 5259 /* get HW VLAN features that can be toggled */ 5260 hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter); 5261 5262 /* Enable HW TC offload if ADQ or tc U32 is supported */ 5263 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ || 5264 TC_U32_SUPPORT(adapter)) 5265 hw_features |= NETIF_F_HW_TC; 5266 5267 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO) 5268 hw_features |= NETIF_F_GSO_UDP_L4; 5269 5270 netdev->hw_features |= hw_features | hw_vlan_features; 5271 vlan_features = iavf_get_netdev_vlan_features(adapter); 5272 5273 netdev->features |= hw_features | vlan_features; 5274 5275 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) 5276 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 5277 5278 if (FDIR_FLTR_SUPPORT(adapter)) { 5279 netdev->hw_features |= NETIF_F_NTUPLE; 5280 netdev->features |= NETIF_F_NTUPLE; 5281 adapter->flags |= IAVF_FLAG_FDIR_ENABLED; 5282 } 5283 5284 netdev->priv_flags |= IFF_UNICAST_FLT; 5285 5286 /* Do not turn on offloads when they are requested to be turned off. 5287 * TSO needs minimum 576 bytes to work correctly. 5288 */ 5289 if (netdev->wanted_features) { 5290 if (!(netdev->wanted_features & NETIF_F_TSO) || 5291 netdev->mtu < 576) 5292 netdev->features &= ~NETIF_F_TSO; 5293 if (!(netdev->wanted_features & NETIF_F_TSO6) || 5294 netdev->mtu < 576) 5295 netdev->features &= ~NETIF_F_TSO6; 5296 if (!(netdev->wanted_features & NETIF_F_TSO_ECN)) 5297 netdev->features &= ~NETIF_F_TSO_ECN; 5298 if (!(netdev->wanted_features & NETIF_F_GRO)) 5299 netdev->features &= ~NETIF_F_GRO; 5300 if (!(netdev->wanted_features & NETIF_F_GSO)) 5301 netdev->features &= ~NETIF_F_GSO; 5302 } 5303 5304 return 0; 5305 } 5306 5307 /** 5308 * iavf_probe - Device Initialization Routine 5309 * @pdev: PCI device information struct 5310 * @ent: entry in iavf_pci_tbl 5311 * 5312 * Returns 0 on success, negative on failure 5313 * 5314 * iavf_probe initializes an adapter identified by a pci_dev structure. 5315 * The OS initialization, configuring of the adapter private structure, 5316 * and a hardware reset occur. 5317 **/ 5318 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 5319 { 5320 struct net_device *netdev; 5321 struct iavf_adapter *adapter = NULL; 5322 struct iavf_hw *hw = NULL; 5323 int err, len; 5324 5325 err = pci_enable_device(pdev); 5326 if (err) 5327 return err; 5328 5329 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 5330 if (err) { 5331 dev_err(&pdev->dev, 5332 "DMA configuration failed: 0x%x\n", err); 5333 goto err_dma; 5334 } 5335 5336 err = pci_request_regions(pdev, iavf_driver_name); 5337 if (err) { 5338 dev_err(&pdev->dev, 5339 "pci_request_regions failed 0x%x\n", err); 5340 goto err_pci_reg; 5341 } 5342 5343 pci_set_master(pdev); 5344 5345 netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter), 5346 IAVF_MAX_REQ_QUEUES); 5347 if (!netdev) { 5348 err = -ENOMEM; 5349 goto err_alloc_etherdev; 5350 } 5351 5352 netif_set_affinity_auto(netdev); 5353 SET_NETDEV_DEV(netdev, &pdev->dev); 5354 5355 pci_set_drvdata(pdev, netdev); 5356 adapter = netdev_priv(netdev); 5357 5358 adapter->netdev = netdev; 5359 adapter->pdev = pdev; 5360 5361 hw = &adapter->hw; 5362 hw->back = adapter; 5363 5364 adapter->wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM, 5365 iavf_driver_name); 5366 if (!adapter->wq) { 5367 err = -ENOMEM; 5368 goto err_alloc_wq; 5369 } 5370 5371 adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1; 5372 iavf_change_state(adapter, __IAVF_STARTUP); 5373 5374 /* Call save state here because it relies on the adapter struct. */ 5375 pci_save_state(pdev); 5376 5377 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 5378 pci_resource_len(pdev, 0)); 5379 if (!hw->hw_addr) { 5380 err = -EIO; 5381 goto err_ioremap; 5382 } 5383 hw->vendor_id = pdev->vendor; 5384 hw->device_id = pdev->device; 5385 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 5386 hw->subsystem_vendor_id = pdev->subsystem_vendor; 5387 hw->subsystem_device_id = pdev->subsystem_device; 5388 hw->bus.device = PCI_SLOT(pdev->devfn); 5389 hw->bus.func = PCI_FUNC(pdev->devfn); 5390 hw->bus.bus_id = pdev->bus->number; 5391 5392 len = struct_size(adapter->qos_caps, cap, IAVF_MAX_QOS_TC_NUM); 5393 adapter->qos_caps = kzalloc(len, GFP_KERNEL); 5394 if (!adapter->qos_caps) { 5395 err = -ENOMEM; 5396 goto err_alloc_qos_cap; 5397 } 5398 5399 mutex_init(&hw->aq.asq_mutex); 5400 mutex_init(&hw->aq.arq_mutex); 5401 5402 spin_lock_init(&adapter->mac_vlan_list_lock); 5403 spin_lock_init(&adapter->cloud_filter_list_lock); 5404 spin_lock_init(&adapter->fdir_fltr_lock); 5405 spin_lock_init(&adapter->adv_rss_lock); 5406 spin_lock_init(&adapter->current_netdev_promisc_flags_lock); 5407 5408 INIT_LIST_HEAD(&adapter->mac_filter_list); 5409 INIT_LIST_HEAD(&adapter->vlan_filter_list); 5410 INIT_LIST_HEAD(&adapter->cloud_filter_list); 5411 INIT_LIST_HEAD(&adapter->fdir_list_head); 5412 INIT_LIST_HEAD(&adapter->adv_rss_list_head); 5413 5414 INIT_WORK(&adapter->reset_task, iavf_reset_task); 5415 INIT_WORK(&adapter->adminq_task, iavf_adminq_task); 5416 INIT_WORK(&adapter->finish_config, iavf_finish_config); 5417 INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task); 5418 5419 /* Setup the wait queue for indicating transition to down status */ 5420 init_waitqueue_head(&adapter->down_waitqueue); 5421 5422 /* Setup the wait queue for indicating transition to running state */ 5423 init_waitqueue_head(&adapter->reset_waitqueue); 5424 5425 /* Setup the wait queue for indicating virtchannel events */ 5426 init_waitqueue_head(&adapter->vc_waitqueue); 5427 5428 INIT_LIST_HEAD(&adapter->ptp.aq_cmds); 5429 init_waitqueue_head(&adapter->ptp.phc_time_waitqueue); 5430 mutex_init(&adapter->ptp.aq_cmd_lock); 5431 5432 queue_delayed_work(adapter->wq, &adapter->watchdog_task, 5433 msecs_to_jiffies(5 * (pdev->devfn & 0x07))); 5434 /* Initialization goes on in the work. Do not add more of it below. */ 5435 return 0; 5436 5437 err_alloc_qos_cap: 5438 iounmap(hw->hw_addr); 5439 err_ioremap: 5440 destroy_workqueue(adapter->wq); 5441 err_alloc_wq: 5442 free_netdev(netdev); 5443 err_alloc_etherdev: 5444 pci_release_regions(pdev); 5445 err_pci_reg: 5446 err_dma: 5447 pci_disable_device(pdev); 5448 return err; 5449 } 5450 5451 /** 5452 * iavf_suspend - Power management suspend routine 5453 * @dev_d: device info pointer 5454 * 5455 * Called when the system (VM) is entering sleep/suspend. 5456 **/ 5457 static int iavf_suspend(struct device *dev_d) 5458 { 5459 struct net_device *netdev = dev_get_drvdata(dev_d); 5460 struct iavf_adapter *adapter = netdev_priv(netdev); 5461 bool running; 5462 5463 netif_device_detach(netdev); 5464 5465 running = netif_running(netdev); 5466 if (running) 5467 rtnl_lock(); 5468 netdev_lock(netdev); 5469 5470 if (running) 5471 iavf_down(adapter); 5472 5473 iavf_free_misc_irq(adapter); 5474 iavf_reset_interrupt_capability(adapter); 5475 5476 netdev_unlock(netdev); 5477 if (running) 5478 rtnl_unlock(); 5479 5480 return 0; 5481 } 5482 5483 /** 5484 * iavf_resume - Power management resume routine 5485 * @dev_d: device info pointer 5486 * 5487 * Called when the system (VM) is resumed from sleep/suspend. 5488 **/ 5489 static int iavf_resume(struct device *dev_d) 5490 { 5491 struct pci_dev *pdev = to_pci_dev(dev_d); 5492 struct iavf_adapter *adapter; 5493 u32 err; 5494 5495 adapter = iavf_pdev_to_adapter(pdev); 5496 5497 pci_set_master(pdev); 5498 5499 rtnl_lock(); 5500 err = iavf_set_interrupt_capability(adapter); 5501 if (err) { 5502 rtnl_unlock(); 5503 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n"); 5504 return err; 5505 } 5506 err = iavf_request_misc_irq(adapter); 5507 rtnl_unlock(); 5508 if (err) { 5509 dev_err(&pdev->dev, "Cannot get interrupt vector.\n"); 5510 return err; 5511 } 5512 5513 queue_work(adapter->wq, &adapter->reset_task); 5514 5515 netif_device_attach(adapter->netdev); 5516 5517 return err; 5518 } 5519 5520 /** 5521 * iavf_remove - Device Removal Routine 5522 * @pdev: PCI device information struct 5523 * 5524 * iavf_remove is called by the PCI subsystem to alert the driver 5525 * that it should release a PCI device. The could be caused by a 5526 * Hot-Plug event, or because the driver is going to be removed from 5527 * memory. 5528 **/ 5529 static void iavf_remove(struct pci_dev *pdev) 5530 { 5531 struct iavf_fdir_fltr *fdir, *fdirtmp; 5532 struct iavf_vlan_filter *vlf, *vlftmp; 5533 struct iavf_cloud_filter *cf, *cftmp; 5534 struct iavf_adv_rss *rss, *rsstmp; 5535 struct iavf_mac_filter *f, *ftmp; 5536 struct iavf_adapter *adapter; 5537 struct net_device *netdev; 5538 struct iavf_hw *hw; 5539 5540 /* Don't proceed with remove if netdev is already freed */ 5541 netdev = pci_get_drvdata(pdev); 5542 if (!netdev) 5543 return; 5544 5545 adapter = iavf_pdev_to_adapter(pdev); 5546 hw = &adapter->hw; 5547 5548 if (test_and_set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) 5549 return; 5550 5551 /* Wait until port initialization is complete. 5552 * There are flows where register/unregister netdev may race. 5553 */ 5554 while (1) { 5555 netdev_lock(netdev); 5556 if (adapter->state == __IAVF_RUNNING || 5557 adapter->state == __IAVF_DOWN || 5558 adapter->state == __IAVF_INIT_FAILED) { 5559 netdev_unlock(netdev); 5560 break; 5561 } 5562 /* Simply return if we already went through iavf_shutdown */ 5563 if (adapter->state == __IAVF_REMOVE) { 5564 netdev_unlock(netdev); 5565 return; 5566 } 5567 5568 netdev_unlock(netdev); 5569 usleep_range(500, 1000); 5570 } 5571 cancel_delayed_work_sync(&adapter->watchdog_task); 5572 cancel_work_sync(&adapter->finish_config); 5573 5574 if (netdev->reg_state == NETREG_REGISTERED) 5575 unregister_netdev(netdev); 5576 5577 netdev_lock(netdev); 5578 dev_info(&adapter->pdev->dev, "Removing device\n"); 5579 iavf_change_state(adapter, __IAVF_REMOVE); 5580 5581 iavf_request_reset(adapter); 5582 msleep(50); 5583 /* If the FW isn't responding, kick it once, but only once. */ 5584 if (!iavf_asq_done(hw)) { 5585 iavf_request_reset(adapter); 5586 msleep(50); 5587 } 5588 5589 iavf_ptp_release(adapter); 5590 5591 iavf_misc_irq_disable(adapter); 5592 /* Shut down all the garbage mashers on the detention level */ 5593 netdev_unlock(netdev); 5594 cancel_work_sync(&adapter->reset_task); 5595 cancel_delayed_work_sync(&adapter->watchdog_task); 5596 cancel_work_sync(&adapter->adminq_task); 5597 netdev_lock(netdev); 5598 5599 adapter->aq_required = 0; 5600 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 5601 5602 iavf_free_all_tx_resources(adapter); 5603 iavf_free_all_rx_resources(adapter); 5604 iavf_free_misc_irq(adapter); 5605 iavf_free_interrupt_scheme(adapter); 5606 5607 iavf_free_rss(adapter); 5608 5609 if (hw->aq.asq.count) 5610 iavf_shutdown_adminq(hw); 5611 5612 /* destroy the locks only once, here */ 5613 mutex_destroy(&hw->aq.arq_mutex); 5614 mutex_destroy(&hw->aq.asq_mutex); 5615 netdev_unlock(netdev); 5616 5617 iounmap(hw->hw_addr); 5618 pci_release_regions(pdev); 5619 kfree(adapter->vf_res); 5620 spin_lock_bh(&adapter->mac_vlan_list_lock); 5621 /* If we got removed before an up/down sequence, we've got a filter 5622 * hanging out there that we need to get rid of. 5623 */ 5624 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 5625 list_del(&f->list); 5626 kfree(f); 5627 } 5628 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list, 5629 list) { 5630 list_del(&vlf->list); 5631 kfree(vlf); 5632 } 5633 5634 spin_unlock_bh(&adapter->mac_vlan_list_lock); 5635 5636 spin_lock_bh(&adapter->cloud_filter_list_lock); 5637 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 5638 list_del(&cf->list); 5639 kfree(cf); 5640 } 5641 spin_unlock_bh(&adapter->cloud_filter_list_lock); 5642 5643 spin_lock_bh(&adapter->fdir_fltr_lock); 5644 list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) { 5645 list_del(&fdir->list); 5646 kfree(fdir); 5647 } 5648 spin_unlock_bh(&adapter->fdir_fltr_lock); 5649 5650 spin_lock_bh(&adapter->adv_rss_lock); 5651 list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head, 5652 list) { 5653 list_del(&rss->list); 5654 kfree(rss); 5655 } 5656 spin_unlock_bh(&adapter->adv_rss_lock); 5657 5658 destroy_workqueue(adapter->wq); 5659 5660 pci_set_drvdata(pdev, NULL); 5661 5662 free_netdev(netdev); 5663 5664 pci_disable_device(pdev); 5665 } 5666 5667 /** 5668 * iavf_shutdown - Shutdown the device in preparation for a reboot 5669 * @pdev: pci device structure 5670 **/ 5671 static void iavf_shutdown(struct pci_dev *pdev) 5672 { 5673 iavf_remove(pdev); 5674 5675 if (system_state == SYSTEM_POWER_OFF) 5676 pci_set_power_state(pdev, PCI_D3hot); 5677 } 5678 5679 static DEFINE_SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume); 5680 5681 static struct pci_driver iavf_driver = { 5682 .name = iavf_driver_name, 5683 .id_table = iavf_pci_tbl, 5684 .probe = iavf_probe, 5685 .remove = iavf_remove, 5686 .driver.pm = pm_sleep_ptr(&iavf_pm_ops), 5687 .shutdown = iavf_shutdown, 5688 }; 5689 5690 /** 5691 * iavf_init_module - Driver Registration Routine 5692 * 5693 * iavf_init_module is the first routine called when the driver is 5694 * loaded. All it does is register with the PCI subsystem. 5695 **/ 5696 static int __init iavf_init_module(void) 5697 { 5698 pr_info("iavf: %s\n", iavf_driver_string); 5699 5700 pr_info("%s\n", iavf_copyright); 5701 5702 return pci_register_driver(&iavf_driver); 5703 } 5704 5705 module_init(iavf_init_module); 5706 5707 /** 5708 * iavf_exit_module - Driver Exit Cleanup Routine 5709 * 5710 * iavf_exit_module is called just before the driver is removed 5711 * from memory. 5712 **/ 5713 static void __exit iavf_exit_module(void) 5714 { 5715 pci_unregister_driver(&iavf_driver); 5716 } 5717 5718 module_exit(iavf_exit_module); 5719 5720 /* iavf_main.c */ 5721