xref: /linux/drivers/net/ethernet/intel/iavf/iavf_main.c (revision cdd30ebb1b9f36159d66f088b61aee264e649d7a)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
3 
4 #include <linux/net/intel/libie/rx.h>
5 
6 #include "iavf.h"
7 #include "iavf_prototype.h"
8 /* All iavf tracepoints are defined by the include below, which must
9  * be included exactly once across the whole kernel with
10  * CREATE_TRACE_POINTS defined
11  */
12 #define CREATE_TRACE_POINTS
13 #include "iavf_trace.h"
14 
15 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
16 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
17 static int iavf_close(struct net_device *netdev);
18 static void iavf_init_get_resources(struct iavf_adapter *adapter);
19 static int iavf_check_reset_complete(struct iavf_hw *hw);
20 
21 char iavf_driver_name[] = "iavf";
22 static const char iavf_driver_string[] =
23 	"Intel(R) Ethernet Adaptive Virtual Function Network Driver";
24 
25 static const char iavf_copyright[] =
26 	"Copyright (c) 2013 - 2018 Intel Corporation.";
27 
28 /* iavf_pci_tbl - PCI Device ID Table
29  *
30  * Wildcard entries (PCI_ANY_ID) should come last
31  * Last entry must be all 0s
32  *
33  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
34  *   Class, Class Mask, private data (not used) }
35  */
36 static const struct pci_device_id iavf_pci_tbl[] = {
37 	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
38 	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
39 	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
40 	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
41 	/* required last entry */
42 	{0, }
43 };
44 
45 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
46 
47 MODULE_ALIAS("i40evf");
48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");
49 MODULE_IMPORT_NS("LIBETH");
50 MODULE_IMPORT_NS("LIBIE");
51 MODULE_LICENSE("GPL v2");
52 
53 static const struct net_device_ops iavf_netdev_ops;
54 
iavf_status_to_errno(enum iavf_status status)55 int iavf_status_to_errno(enum iavf_status status)
56 {
57 	switch (status) {
58 	case IAVF_SUCCESS:
59 		return 0;
60 	case IAVF_ERR_PARAM:
61 	case IAVF_ERR_MAC_TYPE:
62 	case IAVF_ERR_INVALID_MAC_ADDR:
63 	case IAVF_ERR_INVALID_LINK_SETTINGS:
64 	case IAVF_ERR_INVALID_PD_ID:
65 	case IAVF_ERR_INVALID_QP_ID:
66 	case IAVF_ERR_INVALID_CQ_ID:
67 	case IAVF_ERR_INVALID_CEQ_ID:
68 	case IAVF_ERR_INVALID_AEQ_ID:
69 	case IAVF_ERR_INVALID_SIZE:
70 	case IAVF_ERR_INVALID_ARP_INDEX:
71 	case IAVF_ERR_INVALID_FPM_FUNC_ID:
72 	case IAVF_ERR_QP_INVALID_MSG_SIZE:
73 	case IAVF_ERR_INVALID_FRAG_COUNT:
74 	case IAVF_ERR_INVALID_ALIGNMENT:
75 	case IAVF_ERR_INVALID_PUSH_PAGE_INDEX:
76 	case IAVF_ERR_INVALID_IMM_DATA_SIZE:
77 	case IAVF_ERR_INVALID_VF_ID:
78 	case IAVF_ERR_INVALID_HMCFN_ID:
79 	case IAVF_ERR_INVALID_PBLE_INDEX:
80 	case IAVF_ERR_INVALID_SD_INDEX:
81 	case IAVF_ERR_INVALID_PAGE_DESC_INDEX:
82 	case IAVF_ERR_INVALID_SD_TYPE:
83 	case IAVF_ERR_INVALID_HMC_OBJ_INDEX:
84 	case IAVF_ERR_INVALID_HMC_OBJ_COUNT:
85 	case IAVF_ERR_INVALID_SRQ_ARM_LIMIT:
86 		return -EINVAL;
87 	case IAVF_ERR_NVM:
88 	case IAVF_ERR_NVM_CHECKSUM:
89 	case IAVF_ERR_PHY:
90 	case IAVF_ERR_CONFIG:
91 	case IAVF_ERR_UNKNOWN_PHY:
92 	case IAVF_ERR_LINK_SETUP:
93 	case IAVF_ERR_ADAPTER_STOPPED:
94 	case IAVF_ERR_PRIMARY_REQUESTS_PENDING:
95 	case IAVF_ERR_AUTONEG_NOT_COMPLETE:
96 	case IAVF_ERR_RESET_FAILED:
97 	case IAVF_ERR_BAD_PTR:
98 	case IAVF_ERR_SWFW_SYNC:
99 	case IAVF_ERR_QP_TOOMANY_WRS_POSTED:
100 	case IAVF_ERR_QUEUE_EMPTY:
101 	case IAVF_ERR_FLUSHED_QUEUE:
102 	case IAVF_ERR_OPCODE_MISMATCH:
103 	case IAVF_ERR_CQP_COMPL_ERROR:
104 	case IAVF_ERR_BACKING_PAGE_ERROR:
105 	case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE:
106 	case IAVF_ERR_MEMCPY_FAILED:
107 	case IAVF_ERR_SRQ_ENABLED:
108 	case IAVF_ERR_ADMIN_QUEUE_ERROR:
109 	case IAVF_ERR_ADMIN_QUEUE_FULL:
110 	case IAVF_ERR_BAD_RDMA_CQE:
111 	case IAVF_ERR_NVM_BLANK_MODE:
112 	case IAVF_ERR_PE_DOORBELL_NOT_ENABLED:
113 	case IAVF_ERR_DIAG_TEST_FAILED:
114 	case IAVF_ERR_FIRMWARE_API_VERSION:
115 	case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
116 		return -EIO;
117 	case IAVF_ERR_DEVICE_NOT_SUPPORTED:
118 		return -ENODEV;
119 	case IAVF_ERR_NO_AVAILABLE_VSI:
120 	case IAVF_ERR_RING_FULL:
121 		return -ENOSPC;
122 	case IAVF_ERR_NO_MEMORY:
123 		return -ENOMEM;
124 	case IAVF_ERR_TIMEOUT:
125 	case IAVF_ERR_ADMIN_QUEUE_TIMEOUT:
126 		return -ETIMEDOUT;
127 	case IAVF_ERR_NOT_IMPLEMENTED:
128 	case IAVF_NOT_SUPPORTED:
129 		return -EOPNOTSUPP;
130 	case IAVF_ERR_ADMIN_QUEUE_NO_WORK:
131 		return -EALREADY;
132 	case IAVF_ERR_NOT_READY:
133 		return -EBUSY;
134 	case IAVF_ERR_BUF_TOO_SHORT:
135 		return -EMSGSIZE;
136 	}
137 
138 	return -EIO;
139 }
140 
virtchnl_status_to_errno(enum virtchnl_status_code v_status)141 int virtchnl_status_to_errno(enum virtchnl_status_code v_status)
142 {
143 	switch (v_status) {
144 	case VIRTCHNL_STATUS_SUCCESS:
145 		return 0;
146 	case VIRTCHNL_STATUS_ERR_PARAM:
147 	case VIRTCHNL_STATUS_ERR_INVALID_VF_ID:
148 		return -EINVAL;
149 	case VIRTCHNL_STATUS_ERR_NO_MEMORY:
150 		return -ENOMEM;
151 	case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH:
152 	case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR:
153 	case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR:
154 		return -EIO;
155 	case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED:
156 		return -EOPNOTSUPP;
157 	}
158 
159 	return -EIO;
160 }
161 
162 /**
163  * iavf_pdev_to_adapter - go from pci_dev to adapter
164  * @pdev: pci_dev pointer
165  */
iavf_pdev_to_adapter(struct pci_dev * pdev)166 static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev)
167 {
168 	return netdev_priv(pci_get_drvdata(pdev));
169 }
170 
171 /**
172  * iavf_is_reset_in_progress - Check if a reset is in progress
173  * @adapter: board private structure
174  */
iavf_is_reset_in_progress(struct iavf_adapter * adapter)175 static bool iavf_is_reset_in_progress(struct iavf_adapter *adapter)
176 {
177 	if (adapter->state == __IAVF_RESETTING ||
178 	    adapter->flags & (IAVF_FLAG_RESET_PENDING |
179 			      IAVF_FLAG_RESET_NEEDED))
180 		return true;
181 
182 	return false;
183 }
184 
185 /**
186  * iavf_wait_for_reset - Wait for reset to finish.
187  * @adapter: board private structure
188  *
189  * Returns 0 if reset finished successfully, negative on timeout or interrupt.
190  */
iavf_wait_for_reset(struct iavf_adapter * adapter)191 int iavf_wait_for_reset(struct iavf_adapter *adapter)
192 {
193 	int ret = wait_event_interruptible_timeout(adapter->reset_waitqueue,
194 					!iavf_is_reset_in_progress(adapter),
195 					msecs_to_jiffies(5000));
196 
197 	/* If ret < 0 then it means wait was interrupted.
198 	 * If ret == 0 then it means we got a timeout while waiting
199 	 * for reset to finish.
200 	 * If ret > 0 it means reset has finished.
201 	 */
202 	if (ret > 0)
203 		return 0;
204 	else if (ret < 0)
205 		return -EINTR;
206 	else
207 		return -EBUSY;
208 }
209 
210 /**
211  * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
212  * @hw:   pointer to the HW structure
213  * @mem:  ptr to mem struct to fill out
214  * @size: size of memory requested
215  * @alignment: what to align the allocation to
216  **/
iavf_allocate_dma_mem_d(struct iavf_hw * hw,struct iavf_dma_mem * mem,u64 size,u32 alignment)217 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,
218 					 struct iavf_dma_mem *mem,
219 					 u64 size, u32 alignment)
220 {
221 	struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
222 
223 	if (!mem)
224 		return IAVF_ERR_PARAM;
225 
226 	mem->size = ALIGN(size, alignment);
227 	mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
228 				     (dma_addr_t *)&mem->pa, GFP_KERNEL);
229 	if (mem->va)
230 		return 0;
231 	else
232 		return IAVF_ERR_NO_MEMORY;
233 }
234 
235 /**
236  * iavf_free_dma_mem - wrapper for DMA memory freeing
237  * @hw:   pointer to the HW structure
238  * @mem:  ptr to mem struct to free
239  **/
iavf_free_dma_mem(struct iavf_hw * hw,struct iavf_dma_mem * mem)240 enum iavf_status iavf_free_dma_mem(struct iavf_hw *hw, struct iavf_dma_mem *mem)
241 {
242 	struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
243 
244 	if (!mem || !mem->va)
245 		return IAVF_ERR_PARAM;
246 	dma_free_coherent(&adapter->pdev->dev, mem->size,
247 			  mem->va, (dma_addr_t)mem->pa);
248 	return 0;
249 }
250 
251 /**
252  * iavf_allocate_virt_mem - virt memory alloc wrapper
253  * @hw:   pointer to the HW structure
254  * @mem:  ptr to mem struct to fill out
255  * @size: size of memory requested
256  **/
iavf_allocate_virt_mem(struct iavf_hw * hw,struct iavf_virt_mem * mem,u32 size)257 enum iavf_status iavf_allocate_virt_mem(struct iavf_hw *hw,
258 					struct iavf_virt_mem *mem, u32 size)
259 {
260 	if (!mem)
261 		return IAVF_ERR_PARAM;
262 
263 	mem->size = size;
264 	mem->va = kzalloc(size, GFP_KERNEL);
265 
266 	if (mem->va)
267 		return 0;
268 	else
269 		return IAVF_ERR_NO_MEMORY;
270 }
271 
272 /**
273  * iavf_free_virt_mem - virt memory free wrapper
274  * @hw:   pointer to the HW structure
275  * @mem:  ptr to mem struct to free
276  **/
iavf_free_virt_mem(struct iavf_hw * hw,struct iavf_virt_mem * mem)277 void iavf_free_virt_mem(struct iavf_hw *hw, struct iavf_virt_mem *mem)
278 {
279 	kfree(mem->va);
280 }
281 
282 /**
283  * iavf_schedule_reset - Set the flags and schedule a reset event
284  * @adapter: board private structure
285  * @flags: IAVF_FLAG_RESET_PENDING or IAVF_FLAG_RESET_NEEDED
286  **/
iavf_schedule_reset(struct iavf_adapter * adapter,u64 flags)287 void iavf_schedule_reset(struct iavf_adapter *adapter, u64 flags)
288 {
289 	if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section) &&
290 	    !(adapter->flags &
291 	    (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
292 		adapter->flags |= flags;
293 		queue_work(adapter->wq, &adapter->reset_task);
294 	}
295 }
296 
297 /**
298  * iavf_schedule_aq_request - Set the flags and schedule aq request
299  * @adapter: board private structure
300  * @flags: requested aq flags
301  **/
iavf_schedule_aq_request(struct iavf_adapter * adapter,u64 flags)302 void iavf_schedule_aq_request(struct iavf_adapter *adapter, u64 flags)
303 {
304 	adapter->aq_required |= flags;
305 	mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
306 }
307 
308 /**
309  * iavf_tx_timeout - Respond to a Tx Hang
310  * @netdev: network interface device structure
311  * @txqueue: queue number that is timing out
312  **/
iavf_tx_timeout(struct net_device * netdev,unsigned int txqueue)313 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
314 {
315 	struct iavf_adapter *adapter = netdev_priv(netdev);
316 
317 	adapter->tx_timeout_count++;
318 	iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
319 }
320 
321 /**
322  * iavf_misc_irq_disable - Mask off interrupt generation on the NIC
323  * @adapter: board private structure
324  **/
iavf_misc_irq_disable(struct iavf_adapter * adapter)325 static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
326 {
327 	struct iavf_hw *hw = &adapter->hw;
328 
329 	if (!adapter->msix_entries)
330 		return;
331 
332 	wr32(hw, IAVF_VFINT_DYN_CTL01, 0);
333 
334 	iavf_flush(hw);
335 
336 	synchronize_irq(adapter->msix_entries[0].vector);
337 }
338 
339 /**
340  * iavf_misc_irq_enable - Enable default interrupt generation settings
341  * @adapter: board private structure
342  **/
iavf_misc_irq_enable(struct iavf_adapter * adapter)343 static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
344 {
345 	struct iavf_hw *hw = &adapter->hw;
346 
347 	wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |
348 				       IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
349 	wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
350 
351 	iavf_flush(hw);
352 }
353 
354 /**
355  * iavf_irq_disable - Mask off interrupt generation on the NIC
356  * @adapter: board private structure
357  **/
iavf_irq_disable(struct iavf_adapter * adapter)358 static void iavf_irq_disable(struct iavf_adapter *adapter)
359 {
360 	int i;
361 	struct iavf_hw *hw = &adapter->hw;
362 
363 	if (!adapter->msix_entries)
364 		return;
365 
366 	for (i = 1; i < adapter->num_msix_vectors; i++) {
367 		wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);
368 		synchronize_irq(adapter->msix_entries[i].vector);
369 	}
370 	iavf_flush(hw);
371 }
372 
373 /**
374  * iavf_irq_enable_queues - Enable interrupt for all queues
375  * @adapter: board private structure
376  **/
iavf_irq_enable_queues(struct iavf_adapter * adapter)377 static void iavf_irq_enable_queues(struct iavf_adapter *adapter)
378 {
379 	struct iavf_hw *hw = &adapter->hw;
380 	int i;
381 
382 	for (i = 1; i < adapter->num_msix_vectors; i++) {
383 		wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),
384 		     IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
385 		     IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);
386 	}
387 }
388 
389 /**
390  * iavf_irq_enable - Enable default interrupt generation settings
391  * @adapter: board private structure
392  * @flush: boolean value whether to run rd32()
393  **/
iavf_irq_enable(struct iavf_adapter * adapter,bool flush)394 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
395 {
396 	struct iavf_hw *hw = &adapter->hw;
397 
398 	iavf_misc_irq_enable(adapter);
399 	iavf_irq_enable_queues(adapter);
400 
401 	if (flush)
402 		iavf_flush(hw);
403 }
404 
405 /**
406  * iavf_msix_aq - Interrupt handler for vector 0
407  * @irq: interrupt number
408  * @data: pointer to netdev
409  **/
iavf_msix_aq(int irq,void * data)410 static irqreturn_t iavf_msix_aq(int irq, void *data)
411 {
412 	struct net_device *netdev = data;
413 	struct iavf_adapter *adapter = netdev_priv(netdev);
414 	struct iavf_hw *hw = &adapter->hw;
415 
416 	/* handle non-queue interrupts, these reads clear the registers */
417 	rd32(hw, IAVF_VFINT_ICR01);
418 	rd32(hw, IAVF_VFINT_ICR0_ENA1);
419 
420 	if (adapter->state != __IAVF_REMOVE)
421 		/* schedule work on the private workqueue */
422 		queue_work(adapter->wq, &adapter->adminq_task);
423 
424 	return IRQ_HANDLED;
425 }
426 
427 /**
428  * iavf_msix_clean_rings - MSIX mode Interrupt Handler
429  * @irq: interrupt number
430  * @data: pointer to a q_vector
431  **/
iavf_msix_clean_rings(int irq,void * data)432 static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
433 {
434 	struct iavf_q_vector *q_vector = data;
435 
436 	if (!q_vector->tx.ring && !q_vector->rx.ring)
437 		return IRQ_HANDLED;
438 
439 	napi_schedule_irqoff(&q_vector->napi);
440 
441 	return IRQ_HANDLED;
442 }
443 
444 /**
445  * iavf_map_vector_to_rxq - associate irqs with rx queues
446  * @adapter: board private structure
447  * @v_idx: interrupt number
448  * @r_idx: queue number
449  **/
450 static void
iavf_map_vector_to_rxq(struct iavf_adapter * adapter,int v_idx,int r_idx)451 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
452 {
453 	struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
454 	struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];
455 	struct iavf_hw *hw = &adapter->hw;
456 
457 	rx_ring->q_vector = q_vector;
458 	rx_ring->next = q_vector->rx.ring;
459 	rx_ring->vsi = &adapter->vsi;
460 	q_vector->rx.ring = rx_ring;
461 	q_vector->rx.count++;
462 	q_vector->rx.next_update = jiffies + 1;
463 	q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
464 	q_vector->ring_mask |= BIT(r_idx);
465 	wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),
466 	     q_vector->rx.current_itr >> 1);
467 	q_vector->rx.current_itr = q_vector->rx.target_itr;
468 }
469 
470 /**
471  * iavf_map_vector_to_txq - associate irqs with tx queues
472  * @adapter: board private structure
473  * @v_idx: interrupt number
474  * @t_idx: queue number
475  **/
476 static void
iavf_map_vector_to_txq(struct iavf_adapter * adapter,int v_idx,int t_idx)477 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
478 {
479 	struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
480 	struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];
481 	struct iavf_hw *hw = &adapter->hw;
482 
483 	tx_ring->q_vector = q_vector;
484 	tx_ring->next = q_vector->tx.ring;
485 	tx_ring->vsi = &adapter->vsi;
486 	q_vector->tx.ring = tx_ring;
487 	q_vector->tx.count++;
488 	q_vector->tx.next_update = jiffies + 1;
489 	q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
490 	q_vector->num_ringpairs++;
491 	wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),
492 	     q_vector->tx.target_itr >> 1);
493 	q_vector->tx.current_itr = q_vector->tx.target_itr;
494 }
495 
496 /**
497  * iavf_map_rings_to_vectors - Maps descriptor rings to vectors
498  * @adapter: board private structure to initialize
499  *
500  * This function maps descriptor rings to the queue-specific vectors
501  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
502  * one vector per ring/queue, but on a constrained vector budget, we
503  * group the rings as "efficiently" as possible.  You would add new
504  * mapping configurations in here.
505  **/
iavf_map_rings_to_vectors(struct iavf_adapter * adapter)506 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
507 {
508 	int rings_remaining = adapter->num_active_queues;
509 	int ridx = 0, vidx = 0;
510 	int q_vectors;
511 
512 	q_vectors = adapter->num_msix_vectors - NONQ_VECS;
513 
514 	for (; ridx < rings_remaining; ridx++) {
515 		iavf_map_vector_to_rxq(adapter, vidx, ridx);
516 		iavf_map_vector_to_txq(adapter, vidx, ridx);
517 
518 		/* In the case where we have more queues than vectors, continue
519 		 * round-robin on vectors until all queues are mapped.
520 		 */
521 		if (++vidx >= q_vectors)
522 			vidx = 0;
523 	}
524 
525 	adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
526 }
527 
528 /**
529  * iavf_irq_affinity_notify - Callback for affinity changes
530  * @notify: context as to what irq was changed
531  * @mask: the new affinity mask
532  *
533  * This is a callback function used by the irq_set_affinity_notifier function
534  * so that we may register to receive changes to the irq affinity masks.
535  **/
iavf_irq_affinity_notify(struct irq_affinity_notify * notify,const cpumask_t * mask)536 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
537 				     const cpumask_t *mask)
538 {
539 	struct iavf_q_vector *q_vector =
540 		container_of(notify, struct iavf_q_vector, affinity_notify);
541 
542 	cpumask_copy(&q_vector->affinity_mask, mask);
543 }
544 
545 /**
546  * iavf_irq_affinity_release - Callback for affinity notifier release
547  * @ref: internal core kernel usage
548  *
549  * This is a callback function used by the irq_set_affinity_notifier function
550  * to inform the current notification subscriber that they will no longer
551  * receive notifications.
552  **/
iavf_irq_affinity_release(struct kref * ref)553 static void iavf_irq_affinity_release(struct kref *ref) {}
554 
555 /**
556  * iavf_request_traffic_irqs - Initialize MSI-X interrupts
557  * @adapter: board private structure
558  * @basename: device basename
559  *
560  * Allocates MSI-X vectors for tx and rx handling, and requests
561  * interrupts from the kernel.
562  **/
563 static int
iavf_request_traffic_irqs(struct iavf_adapter * adapter,char * basename)564 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
565 {
566 	unsigned int vector, q_vectors;
567 	unsigned int rx_int_idx = 0, tx_int_idx = 0;
568 	int irq_num, err;
569 	int cpu;
570 
571 	iavf_irq_disable(adapter);
572 	/* Decrement for Other and TCP Timer vectors */
573 	q_vectors = adapter->num_msix_vectors - NONQ_VECS;
574 
575 	for (vector = 0; vector < q_vectors; vector++) {
576 		struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];
577 
578 		irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
579 
580 		if (q_vector->tx.ring && q_vector->rx.ring) {
581 			snprintf(q_vector->name, sizeof(q_vector->name),
582 				 "iavf-%s-TxRx-%u", basename, rx_int_idx++);
583 			tx_int_idx++;
584 		} else if (q_vector->rx.ring) {
585 			snprintf(q_vector->name, sizeof(q_vector->name),
586 				 "iavf-%s-rx-%u", basename, rx_int_idx++);
587 		} else if (q_vector->tx.ring) {
588 			snprintf(q_vector->name, sizeof(q_vector->name),
589 				 "iavf-%s-tx-%u", basename, tx_int_idx++);
590 		} else {
591 			/* skip this unused q_vector */
592 			continue;
593 		}
594 		err = request_irq(irq_num,
595 				  iavf_msix_clean_rings,
596 				  0,
597 				  q_vector->name,
598 				  q_vector);
599 		if (err) {
600 			dev_info(&adapter->pdev->dev,
601 				 "Request_irq failed, error: %d\n", err);
602 			goto free_queue_irqs;
603 		}
604 		/* register for affinity change notifications */
605 		q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
606 		q_vector->affinity_notify.release =
607 						   iavf_irq_affinity_release;
608 		irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
609 		/* Spread the IRQ affinity hints across online CPUs. Note that
610 		 * get_cpu_mask returns a mask with a permanent lifetime so
611 		 * it's safe to use as a hint for irq_update_affinity_hint.
612 		 */
613 		cpu = cpumask_local_spread(q_vector->v_idx, -1);
614 		irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
615 	}
616 
617 	return 0;
618 
619 free_queue_irqs:
620 	while (vector) {
621 		vector--;
622 		irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
623 		irq_set_affinity_notifier(irq_num, NULL);
624 		irq_update_affinity_hint(irq_num, NULL);
625 		free_irq(irq_num, &adapter->q_vectors[vector]);
626 	}
627 	return err;
628 }
629 
630 /**
631  * iavf_request_misc_irq - Initialize MSI-X interrupts
632  * @adapter: board private structure
633  *
634  * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
635  * vector is only for the admin queue, and stays active even when the netdev
636  * is closed.
637  **/
iavf_request_misc_irq(struct iavf_adapter * adapter)638 static int iavf_request_misc_irq(struct iavf_adapter *adapter)
639 {
640 	struct net_device *netdev = adapter->netdev;
641 	int err;
642 
643 	snprintf(adapter->misc_vector_name,
644 		 sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",
645 		 dev_name(&adapter->pdev->dev));
646 	err = request_irq(adapter->msix_entries[0].vector,
647 			  &iavf_msix_aq, 0,
648 			  adapter->misc_vector_name, netdev);
649 	if (err) {
650 		dev_err(&adapter->pdev->dev,
651 			"request_irq for %s failed: %d\n",
652 			adapter->misc_vector_name, err);
653 		free_irq(adapter->msix_entries[0].vector, netdev);
654 	}
655 	return err;
656 }
657 
658 /**
659  * iavf_free_traffic_irqs - Free MSI-X interrupts
660  * @adapter: board private structure
661  *
662  * Frees all MSI-X vectors other than 0.
663  **/
iavf_free_traffic_irqs(struct iavf_adapter * adapter)664 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
665 {
666 	int vector, irq_num, q_vectors;
667 
668 	if (!adapter->msix_entries)
669 		return;
670 
671 	q_vectors = adapter->num_msix_vectors - NONQ_VECS;
672 
673 	for (vector = 0; vector < q_vectors; vector++) {
674 		irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
675 		irq_set_affinity_notifier(irq_num, NULL);
676 		irq_update_affinity_hint(irq_num, NULL);
677 		free_irq(irq_num, &adapter->q_vectors[vector]);
678 	}
679 }
680 
681 /**
682  * iavf_free_misc_irq - Free MSI-X miscellaneous vector
683  * @adapter: board private structure
684  *
685  * Frees MSI-X vector 0.
686  **/
iavf_free_misc_irq(struct iavf_adapter * adapter)687 static void iavf_free_misc_irq(struct iavf_adapter *adapter)
688 {
689 	struct net_device *netdev = adapter->netdev;
690 
691 	if (!adapter->msix_entries)
692 		return;
693 
694 	free_irq(adapter->msix_entries[0].vector, netdev);
695 }
696 
697 /**
698  * iavf_configure_tx - Configure Transmit Unit after Reset
699  * @adapter: board private structure
700  *
701  * Configure the Tx unit of the MAC after a reset.
702  **/
iavf_configure_tx(struct iavf_adapter * adapter)703 static void iavf_configure_tx(struct iavf_adapter *adapter)
704 {
705 	struct iavf_hw *hw = &adapter->hw;
706 	int i;
707 
708 	for (i = 0; i < adapter->num_active_queues; i++)
709 		adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);
710 }
711 
712 /**
713  * iavf_configure_rx - Configure Receive Unit after Reset
714  * @adapter: board private structure
715  *
716  * Configure the Rx unit of the MAC after a reset.
717  **/
iavf_configure_rx(struct iavf_adapter * adapter)718 static void iavf_configure_rx(struct iavf_adapter *adapter)
719 {
720 	struct iavf_hw *hw = &adapter->hw;
721 
722 	for (u32 i = 0; i < adapter->num_active_queues; i++)
723 		adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);
724 }
725 
726 /**
727  * iavf_find_vlan - Search filter list for specific vlan filter
728  * @adapter: board private structure
729  * @vlan: vlan tag
730  *
731  * Returns ptr to the filter object or NULL. Must be called while holding the
732  * mac_vlan_list_lock.
733  **/
734 static struct
iavf_find_vlan(struct iavf_adapter * adapter,struct iavf_vlan vlan)735 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter,
736 				 struct iavf_vlan vlan)
737 {
738 	struct iavf_vlan_filter *f;
739 
740 	list_for_each_entry(f, &adapter->vlan_filter_list, list) {
741 		if (f->vlan.vid == vlan.vid &&
742 		    f->vlan.tpid == vlan.tpid)
743 			return f;
744 	}
745 
746 	return NULL;
747 }
748 
749 /**
750  * iavf_add_vlan - Add a vlan filter to the list
751  * @adapter: board private structure
752  * @vlan: VLAN tag
753  *
754  * Returns ptr to the filter object or NULL when no memory available.
755  **/
756 static struct
iavf_add_vlan(struct iavf_adapter * adapter,struct iavf_vlan vlan)757 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,
758 				struct iavf_vlan vlan)
759 {
760 	struct iavf_vlan_filter *f = NULL;
761 
762 	spin_lock_bh(&adapter->mac_vlan_list_lock);
763 
764 	f = iavf_find_vlan(adapter, vlan);
765 	if (!f) {
766 		f = kzalloc(sizeof(*f), GFP_ATOMIC);
767 		if (!f)
768 			goto clearout;
769 
770 		f->vlan = vlan;
771 
772 		list_add_tail(&f->list, &adapter->vlan_filter_list);
773 		f->state = IAVF_VLAN_ADD;
774 		adapter->num_vlan_filters++;
775 		iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER);
776 	}
777 
778 clearout:
779 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
780 	return f;
781 }
782 
783 /**
784  * iavf_del_vlan - Remove a vlan filter from the list
785  * @adapter: board private structure
786  * @vlan: VLAN tag
787  **/
iavf_del_vlan(struct iavf_adapter * adapter,struct iavf_vlan vlan)788 static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan)
789 {
790 	struct iavf_vlan_filter *f;
791 
792 	spin_lock_bh(&adapter->mac_vlan_list_lock);
793 
794 	f = iavf_find_vlan(adapter, vlan);
795 	if (f) {
796 		f->state = IAVF_VLAN_REMOVE;
797 		iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DEL_VLAN_FILTER);
798 	}
799 
800 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
801 }
802 
803 /**
804  * iavf_restore_filters
805  * @adapter: board private structure
806  *
807  * Restore existing non MAC filters when VF netdev comes back up
808  **/
iavf_restore_filters(struct iavf_adapter * adapter)809 static void iavf_restore_filters(struct iavf_adapter *adapter)
810 {
811 	struct iavf_vlan_filter *f;
812 
813 	/* re-add all VLAN filters */
814 	spin_lock_bh(&adapter->mac_vlan_list_lock);
815 
816 	list_for_each_entry(f, &adapter->vlan_filter_list, list) {
817 		if (f->state == IAVF_VLAN_INACTIVE)
818 			f->state = IAVF_VLAN_ADD;
819 	}
820 
821 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
822 	adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
823 }
824 
825 /**
826  * iavf_get_num_vlans_added - get number of VLANs added
827  * @adapter: board private structure
828  */
iavf_get_num_vlans_added(struct iavf_adapter * adapter)829 u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter)
830 {
831 	return adapter->num_vlan_filters;
832 }
833 
834 /**
835  * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF
836  * @adapter: board private structure
837  *
838  * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN,
839  * do not impose a limit as that maintains current behavior and for
840  * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF.
841  **/
iavf_get_max_vlans_allowed(struct iavf_adapter * adapter)842 static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter)
843 {
844 	/* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has
845 	 * never been a limit on the VF driver side
846 	 */
847 	if (VLAN_ALLOWED(adapter))
848 		return VLAN_N_VID;
849 	else if (VLAN_V2_ALLOWED(adapter))
850 		return adapter->vlan_v2_caps.filtering.max_filters;
851 
852 	return 0;
853 }
854 
855 /**
856  * iavf_max_vlans_added - check if maximum VLANs allowed already exist
857  * @adapter: board private structure
858  **/
iavf_max_vlans_added(struct iavf_adapter * adapter)859 static bool iavf_max_vlans_added(struct iavf_adapter *adapter)
860 {
861 	if (iavf_get_num_vlans_added(adapter) <
862 	    iavf_get_max_vlans_allowed(adapter))
863 		return false;
864 
865 	return true;
866 }
867 
868 /**
869  * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
870  * @netdev: network device struct
871  * @proto: unused protocol data
872  * @vid: VLAN tag
873  **/
iavf_vlan_rx_add_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)874 static int iavf_vlan_rx_add_vid(struct net_device *netdev,
875 				__always_unused __be16 proto, u16 vid)
876 {
877 	struct iavf_adapter *adapter = netdev_priv(netdev);
878 
879 	/* Do not track VLAN 0 filter, always added by the PF on VF init */
880 	if (!vid)
881 		return 0;
882 
883 	if (!VLAN_FILTERING_ALLOWED(adapter))
884 		return -EIO;
885 
886 	if (iavf_max_vlans_added(adapter)) {
887 		netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n",
888 			   iavf_get_max_vlans_allowed(adapter));
889 		return -EIO;
890 	}
891 
892 	if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))))
893 		return -ENOMEM;
894 
895 	return 0;
896 }
897 
898 /**
899  * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
900  * @netdev: network device struct
901  * @proto: unused protocol data
902  * @vid: VLAN tag
903  **/
iavf_vlan_rx_kill_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)904 static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
905 				 __always_unused __be16 proto, u16 vid)
906 {
907 	struct iavf_adapter *adapter = netdev_priv(netdev);
908 
909 	/* We do not track VLAN 0 filter */
910 	if (!vid)
911 		return 0;
912 
913 	iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)));
914 	return 0;
915 }
916 
917 /**
918  * iavf_find_filter - Search filter list for specific mac filter
919  * @adapter: board private structure
920  * @macaddr: the MAC address
921  *
922  * Returns ptr to the filter object or NULL. Must be called while holding the
923  * mac_vlan_list_lock.
924  **/
925 static struct
iavf_find_filter(struct iavf_adapter * adapter,const u8 * macaddr)926 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
927 				  const u8 *macaddr)
928 {
929 	struct iavf_mac_filter *f;
930 
931 	if (!macaddr)
932 		return NULL;
933 
934 	list_for_each_entry(f, &adapter->mac_filter_list, list) {
935 		if (ether_addr_equal(macaddr, f->macaddr))
936 			return f;
937 	}
938 	return NULL;
939 }
940 
941 /**
942  * iavf_add_filter - Add a mac filter to the filter list
943  * @adapter: board private structure
944  * @macaddr: the MAC address
945  *
946  * Returns ptr to the filter object or NULL when no memory available.
947  **/
iavf_add_filter(struct iavf_adapter * adapter,const u8 * macaddr)948 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
949 					const u8 *macaddr)
950 {
951 	struct iavf_mac_filter *f;
952 
953 	if (!macaddr)
954 		return NULL;
955 
956 	f = iavf_find_filter(adapter, macaddr);
957 	if (!f) {
958 		f = kzalloc(sizeof(*f), GFP_ATOMIC);
959 		if (!f)
960 			return f;
961 
962 		ether_addr_copy(f->macaddr, macaddr);
963 
964 		list_add_tail(&f->list, &adapter->mac_filter_list);
965 		f->add = true;
966 		f->add_handled = false;
967 		f->is_new_mac = true;
968 		f->is_primary = ether_addr_equal(macaddr, adapter->hw.mac.addr);
969 		adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
970 	} else {
971 		f->remove = false;
972 	}
973 
974 	return f;
975 }
976 
977 /**
978  * iavf_replace_primary_mac - Replace current primary address
979  * @adapter: board private structure
980  * @new_mac: new MAC address to be applied
981  *
982  * Replace current dev_addr and send request to PF for removal of previous
983  * primary MAC address filter and addition of new primary MAC filter.
984  * Return 0 for success, -ENOMEM for failure.
985  *
986  * Do not call this with mac_vlan_list_lock!
987  **/
iavf_replace_primary_mac(struct iavf_adapter * adapter,const u8 * new_mac)988 static int iavf_replace_primary_mac(struct iavf_adapter *adapter,
989 				    const u8 *new_mac)
990 {
991 	struct iavf_hw *hw = &adapter->hw;
992 	struct iavf_mac_filter *new_f;
993 	struct iavf_mac_filter *old_f;
994 
995 	spin_lock_bh(&adapter->mac_vlan_list_lock);
996 
997 	new_f = iavf_add_filter(adapter, new_mac);
998 	if (!new_f) {
999 		spin_unlock_bh(&adapter->mac_vlan_list_lock);
1000 		return -ENOMEM;
1001 	}
1002 
1003 	old_f = iavf_find_filter(adapter, hw->mac.addr);
1004 	if (old_f) {
1005 		old_f->is_primary = false;
1006 		old_f->remove = true;
1007 		adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1008 	}
1009 	/* Always send the request to add if changing primary MAC,
1010 	 * even if filter is already present on the list
1011 	 */
1012 	new_f->is_primary = true;
1013 	new_f->add = true;
1014 	ether_addr_copy(hw->mac.addr, new_mac);
1015 
1016 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
1017 
1018 	/* schedule the watchdog task to immediately process the request */
1019 	iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_MAC_FILTER);
1020 	return 0;
1021 }
1022 
1023 /**
1024  * iavf_is_mac_set_handled - wait for a response to set MAC from PF
1025  * @netdev: network interface device structure
1026  * @macaddr: MAC address to set
1027  *
1028  * Returns true on success, false on failure
1029  */
iavf_is_mac_set_handled(struct net_device * netdev,const u8 * macaddr)1030 static bool iavf_is_mac_set_handled(struct net_device *netdev,
1031 				    const u8 *macaddr)
1032 {
1033 	struct iavf_adapter *adapter = netdev_priv(netdev);
1034 	struct iavf_mac_filter *f;
1035 	bool ret = false;
1036 
1037 	spin_lock_bh(&adapter->mac_vlan_list_lock);
1038 
1039 	f = iavf_find_filter(adapter, macaddr);
1040 
1041 	if (!f || (!f->add && f->add_handled))
1042 		ret = true;
1043 
1044 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
1045 
1046 	return ret;
1047 }
1048 
1049 /**
1050  * iavf_set_mac - NDO callback to set port MAC address
1051  * @netdev: network interface device structure
1052  * @p: pointer to an address structure
1053  *
1054  * Returns 0 on success, negative on failure
1055  */
iavf_set_mac(struct net_device * netdev,void * p)1056 static int iavf_set_mac(struct net_device *netdev, void *p)
1057 {
1058 	struct iavf_adapter *adapter = netdev_priv(netdev);
1059 	struct sockaddr *addr = p;
1060 	int ret;
1061 
1062 	if (!is_valid_ether_addr(addr->sa_data))
1063 		return -EADDRNOTAVAIL;
1064 
1065 	ret = iavf_replace_primary_mac(adapter, addr->sa_data);
1066 
1067 	if (ret)
1068 		return ret;
1069 
1070 	ret = wait_event_interruptible_timeout(adapter->vc_waitqueue,
1071 					       iavf_is_mac_set_handled(netdev, addr->sa_data),
1072 					       msecs_to_jiffies(2500));
1073 
1074 	/* If ret < 0 then it means wait was interrupted.
1075 	 * If ret == 0 then it means we got a timeout.
1076 	 * else it means we got response for set MAC from PF,
1077 	 * check if netdev MAC was updated to requested MAC,
1078 	 * if yes then set MAC succeeded otherwise it failed return -EACCES
1079 	 */
1080 	if (ret < 0)
1081 		return ret;
1082 
1083 	if (!ret)
1084 		return -EAGAIN;
1085 
1086 	if (!ether_addr_equal(netdev->dev_addr, addr->sa_data))
1087 		return -EACCES;
1088 
1089 	return 0;
1090 }
1091 
1092 /**
1093  * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
1094  * @netdev: the netdevice
1095  * @addr: address to add
1096  *
1097  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1098  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1099  */
iavf_addr_sync(struct net_device * netdev,const u8 * addr)1100 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
1101 {
1102 	struct iavf_adapter *adapter = netdev_priv(netdev);
1103 
1104 	if (iavf_add_filter(adapter, addr))
1105 		return 0;
1106 	else
1107 		return -ENOMEM;
1108 }
1109 
1110 /**
1111  * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1112  * @netdev: the netdevice
1113  * @addr: address to add
1114  *
1115  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1116  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1117  */
iavf_addr_unsync(struct net_device * netdev,const u8 * addr)1118 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
1119 {
1120 	struct iavf_adapter *adapter = netdev_priv(netdev);
1121 	struct iavf_mac_filter *f;
1122 
1123 	/* Under some circumstances, we might receive a request to delete
1124 	 * our own device address from our uc list. Because we store the
1125 	 * device address in the VSI's MAC/VLAN filter list, we need to ignore
1126 	 * such requests and not delete our device address from this list.
1127 	 */
1128 	if (ether_addr_equal(addr, netdev->dev_addr))
1129 		return 0;
1130 
1131 	f = iavf_find_filter(adapter, addr);
1132 	if (f) {
1133 		f->remove = true;
1134 		adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1135 	}
1136 	return 0;
1137 }
1138 
1139 /**
1140  * iavf_promiscuous_mode_changed - check if promiscuous mode bits changed
1141  * @adapter: device specific adapter
1142  */
iavf_promiscuous_mode_changed(struct iavf_adapter * adapter)1143 bool iavf_promiscuous_mode_changed(struct iavf_adapter *adapter)
1144 {
1145 	return (adapter->current_netdev_promisc_flags ^ adapter->netdev->flags) &
1146 		(IFF_PROMISC | IFF_ALLMULTI);
1147 }
1148 
1149 /**
1150  * iavf_set_rx_mode - NDO callback to set the netdev filters
1151  * @netdev: network interface device structure
1152  **/
iavf_set_rx_mode(struct net_device * netdev)1153 static void iavf_set_rx_mode(struct net_device *netdev)
1154 {
1155 	struct iavf_adapter *adapter = netdev_priv(netdev);
1156 
1157 	spin_lock_bh(&adapter->mac_vlan_list_lock);
1158 	__dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1159 	__dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1160 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
1161 
1162 	spin_lock_bh(&adapter->current_netdev_promisc_flags_lock);
1163 	if (iavf_promiscuous_mode_changed(adapter))
1164 		adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE;
1165 	spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock);
1166 }
1167 
1168 /**
1169  * iavf_napi_enable_all - enable NAPI on all queue vectors
1170  * @adapter: board private structure
1171  **/
iavf_napi_enable_all(struct iavf_adapter * adapter)1172 static void iavf_napi_enable_all(struct iavf_adapter *adapter)
1173 {
1174 	int q_idx;
1175 	struct iavf_q_vector *q_vector;
1176 	int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1177 
1178 	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1179 		struct napi_struct *napi;
1180 
1181 		q_vector = &adapter->q_vectors[q_idx];
1182 		napi = &q_vector->napi;
1183 		napi_enable(napi);
1184 	}
1185 }
1186 
1187 /**
1188  * iavf_napi_disable_all - disable NAPI on all queue vectors
1189  * @adapter: board private structure
1190  **/
iavf_napi_disable_all(struct iavf_adapter * adapter)1191 static void iavf_napi_disable_all(struct iavf_adapter *adapter)
1192 {
1193 	int q_idx;
1194 	struct iavf_q_vector *q_vector;
1195 	int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1196 
1197 	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1198 		q_vector = &adapter->q_vectors[q_idx];
1199 		napi_disable(&q_vector->napi);
1200 	}
1201 }
1202 
1203 /**
1204  * iavf_configure - set up transmit and receive data structures
1205  * @adapter: board private structure
1206  **/
iavf_configure(struct iavf_adapter * adapter)1207 static void iavf_configure(struct iavf_adapter *adapter)
1208 {
1209 	struct net_device *netdev = adapter->netdev;
1210 	int i;
1211 
1212 	iavf_set_rx_mode(netdev);
1213 
1214 	iavf_configure_tx(adapter);
1215 	iavf_configure_rx(adapter);
1216 	adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
1217 
1218 	for (i = 0; i < adapter->num_active_queues; i++) {
1219 		struct iavf_ring *ring = &adapter->rx_rings[i];
1220 
1221 		iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring));
1222 	}
1223 }
1224 
1225 /**
1226  * iavf_up_complete - Finish the last steps of bringing up a connection
1227  * @adapter: board private structure
1228  *
1229  * Expects to be called while holding crit_lock.
1230  **/
iavf_up_complete(struct iavf_adapter * adapter)1231 static void iavf_up_complete(struct iavf_adapter *adapter)
1232 {
1233 	iavf_change_state(adapter, __IAVF_RUNNING);
1234 	clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1235 
1236 	iavf_napi_enable_all(adapter);
1237 
1238 	iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ENABLE_QUEUES);
1239 }
1240 
1241 /**
1242  * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF
1243  * yet and mark other to be removed.
1244  * @adapter: board private structure
1245  **/
iavf_clear_mac_vlan_filters(struct iavf_adapter * adapter)1246 static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)
1247 {
1248 	struct iavf_vlan_filter *vlf, *vlftmp;
1249 	struct iavf_mac_filter *f, *ftmp;
1250 
1251 	spin_lock_bh(&adapter->mac_vlan_list_lock);
1252 	/* clear the sync flag on all filters */
1253 	__dev_uc_unsync(adapter->netdev, NULL);
1254 	__dev_mc_unsync(adapter->netdev, NULL);
1255 
1256 	/* remove all MAC filters */
1257 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list,
1258 				 list) {
1259 		if (f->add) {
1260 			list_del(&f->list);
1261 			kfree(f);
1262 		} else {
1263 			f->remove = true;
1264 		}
1265 	}
1266 
1267 	/* disable all VLAN filters */
1268 	list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
1269 				 list)
1270 		vlf->state = IAVF_VLAN_DISABLE;
1271 
1272 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
1273 }
1274 
1275 /**
1276  * iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and
1277  * mark other to be removed.
1278  * @adapter: board private structure
1279  **/
iavf_clear_cloud_filters(struct iavf_adapter * adapter)1280 static void iavf_clear_cloud_filters(struct iavf_adapter *adapter)
1281 {
1282 	struct iavf_cloud_filter *cf, *cftmp;
1283 
1284 	/* remove all cloud filters */
1285 	spin_lock_bh(&adapter->cloud_filter_list_lock);
1286 	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
1287 				 list) {
1288 		if (cf->add) {
1289 			list_del(&cf->list);
1290 			kfree(cf);
1291 			adapter->num_cloud_filters--;
1292 		} else {
1293 			cf->del = true;
1294 		}
1295 	}
1296 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
1297 }
1298 
1299 /**
1300  * iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark
1301  * other to be removed.
1302  * @adapter: board private structure
1303  **/
iavf_clear_fdir_filters(struct iavf_adapter * adapter)1304 static void iavf_clear_fdir_filters(struct iavf_adapter *adapter)
1305 {
1306 	struct iavf_fdir_fltr *fdir;
1307 
1308 	/* remove all Flow Director filters */
1309 	spin_lock_bh(&adapter->fdir_fltr_lock);
1310 	list_for_each_entry(fdir, &adapter->fdir_list_head, list) {
1311 		if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) {
1312 			/* Cancel a request, keep filter as inactive */
1313 			fdir->state = IAVF_FDIR_FLTR_INACTIVE;
1314 		} else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING ||
1315 			 fdir->state == IAVF_FDIR_FLTR_ACTIVE) {
1316 			/* Disable filters which are active or have a pending
1317 			 * request to PF to be added
1318 			 */
1319 			fdir->state = IAVF_FDIR_FLTR_DIS_REQUEST;
1320 		}
1321 	}
1322 	spin_unlock_bh(&adapter->fdir_fltr_lock);
1323 }
1324 
1325 /**
1326  * iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark
1327  * other to be removed.
1328  * @adapter: board private structure
1329  **/
iavf_clear_adv_rss_conf(struct iavf_adapter * adapter)1330 static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter)
1331 {
1332 	struct iavf_adv_rss *rss, *rsstmp;
1333 
1334 	/* remove all advance RSS configuration */
1335 	spin_lock_bh(&adapter->adv_rss_lock);
1336 	list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
1337 				 list) {
1338 		if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) {
1339 			list_del(&rss->list);
1340 			kfree(rss);
1341 		} else {
1342 			rss->state = IAVF_ADV_RSS_DEL_REQUEST;
1343 		}
1344 	}
1345 	spin_unlock_bh(&adapter->adv_rss_lock);
1346 }
1347 
1348 /**
1349  * iavf_down - Shutdown the connection processing
1350  * @adapter: board private structure
1351  *
1352  * Expects to be called while holding crit_lock.
1353  **/
iavf_down(struct iavf_adapter * adapter)1354 void iavf_down(struct iavf_adapter *adapter)
1355 {
1356 	struct net_device *netdev = adapter->netdev;
1357 
1358 	if (adapter->state <= __IAVF_DOWN_PENDING)
1359 		return;
1360 
1361 	netif_carrier_off(netdev);
1362 	netif_tx_disable(netdev);
1363 	adapter->link_up = false;
1364 	iavf_napi_disable_all(adapter);
1365 	iavf_irq_disable(adapter);
1366 
1367 	iavf_clear_mac_vlan_filters(adapter);
1368 	iavf_clear_cloud_filters(adapter);
1369 	iavf_clear_fdir_filters(adapter);
1370 	iavf_clear_adv_rss_conf(adapter);
1371 
1372 	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
1373 		return;
1374 
1375 	if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {
1376 		/* cancel any current operation */
1377 		adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1378 		/* Schedule operations to close down the HW. Don't wait
1379 		 * here for this to complete. The watchdog is still running
1380 		 * and it will take care of this.
1381 		 */
1382 		if (!list_empty(&adapter->mac_filter_list))
1383 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1384 		if (!list_empty(&adapter->vlan_filter_list))
1385 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
1386 		if (!list_empty(&adapter->cloud_filter_list))
1387 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
1388 		if (!list_empty(&adapter->fdir_list_head))
1389 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
1390 		if (!list_empty(&adapter->adv_rss_list_head))
1391 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
1392 	}
1393 
1394 	iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DISABLE_QUEUES);
1395 }
1396 
1397 /**
1398  * iavf_acquire_msix_vectors - Setup the MSIX capability
1399  * @adapter: board private structure
1400  * @vectors: number of vectors to request
1401  *
1402  * Work with the OS to set up the MSIX vectors needed.
1403  *
1404  * Returns 0 on success, negative on failure
1405  **/
1406 static int
iavf_acquire_msix_vectors(struct iavf_adapter * adapter,int vectors)1407 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
1408 {
1409 	int err, vector_threshold;
1410 
1411 	/* We'll want at least 3 (vector_threshold):
1412 	 * 0) Other (Admin Queue and link, mostly)
1413 	 * 1) TxQ[0] Cleanup
1414 	 * 2) RxQ[0] Cleanup
1415 	 */
1416 	vector_threshold = MIN_MSIX_COUNT;
1417 
1418 	/* The more we get, the more we will assign to Tx/Rx Cleanup
1419 	 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1420 	 * Right now, we simply care about how many we'll get; we'll
1421 	 * set them up later while requesting irq's.
1422 	 */
1423 	err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1424 				    vector_threshold, vectors);
1425 	if (err < 0) {
1426 		dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1427 		kfree(adapter->msix_entries);
1428 		adapter->msix_entries = NULL;
1429 		return err;
1430 	}
1431 
1432 	/* Adjust for only the vectors we'll use, which is minimum
1433 	 * of max_msix_q_vectors + NONQ_VECS, or the number of
1434 	 * vectors we were allocated.
1435 	 */
1436 	adapter->num_msix_vectors = err;
1437 	return 0;
1438 }
1439 
1440 /**
1441  * iavf_free_queues - Free memory for all rings
1442  * @adapter: board private structure to initialize
1443  *
1444  * Free all of the memory associated with queue pairs.
1445  **/
iavf_free_queues(struct iavf_adapter * adapter)1446 static void iavf_free_queues(struct iavf_adapter *adapter)
1447 {
1448 	if (!adapter->vsi_res)
1449 		return;
1450 	adapter->num_active_queues = 0;
1451 	kfree(adapter->tx_rings);
1452 	adapter->tx_rings = NULL;
1453 	kfree(adapter->rx_rings);
1454 	adapter->rx_rings = NULL;
1455 }
1456 
1457 /**
1458  * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload
1459  * @adapter: board private structure
1460  *
1461  * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or
1462  * stripped in certain descriptor fields. Instead of checking the offload
1463  * capability bits in the hot path, cache the location the ring specific
1464  * flags.
1465  */
iavf_set_queue_vlan_tag_loc(struct iavf_adapter * adapter)1466 void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter)
1467 {
1468 	int i;
1469 
1470 	for (i = 0; i < adapter->num_active_queues; i++) {
1471 		struct iavf_ring *tx_ring = &adapter->tx_rings[i];
1472 		struct iavf_ring *rx_ring = &adapter->rx_rings[i];
1473 
1474 		/* prevent multiple L2TAG bits being set after VFR */
1475 		tx_ring->flags &=
1476 			~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1477 			  IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2);
1478 		rx_ring->flags &=
1479 			~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1480 			  IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2);
1481 
1482 		if (VLAN_ALLOWED(adapter)) {
1483 			tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1484 			rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1485 		} else if (VLAN_V2_ALLOWED(adapter)) {
1486 			struct virtchnl_vlan_supported_caps *stripping_support;
1487 			struct virtchnl_vlan_supported_caps *insertion_support;
1488 
1489 			stripping_support =
1490 				&adapter->vlan_v2_caps.offloads.stripping_support;
1491 			insertion_support =
1492 				&adapter->vlan_v2_caps.offloads.insertion_support;
1493 
1494 			if (stripping_support->outer) {
1495 				if (stripping_support->outer &
1496 				    VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1497 					rx_ring->flags |=
1498 						IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1499 				else if (stripping_support->outer &
1500 					 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1501 					rx_ring->flags |=
1502 						IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1503 			} else if (stripping_support->inner) {
1504 				if (stripping_support->inner &
1505 				    VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1506 					rx_ring->flags |=
1507 						IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1508 				else if (stripping_support->inner &
1509 					 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1510 					rx_ring->flags |=
1511 						IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1512 			}
1513 
1514 			if (insertion_support->outer) {
1515 				if (insertion_support->outer &
1516 				    VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1517 					tx_ring->flags |=
1518 						IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1519 				else if (insertion_support->outer &
1520 					 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1521 					tx_ring->flags |=
1522 						IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1523 			} else if (insertion_support->inner) {
1524 				if (insertion_support->inner &
1525 				    VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1526 					tx_ring->flags |=
1527 						IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1528 				else if (insertion_support->inner &
1529 					 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1530 					tx_ring->flags |=
1531 						IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1532 			}
1533 		}
1534 	}
1535 }
1536 
1537 /**
1538  * iavf_alloc_queues - Allocate memory for all rings
1539  * @adapter: board private structure to initialize
1540  *
1541  * We allocate one ring per queue at run-time since we don't know the
1542  * number of queues at compile-time.  The polling_netdev array is
1543  * intended for Multiqueue, but should work fine with a single queue.
1544  **/
iavf_alloc_queues(struct iavf_adapter * adapter)1545 static int iavf_alloc_queues(struct iavf_adapter *adapter)
1546 {
1547 	int i, num_active_queues;
1548 
1549 	/* If we're in reset reallocating queues we don't actually know yet for
1550 	 * certain the PF gave us the number of queues we asked for but we'll
1551 	 * assume it did.  Once basic reset is finished we'll confirm once we
1552 	 * start negotiating config with PF.
1553 	 */
1554 	if (adapter->num_req_queues)
1555 		num_active_queues = adapter->num_req_queues;
1556 	else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1557 		 adapter->num_tc)
1558 		num_active_queues = adapter->ch_config.total_qps;
1559 	else
1560 		num_active_queues = min_t(int,
1561 					  adapter->vsi_res->num_queue_pairs,
1562 					  (int)(num_online_cpus()));
1563 
1564 
1565 	adapter->tx_rings = kcalloc(num_active_queues,
1566 				    sizeof(struct iavf_ring), GFP_KERNEL);
1567 	if (!adapter->tx_rings)
1568 		goto err_out;
1569 	adapter->rx_rings = kcalloc(num_active_queues,
1570 				    sizeof(struct iavf_ring), GFP_KERNEL);
1571 	if (!adapter->rx_rings)
1572 		goto err_out;
1573 
1574 	for (i = 0; i < num_active_queues; i++) {
1575 		struct iavf_ring *tx_ring;
1576 		struct iavf_ring *rx_ring;
1577 
1578 		tx_ring = &adapter->tx_rings[i];
1579 
1580 		tx_ring->queue_index = i;
1581 		tx_ring->netdev = adapter->netdev;
1582 		tx_ring->dev = &adapter->pdev->dev;
1583 		tx_ring->count = adapter->tx_desc_count;
1584 		tx_ring->itr_setting = IAVF_ITR_TX_DEF;
1585 		if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
1586 			tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;
1587 
1588 		rx_ring = &adapter->rx_rings[i];
1589 		rx_ring->queue_index = i;
1590 		rx_ring->netdev = adapter->netdev;
1591 		rx_ring->count = adapter->rx_desc_count;
1592 		rx_ring->itr_setting = IAVF_ITR_RX_DEF;
1593 	}
1594 
1595 	adapter->num_active_queues = num_active_queues;
1596 
1597 	iavf_set_queue_vlan_tag_loc(adapter);
1598 
1599 	return 0;
1600 
1601 err_out:
1602 	iavf_free_queues(adapter);
1603 	return -ENOMEM;
1604 }
1605 
1606 /**
1607  * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
1608  * @adapter: board private structure to initialize
1609  *
1610  * Attempt to configure the interrupts using the best available
1611  * capabilities of the hardware and the kernel.
1612  **/
iavf_set_interrupt_capability(struct iavf_adapter * adapter)1613 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
1614 {
1615 	int vector, v_budget;
1616 	int pairs = 0;
1617 	int err = 0;
1618 
1619 	if (!adapter->vsi_res) {
1620 		err = -EIO;
1621 		goto out;
1622 	}
1623 	pairs = adapter->num_active_queues;
1624 
1625 	/* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1626 	 * us much good if we have more vectors than CPUs. However, we already
1627 	 * limit the total number of queues by the number of CPUs so we do not
1628 	 * need any further limiting here.
1629 	 */
1630 	v_budget = min_t(int, pairs + NONQ_VECS,
1631 			 (int)adapter->vf_res->max_vectors);
1632 
1633 	adapter->msix_entries = kcalloc(v_budget,
1634 					sizeof(struct msix_entry), GFP_KERNEL);
1635 	if (!adapter->msix_entries) {
1636 		err = -ENOMEM;
1637 		goto out;
1638 	}
1639 
1640 	for (vector = 0; vector < v_budget; vector++)
1641 		adapter->msix_entries[vector].entry = vector;
1642 
1643 	err = iavf_acquire_msix_vectors(adapter, v_budget);
1644 	if (!err)
1645 		iavf_schedule_finish_config(adapter);
1646 
1647 out:
1648 	return err;
1649 }
1650 
1651 /**
1652  * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands
1653  * @adapter: board private structure
1654  *
1655  * Return 0 on success, negative on failure
1656  **/
iavf_config_rss_aq(struct iavf_adapter * adapter)1657 static int iavf_config_rss_aq(struct iavf_adapter *adapter)
1658 {
1659 	struct iavf_aqc_get_set_rss_key_data *rss_key =
1660 		(struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;
1661 	struct iavf_hw *hw = &adapter->hw;
1662 	enum iavf_status status;
1663 
1664 	if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1665 		/* bail because we already have a command pending */
1666 		dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1667 			adapter->current_op);
1668 		return -EBUSY;
1669 	}
1670 
1671 	status = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
1672 	if (status) {
1673 		dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1674 			iavf_stat_str(hw, status),
1675 			iavf_aq_str(hw, hw->aq.asq_last_status));
1676 		return iavf_status_to_errno(status);
1677 
1678 	}
1679 
1680 	status = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,
1681 				     adapter->rss_lut, adapter->rss_lut_size);
1682 	if (status) {
1683 		dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1684 			iavf_stat_str(hw, status),
1685 			iavf_aq_str(hw, hw->aq.asq_last_status));
1686 		return iavf_status_to_errno(status);
1687 	}
1688 
1689 	return 0;
1690 
1691 }
1692 
1693 /**
1694  * iavf_config_rss_reg - Configure RSS keys and lut by writing registers
1695  * @adapter: board private structure
1696  *
1697  * Returns 0 on success, negative on failure
1698  **/
iavf_config_rss_reg(struct iavf_adapter * adapter)1699 static int iavf_config_rss_reg(struct iavf_adapter *adapter)
1700 {
1701 	struct iavf_hw *hw = &adapter->hw;
1702 	u32 *dw;
1703 	u16 i;
1704 
1705 	dw = (u32 *)adapter->rss_key;
1706 	for (i = 0; i <= adapter->rss_key_size / 4; i++)
1707 		wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);
1708 
1709 	dw = (u32 *)adapter->rss_lut;
1710 	for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1711 		wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);
1712 
1713 	iavf_flush(hw);
1714 
1715 	return 0;
1716 }
1717 
1718 /**
1719  * iavf_config_rss - Configure RSS keys and lut
1720  * @adapter: board private structure
1721  *
1722  * Returns 0 on success, negative on failure
1723  **/
iavf_config_rss(struct iavf_adapter * adapter)1724 int iavf_config_rss(struct iavf_adapter *adapter)
1725 {
1726 
1727 	if (RSS_PF(adapter)) {
1728 		adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
1729 					IAVF_FLAG_AQ_SET_RSS_KEY;
1730 		return 0;
1731 	} else if (RSS_AQ(adapter)) {
1732 		return iavf_config_rss_aq(adapter);
1733 	} else {
1734 		return iavf_config_rss_reg(adapter);
1735 	}
1736 }
1737 
1738 /**
1739  * iavf_fill_rss_lut - Fill the lut with default values
1740  * @adapter: board private structure
1741  **/
iavf_fill_rss_lut(struct iavf_adapter * adapter)1742 static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
1743 {
1744 	u16 i;
1745 
1746 	for (i = 0; i < adapter->rss_lut_size; i++)
1747 		adapter->rss_lut[i] = i % adapter->num_active_queues;
1748 }
1749 
1750 /**
1751  * iavf_init_rss - Prepare for RSS
1752  * @adapter: board private structure
1753  *
1754  * Return 0 on success, negative on failure
1755  **/
iavf_init_rss(struct iavf_adapter * adapter)1756 static int iavf_init_rss(struct iavf_adapter *adapter)
1757 {
1758 	struct iavf_hw *hw = &adapter->hw;
1759 
1760 	if (!RSS_PF(adapter)) {
1761 		/* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1762 		if (adapter->vf_res->vf_cap_flags &
1763 		    VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1764 			adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;
1765 		else
1766 			adapter->hena = IAVF_DEFAULT_RSS_HENA;
1767 
1768 		wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);
1769 		wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1770 	}
1771 
1772 	iavf_fill_rss_lut(adapter);
1773 	netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
1774 
1775 	return iavf_config_rss(adapter);
1776 }
1777 
1778 /**
1779  * iavf_alloc_q_vectors - Allocate memory for interrupt vectors
1780  * @adapter: board private structure to initialize
1781  *
1782  * We allocate one q_vector per queue interrupt.  If allocation fails we
1783  * return -ENOMEM.
1784  **/
iavf_alloc_q_vectors(struct iavf_adapter * adapter)1785 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
1786 {
1787 	int q_idx = 0, num_q_vectors;
1788 	struct iavf_q_vector *q_vector;
1789 
1790 	num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1791 	adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
1792 				     GFP_KERNEL);
1793 	if (!adapter->q_vectors)
1794 		return -ENOMEM;
1795 
1796 	for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1797 		q_vector = &adapter->q_vectors[q_idx];
1798 		q_vector->adapter = adapter;
1799 		q_vector->vsi = &adapter->vsi;
1800 		q_vector->v_idx = q_idx;
1801 		q_vector->reg_idx = q_idx;
1802 		cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
1803 		netif_napi_add(adapter->netdev, &q_vector->napi,
1804 			       iavf_napi_poll);
1805 	}
1806 
1807 	return 0;
1808 }
1809 
1810 /**
1811  * iavf_free_q_vectors - Free memory allocated for interrupt vectors
1812  * @adapter: board private structure to initialize
1813  *
1814  * This function frees the memory allocated to the q_vectors.  In addition if
1815  * NAPI is enabled it will delete any references to the NAPI struct prior
1816  * to freeing the q_vector.
1817  **/
iavf_free_q_vectors(struct iavf_adapter * adapter)1818 static void iavf_free_q_vectors(struct iavf_adapter *adapter)
1819 {
1820 	int q_idx, num_q_vectors;
1821 
1822 	if (!adapter->q_vectors)
1823 		return;
1824 
1825 	num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1826 
1827 	for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1828 		struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];
1829 
1830 		netif_napi_del(&q_vector->napi);
1831 	}
1832 	kfree(adapter->q_vectors);
1833 	adapter->q_vectors = NULL;
1834 }
1835 
1836 /**
1837  * iavf_reset_interrupt_capability - Reset MSIX setup
1838  * @adapter: board private structure
1839  *
1840  **/
iavf_reset_interrupt_capability(struct iavf_adapter * adapter)1841 static void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
1842 {
1843 	if (!adapter->msix_entries)
1844 		return;
1845 
1846 	pci_disable_msix(adapter->pdev);
1847 	kfree(adapter->msix_entries);
1848 	adapter->msix_entries = NULL;
1849 }
1850 
1851 /**
1852  * iavf_init_interrupt_scheme - Determine if MSIX is supported and init
1853  * @adapter: board private structure to initialize
1854  *
1855  **/
iavf_init_interrupt_scheme(struct iavf_adapter * adapter)1856 static int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
1857 {
1858 	int err;
1859 
1860 	err = iavf_alloc_queues(adapter);
1861 	if (err) {
1862 		dev_err(&adapter->pdev->dev,
1863 			"Unable to allocate memory for queues\n");
1864 		goto err_alloc_queues;
1865 	}
1866 
1867 	err = iavf_set_interrupt_capability(adapter);
1868 	if (err) {
1869 		dev_err(&adapter->pdev->dev,
1870 			"Unable to setup interrupt capabilities\n");
1871 		goto err_set_interrupt;
1872 	}
1873 
1874 	err = iavf_alloc_q_vectors(adapter);
1875 	if (err) {
1876 		dev_err(&adapter->pdev->dev,
1877 			"Unable to allocate memory for queue vectors\n");
1878 		goto err_alloc_q_vectors;
1879 	}
1880 
1881 	/* If we've made it so far while ADq flag being ON, then we haven't
1882 	 * bailed out anywhere in middle. And ADq isn't just enabled but actual
1883 	 * resources have been allocated in the reset path.
1884 	 * Now we can truly claim that ADq is enabled.
1885 	 */
1886 	if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1887 	    adapter->num_tc)
1888 		dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1889 			 adapter->num_tc);
1890 
1891 	dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1892 		 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1893 		 adapter->num_active_queues);
1894 
1895 	return 0;
1896 err_alloc_q_vectors:
1897 	iavf_reset_interrupt_capability(adapter);
1898 err_set_interrupt:
1899 	iavf_free_queues(adapter);
1900 err_alloc_queues:
1901 	return err;
1902 }
1903 
1904 /**
1905  * iavf_free_interrupt_scheme - Undo what iavf_init_interrupt_scheme does
1906  * @adapter: board private structure
1907  **/
iavf_free_interrupt_scheme(struct iavf_adapter * adapter)1908 static void iavf_free_interrupt_scheme(struct iavf_adapter *adapter)
1909 {
1910 	iavf_free_q_vectors(adapter);
1911 	iavf_reset_interrupt_capability(adapter);
1912 	iavf_free_queues(adapter);
1913 }
1914 
1915 /**
1916  * iavf_free_rss - Free memory used by RSS structs
1917  * @adapter: board private structure
1918  **/
iavf_free_rss(struct iavf_adapter * adapter)1919 static void iavf_free_rss(struct iavf_adapter *adapter)
1920 {
1921 	kfree(adapter->rss_key);
1922 	adapter->rss_key = NULL;
1923 
1924 	kfree(adapter->rss_lut);
1925 	adapter->rss_lut = NULL;
1926 }
1927 
1928 /**
1929  * iavf_reinit_interrupt_scheme - Reallocate queues and vectors
1930  * @adapter: board private structure
1931  * @running: true if adapter->state == __IAVF_RUNNING
1932  *
1933  * Returns 0 on success, negative on failure
1934  **/
iavf_reinit_interrupt_scheme(struct iavf_adapter * adapter,bool running)1935 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter, bool running)
1936 {
1937 	struct net_device *netdev = adapter->netdev;
1938 	int err;
1939 
1940 	if (running)
1941 		iavf_free_traffic_irqs(adapter);
1942 	iavf_free_misc_irq(adapter);
1943 	iavf_free_interrupt_scheme(adapter);
1944 
1945 	err = iavf_init_interrupt_scheme(adapter);
1946 	if (err)
1947 		goto err;
1948 
1949 	netif_tx_stop_all_queues(netdev);
1950 
1951 	err = iavf_request_misc_irq(adapter);
1952 	if (err)
1953 		goto err;
1954 
1955 	set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1956 
1957 	iavf_map_rings_to_vectors(adapter);
1958 err:
1959 	return err;
1960 }
1961 
1962 /**
1963  * iavf_finish_config - do all netdev work that needs RTNL
1964  * @work: our work_struct
1965  *
1966  * Do work that needs both RTNL and crit_lock.
1967  **/
iavf_finish_config(struct work_struct * work)1968 static void iavf_finish_config(struct work_struct *work)
1969 {
1970 	struct iavf_adapter *adapter;
1971 	int pairs, err;
1972 
1973 	adapter = container_of(work, struct iavf_adapter, finish_config);
1974 
1975 	/* Always take RTNL first to prevent circular lock dependency;
1976 	 * The dev->lock is needed to update the queue number
1977 	 */
1978 	rtnl_lock();
1979 	mutex_lock(&adapter->netdev->lock);
1980 	mutex_lock(&adapter->crit_lock);
1981 
1982 	if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES) &&
1983 	    adapter->netdev->reg_state == NETREG_REGISTERED &&
1984 	    !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {
1985 		netdev_update_features(adapter->netdev);
1986 		adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES;
1987 	}
1988 
1989 	switch (adapter->state) {
1990 	case __IAVF_DOWN:
1991 		if (adapter->netdev->reg_state != NETREG_REGISTERED) {
1992 			err = register_netdevice(adapter->netdev);
1993 			if (err) {
1994 				dev_err(&adapter->pdev->dev, "Unable to register netdev (%d)\n",
1995 					err);
1996 
1997 				/* go back and try again.*/
1998 				iavf_free_rss(adapter);
1999 				iavf_free_misc_irq(adapter);
2000 				iavf_reset_interrupt_capability(adapter);
2001 				iavf_change_state(adapter,
2002 						  __IAVF_INIT_CONFIG_ADAPTER);
2003 				goto out;
2004 			}
2005 		}
2006 
2007 		/* Set the real number of queues when reset occurs while
2008 		 * state == __IAVF_DOWN
2009 		 */
2010 		fallthrough;
2011 	case __IAVF_RUNNING:
2012 		pairs = adapter->num_active_queues;
2013 		netif_set_real_num_rx_queues(adapter->netdev, pairs);
2014 		netif_set_real_num_tx_queues(adapter->netdev, pairs);
2015 		break;
2016 
2017 	default:
2018 		break;
2019 	}
2020 
2021 out:
2022 	mutex_unlock(&adapter->crit_lock);
2023 	mutex_unlock(&adapter->netdev->lock);
2024 	rtnl_unlock();
2025 }
2026 
2027 /**
2028  * iavf_schedule_finish_config - Set the flags and schedule a reset event
2029  * @adapter: board private structure
2030  **/
iavf_schedule_finish_config(struct iavf_adapter * adapter)2031 void iavf_schedule_finish_config(struct iavf_adapter *adapter)
2032 {
2033 	if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
2034 		queue_work(adapter->wq, &adapter->finish_config);
2035 }
2036 
2037 /**
2038  * iavf_process_aq_command - process aq_required flags
2039  * and sends aq command
2040  * @adapter: pointer to iavf adapter structure
2041  *
2042  * Returns 0 on success
2043  * Returns error code if no command was sent
2044  * or error code if the command failed.
2045  **/
iavf_process_aq_command(struct iavf_adapter * adapter)2046 static int iavf_process_aq_command(struct iavf_adapter *adapter)
2047 {
2048 	if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)
2049 		return iavf_send_vf_config_msg(adapter);
2050 	if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS)
2051 		return iavf_send_vf_offload_vlan_v2_msg(adapter);
2052 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
2053 		iavf_disable_queues(adapter);
2054 		return 0;
2055 	}
2056 
2057 	if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
2058 		iavf_map_queues(adapter);
2059 		return 0;
2060 	}
2061 
2062 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
2063 		iavf_add_ether_addrs(adapter);
2064 		return 0;
2065 	}
2066 
2067 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
2068 		iavf_add_vlans(adapter);
2069 		return 0;
2070 	}
2071 
2072 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
2073 		iavf_del_ether_addrs(adapter);
2074 		return 0;
2075 	}
2076 
2077 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
2078 		iavf_del_vlans(adapter);
2079 		return 0;
2080 	}
2081 
2082 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
2083 		iavf_enable_vlan_stripping(adapter);
2084 		return 0;
2085 	}
2086 
2087 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
2088 		iavf_disable_vlan_stripping(adapter);
2089 		return 0;
2090 	}
2091 
2092 	if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES_BW) {
2093 		iavf_cfg_queues_bw(adapter);
2094 		return 0;
2095 	}
2096 
2097 	if (adapter->aq_required & IAVF_FLAG_AQ_GET_QOS_CAPS) {
2098 		iavf_get_qos_caps(adapter);
2099 		return 0;
2100 	}
2101 
2102 	if (adapter->aq_required & IAVF_FLAG_AQ_CFG_QUEUES_QUANTA_SIZE) {
2103 		iavf_cfg_queues_quanta_size(adapter);
2104 		return 0;
2105 	}
2106 
2107 	if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
2108 		iavf_configure_queues(adapter);
2109 		return 0;
2110 	}
2111 
2112 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
2113 		iavf_enable_queues(adapter);
2114 		return 0;
2115 	}
2116 
2117 	if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
2118 		/* This message goes straight to the firmware, not the
2119 		 * PF, so we don't have to set current_op as we will
2120 		 * not get a response through the ARQ.
2121 		 */
2122 		adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
2123 		return 0;
2124 	}
2125 	if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
2126 		iavf_get_hena(adapter);
2127 		return 0;
2128 	}
2129 	if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
2130 		iavf_set_hena(adapter);
2131 		return 0;
2132 	}
2133 	if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
2134 		iavf_set_rss_key(adapter);
2135 		return 0;
2136 	}
2137 	if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
2138 		iavf_set_rss_lut(adapter);
2139 		return 0;
2140 	}
2141 	if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_HFUNC) {
2142 		iavf_set_rss_hfunc(adapter);
2143 		return 0;
2144 	}
2145 
2146 	if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE) {
2147 		iavf_set_promiscuous(adapter);
2148 		return 0;
2149 	}
2150 
2151 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
2152 		iavf_enable_channels(adapter);
2153 		return 0;
2154 	}
2155 
2156 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
2157 		iavf_disable_channels(adapter);
2158 		return 0;
2159 	}
2160 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
2161 		iavf_add_cloud_filter(adapter);
2162 		return 0;
2163 	}
2164 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
2165 		iavf_del_cloud_filter(adapter);
2166 		return 0;
2167 	}
2168 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {
2169 		iavf_add_fdir_filter(adapter);
2170 		return IAVF_SUCCESS;
2171 	}
2172 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {
2173 		iavf_del_fdir_filter(adapter);
2174 		return IAVF_SUCCESS;
2175 	}
2176 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {
2177 		iavf_add_adv_rss_cfg(adapter);
2178 		return 0;
2179 	}
2180 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {
2181 		iavf_del_adv_rss_cfg(adapter);
2182 		return 0;
2183 	}
2184 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) {
2185 		iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2186 		return 0;
2187 	}
2188 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) {
2189 		iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2190 		return 0;
2191 	}
2192 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) {
2193 		iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2194 		return 0;
2195 	}
2196 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) {
2197 		iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2198 		return 0;
2199 	}
2200 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) {
2201 		iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2202 		return 0;
2203 	}
2204 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) {
2205 		iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2206 		return 0;
2207 	}
2208 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) {
2209 		iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2210 		return 0;
2211 	}
2212 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) {
2213 		iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2214 		return 0;
2215 	}
2216 
2217 	if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) {
2218 		iavf_request_stats(adapter);
2219 		return 0;
2220 	}
2221 
2222 	return -EAGAIN;
2223 }
2224 
2225 /**
2226  * iavf_set_vlan_offload_features - set VLAN offload configuration
2227  * @adapter: board private structure
2228  * @prev_features: previous features used for comparison
2229  * @features: updated features used for configuration
2230  *
2231  * Set the aq_required bit(s) based on the requested features passed in to
2232  * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule
2233  * the watchdog if any changes are requested to expedite the request via
2234  * virtchnl.
2235  **/
2236 static void
iavf_set_vlan_offload_features(struct iavf_adapter * adapter,netdev_features_t prev_features,netdev_features_t features)2237 iavf_set_vlan_offload_features(struct iavf_adapter *adapter,
2238 			       netdev_features_t prev_features,
2239 			       netdev_features_t features)
2240 {
2241 	bool enable_stripping = true, enable_insertion = true;
2242 	u16 vlan_ethertype = 0;
2243 	u64 aq_required = 0;
2244 
2245 	/* keep cases separate because one ethertype for offloads can be
2246 	 * disabled at the same time as another is disabled, so check for an
2247 	 * enabled ethertype first, then check for disabled. Default to
2248 	 * ETH_P_8021Q so an ethertype is specified if disabling insertion and
2249 	 * stripping.
2250 	 */
2251 	if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2252 		vlan_ethertype = ETH_P_8021AD;
2253 	else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2254 		vlan_ethertype = ETH_P_8021Q;
2255 	else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2256 		vlan_ethertype = ETH_P_8021AD;
2257 	else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2258 		vlan_ethertype = ETH_P_8021Q;
2259 	else
2260 		vlan_ethertype = ETH_P_8021Q;
2261 
2262 	if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
2263 		enable_stripping = false;
2264 	if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
2265 		enable_insertion = false;
2266 
2267 	if (VLAN_ALLOWED(adapter)) {
2268 		/* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN
2269 		 * stripping via virtchnl. VLAN insertion can be toggled on the
2270 		 * netdev, but it doesn't require a virtchnl message
2271 		 */
2272 		if (enable_stripping)
2273 			aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
2274 		else
2275 			aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
2276 
2277 	} else if (VLAN_V2_ALLOWED(adapter)) {
2278 		switch (vlan_ethertype) {
2279 		case ETH_P_8021Q:
2280 			if (enable_stripping)
2281 				aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING;
2282 			else
2283 				aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING;
2284 
2285 			if (enable_insertion)
2286 				aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION;
2287 			else
2288 				aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION;
2289 			break;
2290 		case ETH_P_8021AD:
2291 			if (enable_stripping)
2292 				aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING;
2293 			else
2294 				aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING;
2295 
2296 			if (enable_insertion)
2297 				aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION;
2298 			else
2299 				aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION;
2300 			break;
2301 		}
2302 	}
2303 
2304 	if (aq_required)
2305 		iavf_schedule_aq_request(adapter, aq_required);
2306 }
2307 
2308 /**
2309  * iavf_startup - first step of driver startup
2310  * @adapter: board private structure
2311  *
2312  * Function process __IAVF_STARTUP driver state.
2313  * When success the state is changed to __IAVF_INIT_VERSION_CHECK
2314  * when fails the state is changed to __IAVF_INIT_FAILED
2315  **/
iavf_startup(struct iavf_adapter * adapter)2316 static void iavf_startup(struct iavf_adapter *adapter)
2317 {
2318 	struct pci_dev *pdev = adapter->pdev;
2319 	struct iavf_hw *hw = &adapter->hw;
2320 	enum iavf_status status;
2321 	int ret;
2322 
2323 	WARN_ON(adapter->state != __IAVF_STARTUP);
2324 
2325 	/* driver loaded, probe complete */
2326 	adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2327 	adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2328 
2329 	ret = iavf_check_reset_complete(hw);
2330 	if (ret) {
2331 		dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
2332 			 ret);
2333 		goto err;
2334 	}
2335 	hw->aq.num_arq_entries = IAVF_AQ_LEN;
2336 	hw->aq.num_asq_entries = IAVF_AQ_LEN;
2337 	hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2338 	hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2339 
2340 	status = iavf_init_adminq(hw);
2341 	if (status) {
2342 		dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
2343 			status);
2344 		goto err;
2345 	}
2346 	ret = iavf_send_api_ver(adapter);
2347 	if (ret) {
2348 		dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret);
2349 		iavf_shutdown_adminq(hw);
2350 		goto err;
2351 	}
2352 	iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK);
2353 	return;
2354 err:
2355 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2356 }
2357 
2358 /**
2359  * iavf_init_version_check - second step of driver startup
2360  * @adapter: board private structure
2361  *
2362  * Function process __IAVF_INIT_VERSION_CHECK driver state.
2363  * When success the state is changed to __IAVF_INIT_GET_RESOURCES
2364  * when fails the state is changed to __IAVF_INIT_FAILED
2365  **/
iavf_init_version_check(struct iavf_adapter * adapter)2366 static void iavf_init_version_check(struct iavf_adapter *adapter)
2367 {
2368 	struct pci_dev *pdev = adapter->pdev;
2369 	struct iavf_hw *hw = &adapter->hw;
2370 	int err = -EAGAIN;
2371 
2372 	WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);
2373 
2374 	if (!iavf_asq_done(hw)) {
2375 		dev_err(&pdev->dev, "Admin queue command never completed\n");
2376 		iavf_shutdown_adminq(hw);
2377 		iavf_change_state(adapter, __IAVF_STARTUP);
2378 		goto err;
2379 	}
2380 
2381 	/* aq msg sent, awaiting reply */
2382 	err = iavf_verify_api_ver(adapter);
2383 	if (err) {
2384 		if (err == -EALREADY)
2385 			err = iavf_send_api_ver(adapter);
2386 		else
2387 			dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
2388 				adapter->pf_version.major,
2389 				adapter->pf_version.minor,
2390 				VIRTCHNL_VERSION_MAJOR,
2391 				VIRTCHNL_VERSION_MINOR);
2392 		goto err;
2393 	}
2394 	err = iavf_send_vf_config_msg(adapter);
2395 	if (err) {
2396 		dev_err(&pdev->dev, "Unable to send config request (%d)\n",
2397 			err);
2398 		goto err;
2399 	}
2400 	iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES);
2401 	return;
2402 err:
2403 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2404 }
2405 
2406 /**
2407  * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES
2408  * @adapter: board private structure
2409  */
iavf_parse_vf_resource_msg(struct iavf_adapter * adapter)2410 int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter)
2411 {
2412 	int i, num_req_queues = adapter->num_req_queues;
2413 	struct iavf_vsi *vsi = &adapter->vsi;
2414 
2415 	for (i = 0; i < adapter->vf_res->num_vsis; i++) {
2416 		if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
2417 			adapter->vsi_res = &adapter->vf_res->vsi_res[i];
2418 	}
2419 	if (!adapter->vsi_res) {
2420 		dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
2421 		return -ENODEV;
2422 	}
2423 
2424 	if (num_req_queues &&
2425 	    num_req_queues > adapter->vsi_res->num_queue_pairs) {
2426 		/* Problem.  The PF gave us fewer queues than what we had
2427 		 * negotiated in our request.  Need a reset to see if we can't
2428 		 * get back to a working state.
2429 		 */
2430 		dev_err(&adapter->pdev->dev,
2431 			"Requested %d queues, but PF only gave us %d.\n",
2432 			num_req_queues,
2433 			adapter->vsi_res->num_queue_pairs);
2434 		adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED;
2435 		adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
2436 		iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
2437 
2438 		return -EAGAIN;
2439 	}
2440 	adapter->num_req_queues = 0;
2441 	adapter->vsi.id = adapter->vsi_res->vsi_id;
2442 
2443 	adapter->vsi.back = adapter;
2444 	adapter->vsi.base_vector = 1;
2445 	vsi->netdev = adapter->netdev;
2446 	vsi->qs_handle = adapter->vsi_res->qset_handle;
2447 	if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
2448 		adapter->rss_key_size = adapter->vf_res->rss_key_size;
2449 		adapter->rss_lut_size = adapter->vf_res->rss_lut_size;
2450 	} else {
2451 		adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
2452 		adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
2453 	}
2454 
2455 	return 0;
2456 }
2457 
2458 /**
2459  * iavf_init_get_resources - third step of driver startup
2460  * @adapter: board private structure
2461  *
2462  * Function process __IAVF_INIT_GET_RESOURCES driver state and
2463  * finishes driver initialization procedure.
2464  * When success the state is changed to __IAVF_DOWN
2465  * when fails the state is changed to __IAVF_INIT_FAILED
2466  **/
iavf_init_get_resources(struct iavf_adapter * adapter)2467 static void iavf_init_get_resources(struct iavf_adapter *adapter)
2468 {
2469 	struct pci_dev *pdev = adapter->pdev;
2470 	struct iavf_hw *hw = &adapter->hw;
2471 	int err;
2472 
2473 	WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
2474 	/* aq msg sent, awaiting reply */
2475 	if (!adapter->vf_res) {
2476 		adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
2477 					  GFP_KERNEL);
2478 		if (!adapter->vf_res) {
2479 			err = -ENOMEM;
2480 			goto err;
2481 		}
2482 	}
2483 	err = iavf_get_vf_config(adapter);
2484 	if (err == -EALREADY) {
2485 		err = iavf_send_vf_config_msg(adapter);
2486 		goto err;
2487 	} else if (err == -EINVAL) {
2488 		/* We only get -EINVAL if the device is in a very bad
2489 		 * state or if we've been disabled for previous bad
2490 		 * behavior. Either way, we're done now.
2491 		 */
2492 		iavf_shutdown_adminq(hw);
2493 		dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
2494 		return;
2495 	}
2496 	if (err) {
2497 		dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
2498 		goto err_alloc;
2499 	}
2500 
2501 	err = iavf_parse_vf_resource_msg(adapter);
2502 	if (err) {
2503 		dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n",
2504 			err);
2505 		goto err_alloc;
2506 	}
2507 	/* Some features require additional messages to negotiate extended
2508 	 * capabilities. These are processed in sequence by the
2509 	 * __IAVF_INIT_EXTENDED_CAPS driver state.
2510 	 */
2511 	adapter->extended_caps = IAVF_EXTENDED_CAPS;
2512 
2513 	iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS);
2514 	return;
2515 
2516 err_alloc:
2517 	kfree(adapter->vf_res);
2518 	adapter->vf_res = NULL;
2519 err:
2520 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2521 }
2522 
2523 /**
2524  * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2525  * @adapter: board private structure
2526  *
2527  * Function processes send of the extended VLAN V2 capability message to the
2528  * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent,
2529  * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2.
2530  */
iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter * adapter)2531 static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2532 {
2533 	int ret;
2534 
2535 	WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2));
2536 
2537 	ret = iavf_send_vf_offload_vlan_v2_msg(adapter);
2538 	if (ret && ret == -EOPNOTSUPP) {
2539 		/* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case,
2540 		 * we did not send the capability exchange message and do not
2541 		 * expect a response.
2542 		 */
2543 		adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2544 	}
2545 
2546 	/* We sent the message, so move on to the next step */
2547 	adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2548 }
2549 
2550 /**
2551  * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2552  * @adapter: board private structure
2553  *
2554  * Function processes receipt of the extended VLAN V2 capability message from
2555  * the PF.
2556  **/
iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter * adapter)2557 static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2558 {
2559 	int ret;
2560 
2561 	WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2));
2562 
2563 	memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps));
2564 
2565 	ret = iavf_get_vf_vlan_v2_caps(adapter);
2566 	if (ret)
2567 		goto err;
2568 
2569 	/* We've processed receipt of the VLAN V2 caps message */
2570 	adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2571 	return;
2572 err:
2573 	/* We didn't receive a reply. Make sure we try sending again when
2574 	 * __IAVF_INIT_FAILED attempts to recover.
2575 	 */
2576 	adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2577 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2578 }
2579 
2580 /**
2581  * iavf_init_process_extended_caps - Part of driver startup
2582  * @adapter: board private structure
2583  *
2584  * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state
2585  * handles negotiating capabilities for features which require an additional
2586  * message.
2587  *
2588  * Once all extended capabilities exchanges are finished, the driver will
2589  * transition into __IAVF_INIT_CONFIG_ADAPTER.
2590  */
iavf_init_process_extended_caps(struct iavf_adapter * adapter)2591 static void iavf_init_process_extended_caps(struct iavf_adapter *adapter)
2592 {
2593 	WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS);
2594 
2595 	/* Process capability exchange for VLAN V2 */
2596 	if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) {
2597 		iavf_init_send_offload_vlan_v2_caps(adapter);
2598 		return;
2599 	} else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) {
2600 		iavf_init_recv_offload_vlan_v2_caps(adapter);
2601 		return;
2602 	}
2603 
2604 	/* When we reach here, no further extended capabilities exchanges are
2605 	 * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER
2606 	 */
2607 	iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER);
2608 }
2609 
2610 /**
2611  * iavf_init_config_adapter - last part of driver startup
2612  * @adapter: board private structure
2613  *
2614  * After all the supported capabilities are negotiated, then the
2615  * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization.
2616  */
iavf_init_config_adapter(struct iavf_adapter * adapter)2617 static void iavf_init_config_adapter(struct iavf_adapter *adapter)
2618 {
2619 	struct net_device *netdev = adapter->netdev;
2620 	struct pci_dev *pdev = adapter->pdev;
2621 	int err;
2622 
2623 	WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER);
2624 
2625 	if (iavf_process_config(adapter))
2626 		goto err;
2627 
2628 	adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2629 
2630 	adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
2631 
2632 	netdev->netdev_ops = &iavf_netdev_ops;
2633 	iavf_set_ethtool_ops(netdev);
2634 	netdev->watchdog_timeo = 5 * HZ;
2635 
2636 	netdev->min_mtu = ETH_MIN_MTU;
2637 	netdev->max_mtu = LIBIE_MAX_MTU;
2638 
2639 	if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
2640 		dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
2641 			 adapter->hw.mac.addr);
2642 		eth_hw_addr_random(netdev);
2643 		ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
2644 	} else {
2645 		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2646 		ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
2647 	}
2648 
2649 	adapter->tx_desc_count = IAVF_DEFAULT_TXD;
2650 	adapter->rx_desc_count = IAVF_DEFAULT_RXD;
2651 	err = iavf_init_interrupt_scheme(adapter);
2652 	if (err)
2653 		goto err_sw_init;
2654 	iavf_map_rings_to_vectors(adapter);
2655 	if (adapter->vf_res->vf_cap_flags &
2656 		VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
2657 		adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
2658 
2659 	err = iavf_request_misc_irq(adapter);
2660 	if (err)
2661 		goto err_sw_init;
2662 
2663 	netif_carrier_off(netdev);
2664 	adapter->link_up = false;
2665 	netif_tx_stop_all_queues(netdev);
2666 
2667 	dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
2668 	if (netdev->features & NETIF_F_GRO)
2669 		dev_info(&pdev->dev, "GRO is enabled\n");
2670 
2671 	iavf_change_state(adapter, __IAVF_DOWN);
2672 	set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2673 
2674 	iavf_misc_irq_enable(adapter);
2675 	wake_up(&adapter->down_waitqueue);
2676 
2677 	adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
2678 	adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
2679 	if (!adapter->rss_key || !adapter->rss_lut) {
2680 		err = -ENOMEM;
2681 		goto err_mem;
2682 	}
2683 	if (RSS_AQ(adapter))
2684 		adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
2685 	else
2686 		iavf_init_rss(adapter);
2687 
2688 	if (VLAN_V2_ALLOWED(adapter))
2689 		/* request initial VLAN offload settings */
2690 		iavf_set_vlan_offload_features(adapter, 0, netdev->features);
2691 
2692 	if (QOS_ALLOWED(adapter))
2693 		adapter->aq_required |= IAVF_FLAG_AQ_GET_QOS_CAPS;
2694 
2695 	iavf_schedule_finish_config(adapter);
2696 	return;
2697 
2698 err_mem:
2699 	iavf_free_rss(adapter);
2700 	iavf_free_misc_irq(adapter);
2701 err_sw_init:
2702 	iavf_reset_interrupt_capability(adapter);
2703 err:
2704 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2705 }
2706 
2707 /**
2708  * iavf_watchdog_task - Periodic call-back task
2709  * @work: pointer to work_struct
2710  **/
iavf_watchdog_task(struct work_struct * work)2711 static void iavf_watchdog_task(struct work_struct *work)
2712 {
2713 	struct iavf_adapter *adapter = container_of(work,
2714 						    struct iavf_adapter,
2715 						    watchdog_task.work);
2716 	struct iavf_hw *hw = &adapter->hw;
2717 	u32 reg_val;
2718 
2719 	if (!mutex_trylock(&adapter->crit_lock)) {
2720 		if (adapter->state == __IAVF_REMOVE)
2721 			return;
2722 
2723 		goto restart_watchdog;
2724 	}
2725 
2726 	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
2727 		iavf_change_state(adapter, __IAVF_COMM_FAILED);
2728 
2729 	switch (adapter->state) {
2730 	case __IAVF_STARTUP:
2731 		iavf_startup(adapter);
2732 		mutex_unlock(&adapter->crit_lock);
2733 		queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2734 				   msecs_to_jiffies(30));
2735 		return;
2736 	case __IAVF_INIT_VERSION_CHECK:
2737 		iavf_init_version_check(adapter);
2738 		mutex_unlock(&adapter->crit_lock);
2739 		queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2740 				   msecs_to_jiffies(30));
2741 		return;
2742 	case __IAVF_INIT_GET_RESOURCES:
2743 		iavf_init_get_resources(adapter);
2744 		mutex_unlock(&adapter->crit_lock);
2745 		queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2746 				   msecs_to_jiffies(1));
2747 		return;
2748 	case __IAVF_INIT_EXTENDED_CAPS:
2749 		iavf_init_process_extended_caps(adapter);
2750 		mutex_unlock(&adapter->crit_lock);
2751 		queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2752 				   msecs_to_jiffies(1));
2753 		return;
2754 	case __IAVF_INIT_CONFIG_ADAPTER:
2755 		iavf_init_config_adapter(adapter);
2756 		mutex_unlock(&adapter->crit_lock);
2757 		queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2758 				   msecs_to_jiffies(1));
2759 		return;
2760 	case __IAVF_INIT_FAILED:
2761 		if (test_bit(__IAVF_IN_REMOVE_TASK,
2762 			     &adapter->crit_section)) {
2763 			/* Do not update the state and do not reschedule
2764 			 * watchdog task, iavf_remove should handle this state
2765 			 * as it can loop forever
2766 			 */
2767 			mutex_unlock(&adapter->crit_lock);
2768 			return;
2769 		}
2770 		if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
2771 			dev_err(&adapter->pdev->dev,
2772 				"Failed to communicate with PF; waiting before retry\n");
2773 			adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2774 			iavf_shutdown_adminq(hw);
2775 			mutex_unlock(&adapter->crit_lock);
2776 			queue_delayed_work(adapter->wq,
2777 					   &adapter->watchdog_task, (5 * HZ));
2778 			return;
2779 		}
2780 		/* Try again from failed step*/
2781 		iavf_change_state(adapter, adapter->last_state);
2782 		mutex_unlock(&adapter->crit_lock);
2783 		queue_delayed_work(adapter->wq, &adapter->watchdog_task, HZ);
2784 		return;
2785 	case __IAVF_COMM_FAILED:
2786 		if (test_bit(__IAVF_IN_REMOVE_TASK,
2787 			     &adapter->crit_section)) {
2788 			/* Set state to __IAVF_INIT_FAILED and perform remove
2789 			 * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task
2790 			 * doesn't bring the state back to __IAVF_COMM_FAILED.
2791 			 */
2792 			iavf_change_state(adapter, __IAVF_INIT_FAILED);
2793 			adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2794 			mutex_unlock(&adapter->crit_lock);
2795 			return;
2796 		}
2797 		reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
2798 			  IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
2799 		if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
2800 		    reg_val == VIRTCHNL_VFR_COMPLETED) {
2801 			/* A chance for redemption! */
2802 			dev_err(&adapter->pdev->dev,
2803 				"Hardware came out of reset. Attempting reinit.\n");
2804 			/* When init task contacts the PF and
2805 			 * gets everything set up again, it'll restart the
2806 			 * watchdog for us. Down, boy. Sit. Stay. Woof.
2807 			 */
2808 			iavf_change_state(adapter, __IAVF_STARTUP);
2809 			adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2810 		}
2811 		adapter->aq_required = 0;
2812 		adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2813 		mutex_unlock(&adapter->crit_lock);
2814 		queue_delayed_work(adapter->wq,
2815 				   &adapter->watchdog_task,
2816 				   msecs_to_jiffies(10));
2817 		return;
2818 	case __IAVF_RESETTING:
2819 		mutex_unlock(&adapter->crit_lock);
2820 		queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2821 				   HZ * 2);
2822 		return;
2823 	case __IAVF_DOWN:
2824 	case __IAVF_DOWN_PENDING:
2825 	case __IAVF_TESTING:
2826 	case __IAVF_RUNNING:
2827 		if (adapter->current_op) {
2828 			if (!iavf_asq_done(hw)) {
2829 				dev_dbg(&adapter->pdev->dev,
2830 					"Admin queue timeout\n");
2831 				iavf_send_api_ver(adapter);
2832 			}
2833 		} else {
2834 			int ret = iavf_process_aq_command(adapter);
2835 
2836 			/* An error will be returned if no commands were
2837 			 * processed; use this opportunity to update stats
2838 			 * if the error isn't -ENOTSUPP
2839 			 */
2840 			if (ret && ret != -EOPNOTSUPP &&
2841 			    adapter->state == __IAVF_RUNNING)
2842 				iavf_request_stats(adapter);
2843 		}
2844 		if (adapter->state == __IAVF_RUNNING)
2845 			iavf_detect_recover_hung(&adapter->vsi);
2846 		break;
2847 	case __IAVF_REMOVE:
2848 	default:
2849 		mutex_unlock(&adapter->crit_lock);
2850 		return;
2851 	}
2852 
2853 	/* check for hw reset */
2854 	reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2855 	if (!reg_val) {
2856 		adapter->aq_required = 0;
2857 		adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2858 		dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
2859 		iavf_schedule_reset(adapter, IAVF_FLAG_RESET_PENDING);
2860 		mutex_unlock(&adapter->crit_lock);
2861 		queue_delayed_work(adapter->wq,
2862 				   &adapter->watchdog_task, HZ * 2);
2863 		return;
2864 	}
2865 
2866 	mutex_unlock(&adapter->crit_lock);
2867 restart_watchdog:
2868 	if (adapter->state >= __IAVF_DOWN)
2869 		queue_work(adapter->wq, &adapter->adminq_task);
2870 	if (adapter->aq_required)
2871 		queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2872 				   msecs_to_jiffies(20));
2873 	else
2874 		queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2875 				   HZ * 2);
2876 }
2877 
2878 /**
2879  * iavf_disable_vf - disable VF
2880  * @adapter: board private structure
2881  *
2882  * Set communication failed flag and free all resources.
2883  * NOTE: This function is expected to be called with crit_lock being held.
2884  **/
iavf_disable_vf(struct iavf_adapter * adapter)2885 static void iavf_disable_vf(struct iavf_adapter *adapter)
2886 {
2887 	struct iavf_mac_filter *f, *ftmp;
2888 	struct iavf_vlan_filter *fv, *fvtmp;
2889 	struct iavf_cloud_filter *cf, *cftmp;
2890 
2891 	adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2892 
2893 	/* We don't use netif_running() because it may be true prior to
2894 	 * ndo_open() returning, so we can't assume it means all our open
2895 	 * tasks have finished, since we're not holding the rtnl_lock here.
2896 	 */
2897 	if (adapter->state == __IAVF_RUNNING) {
2898 		set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2899 		netif_carrier_off(adapter->netdev);
2900 		netif_tx_disable(adapter->netdev);
2901 		adapter->link_up = false;
2902 		iavf_napi_disable_all(adapter);
2903 		iavf_irq_disable(adapter);
2904 		iavf_free_traffic_irqs(adapter);
2905 		iavf_free_all_tx_resources(adapter);
2906 		iavf_free_all_rx_resources(adapter);
2907 	}
2908 
2909 	spin_lock_bh(&adapter->mac_vlan_list_lock);
2910 
2911 	/* Delete all of the filters */
2912 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2913 		list_del(&f->list);
2914 		kfree(f);
2915 	}
2916 
2917 	list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
2918 		list_del(&fv->list);
2919 		kfree(fv);
2920 	}
2921 	adapter->num_vlan_filters = 0;
2922 
2923 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
2924 
2925 	spin_lock_bh(&adapter->cloud_filter_list_lock);
2926 	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
2927 		list_del(&cf->list);
2928 		kfree(cf);
2929 		adapter->num_cloud_filters--;
2930 	}
2931 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
2932 
2933 	iavf_free_misc_irq(adapter);
2934 	iavf_free_interrupt_scheme(adapter);
2935 	memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
2936 	iavf_shutdown_adminq(&adapter->hw);
2937 	adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2938 	iavf_change_state(adapter, __IAVF_DOWN);
2939 	wake_up(&adapter->down_waitqueue);
2940 	dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
2941 }
2942 
2943 /**
2944  * iavf_reconfig_qs_bw - Call-back task to handle hardware reset
2945  * @adapter: board private structure
2946  *
2947  * After a reset, the shaper parameters of queues need to be replayed again.
2948  * Since the net_shaper object inside TX rings persists across reset,
2949  * set the update flag for all queues so that the virtchnl message is triggered
2950  * for all queues.
2951  **/
iavf_reconfig_qs_bw(struct iavf_adapter * adapter)2952 static void iavf_reconfig_qs_bw(struct iavf_adapter *adapter)
2953 {
2954 	int i, num = 0;
2955 
2956 	for (i = 0; i < adapter->num_active_queues; i++)
2957 		if (adapter->tx_rings[i].q_shaper.bw_min ||
2958 		    adapter->tx_rings[i].q_shaper.bw_max) {
2959 			adapter->tx_rings[i].q_shaper_update = true;
2960 			num++;
2961 		}
2962 
2963 	if (num)
2964 		adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES_BW;
2965 }
2966 
2967 /**
2968  * iavf_reset_task - Call-back task to handle hardware reset
2969  * @work: pointer to work_struct
2970  *
2971  * During reset we need to shut down and reinitialize the admin queue
2972  * before we can use it to communicate with the PF again. We also clear
2973  * and reinit the rings because that context is lost as well.
2974  **/
iavf_reset_task(struct work_struct * work)2975 static void iavf_reset_task(struct work_struct *work)
2976 {
2977 	struct iavf_adapter *adapter = container_of(work,
2978 						      struct iavf_adapter,
2979 						      reset_task);
2980 	struct virtchnl_vf_resource *vfres = adapter->vf_res;
2981 	struct net_device *netdev = adapter->netdev;
2982 	struct iavf_hw *hw = &adapter->hw;
2983 	struct iavf_mac_filter *f, *ftmp;
2984 	struct iavf_cloud_filter *cf;
2985 	enum iavf_status status;
2986 	u32 reg_val;
2987 	int i = 0, err;
2988 	bool running;
2989 
2990 	/* When device is being removed it doesn't make sense to run the reset
2991 	 * task, just return in such a case.
2992 	 */
2993 	mutex_lock(&netdev->lock);
2994 	if (!mutex_trylock(&adapter->crit_lock)) {
2995 		if (adapter->state != __IAVF_REMOVE)
2996 			queue_work(adapter->wq, &adapter->reset_task);
2997 
2998 		mutex_unlock(&netdev->lock);
2999 		return;
3000 	}
3001 
3002 	iavf_misc_irq_disable(adapter);
3003 	if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
3004 		adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
3005 		/* Restart the AQ here. If we have been reset but didn't
3006 		 * detect it, or if the PF had to reinit, our AQ will be hosed.
3007 		 */
3008 		iavf_shutdown_adminq(hw);
3009 		iavf_init_adminq(hw);
3010 		iavf_request_reset(adapter);
3011 	}
3012 	adapter->flags |= IAVF_FLAG_RESET_PENDING;
3013 
3014 	/* poll until we see the reset actually happen */
3015 	for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
3016 		reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
3017 			  IAVF_VF_ARQLEN1_ARQENABLE_MASK;
3018 		if (!reg_val)
3019 			break;
3020 		usleep_range(5000, 10000);
3021 	}
3022 	if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
3023 		dev_info(&adapter->pdev->dev, "Never saw reset\n");
3024 		goto continue_reset; /* act like the reset happened */
3025 	}
3026 
3027 	/* wait until the reset is complete and the PF is responding to us */
3028 	for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
3029 		/* sleep first to make sure a minimum wait time is met */
3030 		msleep(IAVF_RESET_WAIT_MS);
3031 
3032 		reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
3033 			  IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
3034 		if (reg_val == VIRTCHNL_VFR_VFACTIVE)
3035 			break;
3036 	}
3037 
3038 	pci_set_master(adapter->pdev);
3039 	pci_restore_msi_state(adapter->pdev);
3040 
3041 	if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
3042 		dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
3043 			reg_val);
3044 		iavf_disable_vf(adapter);
3045 		mutex_unlock(&adapter->crit_lock);
3046 		mutex_unlock(&netdev->lock);
3047 		return; /* Do not attempt to reinit. It's dead, Jim. */
3048 	}
3049 
3050 continue_reset:
3051 	/* We don't use netif_running() because it may be true prior to
3052 	 * ndo_open() returning, so we can't assume it means all our open
3053 	 * tasks have finished, since we're not holding the rtnl_lock here.
3054 	 */
3055 	running = adapter->state == __IAVF_RUNNING;
3056 
3057 	if (running) {
3058 		netif_carrier_off(netdev);
3059 		netif_tx_stop_all_queues(netdev);
3060 		adapter->link_up = false;
3061 		iavf_napi_disable_all(adapter);
3062 	}
3063 	iavf_irq_disable(adapter);
3064 
3065 	iavf_change_state(adapter, __IAVF_RESETTING);
3066 	adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
3067 
3068 	/* free the Tx/Rx rings and descriptors, might be better to just
3069 	 * re-use them sometime in the future
3070 	 */
3071 	iavf_free_all_rx_resources(adapter);
3072 	iavf_free_all_tx_resources(adapter);
3073 
3074 	adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
3075 	/* kill and reinit the admin queue */
3076 	iavf_shutdown_adminq(hw);
3077 	adapter->current_op = VIRTCHNL_OP_UNKNOWN;
3078 	status = iavf_init_adminq(hw);
3079 	if (status) {
3080 		dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
3081 			 status);
3082 		goto reset_err;
3083 	}
3084 	adapter->aq_required = 0;
3085 
3086 	if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3087 	    (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3088 		err = iavf_reinit_interrupt_scheme(adapter, running);
3089 		if (err)
3090 			goto reset_err;
3091 	}
3092 
3093 	if (RSS_AQ(adapter)) {
3094 		adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
3095 	} else {
3096 		err = iavf_init_rss(adapter);
3097 		if (err)
3098 			goto reset_err;
3099 	}
3100 
3101 	adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
3102 	/* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been
3103 	 * sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here,
3104 	 * however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until
3105 	 * VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have
3106 	 * been successfully sent and negotiated
3107 	 */
3108 	adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS;
3109 	adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
3110 
3111 	spin_lock_bh(&adapter->mac_vlan_list_lock);
3112 
3113 	/* Delete filter for the current MAC address, it could have
3114 	 * been changed by the PF via administratively set MAC.
3115 	 * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.
3116 	 */
3117 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
3118 		if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) {
3119 			list_del(&f->list);
3120 			kfree(f);
3121 		}
3122 	}
3123 	/* re-add all MAC filters */
3124 	list_for_each_entry(f, &adapter->mac_filter_list, list) {
3125 		f->add = true;
3126 	}
3127 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
3128 
3129 	/* check if TCs are running and re-add all cloud filters */
3130 	spin_lock_bh(&adapter->cloud_filter_list_lock);
3131 	if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
3132 	    adapter->num_tc) {
3133 		list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
3134 			cf->add = true;
3135 		}
3136 	}
3137 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
3138 
3139 	adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
3140 	adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
3141 	iavf_misc_irq_enable(adapter);
3142 
3143 	mod_delayed_work(adapter->wq, &adapter->watchdog_task, 2);
3144 
3145 	/* We were running when the reset started, so we need to restore some
3146 	 * state here.
3147 	 */
3148 	if (running) {
3149 		/* allocate transmit descriptors */
3150 		err = iavf_setup_all_tx_resources(adapter);
3151 		if (err)
3152 			goto reset_err;
3153 
3154 		/* allocate receive descriptors */
3155 		err = iavf_setup_all_rx_resources(adapter);
3156 		if (err)
3157 			goto reset_err;
3158 
3159 		if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3160 		    (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3161 			err = iavf_request_traffic_irqs(adapter, netdev->name);
3162 			if (err)
3163 				goto reset_err;
3164 
3165 			adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED;
3166 		}
3167 
3168 		iavf_configure(adapter);
3169 
3170 		/* iavf_up_complete() will switch device back
3171 		 * to __IAVF_RUNNING
3172 		 */
3173 		iavf_up_complete(adapter);
3174 
3175 		iavf_irq_enable(adapter, true);
3176 
3177 		iavf_reconfig_qs_bw(adapter);
3178 	} else {
3179 		iavf_change_state(adapter, __IAVF_DOWN);
3180 		wake_up(&adapter->down_waitqueue);
3181 	}
3182 
3183 	adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
3184 
3185 	wake_up(&adapter->reset_waitqueue);
3186 	mutex_unlock(&adapter->crit_lock);
3187 	mutex_unlock(&netdev->lock);
3188 
3189 	return;
3190 reset_err:
3191 	if (running) {
3192 		set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
3193 		iavf_free_traffic_irqs(adapter);
3194 	}
3195 	iavf_disable_vf(adapter);
3196 
3197 	mutex_unlock(&adapter->crit_lock);
3198 	mutex_unlock(&netdev->lock);
3199 	dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
3200 }
3201 
3202 /**
3203  * iavf_adminq_task - worker thread to clean the admin queue
3204  * @work: pointer to work_struct containing our data
3205  **/
iavf_adminq_task(struct work_struct * work)3206 static void iavf_adminq_task(struct work_struct *work)
3207 {
3208 	struct iavf_adapter *adapter =
3209 		container_of(work, struct iavf_adapter, adminq_task);
3210 	struct iavf_hw *hw = &adapter->hw;
3211 	struct iavf_arq_event_info event;
3212 	enum virtchnl_ops v_op;
3213 	enum iavf_status ret, v_ret;
3214 	u32 val, oldval;
3215 	u16 pending;
3216 
3217 	if (!mutex_trylock(&adapter->crit_lock)) {
3218 		if (adapter->state == __IAVF_REMOVE)
3219 			return;
3220 
3221 		queue_work(adapter->wq, &adapter->adminq_task);
3222 		goto out;
3223 	}
3224 
3225 	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
3226 		goto unlock;
3227 
3228 	event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
3229 	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
3230 	if (!event.msg_buf)
3231 		goto unlock;
3232 
3233 	do {
3234 		ret = iavf_clean_arq_element(hw, &event, &pending);
3235 		v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
3236 		v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);
3237 
3238 		if (ret || !v_op)
3239 			break; /* No event to process or error cleaning ARQ */
3240 
3241 		iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
3242 					 event.msg_len);
3243 		if (pending != 0)
3244 			memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
3245 	} while (pending);
3246 
3247 	if (iavf_is_reset_in_progress(adapter))
3248 		goto freedom;
3249 
3250 	/* check for error indications */
3251 	val = rd32(hw, IAVF_VF_ARQLEN1);
3252 	if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */
3253 		goto freedom;
3254 	oldval = val;
3255 	if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
3256 		dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
3257 		val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;
3258 	}
3259 	if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {
3260 		dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
3261 		val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;
3262 	}
3263 	if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {
3264 		dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
3265 		val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;
3266 	}
3267 	if (oldval != val)
3268 		wr32(hw, IAVF_VF_ARQLEN1, val);
3269 
3270 	val = rd32(hw, IAVF_VF_ATQLEN1);
3271 	oldval = val;
3272 	if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {
3273 		dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
3274 		val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;
3275 	}
3276 	if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {
3277 		dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
3278 		val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;
3279 	}
3280 	if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
3281 		dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
3282 		val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;
3283 	}
3284 	if (oldval != val)
3285 		wr32(hw, IAVF_VF_ATQLEN1, val);
3286 
3287 freedom:
3288 	kfree(event.msg_buf);
3289 unlock:
3290 	mutex_unlock(&adapter->crit_lock);
3291 out:
3292 	/* re-enable Admin queue interrupt cause */
3293 	iavf_misc_irq_enable(adapter);
3294 }
3295 
3296 /**
3297  * iavf_free_all_tx_resources - Free Tx Resources for All Queues
3298  * @adapter: board private structure
3299  *
3300  * Free all transmit software resources
3301  **/
iavf_free_all_tx_resources(struct iavf_adapter * adapter)3302 void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
3303 {
3304 	int i;
3305 
3306 	if (!adapter->tx_rings)
3307 		return;
3308 
3309 	for (i = 0; i < adapter->num_active_queues; i++)
3310 		if (adapter->tx_rings[i].desc)
3311 			iavf_free_tx_resources(&adapter->tx_rings[i]);
3312 }
3313 
3314 /**
3315  * iavf_setup_all_tx_resources - allocate all queues Tx resources
3316  * @adapter: board private structure
3317  *
3318  * If this function returns with an error, then it's possible one or
3319  * more of the rings is populated (while the rest are not).  It is the
3320  * callers duty to clean those orphaned rings.
3321  *
3322  * Return 0 on success, negative on failure
3323  **/
iavf_setup_all_tx_resources(struct iavf_adapter * adapter)3324 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
3325 {
3326 	int i, err = 0;
3327 
3328 	for (i = 0; i < adapter->num_active_queues; i++) {
3329 		adapter->tx_rings[i].count = adapter->tx_desc_count;
3330 		err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);
3331 		if (!err)
3332 			continue;
3333 		dev_err(&adapter->pdev->dev,
3334 			"Allocation for Tx Queue %u failed\n", i);
3335 		break;
3336 	}
3337 
3338 	return err;
3339 }
3340 
3341 /**
3342  * iavf_setup_all_rx_resources - allocate all queues Rx resources
3343  * @adapter: board private structure
3344  *
3345  * If this function returns with an error, then it's possible one or
3346  * more of the rings is populated (while the rest are not).  It is the
3347  * callers duty to clean those orphaned rings.
3348  *
3349  * Return 0 on success, negative on failure
3350  **/
iavf_setup_all_rx_resources(struct iavf_adapter * adapter)3351 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
3352 {
3353 	int i, err = 0;
3354 
3355 	for (i = 0; i < adapter->num_active_queues; i++) {
3356 		adapter->rx_rings[i].count = adapter->rx_desc_count;
3357 		err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
3358 		if (!err)
3359 			continue;
3360 		dev_err(&adapter->pdev->dev,
3361 			"Allocation for Rx Queue %u failed\n", i);
3362 		break;
3363 	}
3364 	return err;
3365 }
3366 
3367 /**
3368  * iavf_free_all_rx_resources - Free Rx Resources for All Queues
3369  * @adapter: board private structure
3370  *
3371  * Free all receive software resources
3372  **/
iavf_free_all_rx_resources(struct iavf_adapter * adapter)3373 void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
3374 {
3375 	int i;
3376 
3377 	if (!adapter->rx_rings)
3378 		return;
3379 
3380 	for (i = 0; i < adapter->num_active_queues; i++)
3381 		if (adapter->rx_rings[i].desc)
3382 			iavf_free_rx_resources(&adapter->rx_rings[i]);
3383 }
3384 
3385 /**
3386  * iavf_validate_tx_bandwidth - validate the max Tx bandwidth
3387  * @adapter: board private structure
3388  * @max_tx_rate: max Tx bw for a tc
3389  **/
iavf_validate_tx_bandwidth(struct iavf_adapter * adapter,u64 max_tx_rate)3390 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
3391 				      u64 max_tx_rate)
3392 {
3393 	int speed = 0, ret = 0;
3394 
3395 	if (ADV_LINK_SUPPORT(adapter)) {
3396 		if (adapter->link_speed_mbps < U32_MAX) {
3397 			speed = adapter->link_speed_mbps;
3398 			goto validate_bw;
3399 		} else {
3400 			dev_err(&adapter->pdev->dev, "Unknown link speed\n");
3401 			return -EINVAL;
3402 		}
3403 	}
3404 
3405 	switch (adapter->link_speed) {
3406 	case VIRTCHNL_LINK_SPEED_40GB:
3407 		speed = SPEED_40000;
3408 		break;
3409 	case VIRTCHNL_LINK_SPEED_25GB:
3410 		speed = SPEED_25000;
3411 		break;
3412 	case VIRTCHNL_LINK_SPEED_20GB:
3413 		speed = SPEED_20000;
3414 		break;
3415 	case VIRTCHNL_LINK_SPEED_10GB:
3416 		speed = SPEED_10000;
3417 		break;
3418 	case VIRTCHNL_LINK_SPEED_5GB:
3419 		speed = SPEED_5000;
3420 		break;
3421 	case VIRTCHNL_LINK_SPEED_2_5GB:
3422 		speed = SPEED_2500;
3423 		break;
3424 	case VIRTCHNL_LINK_SPEED_1GB:
3425 		speed = SPEED_1000;
3426 		break;
3427 	case VIRTCHNL_LINK_SPEED_100MB:
3428 		speed = SPEED_100;
3429 		break;
3430 	default:
3431 		break;
3432 	}
3433 
3434 validate_bw:
3435 	if (max_tx_rate > speed) {
3436 		dev_err(&adapter->pdev->dev,
3437 			"Invalid tx rate specified\n");
3438 		ret = -EINVAL;
3439 	}
3440 
3441 	return ret;
3442 }
3443 
3444 /**
3445  * iavf_validate_ch_config - validate queue mapping info
3446  * @adapter: board private structure
3447  * @mqprio_qopt: queue parameters
3448  *
3449  * This function validates if the config provided by the user to
3450  * configure queue channels is valid or not. Returns 0 on a valid
3451  * config.
3452  **/
iavf_validate_ch_config(struct iavf_adapter * adapter,struct tc_mqprio_qopt_offload * mqprio_qopt)3453 static int iavf_validate_ch_config(struct iavf_adapter *adapter,
3454 				   struct tc_mqprio_qopt_offload *mqprio_qopt)
3455 {
3456 	u64 total_max_rate = 0;
3457 	u32 tx_rate_rem = 0;
3458 	int i, num_qps = 0;
3459 	u64 tx_rate = 0;
3460 	int ret = 0;
3461 
3462 	if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
3463 	    mqprio_qopt->qopt.num_tc < 1)
3464 		return -EINVAL;
3465 
3466 	for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
3467 		if (!mqprio_qopt->qopt.count[i] ||
3468 		    mqprio_qopt->qopt.offset[i] != num_qps)
3469 			return -EINVAL;
3470 		if (mqprio_qopt->min_rate[i]) {
3471 			dev_err(&adapter->pdev->dev,
3472 				"Invalid min tx rate (greater than 0) specified for TC%d\n",
3473 				i);
3474 			return -EINVAL;
3475 		}
3476 
3477 		/* convert to Mbps */
3478 		tx_rate = div_u64(mqprio_qopt->max_rate[i],
3479 				  IAVF_MBPS_DIVISOR);
3480 
3481 		if (mqprio_qopt->max_rate[i] &&
3482 		    tx_rate < IAVF_MBPS_QUANTA) {
3483 			dev_err(&adapter->pdev->dev,
3484 				"Invalid max tx rate for TC%d, minimum %dMbps\n",
3485 				i, IAVF_MBPS_QUANTA);
3486 			return -EINVAL;
3487 		}
3488 
3489 		(void)div_u64_rem(tx_rate, IAVF_MBPS_QUANTA, &tx_rate_rem);
3490 
3491 		if (tx_rate_rem != 0) {
3492 			dev_err(&adapter->pdev->dev,
3493 				"Invalid max tx rate for TC%d, not divisible by %d\n",
3494 				i, IAVF_MBPS_QUANTA);
3495 			return -EINVAL;
3496 		}
3497 
3498 		total_max_rate += tx_rate;
3499 		num_qps += mqprio_qopt->qopt.count[i];
3500 	}
3501 	if (num_qps > adapter->num_active_queues) {
3502 		dev_err(&adapter->pdev->dev,
3503 			"Cannot support requested number of queues\n");
3504 		return -EINVAL;
3505 	}
3506 
3507 	ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
3508 	return ret;
3509 }
3510 
3511 /**
3512  * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes
3513  * @adapter: board private structure
3514  **/
iavf_del_all_cloud_filters(struct iavf_adapter * adapter)3515 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
3516 {
3517 	struct iavf_cloud_filter *cf, *cftmp;
3518 
3519 	spin_lock_bh(&adapter->cloud_filter_list_lock);
3520 	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
3521 				 list) {
3522 		list_del(&cf->list);
3523 		kfree(cf);
3524 		adapter->num_cloud_filters--;
3525 	}
3526 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
3527 }
3528 
3529 /**
3530  * iavf_is_tc_config_same - Compare the mqprio TC config with the
3531  * TC config already configured on this adapter.
3532  * @adapter: board private structure
3533  * @mqprio_qopt: TC config received from kernel.
3534  *
3535  * This function compares the TC config received from the kernel
3536  * with the config already configured on the adapter.
3537  *
3538  * Return: True if configuration is same, false otherwise.
3539  **/
iavf_is_tc_config_same(struct iavf_adapter * adapter,struct tc_mqprio_qopt * mqprio_qopt)3540 static bool iavf_is_tc_config_same(struct iavf_adapter *adapter,
3541 				   struct tc_mqprio_qopt *mqprio_qopt)
3542 {
3543 	struct virtchnl_channel_info *ch = &adapter->ch_config.ch_info[0];
3544 	int i;
3545 
3546 	if (adapter->num_tc != mqprio_qopt->num_tc)
3547 		return false;
3548 
3549 	for (i = 0; i < adapter->num_tc; i++) {
3550 		if (ch[i].count != mqprio_qopt->count[i] ||
3551 		    ch[i].offset != mqprio_qopt->offset[i])
3552 			return false;
3553 	}
3554 	return true;
3555 }
3556 
3557 /**
3558  * __iavf_setup_tc - configure multiple traffic classes
3559  * @netdev: network interface device structure
3560  * @type_data: tc offload data
3561  *
3562  * This function processes the config information provided by the
3563  * user to configure traffic classes/queue channels and packages the
3564  * information to request the PF to setup traffic classes.
3565  *
3566  * Returns 0 on success.
3567  **/
__iavf_setup_tc(struct net_device * netdev,void * type_data)3568 static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
3569 {
3570 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
3571 	struct iavf_adapter *adapter = netdev_priv(netdev);
3572 	struct virtchnl_vf_resource *vfres = adapter->vf_res;
3573 	u8 num_tc = 0, total_qps = 0;
3574 	int ret = 0, netdev_tc = 0;
3575 	u64 max_tx_rate;
3576 	u16 mode;
3577 	int i;
3578 
3579 	num_tc = mqprio_qopt->qopt.num_tc;
3580 	mode = mqprio_qopt->mode;
3581 
3582 	/* delete queue_channel */
3583 	if (!mqprio_qopt->qopt.hw) {
3584 		if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
3585 			/* reset the tc configuration */
3586 			netdev_reset_tc(netdev);
3587 			adapter->num_tc = 0;
3588 			netif_tx_stop_all_queues(netdev);
3589 			netif_tx_disable(netdev);
3590 			iavf_del_all_cloud_filters(adapter);
3591 			adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
3592 			total_qps = adapter->orig_num_active_queues;
3593 			goto exit;
3594 		} else {
3595 			return -EINVAL;
3596 		}
3597 	}
3598 
3599 	/* add queue channel */
3600 	if (mode == TC_MQPRIO_MODE_CHANNEL) {
3601 		if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
3602 			dev_err(&adapter->pdev->dev, "ADq not supported\n");
3603 			return -EOPNOTSUPP;
3604 		}
3605 		if (adapter->ch_config.state != __IAVF_TC_INVALID) {
3606 			dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
3607 			return -EINVAL;
3608 		}
3609 
3610 		ret = iavf_validate_ch_config(adapter, mqprio_qopt);
3611 		if (ret)
3612 			return ret;
3613 		/* Return if same TC config is requested */
3614 		if (iavf_is_tc_config_same(adapter, &mqprio_qopt->qopt))
3615 			return 0;
3616 		adapter->num_tc = num_tc;
3617 
3618 		for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3619 			if (i < num_tc) {
3620 				adapter->ch_config.ch_info[i].count =
3621 					mqprio_qopt->qopt.count[i];
3622 				adapter->ch_config.ch_info[i].offset =
3623 					mqprio_qopt->qopt.offset[i];
3624 				total_qps += mqprio_qopt->qopt.count[i];
3625 				max_tx_rate = mqprio_qopt->max_rate[i];
3626 				/* convert to Mbps */
3627 				max_tx_rate = div_u64(max_tx_rate,
3628 						      IAVF_MBPS_DIVISOR);
3629 				adapter->ch_config.ch_info[i].max_tx_rate =
3630 					max_tx_rate;
3631 			} else {
3632 				adapter->ch_config.ch_info[i].count = 1;
3633 				adapter->ch_config.ch_info[i].offset = 0;
3634 			}
3635 		}
3636 
3637 		/* Take snapshot of original config such as "num_active_queues"
3638 		 * It is used later when delete ADQ flow is exercised, so that
3639 		 * once delete ADQ flow completes, VF shall go back to its
3640 		 * original queue configuration
3641 		 */
3642 
3643 		adapter->orig_num_active_queues = adapter->num_active_queues;
3644 
3645 		/* Store queue info based on TC so that VF gets configured
3646 		 * with correct number of queues when VF completes ADQ config
3647 		 * flow
3648 		 */
3649 		adapter->ch_config.total_qps = total_qps;
3650 
3651 		netif_tx_stop_all_queues(netdev);
3652 		netif_tx_disable(netdev);
3653 		adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
3654 		netdev_reset_tc(netdev);
3655 		/* Report the tc mapping up the stack */
3656 		netdev_set_num_tc(adapter->netdev, num_tc);
3657 		for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3658 			u16 qcount = mqprio_qopt->qopt.count[i];
3659 			u16 qoffset = mqprio_qopt->qopt.offset[i];
3660 
3661 			if (i < num_tc)
3662 				netdev_set_tc_queue(netdev, netdev_tc++, qcount,
3663 						    qoffset);
3664 		}
3665 	}
3666 exit:
3667 	if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
3668 		return 0;
3669 
3670 	mutex_lock(&netdev->lock);
3671 	netif_set_real_num_rx_queues(netdev, total_qps);
3672 	netif_set_real_num_tx_queues(netdev, total_qps);
3673 	mutex_unlock(&netdev->lock);
3674 
3675 	return ret;
3676 }
3677 
3678 /**
3679  * iavf_parse_cls_flower - Parse tc flower filters provided by kernel
3680  * @adapter: board private structure
3681  * @f: pointer to struct flow_cls_offload
3682  * @filter: pointer to cloud filter structure
3683  */
iavf_parse_cls_flower(struct iavf_adapter * adapter,struct flow_cls_offload * f,struct iavf_cloud_filter * filter)3684 static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
3685 				 struct flow_cls_offload *f,
3686 				 struct iavf_cloud_filter *filter)
3687 {
3688 	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
3689 	struct flow_dissector *dissector = rule->match.dissector;
3690 	u16 n_proto_mask = 0;
3691 	u16 n_proto_key = 0;
3692 	u8 field_flags = 0;
3693 	u16 addr_type = 0;
3694 	u16 n_proto = 0;
3695 	int i = 0;
3696 	struct virtchnl_filter *vf = &filter->f;
3697 
3698 	if (dissector->used_keys &
3699 	    ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
3700 	      BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
3701 	      BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
3702 	      BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
3703 	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
3704 	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
3705 	      BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
3706 	      BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
3707 		dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%llx\n",
3708 			dissector->used_keys);
3709 		return -EOPNOTSUPP;
3710 	}
3711 
3712 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
3713 		struct flow_match_enc_keyid match;
3714 
3715 		flow_rule_match_enc_keyid(rule, &match);
3716 		if (match.mask->keyid != 0)
3717 			field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
3718 	}
3719 
3720 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
3721 		struct flow_match_basic match;
3722 
3723 		flow_rule_match_basic(rule, &match);
3724 		n_proto_key = ntohs(match.key->n_proto);
3725 		n_proto_mask = ntohs(match.mask->n_proto);
3726 
3727 		if (n_proto_key == ETH_P_ALL) {
3728 			n_proto_key = 0;
3729 			n_proto_mask = 0;
3730 		}
3731 		n_proto = n_proto_key & n_proto_mask;
3732 		if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
3733 			return -EINVAL;
3734 		if (n_proto == ETH_P_IPV6) {
3735 			/* specify flow type as TCP IPv6 */
3736 			vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
3737 		}
3738 
3739 		if (match.key->ip_proto != IPPROTO_TCP) {
3740 			dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
3741 			return -EINVAL;
3742 		}
3743 	}
3744 
3745 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
3746 		struct flow_match_eth_addrs match;
3747 
3748 		flow_rule_match_eth_addrs(rule, &match);
3749 
3750 		/* use is_broadcast and is_zero to check for all 0xf or 0 */
3751 		if (!is_zero_ether_addr(match.mask->dst)) {
3752 			if (is_broadcast_ether_addr(match.mask->dst)) {
3753 				field_flags |= IAVF_CLOUD_FIELD_OMAC;
3754 			} else {
3755 				dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
3756 					match.mask->dst);
3757 				return -EINVAL;
3758 			}
3759 		}
3760 
3761 		if (!is_zero_ether_addr(match.mask->src)) {
3762 			if (is_broadcast_ether_addr(match.mask->src)) {
3763 				field_flags |= IAVF_CLOUD_FIELD_IMAC;
3764 			} else {
3765 				dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
3766 					match.mask->src);
3767 				return -EINVAL;
3768 			}
3769 		}
3770 
3771 		if (!is_zero_ether_addr(match.key->dst))
3772 			if (is_valid_ether_addr(match.key->dst) ||
3773 			    is_multicast_ether_addr(match.key->dst)) {
3774 				/* set the mask if a valid dst_mac address */
3775 				for (i = 0; i < ETH_ALEN; i++)
3776 					vf->mask.tcp_spec.dst_mac[i] |= 0xff;
3777 				ether_addr_copy(vf->data.tcp_spec.dst_mac,
3778 						match.key->dst);
3779 			}
3780 
3781 		if (!is_zero_ether_addr(match.key->src))
3782 			if (is_valid_ether_addr(match.key->src) ||
3783 			    is_multicast_ether_addr(match.key->src)) {
3784 				/* set the mask if a valid dst_mac address */
3785 				for (i = 0; i < ETH_ALEN; i++)
3786 					vf->mask.tcp_spec.src_mac[i] |= 0xff;
3787 				ether_addr_copy(vf->data.tcp_spec.src_mac,
3788 						match.key->src);
3789 		}
3790 	}
3791 
3792 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
3793 		struct flow_match_vlan match;
3794 
3795 		flow_rule_match_vlan(rule, &match);
3796 		if (match.mask->vlan_id) {
3797 			if (match.mask->vlan_id == VLAN_VID_MASK) {
3798 				field_flags |= IAVF_CLOUD_FIELD_IVLAN;
3799 			} else {
3800 				dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
3801 					match.mask->vlan_id);
3802 				return -EINVAL;
3803 			}
3804 		}
3805 		vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
3806 		vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);
3807 	}
3808 
3809 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
3810 		struct flow_match_control match;
3811 
3812 		flow_rule_match_control(rule, &match);
3813 		addr_type = match.key->addr_type;
3814 
3815 		if (flow_rule_has_control_flags(match.mask->flags,
3816 						f->common.extack))
3817 			return -EOPNOTSUPP;
3818 	}
3819 
3820 	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
3821 		struct flow_match_ipv4_addrs match;
3822 
3823 		flow_rule_match_ipv4_addrs(rule, &match);
3824 		if (match.mask->dst) {
3825 			if (match.mask->dst == cpu_to_be32(0xffffffff)) {
3826 				field_flags |= IAVF_CLOUD_FIELD_IIP;
3827 			} else {
3828 				dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
3829 					be32_to_cpu(match.mask->dst));
3830 				return -EINVAL;
3831 			}
3832 		}
3833 
3834 		if (match.mask->src) {
3835 			if (match.mask->src == cpu_to_be32(0xffffffff)) {
3836 				field_flags |= IAVF_CLOUD_FIELD_IIP;
3837 			} else {
3838 				dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
3839 					be32_to_cpu(match.mask->src));
3840 				return -EINVAL;
3841 			}
3842 		}
3843 
3844 		if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
3845 			dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
3846 			return -EINVAL;
3847 		}
3848 		if (match.key->dst) {
3849 			vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
3850 			vf->data.tcp_spec.dst_ip[0] = match.key->dst;
3851 		}
3852 		if (match.key->src) {
3853 			vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
3854 			vf->data.tcp_spec.src_ip[0] = match.key->src;
3855 		}
3856 	}
3857 
3858 	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
3859 		struct flow_match_ipv6_addrs match;
3860 
3861 		flow_rule_match_ipv6_addrs(rule, &match);
3862 
3863 		/* validate mask, make sure it is not IPV6_ADDR_ANY */
3864 		if (ipv6_addr_any(&match.mask->dst)) {
3865 			dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
3866 				IPV6_ADDR_ANY);
3867 			return -EINVAL;
3868 		}
3869 
3870 		/* src and dest IPv6 address should not be LOOPBACK
3871 		 * (0:0:0:0:0:0:0:1) which can be represented as ::1
3872 		 */
3873 		if (ipv6_addr_loopback(&match.key->dst) ||
3874 		    ipv6_addr_loopback(&match.key->src)) {
3875 			dev_err(&adapter->pdev->dev,
3876 				"ipv6 addr should not be loopback\n");
3877 			return -EINVAL;
3878 		}
3879 		if (!ipv6_addr_any(&match.mask->dst) ||
3880 		    !ipv6_addr_any(&match.mask->src))
3881 			field_flags |= IAVF_CLOUD_FIELD_IIP;
3882 
3883 		for (i = 0; i < 4; i++)
3884 			vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
3885 		memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,
3886 		       sizeof(vf->data.tcp_spec.dst_ip));
3887 		for (i = 0; i < 4; i++)
3888 			vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
3889 		memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,
3890 		       sizeof(vf->data.tcp_spec.src_ip));
3891 	}
3892 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
3893 		struct flow_match_ports match;
3894 
3895 		flow_rule_match_ports(rule, &match);
3896 		if (match.mask->src) {
3897 			if (match.mask->src == cpu_to_be16(0xffff)) {
3898 				field_flags |= IAVF_CLOUD_FIELD_IIP;
3899 			} else {
3900 				dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
3901 					be16_to_cpu(match.mask->src));
3902 				return -EINVAL;
3903 			}
3904 		}
3905 
3906 		if (match.mask->dst) {
3907 			if (match.mask->dst == cpu_to_be16(0xffff)) {
3908 				field_flags |= IAVF_CLOUD_FIELD_IIP;
3909 			} else {
3910 				dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
3911 					be16_to_cpu(match.mask->dst));
3912 				return -EINVAL;
3913 			}
3914 		}
3915 		if (match.key->dst) {
3916 			vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
3917 			vf->data.tcp_spec.dst_port = match.key->dst;
3918 		}
3919 
3920 		if (match.key->src) {
3921 			vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
3922 			vf->data.tcp_spec.src_port = match.key->src;
3923 		}
3924 	}
3925 	vf->field_flags = field_flags;
3926 
3927 	return 0;
3928 }
3929 
3930 /**
3931  * iavf_handle_tclass - Forward to a traffic class on the device
3932  * @adapter: board private structure
3933  * @tc: traffic class index on the device
3934  * @filter: pointer to cloud filter structure
3935  */
iavf_handle_tclass(struct iavf_adapter * adapter,u32 tc,struct iavf_cloud_filter * filter)3936 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
3937 			      struct iavf_cloud_filter *filter)
3938 {
3939 	if (tc == 0)
3940 		return 0;
3941 	if (tc < adapter->num_tc) {
3942 		if (!filter->f.data.tcp_spec.dst_port) {
3943 			dev_err(&adapter->pdev->dev,
3944 				"Specify destination port to redirect to traffic class other than TC0\n");
3945 			return -EINVAL;
3946 		}
3947 	}
3948 	/* redirect to a traffic class on the same device */
3949 	filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
3950 	filter->f.action_meta = tc;
3951 	return 0;
3952 }
3953 
3954 /**
3955  * iavf_find_cf - Find the cloud filter in the list
3956  * @adapter: Board private structure
3957  * @cookie: filter specific cookie
3958  *
3959  * Returns ptr to the filter object or NULL. Must be called while holding the
3960  * cloud_filter_list_lock.
3961  */
iavf_find_cf(struct iavf_adapter * adapter,unsigned long * cookie)3962 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
3963 					      unsigned long *cookie)
3964 {
3965 	struct iavf_cloud_filter *filter = NULL;
3966 
3967 	if (!cookie)
3968 		return NULL;
3969 
3970 	list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
3971 		if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
3972 			return filter;
3973 	}
3974 	return NULL;
3975 }
3976 
3977 /**
3978  * iavf_configure_clsflower - Add tc flower filters
3979  * @adapter: board private structure
3980  * @cls_flower: Pointer to struct flow_cls_offload
3981  */
iavf_configure_clsflower(struct iavf_adapter * adapter,struct flow_cls_offload * cls_flower)3982 static int iavf_configure_clsflower(struct iavf_adapter *adapter,
3983 				    struct flow_cls_offload *cls_flower)
3984 {
3985 	int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
3986 	struct iavf_cloud_filter *filter = NULL;
3987 	int err = -EINVAL, count = 50;
3988 
3989 	if (tc < 0) {
3990 		dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
3991 		return -EINVAL;
3992 	}
3993 
3994 	filter = kzalloc(sizeof(*filter), GFP_KERNEL);
3995 	if (!filter)
3996 		return -ENOMEM;
3997 
3998 	while (!mutex_trylock(&adapter->crit_lock)) {
3999 		if (--count == 0) {
4000 			kfree(filter);
4001 			return err;
4002 		}
4003 		udelay(1);
4004 	}
4005 
4006 	filter->cookie = cls_flower->cookie;
4007 
4008 	/* bail out here if filter already exists */
4009 	spin_lock_bh(&adapter->cloud_filter_list_lock);
4010 	if (iavf_find_cf(adapter, &cls_flower->cookie)) {
4011 		dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n");
4012 		err = -EEXIST;
4013 		goto spin_unlock;
4014 	}
4015 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
4016 
4017 	/* set the mask to all zeroes to begin with */
4018 	memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
4019 	/* start out with flow type and eth type IPv4 to begin with */
4020 	filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
4021 	err = iavf_parse_cls_flower(adapter, cls_flower, filter);
4022 	if (err)
4023 		goto err;
4024 
4025 	err = iavf_handle_tclass(adapter, tc, filter);
4026 	if (err)
4027 		goto err;
4028 
4029 	/* add filter to the list */
4030 	spin_lock_bh(&adapter->cloud_filter_list_lock);
4031 	list_add_tail(&filter->list, &adapter->cloud_filter_list);
4032 	adapter->num_cloud_filters++;
4033 	filter->add = true;
4034 	adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
4035 spin_unlock:
4036 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
4037 err:
4038 	if (err)
4039 		kfree(filter);
4040 
4041 	mutex_unlock(&adapter->crit_lock);
4042 	return err;
4043 }
4044 
4045 /**
4046  * iavf_delete_clsflower - Remove tc flower filters
4047  * @adapter: board private structure
4048  * @cls_flower: Pointer to struct flow_cls_offload
4049  */
iavf_delete_clsflower(struct iavf_adapter * adapter,struct flow_cls_offload * cls_flower)4050 static int iavf_delete_clsflower(struct iavf_adapter *adapter,
4051 				 struct flow_cls_offload *cls_flower)
4052 {
4053 	struct iavf_cloud_filter *filter = NULL;
4054 	int err = 0;
4055 
4056 	spin_lock_bh(&adapter->cloud_filter_list_lock);
4057 	filter = iavf_find_cf(adapter, &cls_flower->cookie);
4058 	if (filter) {
4059 		filter->del = true;
4060 		adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
4061 	} else {
4062 		err = -EINVAL;
4063 	}
4064 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
4065 
4066 	return err;
4067 }
4068 
4069 /**
4070  * iavf_setup_tc_cls_flower - flower classifier offloads
4071  * @adapter: pointer to iavf adapter structure
4072  * @cls_flower: pointer to flow_cls_offload struct with flow info
4073  */
iavf_setup_tc_cls_flower(struct iavf_adapter * adapter,struct flow_cls_offload * cls_flower)4074 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
4075 				    struct flow_cls_offload *cls_flower)
4076 {
4077 	switch (cls_flower->command) {
4078 	case FLOW_CLS_REPLACE:
4079 		return iavf_configure_clsflower(adapter, cls_flower);
4080 	case FLOW_CLS_DESTROY:
4081 		return iavf_delete_clsflower(adapter, cls_flower);
4082 	case FLOW_CLS_STATS:
4083 		return -EOPNOTSUPP;
4084 	default:
4085 		return -EOPNOTSUPP;
4086 	}
4087 }
4088 
4089 /**
4090  * iavf_add_cls_u32 - Add U32 classifier offloads
4091  * @adapter: pointer to iavf adapter structure
4092  * @cls_u32: pointer to tc_cls_u32_offload struct with flow info
4093  *
4094  * Return: 0 on success or negative errno on failure.
4095  */
iavf_add_cls_u32(struct iavf_adapter * adapter,struct tc_cls_u32_offload * cls_u32)4096 static int iavf_add_cls_u32(struct iavf_adapter *adapter,
4097 			    struct tc_cls_u32_offload *cls_u32)
4098 {
4099 	struct netlink_ext_ack *extack = cls_u32->common.extack;
4100 	struct virtchnl_fdir_rule *rule_cfg;
4101 	struct virtchnl_filter_action *vact;
4102 	struct virtchnl_proto_hdrs *hdrs;
4103 	struct ethhdr *spec_h, *mask_h;
4104 	const struct tc_action *act;
4105 	struct iavf_fdir_fltr *fltr;
4106 	struct tcf_exts *exts;
4107 	unsigned int q_index;
4108 	int i, status = 0;
4109 	int off_base = 0;
4110 
4111 	if (cls_u32->knode.link_handle) {
4112 		NL_SET_ERR_MSG_MOD(extack, "Linking not supported");
4113 		return -EOPNOTSUPP;
4114 	}
4115 
4116 	fltr = kzalloc(sizeof(*fltr), GFP_KERNEL);
4117 	if (!fltr)
4118 		return -ENOMEM;
4119 
4120 	rule_cfg = &fltr->vc_add_msg.rule_cfg;
4121 	hdrs = &rule_cfg->proto_hdrs;
4122 	hdrs->count = 0;
4123 
4124 	/* The parser lib at the PF expects the packet starting with MAC hdr */
4125 	switch (ntohs(cls_u32->common.protocol)) {
4126 	case ETH_P_802_3:
4127 		break;
4128 	case ETH_P_IP:
4129 		spec_h = (struct ethhdr *)hdrs->raw.spec;
4130 		mask_h = (struct ethhdr *)hdrs->raw.mask;
4131 		spec_h->h_proto = htons(ETH_P_IP);
4132 		mask_h->h_proto = htons(0xFFFF);
4133 		off_base += ETH_HLEN;
4134 		break;
4135 	default:
4136 		NL_SET_ERR_MSG_MOD(extack, "Only 802_3 and ip filter protocols are supported");
4137 		status = -EOPNOTSUPP;
4138 		goto free_alloc;
4139 	}
4140 
4141 	for (i = 0; i < cls_u32->knode.sel->nkeys; i++) {
4142 		__be32 val, mask;
4143 		int off;
4144 
4145 		off = off_base + cls_u32->knode.sel->keys[i].off;
4146 		val = cls_u32->knode.sel->keys[i].val;
4147 		mask = cls_u32->knode.sel->keys[i].mask;
4148 
4149 		if (off >= sizeof(hdrs->raw.spec)) {
4150 			NL_SET_ERR_MSG_MOD(extack, "Input exceeds maximum allowed.");
4151 			status = -EINVAL;
4152 			goto free_alloc;
4153 		}
4154 
4155 		memcpy(&hdrs->raw.spec[off], &val, sizeof(val));
4156 		memcpy(&hdrs->raw.mask[off], &mask, sizeof(mask));
4157 		hdrs->raw.pkt_len = off + sizeof(val);
4158 	}
4159 
4160 	/* Only one action is allowed */
4161 	rule_cfg->action_set.count = 1;
4162 	vact = &rule_cfg->action_set.actions[0];
4163 	exts = cls_u32->knode.exts;
4164 
4165 	tcf_exts_for_each_action(i, act, exts) {
4166 		/* FDIR queue */
4167 		if (is_tcf_skbedit_rx_queue_mapping(act)) {
4168 			q_index = tcf_skbedit_rx_queue_mapping(act);
4169 			if (q_index >= adapter->num_active_queues) {
4170 				status = -EINVAL;
4171 				goto free_alloc;
4172 			}
4173 
4174 			vact->type = VIRTCHNL_ACTION_QUEUE;
4175 			vact->act_conf.queue.index = q_index;
4176 			break;
4177 		}
4178 
4179 		/* Drop */
4180 		if (is_tcf_gact_shot(act)) {
4181 			vact->type = VIRTCHNL_ACTION_DROP;
4182 			break;
4183 		}
4184 
4185 		/* Unsupported action */
4186 		NL_SET_ERR_MSG_MOD(extack, "Unsupported action.");
4187 		status = -EOPNOTSUPP;
4188 		goto free_alloc;
4189 	}
4190 
4191 	fltr->vc_add_msg.vsi_id = adapter->vsi.id;
4192 	fltr->cls_u32_handle = cls_u32->knode.handle;
4193 	return iavf_fdir_add_fltr(adapter, fltr);
4194 
4195 free_alloc:
4196 	kfree(fltr);
4197 	return status;
4198 }
4199 
4200 /**
4201  * iavf_del_cls_u32 - Delete U32 classifier offloads
4202  * @adapter: pointer to iavf adapter structure
4203  * @cls_u32: pointer to tc_cls_u32_offload struct with flow info
4204  *
4205  * Return: 0 on success or negative errno on failure.
4206  */
iavf_del_cls_u32(struct iavf_adapter * adapter,struct tc_cls_u32_offload * cls_u32)4207 static int iavf_del_cls_u32(struct iavf_adapter *adapter,
4208 			    struct tc_cls_u32_offload *cls_u32)
4209 {
4210 	return iavf_fdir_del_fltr(adapter, true, cls_u32->knode.handle);
4211 }
4212 
4213 /**
4214  * iavf_setup_tc_cls_u32 - U32 filter offloads
4215  * @adapter: pointer to iavf adapter structure
4216  * @cls_u32: pointer to tc_cls_u32_offload struct with flow info
4217  *
4218  * Return: 0 on success or negative errno on failure.
4219  */
iavf_setup_tc_cls_u32(struct iavf_adapter * adapter,struct tc_cls_u32_offload * cls_u32)4220 static int iavf_setup_tc_cls_u32(struct iavf_adapter *adapter,
4221 				 struct tc_cls_u32_offload *cls_u32)
4222 {
4223 	if (!TC_U32_SUPPORT(adapter) || !FDIR_FLTR_SUPPORT(adapter))
4224 		return -EOPNOTSUPP;
4225 
4226 	switch (cls_u32->command) {
4227 	case TC_CLSU32_NEW_KNODE:
4228 	case TC_CLSU32_REPLACE_KNODE:
4229 		return iavf_add_cls_u32(adapter, cls_u32);
4230 	case TC_CLSU32_DELETE_KNODE:
4231 		return iavf_del_cls_u32(adapter, cls_u32);
4232 	default:
4233 		return -EOPNOTSUPP;
4234 	}
4235 }
4236 
4237 /**
4238  * iavf_setup_tc_block_cb - block callback for tc
4239  * @type: type of offload
4240  * @type_data: offload data
4241  * @cb_priv:
4242  *
4243  * This function is the block callback for traffic classes
4244  **/
iavf_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)4245 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4246 				  void *cb_priv)
4247 {
4248 	struct iavf_adapter *adapter = cb_priv;
4249 
4250 	if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
4251 		return -EOPNOTSUPP;
4252 
4253 	switch (type) {
4254 	case TC_SETUP_CLSFLOWER:
4255 		return iavf_setup_tc_cls_flower(cb_priv, type_data);
4256 	case TC_SETUP_CLSU32:
4257 		return iavf_setup_tc_cls_u32(cb_priv, type_data);
4258 	default:
4259 		return -EOPNOTSUPP;
4260 	}
4261 }
4262 
4263 static LIST_HEAD(iavf_block_cb_list);
4264 
4265 /**
4266  * iavf_setup_tc - configure multiple traffic classes
4267  * @netdev: network interface device structure
4268  * @type: type of offload
4269  * @type_data: tc offload data
4270  *
4271  * This function is the callback to ndo_setup_tc in the
4272  * netdev_ops.
4273  *
4274  * Returns 0 on success
4275  **/
iavf_setup_tc(struct net_device * netdev,enum tc_setup_type type,void * type_data)4276 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
4277 			 void *type_data)
4278 {
4279 	struct iavf_adapter *adapter = netdev_priv(netdev);
4280 
4281 	switch (type) {
4282 	case TC_SETUP_QDISC_MQPRIO:
4283 		return __iavf_setup_tc(netdev, type_data);
4284 	case TC_SETUP_BLOCK:
4285 		return flow_block_cb_setup_simple(type_data,
4286 						  &iavf_block_cb_list,
4287 						  iavf_setup_tc_block_cb,
4288 						  adapter, adapter, true);
4289 	default:
4290 		return -EOPNOTSUPP;
4291 	}
4292 }
4293 
4294 /**
4295  * iavf_restore_fdir_filters
4296  * @adapter: board private structure
4297  *
4298  * Restore existing FDIR filters when VF netdev comes back up.
4299  **/
iavf_restore_fdir_filters(struct iavf_adapter * adapter)4300 static void iavf_restore_fdir_filters(struct iavf_adapter *adapter)
4301 {
4302 	struct iavf_fdir_fltr *f;
4303 
4304 	spin_lock_bh(&adapter->fdir_fltr_lock);
4305 	list_for_each_entry(f, &adapter->fdir_list_head, list) {
4306 		if (f->state == IAVF_FDIR_FLTR_DIS_REQUEST) {
4307 			/* Cancel a request, keep filter as active */
4308 			f->state = IAVF_FDIR_FLTR_ACTIVE;
4309 		} else if (f->state == IAVF_FDIR_FLTR_DIS_PENDING ||
4310 			   f->state == IAVF_FDIR_FLTR_INACTIVE) {
4311 			/* Add filters which are inactive or have a pending
4312 			 * request to PF to be deleted
4313 			 */
4314 			f->state = IAVF_FDIR_FLTR_ADD_REQUEST;
4315 			adapter->aq_required |= IAVF_FLAG_AQ_ADD_FDIR_FILTER;
4316 		}
4317 	}
4318 	spin_unlock_bh(&adapter->fdir_fltr_lock);
4319 }
4320 
4321 /**
4322  * iavf_open - Called when a network interface is made active
4323  * @netdev: network interface device structure
4324  *
4325  * Returns 0 on success, negative value on failure
4326  *
4327  * The open entry point is called when a network interface is made
4328  * active by the system (IFF_UP).  At this point all resources needed
4329  * for transmit and receive operations are allocated, the interrupt
4330  * handler is registered with the OS, the watchdog is started,
4331  * and the stack is notified that the interface is ready.
4332  **/
iavf_open(struct net_device * netdev)4333 static int iavf_open(struct net_device *netdev)
4334 {
4335 	struct iavf_adapter *adapter = netdev_priv(netdev);
4336 	int err;
4337 
4338 	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
4339 		dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
4340 		return -EIO;
4341 	}
4342 
4343 	while (!mutex_trylock(&adapter->crit_lock)) {
4344 		/* If we are in __IAVF_INIT_CONFIG_ADAPTER state the crit_lock
4345 		 * is already taken and iavf_open is called from an upper
4346 		 * device's notifier reacting on NETDEV_REGISTER event.
4347 		 * We have to leave here to avoid dead lock.
4348 		 */
4349 		if (adapter->state == __IAVF_INIT_CONFIG_ADAPTER)
4350 			return -EBUSY;
4351 
4352 		usleep_range(500, 1000);
4353 	}
4354 
4355 	if (adapter->state != __IAVF_DOWN) {
4356 		err = -EBUSY;
4357 		goto err_unlock;
4358 	}
4359 
4360 	if (adapter->state == __IAVF_RUNNING &&
4361 	    !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) {
4362 		dev_dbg(&adapter->pdev->dev, "VF is already open.\n");
4363 		err = 0;
4364 		goto err_unlock;
4365 	}
4366 
4367 	/* allocate transmit descriptors */
4368 	err = iavf_setup_all_tx_resources(adapter);
4369 	if (err)
4370 		goto err_setup_tx;
4371 
4372 	/* allocate receive descriptors */
4373 	err = iavf_setup_all_rx_resources(adapter);
4374 	if (err)
4375 		goto err_setup_rx;
4376 
4377 	/* clear any pending interrupts, may auto mask */
4378 	err = iavf_request_traffic_irqs(adapter, netdev->name);
4379 	if (err)
4380 		goto err_req_irq;
4381 
4382 	spin_lock_bh(&adapter->mac_vlan_list_lock);
4383 
4384 	iavf_add_filter(adapter, adapter->hw.mac.addr);
4385 
4386 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
4387 
4388 	/* Restore filters that were removed with IFF_DOWN */
4389 	iavf_restore_filters(adapter);
4390 	iavf_restore_fdir_filters(adapter);
4391 
4392 	iavf_configure(adapter);
4393 
4394 	iavf_up_complete(adapter);
4395 
4396 	iavf_irq_enable(adapter, true);
4397 
4398 	mutex_unlock(&adapter->crit_lock);
4399 
4400 	return 0;
4401 
4402 err_req_irq:
4403 	iavf_down(adapter);
4404 	iavf_free_traffic_irqs(adapter);
4405 err_setup_rx:
4406 	iavf_free_all_rx_resources(adapter);
4407 err_setup_tx:
4408 	iavf_free_all_tx_resources(adapter);
4409 err_unlock:
4410 	mutex_unlock(&adapter->crit_lock);
4411 
4412 	return err;
4413 }
4414 
4415 /**
4416  * iavf_close - Disables a network interface
4417  * @netdev: network interface device structure
4418  *
4419  * Returns 0, this is not allowed to fail
4420  *
4421  * The close entry point is called when an interface is de-activated
4422  * by the OS.  The hardware is still under the drivers control, but
4423  * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
4424  * are freed, along with all transmit and receive resources.
4425  **/
iavf_close(struct net_device * netdev)4426 static int iavf_close(struct net_device *netdev)
4427 {
4428 	struct iavf_adapter *adapter = netdev_priv(netdev);
4429 	u64 aq_to_restore;
4430 	int status;
4431 
4432 	mutex_lock(&adapter->crit_lock);
4433 
4434 	if (adapter->state <= __IAVF_DOWN_PENDING) {
4435 		mutex_unlock(&adapter->crit_lock);
4436 		return 0;
4437 	}
4438 
4439 	set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
4440 	/* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before
4441 	 * IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl
4442 	 * deadlock with adminq_task() until iavf_close timeouts. We must send
4443 	 * IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make
4444 	 * disable queues possible for vf. Give only necessary flags to
4445 	 * iavf_down and save other to set them right before iavf_close()
4446 	 * returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and
4447 	 * iavf will be in DOWN state.
4448 	 */
4449 	aq_to_restore = adapter->aq_required;
4450 	adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG;
4451 
4452 	/* Remove flags which we do not want to send after close or we want to
4453 	 * send before disable queues.
4454 	 */
4455 	aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG		|
4456 			   IAVF_FLAG_AQ_ENABLE_QUEUES		|
4457 			   IAVF_FLAG_AQ_CONFIGURE_QUEUES	|
4458 			   IAVF_FLAG_AQ_ADD_VLAN_FILTER		|
4459 			   IAVF_FLAG_AQ_ADD_MAC_FILTER		|
4460 			   IAVF_FLAG_AQ_ADD_CLOUD_FILTER	|
4461 			   IAVF_FLAG_AQ_ADD_FDIR_FILTER		|
4462 			   IAVF_FLAG_AQ_ADD_ADV_RSS_CFG);
4463 
4464 	iavf_down(adapter);
4465 	iavf_change_state(adapter, __IAVF_DOWN_PENDING);
4466 	iavf_free_traffic_irqs(adapter);
4467 
4468 	mutex_unlock(&adapter->crit_lock);
4469 
4470 	/* We explicitly don't free resources here because the hardware is
4471 	 * still active and can DMA into memory. Resources are cleared in
4472 	 * iavf_virtchnl_completion() after we get confirmation from the PF
4473 	 * driver that the rings have been stopped.
4474 	 *
4475 	 * Also, we wait for state to transition to __IAVF_DOWN before
4476 	 * returning. State change occurs in iavf_virtchnl_completion() after
4477 	 * VF resources are released (which occurs after PF driver processes and
4478 	 * responds to admin queue commands).
4479 	 */
4480 
4481 	status = wait_event_timeout(adapter->down_waitqueue,
4482 				    adapter->state == __IAVF_DOWN,
4483 				    msecs_to_jiffies(500));
4484 	if (!status)
4485 		netdev_warn(netdev, "Device resources not yet released\n");
4486 
4487 	mutex_lock(&adapter->crit_lock);
4488 	adapter->aq_required |= aq_to_restore;
4489 	mutex_unlock(&adapter->crit_lock);
4490 	return 0;
4491 }
4492 
4493 /**
4494  * iavf_change_mtu - Change the Maximum Transfer Unit
4495  * @netdev: network interface device structure
4496  * @new_mtu: new value for maximum frame size
4497  *
4498  * Returns 0 on success, negative on failure
4499  **/
iavf_change_mtu(struct net_device * netdev,int new_mtu)4500 static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
4501 {
4502 	struct iavf_adapter *adapter = netdev_priv(netdev);
4503 	int ret = 0;
4504 
4505 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
4506 		   netdev->mtu, new_mtu);
4507 	WRITE_ONCE(netdev->mtu, new_mtu);
4508 
4509 	if (netif_running(netdev)) {
4510 		iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
4511 		ret = iavf_wait_for_reset(adapter);
4512 		if (ret < 0)
4513 			netdev_warn(netdev, "MTU change interrupted waiting for reset");
4514 		else if (ret)
4515 			netdev_warn(netdev, "MTU change timed out waiting for reset");
4516 	}
4517 
4518 	return ret;
4519 }
4520 
4521 /**
4522  * iavf_disable_fdir - disable Flow Director and clear existing filters
4523  * @adapter: board private structure
4524  **/
iavf_disable_fdir(struct iavf_adapter * adapter)4525 static void iavf_disable_fdir(struct iavf_adapter *adapter)
4526 {
4527 	struct iavf_fdir_fltr *fdir, *fdirtmp;
4528 	bool del_filters = false;
4529 
4530 	adapter->flags &= ~IAVF_FLAG_FDIR_ENABLED;
4531 
4532 	/* remove all Flow Director filters */
4533 	spin_lock_bh(&adapter->fdir_fltr_lock);
4534 	list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head,
4535 				 list) {
4536 		if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST ||
4537 		    fdir->state == IAVF_FDIR_FLTR_INACTIVE) {
4538 			/* Delete filters not registered in PF */
4539 			list_del(&fdir->list);
4540 			iavf_dec_fdir_active_fltr(adapter, fdir);
4541 			kfree(fdir);
4542 		} else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING ||
4543 			   fdir->state == IAVF_FDIR_FLTR_DIS_REQUEST ||
4544 			   fdir->state == IAVF_FDIR_FLTR_ACTIVE) {
4545 			/* Filters registered in PF, schedule their deletion */
4546 			fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
4547 			del_filters = true;
4548 		} else if (fdir->state == IAVF_FDIR_FLTR_DIS_PENDING) {
4549 			/* Request to delete filter already sent to PF, change
4550 			 * state to DEL_PENDING to delete filter after PF's
4551 			 * response, not set as INACTIVE
4552 			 */
4553 			fdir->state = IAVF_FDIR_FLTR_DEL_PENDING;
4554 		}
4555 	}
4556 	spin_unlock_bh(&adapter->fdir_fltr_lock);
4557 
4558 	if (del_filters) {
4559 		adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
4560 		mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
4561 	}
4562 }
4563 
4564 #define NETIF_VLAN_OFFLOAD_FEATURES	(NETIF_F_HW_VLAN_CTAG_RX | \
4565 					 NETIF_F_HW_VLAN_CTAG_TX | \
4566 					 NETIF_F_HW_VLAN_STAG_RX | \
4567 					 NETIF_F_HW_VLAN_STAG_TX)
4568 
4569 /**
4570  * iavf_set_features - set the netdev feature flags
4571  * @netdev: ptr to the netdev being adjusted
4572  * @features: the feature set that the stack is suggesting
4573  * Note: expects to be called while under rtnl_lock()
4574  **/
iavf_set_features(struct net_device * netdev,netdev_features_t features)4575 static int iavf_set_features(struct net_device *netdev,
4576 			     netdev_features_t features)
4577 {
4578 	struct iavf_adapter *adapter = netdev_priv(netdev);
4579 
4580 	/* trigger update on any VLAN feature change */
4581 	if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^
4582 	    (features & NETIF_VLAN_OFFLOAD_FEATURES))
4583 		iavf_set_vlan_offload_features(adapter, netdev->features,
4584 					       features);
4585 	if (CRC_OFFLOAD_ALLOWED(adapter) &&
4586 	    ((netdev->features & NETIF_F_RXFCS) ^ (features & NETIF_F_RXFCS)))
4587 		iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
4588 
4589 	if ((netdev->features & NETIF_F_NTUPLE) ^ (features & NETIF_F_NTUPLE)) {
4590 		if (features & NETIF_F_NTUPLE)
4591 			adapter->flags |= IAVF_FLAG_FDIR_ENABLED;
4592 		else
4593 			iavf_disable_fdir(adapter);
4594 	}
4595 
4596 	return 0;
4597 }
4598 
4599 /**
4600  * iavf_features_check - Validate encapsulated packet conforms to limits
4601  * @skb: skb buff
4602  * @dev: This physical port's netdev
4603  * @features: Offload features that the stack believes apply
4604  **/
iavf_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)4605 static netdev_features_t iavf_features_check(struct sk_buff *skb,
4606 					     struct net_device *dev,
4607 					     netdev_features_t features)
4608 {
4609 	size_t len;
4610 
4611 	/* No point in doing any of this if neither checksum nor GSO are
4612 	 * being requested for this frame.  We can rule out both by just
4613 	 * checking for CHECKSUM_PARTIAL
4614 	 */
4615 	if (skb->ip_summed != CHECKSUM_PARTIAL)
4616 		return features;
4617 
4618 	/* We cannot support GSO if the MSS is going to be less than
4619 	 * 64 bytes.  If it is then we need to drop support for GSO.
4620 	 */
4621 	if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
4622 		features &= ~NETIF_F_GSO_MASK;
4623 
4624 	/* MACLEN can support at most 63 words */
4625 	len = skb_network_offset(skb);
4626 	if (len & ~(63 * 2))
4627 		goto out_err;
4628 
4629 	/* IPLEN and EIPLEN can support at most 127 dwords */
4630 	len = skb_network_header_len(skb);
4631 	if (len & ~(127 * 4))
4632 		goto out_err;
4633 
4634 	if (skb->encapsulation) {
4635 		/* L4TUNLEN can support 127 words */
4636 		len = skb_inner_network_header(skb) - skb_transport_header(skb);
4637 		if (len & ~(127 * 2))
4638 			goto out_err;
4639 
4640 		/* IPLEN can support at most 127 dwords */
4641 		len = skb_inner_transport_header(skb) -
4642 		      skb_inner_network_header(skb);
4643 		if (len & ~(127 * 4))
4644 			goto out_err;
4645 	}
4646 
4647 	/* No need to validate L4LEN as TCP is the only protocol with a
4648 	 * flexible value and we support all possible values supported
4649 	 * by TCP, which is at most 15 dwords
4650 	 */
4651 
4652 	return features;
4653 out_err:
4654 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4655 }
4656 
4657 /**
4658  * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off
4659  * @adapter: board private structure
4660  *
4661  * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4662  * were negotiated determine the VLAN features that can be toggled on and off.
4663  **/
4664 static netdev_features_t
iavf_get_netdev_vlan_hw_features(struct iavf_adapter * adapter)4665 iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter)
4666 {
4667 	netdev_features_t hw_features = 0;
4668 
4669 	if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4670 		return hw_features;
4671 
4672 	/* Enable VLAN features if supported */
4673 	if (VLAN_ALLOWED(adapter)) {
4674 		hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
4675 				NETIF_F_HW_VLAN_CTAG_RX);
4676 	} else if (VLAN_V2_ALLOWED(adapter)) {
4677 		struct virtchnl_vlan_caps *vlan_v2_caps =
4678 			&adapter->vlan_v2_caps;
4679 		struct virtchnl_vlan_supported_caps *stripping_support =
4680 			&vlan_v2_caps->offloads.stripping_support;
4681 		struct virtchnl_vlan_supported_caps *insertion_support =
4682 			&vlan_v2_caps->offloads.insertion_support;
4683 
4684 		if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4685 		    stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4686 			if (stripping_support->outer &
4687 			    VIRTCHNL_VLAN_ETHERTYPE_8100)
4688 				hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4689 			if (stripping_support->outer &
4690 			    VIRTCHNL_VLAN_ETHERTYPE_88A8)
4691 				hw_features |= NETIF_F_HW_VLAN_STAG_RX;
4692 		} else if (stripping_support->inner !=
4693 			   VIRTCHNL_VLAN_UNSUPPORTED &&
4694 			   stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4695 			if (stripping_support->inner &
4696 			    VIRTCHNL_VLAN_ETHERTYPE_8100)
4697 				hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4698 		}
4699 
4700 		if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4701 		    insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4702 			if (insertion_support->outer &
4703 			    VIRTCHNL_VLAN_ETHERTYPE_8100)
4704 				hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4705 			if (insertion_support->outer &
4706 			    VIRTCHNL_VLAN_ETHERTYPE_88A8)
4707 				hw_features |= NETIF_F_HW_VLAN_STAG_TX;
4708 		} else if (insertion_support->inner &&
4709 			   insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4710 			if (insertion_support->inner &
4711 			    VIRTCHNL_VLAN_ETHERTYPE_8100)
4712 				hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4713 		}
4714 	}
4715 
4716 	if (CRC_OFFLOAD_ALLOWED(adapter))
4717 		hw_features |= NETIF_F_RXFCS;
4718 
4719 	return hw_features;
4720 }
4721 
4722 /**
4723  * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures
4724  * @adapter: board private structure
4725  *
4726  * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4727  * were negotiated determine the VLAN features that are enabled by default.
4728  **/
4729 static netdev_features_t
iavf_get_netdev_vlan_features(struct iavf_adapter * adapter)4730 iavf_get_netdev_vlan_features(struct iavf_adapter *adapter)
4731 {
4732 	netdev_features_t features = 0;
4733 
4734 	if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4735 		return features;
4736 
4737 	if (VLAN_ALLOWED(adapter)) {
4738 		features |= NETIF_F_HW_VLAN_CTAG_FILTER |
4739 			NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX;
4740 	} else if (VLAN_V2_ALLOWED(adapter)) {
4741 		struct virtchnl_vlan_caps *vlan_v2_caps =
4742 			&adapter->vlan_v2_caps;
4743 		struct virtchnl_vlan_supported_caps *filtering_support =
4744 			&vlan_v2_caps->filtering.filtering_support;
4745 		struct virtchnl_vlan_supported_caps *stripping_support =
4746 			&vlan_v2_caps->offloads.stripping_support;
4747 		struct virtchnl_vlan_supported_caps *insertion_support =
4748 			&vlan_v2_caps->offloads.insertion_support;
4749 		u32 ethertype_init;
4750 
4751 		/* give priority to outer stripping and don't support both outer
4752 		 * and inner stripping
4753 		 */
4754 		ethertype_init = vlan_v2_caps->offloads.ethertype_init;
4755 		if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4756 			if (stripping_support->outer &
4757 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4758 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4759 				features |= NETIF_F_HW_VLAN_CTAG_RX;
4760 			else if (stripping_support->outer &
4761 				 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4762 				 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4763 				features |= NETIF_F_HW_VLAN_STAG_RX;
4764 		} else if (stripping_support->inner !=
4765 			   VIRTCHNL_VLAN_UNSUPPORTED) {
4766 			if (stripping_support->inner &
4767 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4768 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4769 				features |= NETIF_F_HW_VLAN_CTAG_RX;
4770 		}
4771 
4772 		/* give priority to outer insertion and don't support both outer
4773 		 * and inner insertion
4774 		 */
4775 		if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4776 			if (insertion_support->outer &
4777 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4778 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4779 				features |= NETIF_F_HW_VLAN_CTAG_TX;
4780 			else if (insertion_support->outer &
4781 				 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4782 				 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4783 				features |= NETIF_F_HW_VLAN_STAG_TX;
4784 		} else if (insertion_support->inner !=
4785 			   VIRTCHNL_VLAN_UNSUPPORTED) {
4786 			if (insertion_support->inner &
4787 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4788 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4789 				features |= NETIF_F_HW_VLAN_CTAG_TX;
4790 		}
4791 
4792 		/* give priority to outer filtering and don't bother if both
4793 		 * outer and inner filtering are enabled
4794 		 */
4795 		ethertype_init = vlan_v2_caps->filtering.ethertype_init;
4796 		if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4797 			if (filtering_support->outer &
4798 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4799 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4800 				features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4801 			if (filtering_support->outer &
4802 			    VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4803 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4804 				features |= NETIF_F_HW_VLAN_STAG_FILTER;
4805 		} else if (filtering_support->inner !=
4806 			   VIRTCHNL_VLAN_UNSUPPORTED) {
4807 			if (filtering_support->inner &
4808 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4809 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4810 				features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4811 			if (filtering_support->inner &
4812 			    VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4813 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4814 				features |= NETIF_F_HW_VLAN_STAG_FILTER;
4815 		}
4816 	}
4817 
4818 	return features;
4819 }
4820 
4821 #define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \
4822 	(!(((requested) & (feature_bit)) && \
4823 	   !((allowed) & (feature_bit))))
4824 
4825 /**
4826  * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support
4827  * @adapter: board private structure
4828  * @requested_features: stack requested NETDEV features
4829  **/
4830 static netdev_features_t
iavf_fix_netdev_vlan_features(struct iavf_adapter * adapter,netdev_features_t requested_features)4831 iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter,
4832 			      netdev_features_t requested_features)
4833 {
4834 	netdev_features_t allowed_features;
4835 
4836 	allowed_features = iavf_get_netdev_vlan_hw_features(adapter) |
4837 		iavf_get_netdev_vlan_features(adapter);
4838 
4839 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4840 					      allowed_features,
4841 					      NETIF_F_HW_VLAN_CTAG_TX))
4842 		requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX;
4843 
4844 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4845 					      allowed_features,
4846 					      NETIF_F_HW_VLAN_CTAG_RX))
4847 		requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
4848 
4849 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4850 					      allowed_features,
4851 					      NETIF_F_HW_VLAN_STAG_TX))
4852 		requested_features &= ~NETIF_F_HW_VLAN_STAG_TX;
4853 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4854 					      allowed_features,
4855 					      NETIF_F_HW_VLAN_STAG_RX))
4856 		requested_features &= ~NETIF_F_HW_VLAN_STAG_RX;
4857 
4858 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4859 					      allowed_features,
4860 					      NETIF_F_HW_VLAN_CTAG_FILTER))
4861 		requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
4862 
4863 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4864 					      allowed_features,
4865 					      NETIF_F_HW_VLAN_STAG_FILTER))
4866 		requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER;
4867 
4868 	if ((requested_features &
4869 	     (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
4870 	    (requested_features &
4871 	     (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) &&
4872 	    adapter->vlan_v2_caps.offloads.ethertype_match ==
4873 	    VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) {
4874 		netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
4875 		requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX |
4876 					NETIF_F_HW_VLAN_STAG_TX);
4877 	}
4878 
4879 	return requested_features;
4880 }
4881 
4882 /**
4883  * iavf_fix_strip_features - fix NETDEV CRC and VLAN strip features
4884  * @adapter: board private structure
4885  * @requested_features: stack requested NETDEV features
4886  *
4887  * Returns fixed-up features bits
4888  **/
4889 static netdev_features_t
iavf_fix_strip_features(struct iavf_adapter * adapter,netdev_features_t requested_features)4890 iavf_fix_strip_features(struct iavf_adapter *adapter,
4891 			netdev_features_t requested_features)
4892 {
4893 	struct net_device *netdev = adapter->netdev;
4894 	bool crc_offload_req, is_vlan_strip;
4895 	netdev_features_t vlan_strip;
4896 	int num_non_zero_vlan;
4897 
4898 	crc_offload_req = CRC_OFFLOAD_ALLOWED(adapter) &&
4899 			  (requested_features & NETIF_F_RXFCS);
4900 	num_non_zero_vlan = iavf_get_num_vlans_added(adapter);
4901 	vlan_strip = (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX);
4902 	is_vlan_strip = requested_features & vlan_strip;
4903 
4904 	if (!crc_offload_req)
4905 		return requested_features;
4906 
4907 	if (!num_non_zero_vlan && (netdev->features & vlan_strip) &&
4908 	    !(netdev->features & NETIF_F_RXFCS) && is_vlan_strip) {
4909 		requested_features &= ~vlan_strip;
4910 		netdev_info(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n");
4911 		return requested_features;
4912 	}
4913 
4914 	if ((netdev->features & NETIF_F_RXFCS) && is_vlan_strip) {
4915 		requested_features &= ~vlan_strip;
4916 		if (!(netdev->features & vlan_strip))
4917 			netdev_info(netdev, "To enable VLAN stripping, first need to enable FCS/CRC stripping");
4918 
4919 		return requested_features;
4920 	}
4921 
4922 	if (num_non_zero_vlan && is_vlan_strip &&
4923 	    !(netdev->features & NETIF_F_RXFCS)) {
4924 		requested_features &= ~NETIF_F_RXFCS;
4925 		netdev_info(netdev, "To disable FCS/CRC stripping, first need to disable VLAN stripping");
4926 	}
4927 
4928 	return requested_features;
4929 }
4930 
4931 /**
4932  * iavf_fix_features - fix up the netdev feature bits
4933  * @netdev: our net device
4934  * @features: desired feature bits
4935  *
4936  * Returns fixed-up features bits
4937  **/
iavf_fix_features(struct net_device * netdev,netdev_features_t features)4938 static netdev_features_t iavf_fix_features(struct net_device *netdev,
4939 					   netdev_features_t features)
4940 {
4941 	struct iavf_adapter *adapter = netdev_priv(netdev);
4942 
4943 	features = iavf_fix_netdev_vlan_features(adapter, features);
4944 
4945 	if (!FDIR_FLTR_SUPPORT(adapter))
4946 		features &= ~NETIF_F_NTUPLE;
4947 
4948 	return iavf_fix_strip_features(adapter, features);
4949 }
4950 
4951 static int
iavf_verify_shaper(struct net_shaper_binding * binding,const struct net_shaper * shaper,struct netlink_ext_ack * extack)4952 iavf_verify_shaper(struct net_shaper_binding *binding,
4953 		   const struct net_shaper *shaper,
4954 		   struct netlink_ext_ack *extack)
4955 {
4956 	struct iavf_adapter *adapter = netdev_priv(binding->netdev);
4957 	u64 vf_max;
4958 
4959 	if (shaper->handle.scope == NET_SHAPER_SCOPE_QUEUE) {
4960 		vf_max = adapter->qos_caps->cap[0].shaper.peak;
4961 		if (vf_max && shaper->bw_max > vf_max) {
4962 			NL_SET_ERR_MSG_FMT(extack, "Max rate (%llu) of queue %d can't exceed max TX rate of VF (%llu kbps)",
4963 					   shaper->bw_max, shaper->handle.id,
4964 					   vf_max);
4965 			return -EINVAL;
4966 		}
4967 	}
4968 	return 0;
4969 }
4970 
4971 static int
iavf_shaper_set(struct net_shaper_binding * binding,const struct net_shaper * shaper,struct netlink_ext_ack * extack)4972 iavf_shaper_set(struct net_shaper_binding *binding,
4973 		const struct net_shaper *shaper,
4974 		struct netlink_ext_ack *extack)
4975 {
4976 	struct iavf_adapter *adapter = netdev_priv(binding->netdev);
4977 	const struct net_shaper_handle *handle = &shaper->handle;
4978 	struct iavf_ring *tx_ring;
4979 	int ret = 0;
4980 
4981 	mutex_lock(&adapter->crit_lock);
4982 	if (handle->id >= adapter->num_active_queues)
4983 		goto unlock;
4984 
4985 	ret = iavf_verify_shaper(binding, shaper, extack);
4986 	if (ret)
4987 		goto unlock;
4988 
4989 	tx_ring = &adapter->tx_rings[handle->id];
4990 
4991 	tx_ring->q_shaper.bw_min = div_u64(shaper->bw_min, 1000);
4992 	tx_ring->q_shaper.bw_max = div_u64(shaper->bw_max, 1000);
4993 	tx_ring->q_shaper_update = true;
4994 
4995 	adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES_BW;
4996 
4997 unlock:
4998 	mutex_unlock(&adapter->crit_lock);
4999 	return ret;
5000 }
5001 
iavf_shaper_del(struct net_shaper_binding * binding,const struct net_shaper_handle * handle,struct netlink_ext_ack * extack)5002 static int iavf_shaper_del(struct net_shaper_binding *binding,
5003 			   const struct net_shaper_handle *handle,
5004 			   struct netlink_ext_ack *extack)
5005 {
5006 	struct iavf_adapter *adapter = netdev_priv(binding->netdev);
5007 	struct iavf_ring *tx_ring;
5008 
5009 	mutex_lock(&adapter->crit_lock);
5010 	if (handle->id >= adapter->num_active_queues)
5011 		goto unlock;
5012 
5013 	tx_ring = &adapter->tx_rings[handle->id];
5014 	tx_ring->q_shaper.bw_min = 0;
5015 	tx_ring->q_shaper.bw_max = 0;
5016 	tx_ring->q_shaper_update = true;
5017 
5018 	adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES_BW;
5019 
5020 unlock:
5021 	mutex_unlock(&adapter->crit_lock);
5022 	return 0;
5023 }
5024 
iavf_shaper_cap(struct net_shaper_binding * binding,enum net_shaper_scope scope,unsigned long * flags)5025 static void iavf_shaper_cap(struct net_shaper_binding *binding,
5026 			    enum net_shaper_scope scope,
5027 			    unsigned long *flags)
5028 {
5029 	if (scope != NET_SHAPER_SCOPE_QUEUE)
5030 		return;
5031 
5032 	*flags = BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MIN) |
5033 		 BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MAX) |
5034 		 BIT(NET_SHAPER_A_CAPS_SUPPORT_METRIC_BPS);
5035 }
5036 
5037 static const struct net_shaper_ops iavf_shaper_ops = {
5038 	.set = iavf_shaper_set,
5039 	.delete = iavf_shaper_del,
5040 	.capabilities = iavf_shaper_cap,
5041 };
5042 
5043 static const struct net_device_ops iavf_netdev_ops = {
5044 	.ndo_open		= iavf_open,
5045 	.ndo_stop		= iavf_close,
5046 	.ndo_start_xmit		= iavf_xmit_frame,
5047 	.ndo_set_rx_mode	= iavf_set_rx_mode,
5048 	.ndo_validate_addr	= eth_validate_addr,
5049 	.ndo_set_mac_address	= iavf_set_mac,
5050 	.ndo_change_mtu		= iavf_change_mtu,
5051 	.ndo_tx_timeout		= iavf_tx_timeout,
5052 	.ndo_vlan_rx_add_vid	= iavf_vlan_rx_add_vid,
5053 	.ndo_vlan_rx_kill_vid	= iavf_vlan_rx_kill_vid,
5054 	.ndo_features_check	= iavf_features_check,
5055 	.ndo_fix_features	= iavf_fix_features,
5056 	.ndo_set_features	= iavf_set_features,
5057 	.ndo_setup_tc		= iavf_setup_tc,
5058 	.net_shaper_ops		= &iavf_shaper_ops,
5059 };
5060 
5061 /**
5062  * iavf_check_reset_complete - check that VF reset is complete
5063  * @hw: pointer to hw struct
5064  *
5065  * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
5066  **/
iavf_check_reset_complete(struct iavf_hw * hw)5067 static int iavf_check_reset_complete(struct iavf_hw *hw)
5068 {
5069 	u32 rstat;
5070 	int i;
5071 
5072 	for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
5073 		rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
5074 			     IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
5075 		if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
5076 		    (rstat == VIRTCHNL_VFR_COMPLETED))
5077 			return 0;
5078 		msleep(IAVF_RESET_WAIT_MS);
5079 	}
5080 	return -EBUSY;
5081 }
5082 
5083 /**
5084  * iavf_process_config - Process the config information we got from the PF
5085  * @adapter: board private structure
5086  *
5087  * Verify that we have a valid config struct, and set up our netdev features
5088  * and our VSI struct.
5089  **/
iavf_process_config(struct iavf_adapter * adapter)5090 int iavf_process_config(struct iavf_adapter *adapter)
5091 {
5092 	struct virtchnl_vf_resource *vfres = adapter->vf_res;
5093 	netdev_features_t hw_vlan_features, vlan_features;
5094 	struct net_device *netdev = adapter->netdev;
5095 	netdev_features_t hw_enc_features;
5096 	netdev_features_t hw_features;
5097 
5098 	hw_enc_features = NETIF_F_SG			|
5099 			  NETIF_F_IP_CSUM		|
5100 			  NETIF_F_IPV6_CSUM		|
5101 			  NETIF_F_HIGHDMA		|
5102 			  NETIF_F_SOFT_FEATURES	|
5103 			  NETIF_F_TSO			|
5104 			  NETIF_F_TSO_ECN		|
5105 			  NETIF_F_TSO6			|
5106 			  NETIF_F_SCTP_CRC		|
5107 			  NETIF_F_RXHASH		|
5108 			  NETIF_F_RXCSUM		|
5109 			  0;
5110 
5111 	/* advertise to stack only if offloads for encapsulated packets is
5112 	 * supported
5113 	 */
5114 	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
5115 		hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL	|
5116 				   NETIF_F_GSO_GRE		|
5117 				   NETIF_F_GSO_GRE_CSUM		|
5118 				   NETIF_F_GSO_IPXIP4		|
5119 				   NETIF_F_GSO_IPXIP6		|
5120 				   NETIF_F_GSO_UDP_TUNNEL_CSUM	|
5121 				   NETIF_F_GSO_PARTIAL		|
5122 				   0;
5123 
5124 		if (!(vfres->vf_cap_flags &
5125 		      VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
5126 			netdev->gso_partial_features |=
5127 				NETIF_F_GSO_UDP_TUNNEL_CSUM;
5128 
5129 		netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
5130 		netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
5131 		netdev->hw_enc_features |= hw_enc_features;
5132 	}
5133 	/* record features VLANs can make use of */
5134 	netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
5135 
5136 	/* Write features and hw_features separately to avoid polluting
5137 	 * with, or dropping, features that are set when we registered.
5138 	 */
5139 	hw_features = hw_enc_features;
5140 
5141 	/* get HW VLAN features that can be toggled */
5142 	hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter);
5143 
5144 	/* Enable HW TC offload if ADQ or tc U32 is supported */
5145 	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ ||
5146 	    TC_U32_SUPPORT(adapter))
5147 		hw_features |= NETIF_F_HW_TC;
5148 
5149 	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)
5150 		hw_features |= NETIF_F_GSO_UDP_L4;
5151 
5152 	netdev->hw_features |= hw_features | hw_vlan_features;
5153 	vlan_features = iavf_get_netdev_vlan_features(adapter);
5154 
5155 	netdev->features |= hw_features | vlan_features;
5156 
5157 	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
5158 		netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
5159 
5160 	if (FDIR_FLTR_SUPPORT(adapter)) {
5161 		netdev->hw_features |= NETIF_F_NTUPLE;
5162 		netdev->features |= NETIF_F_NTUPLE;
5163 		adapter->flags |= IAVF_FLAG_FDIR_ENABLED;
5164 	}
5165 
5166 	netdev->priv_flags |= IFF_UNICAST_FLT;
5167 
5168 	/* Do not turn on offloads when they are requested to be turned off.
5169 	 * TSO needs minimum 576 bytes to work correctly.
5170 	 */
5171 	if (netdev->wanted_features) {
5172 		if (!(netdev->wanted_features & NETIF_F_TSO) ||
5173 		    netdev->mtu < 576)
5174 			netdev->features &= ~NETIF_F_TSO;
5175 		if (!(netdev->wanted_features & NETIF_F_TSO6) ||
5176 		    netdev->mtu < 576)
5177 			netdev->features &= ~NETIF_F_TSO6;
5178 		if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
5179 			netdev->features &= ~NETIF_F_TSO_ECN;
5180 		if (!(netdev->wanted_features & NETIF_F_GRO))
5181 			netdev->features &= ~NETIF_F_GRO;
5182 		if (!(netdev->wanted_features & NETIF_F_GSO))
5183 			netdev->features &= ~NETIF_F_GSO;
5184 	}
5185 
5186 	return 0;
5187 }
5188 
5189 /**
5190  * iavf_probe - Device Initialization Routine
5191  * @pdev: PCI device information struct
5192  * @ent: entry in iavf_pci_tbl
5193  *
5194  * Returns 0 on success, negative on failure
5195  *
5196  * iavf_probe initializes an adapter identified by a pci_dev structure.
5197  * The OS initialization, configuring of the adapter private structure,
5198  * and a hardware reset occur.
5199  **/
iavf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)5200 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
5201 {
5202 	struct net_device *netdev;
5203 	struct iavf_adapter *adapter = NULL;
5204 	struct iavf_hw *hw = NULL;
5205 	int err, len;
5206 
5207 	err = pci_enable_device(pdev);
5208 	if (err)
5209 		return err;
5210 
5211 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
5212 	if (err) {
5213 		dev_err(&pdev->dev,
5214 			"DMA configuration failed: 0x%x\n", err);
5215 		goto err_dma;
5216 	}
5217 
5218 	err = pci_request_regions(pdev, iavf_driver_name);
5219 	if (err) {
5220 		dev_err(&pdev->dev,
5221 			"pci_request_regions failed 0x%x\n", err);
5222 		goto err_pci_reg;
5223 	}
5224 
5225 	pci_set_master(pdev);
5226 
5227 	netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
5228 				   IAVF_MAX_REQ_QUEUES);
5229 	if (!netdev) {
5230 		err = -ENOMEM;
5231 		goto err_alloc_etherdev;
5232 	}
5233 
5234 	SET_NETDEV_DEV(netdev, &pdev->dev);
5235 
5236 	pci_set_drvdata(pdev, netdev);
5237 	adapter = netdev_priv(netdev);
5238 
5239 	adapter->netdev = netdev;
5240 	adapter->pdev = pdev;
5241 
5242 	hw = &adapter->hw;
5243 	hw->back = adapter;
5244 
5245 	adapter->wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM,
5246 					      iavf_driver_name);
5247 	if (!adapter->wq) {
5248 		err = -ENOMEM;
5249 		goto err_alloc_wq;
5250 	}
5251 
5252 	adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
5253 	iavf_change_state(adapter, __IAVF_STARTUP);
5254 
5255 	/* Call save state here because it relies on the adapter struct. */
5256 	pci_save_state(pdev);
5257 
5258 	hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
5259 			      pci_resource_len(pdev, 0));
5260 	if (!hw->hw_addr) {
5261 		err = -EIO;
5262 		goto err_ioremap;
5263 	}
5264 	hw->vendor_id = pdev->vendor;
5265 	hw->device_id = pdev->device;
5266 	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
5267 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
5268 	hw->subsystem_device_id = pdev->subsystem_device;
5269 	hw->bus.device = PCI_SLOT(pdev->devfn);
5270 	hw->bus.func = PCI_FUNC(pdev->devfn);
5271 	hw->bus.bus_id = pdev->bus->number;
5272 
5273 	len = struct_size(adapter->qos_caps, cap, IAVF_MAX_QOS_TC_NUM);
5274 	adapter->qos_caps = kzalloc(len, GFP_KERNEL);
5275 	if (!adapter->qos_caps) {
5276 		err = -ENOMEM;
5277 		goto err_alloc_qos_cap;
5278 	}
5279 
5280 	/* set up the locks for the AQ, do this only once in probe
5281 	 * and destroy them only once in remove
5282 	 */
5283 	mutex_init(&adapter->crit_lock);
5284 	mutex_init(&hw->aq.asq_mutex);
5285 	mutex_init(&hw->aq.arq_mutex);
5286 
5287 	spin_lock_init(&adapter->mac_vlan_list_lock);
5288 	spin_lock_init(&adapter->cloud_filter_list_lock);
5289 	spin_lock_init(&adapter->fdir_fltr_lock);
5290 	spin_lock_init(&adapter->adv_rss_lock);
5291 	spin_lock_init(&adapter->current_netdev_promisc_flags_lock);
5292 
5293 	INIT_LIST_HEAD(&adapter->mac_filter_list);
5294 	INIT_LIST_HEAD(&adapter->vlan_filter_list);
5295 	INIT_LIST_HEAD(&adapter->cloud_filter_list);
5296 	INIT_LIST_HEAD(&adapter->fdir_list_head);
5297 	INIT_LIST_HEAD(&adapter->adv_rss_list_head);
5298 
5299 	INIT_WORK(&adapter->reset_task, iavf_reset_task);
5300 	INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
5301 	INIT_WORK(&adapter->finish_config, iavf_finish_config);
5302 	INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
5303 
5304 	/* Setup the wait queue for indicating transition to down status */
5305 	init_waitqueue_head(&adapter->down_waitqueue);
5306 
5307 	/* Setup the wait queue for indicating transition to running state */
5308 	init_waitqueue_head(&adapter->reset_waitqueue);
5309 
5310 	/* Setup the wait queue for indicating virtchannel events */
5311 	init_waitqueue_head(&adapter->vc_waitqueue);
5312 
5313 	queue_delayed_work(adapter->wq, &adapter->watchdog_task,
5314 			   msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
5315 	/* Initialization goes on in the work. Do not add more of it below. */
5316 	return 0;
5317 
5318 err_alloc_qos_cap:
5319 	iounmap(hw->hw_addr);
5320 err_ioremap:
5321 	destroy_workqueue(adapter->wq);
5322 err_alloc_wq:
5323 	free_netdev(netdev);
5324 err_alloc_etherdev:
5325 	pci_release_regions(pdev);
5326 err_pci_reg:
5327 err_dma:
5328 	pci_disable_device(pdev);
5329 	return err;
5330 }
5331 
5332 /**
5333  * iavf_suspend - Power management suspend routine
5334  * @dev_d: device info pointer
5335  *
5336  * Called when the system (VM) is entering sleep/suspend.
5337  **/
iavf_suspend(struct device * dev_d)5338 static int iavf_suspend(struct device *dev_d)
5339 {
5340 	struct net_device *netdev = dev_get_drvdata(dev_d);
5341 	struct iavf_adapter *adapter = netdev_priv(netdev);
5342 
5343 	netif_device_detach(netdev);
5344 
5345 	mutex_lock(&adapter->crit_lock);
5346 
5347 	if (netif_running(netdev)) {
5348 		rtnl_lock();
5349 		iavf_down(adapter);
5350 		rtnl_unlock();
5351 	}
5352 	iavf_free_misc_irq(adapter);
5353 	iavf_reset_interrupt_capability(adapter);
5354 
5355 	mutex_unlock(&adapter->crit_lock);
5356 
5357 	return 0;
5358 }
5359 
5360 /**
5361  * iavf_resume - Power management resume routine
5362  * @dev_d: device info pointer
5363  *
5364  * Called when the system (VM) is resumed from sleep/suspend.
5365  **/
iavf_resume(struct device * dev_d)5366 static int iavf_resume(struct device *dev_d)
5367 {
5368 	struct pci_dev *pdev = to_pci_dev(dev_d);
5369 	struct iavf_adapter *adapter;
5370 	u32 err;
5371 
5372 	adapter = iavf_pdev_to_adapter(pdev);
5373 
5374 	pci_set_master(pdev);
5375 
5376 	rtnl_lock();
5377 	err = iavf_set_interrupt_capability(adapter);
5378 	if (err) {
5379 		rtnl_unlock();
5380 		dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
5381 		return err;
5382 	}
5383 	err = iavf_request_misc_irq(adapter);
5384 	rtnl_unlock();
5385 	if (err) {
5386 		dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
5387 		return err;
5388 	}
5389 
5390 	queue_work(adapter->wq, &adapter->reset_task);
5391 
5392 	netif_device_attach(adapter->netdev);
5393 
5394 	return err;
5395 }
5396 
5397 /**
5398  * iavf_remove - Device Removal Routine
5399  * @pdev: PCI device information struct
5400  *
5401  * iavf_remove is called by the PCI subsystem to alert the driver
5402  * that it should release a PCI device.  The could be caused by a
5403  * Hot-Plug event, or because the driver is going to be removed from
5404  * memory.
5405  **/
iavf_remove(struct pci_dev * pdev)5406 static void iavf_remove(struct pci_dev *pdev)
5407 {
5408 	struct iavf_fdir_fltr *fdir, *fdirtmp;
5409 	struct iavf_vlan_filter *vlf, *vlftmp;
5410 	struct iavf_cloud_filter *cf, *cftmp;
5411 	struct iavf_adv_rss *rss, *rsstmp;
5412 	struct iavf_mac_filter *f, *ftmp;
5413 	struct iavf_adapter *adapter;
5414 	struct net_device *netdev;
5415 	struct iavf_hw *hw;
5416 
5417 	/* Don't proceed with remove if netdev is already freed */
5418 	netdev = pci_get_drvdata(pdev);
5419 	if (!netdev)
5420 		return;
5421 
5422 	adapter = iavf_pdev_to_adapter(pdev);
5423 	hw = &adapter->hw;
5424 
5425 	if (test_and_set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
5426 		return;
5427 
5428 	/* Wait until port initialization is complete.
5429 	 * There are flows where register/unregister netdev may race.
5430 	 */
5431 	while (1) {
5432 		mutex_lock(&adapter->crit_lock);
5433 		if (adapter->state == __IAVF_RUNNING ||
5434 		    adapter->state == __IAVF_DOWN ||
5435 		    adapter->state == __IAVF_INIT_FAILED) {
5436 			mutex_unlock(&adapter->crit_lock);
5437 			break;
5438 		}
5439 		/* Simply return if we already went through iavf_shutdown */
5440 		if (adapter->state == __IAVF_REMOVE) {
5441 			mutex_unlock(&adapter->crit_lock);
5442 			return;
5443 		}
5444 
5445 		mutex_unlock(&adapter->crit_lock);
5446 		usleep_range(500, 1000);
5447 	}
5448 	cancel_delayed_work_sync(&adapter->watchdog_task);
5449 	cancel_work_sync(&adapter->finish_config);
5450 
5451 	if (netdev->reg_state == NETREG_REGISTERED)
5452 		unregister_netdev(netdev);
5453 
5454 	mutex_lock(&adapter->crit_lock);
5455 	dev_info(&adapter->pdev->dev, "Removing device\n");
5456 	iavf_change_state(adapter, __IAVF_REMOVE);
5457 
5458 	iavf_request_reset(adapter);
5459 	msleep(50);
5460 	/* If the FW isn't responding, kick it once, but only once. */
5461 	if (!iavf_asq_done(hw)) {
5462 		iavf_request_reset(adapter);
5463 		msleep(50);
5464 	}
5465 
5466 	iavf_misc_irq_disable(adapter);
5467 	/* Shut down all the garbage mashers on the detention level */
5468 	cancel_work_sync(&adapter->reset_task);
5469 	cancel_delayed_work_sync(&adapter->watchdog_task);
5470 	cancel_work_sync(&adapter->adminq_task);
5471 
5472 	adapter->aq_required = 0;
5473 	adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
5474 
5475 	iavf_free_all_tx_resources(adapter);
5476 	iavf_free_all_rx_resources(adapter);
5477 	iavf_free_misc_irq(adapter);
5478 	iavf_free_interrupt_scheme(adapter);
5479 
5480 	iavf_free_rss(adapter);
5481 
5482 	if (hw->aq.asq.count)
5483 		iavf_shutdown_adminq(hw);
5484 
5485 	/* destroy the locks only once, here */
5486 	mutex_destroy(&hw->aq.arq_mutex);
5487 	mutex_destroy(&hw->aq.asq_mutex);
5488 	mutex_unlock(&adapter->crit_lock);
5489 	mutex_destroy(&adapter->crit_lock);
5490 
5491 	iounmap(hw->hw_addr);
5492 	pci_release_regions(pdev);
5493 	kfree(adapter->vf_res);
5494 	spin_lock_bh(&adapter->mac_vlan_list_lock);
5495 	/* If we got removed before an up/down sequence, we've got a filter
5496 	 * hanging out there that we need to get rid of.
5497 	 */
5498 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
5499 		list_del(&f->list);
5500 		kfree(f);
5501 	}
5502 	list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
5503 				 list) {
5504 		list_del(&vlf->list);
5505 		kfree(vlf);
5506 	}
5507 
5508 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
5509 
5510 	spin_lock_bh(&adapter->cloud_filter_list_lock);
5511 	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
5512 		list_del(&cf->list);
5513 		kfree(cf);
5514 	}
5515 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
5516 
5517 	spin_lock_bh(&adapter->fdir_fltr_lock);
5518 	list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {
5519 		list_del(&fdir->list);
5520 		kfree(fdir);
5521 	}
5522 	spin_unlock_bh(&adapter->fdir_fltr_lock);
5523 
5524 	spin_lock_bh(&adapter->adv_rss_lock);
5525 	list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
5526 				 list) {
5527 		list_del(&rss->list);
5528 		kfree(rss);
5529 	}
5530 	spin_unlock_bh(&adapter->adv_rss_lock);
5531 
5532 	destroy_workqueue(adapter->wq);
5533 
5534 	pci_set_drvdata(pdev, NULL);
5535 
5536 	free_netdev(netdev);
5537 
5538 	pci_disable_device(pdev);
5539 }
5540 
5541 /**
5542  * iavf_shutdown - Shutdown the device in preparation for a reboot
5543  * @pdev: pci device structure
5544  **/
iavf_shutdown(struct pci_dev * pdev)5545 static void iavf_shutdown(struct pci_dev *pdev)
5546 {
5547 	iavf_remove(pdev);
5548 
5549 	if (system_state == SYSTEM_POWER_OFF)
5550 		pci_set_power_state(pdev, PCI_D3hot);
5551 }
5552 
5553 static DEFINE_SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);
5554 
5555 static struct pci_driver iavf_driver = {
5556 	.name      = iavf_driver_name,
5557 	.id_table  = iavf_pci_tbl,
5558 	.probe     = iavf_probe,
5559 	.remove    = iavf_remove,
5560 	.driver.pm = pm_sleep_ptr(&iavf_pm_ops),
5561 	.shutdown  = iavf_shutdown,
5562 };
5563 
5564 /**
5565  * iavf_init_module - Driver Registration Routine
5566  *
5567  * iavf_init_module is the first routine called when the driver is
5568  * loaded. All it does is register with the PCI subsystem.
5569  **/
iavf_init_module(void)5570 static int __init iavf_init_module(void)
5571 {
5572 	pr_info("iavf: %s\n", iavf_driver_string);
5573 
5574 	pr_info("%s\n", iavf_copyright);
5575 
5576 	return pci_register_driver(&iavf_driver);
5577 }
5578 
5579 module_init(iavf_init_module);
5580 
5581 /**
5582  * iavf_exit_module - Driver Exit Cleanup Routine
5583  *
5584  * iavf_exit_module is called just before the driver is removed
5585  * from memory.
5586  **/
iavf_exit_module(void)5587 static void __exit iavf_exit_module(void)
5588 {
5589 	pci_unregister_driver(&iavf_driver);
5590 }
5591 
5592 module_exit(iavf_exit_module);
5593 
5594 /* iavf_main.c */
5595