1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
3
4 #include <generated/utsrelease.h>
5 #include <linux/crash_dump.h>
6 #include <linux/if_bridge.h>
7 #include <linux/if_macvlan.h>
8 #include <linux/module.h>
9 #include <net/pkt_cls.h>
10 #include <net/xdp_sock_drv.h>
11
12 /* Local includes */
13 #include "i40e.h"
14 #include "i40e_devids.h"
15 #include "i40e_diag.h"
16 #include "i40e_lan_hmc.h"
17 #include "i40e_virtchnl_pf.h"
18 #include "i40e_xsk.h"
19
20 /* All i40e tracepoints are defined by the include below, which
21 * must be included exactly once across the whole kernel with
22 * CREATE_TRACE_POINTS defined
23 */
24 #define CREATE_TRACE_POINTS
25 #include "i40e_trace.h"
26
27 const char i40e_driver_name[] = "i40e";
28 static const char i40e_driver_string[] =
29 "Intel(R) Ethernet Connection XL710 Network Driver";
30
31 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
32
33 /* a bit of forward declarations */
34 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
35 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
36 static int i40e_add_vsi(struct i40e_vsi *vsi);
37 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
38 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
39 static int i40e_setup_misc_vector(struct i40e_pf *pf);
40 static void i40e_determine_queue_usage(struct i40e_pf *pf);
41 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
42 static void i40e_prep_for_reset(struct i40e_pf *pf);
43 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
44 bool lock_acquired);
45 static int i40e_reset(struct i40e_pf *pf);
46 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
47 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
48 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
49 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
50 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
51 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
52 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
53 static int i40e_get_capabilities(struct i40e_pf *pf,
54 enum i40e_admin_queue_opc list_type);
55 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
56
57 /* i40e_pci_tbl - PCI Device ID Table
58 *
59 * Last entry must be all 0s
60 *
61 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
62 * Class, Class Mask, private data (not used) }
63 */
64 static const struct pci_device_id i40e_pci_tbl[] = {
65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0},
73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0},
85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
91 /* required last entry */
92 {0, }
93 };
94 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
95
96 #define I40E_MAX_VF_COUNT 128
97 static int debug = -1;
98 module_param(debug, uint, 0);
99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
100
101 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
102 MODULE_IMPORT_NS(LIBIE);
103 MODULE_LICENSE("GPL v2");
104
105 static struct workqueue_struct *i40e_wq;
106
netdev_hw_addr_refcnt(struct i40e_mac_filter * f,struct net_device * netdev,int delta)107 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
108 struct net_device *netdev, int delta)
109 {
110 struct netdev_hw_addr_list *ha_list;
111 struct netdev_hw_addr *ha;
112
113 if (!f || !netdev)
114 return;
115
116 if (is_unicast_ether_addr(f->macaddr) || is_link_local_ether_addr(f->macaddr))
117 ha_list = &netdev->uc;
118 else
119 ha_list = &netdev->mc;
120
121 netdev_hw_addr_list_for_each(ha, ha_list) {
122 if (ether_addr_equal(ha->addr, f->macaddr)) {
123 ha->refcount += delta;
124 if (ha->refcount <= 0)
125 ha->refcount = 1;
126 break;
127 }
128 }
129 }
130
131 /**
132 * i40e_hw_to_dev - get device pointer from the hardware structure
133 * @hw: pointer to the device HW structure
134 **/
i40e_hw_to_dev(struct i40e_hw * hw)135 struct device *i40e_hw_to_dev(struct i40e_hw *hw)
136 {
137 struct i40e_pf *pf = i40e_hw_to_pf(hw);
138
139 return &pf->pdev->dev;
140 }
141
142 /**
143 * i40e_allocate_dma_mem - OS specific memory alloc for shared code
144 * @hw: pointer to the HW structure
145 * @mem: ptr to mem struct to fill out
146 * @size: size of memory requested
147 * @alignment: what to align the allocation to
148 **/
i40e_allocate_dma_mem(struct i40e_hw * hw,struct i40e_dma_mem * mem,u64 size,u32 alignment)149 int i40e_allocate_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem,
150 u64 size, u32 alignment)
151 {
152 struct i40e_pf *pf = i40e_hw_to_pf(hw);
153
154 mem->size = ALIGN(size, alignment);
155 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
156 GFP_KERNEL);
157 if (!mem->va)
158 return -ENOMEM;
159
160 return 0;
161 }
162
163 /**
164 * i40e_free_dma_mem - OS specific memory free for shared code
165 * @hw: pointer to the HW structure
166 * @mem: ptr to mem struct to free
167 **/
i40e_free_dma_mem(struct i40e_hw * hw,struct i40e_dma_mem * mem)168 int i40e_free_dma_mem(struct i40e_hw *hw, struct i40e_dma_mem *mem)
169 {
170 struct i40e_pf *pf = i40e_hw_to_pf(hw);
171
172 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
173 mem->va = NULL;
174 mem->pa = 0;
175 mem->size = 0;
176
177 return 0;
178 }
179
180 /**
181 * i40e_allocate_virt_mem - OS specific memory alloc for shared code
182 * @hw: pointer to the HW structure
183 * @mem: ptr to mem struct to fill out
184 * @size: size of memory requested
185 **/
i40e_allocate_virt_mem(struct i40e_hw * hw,struct i40e_virt_mem * mem,u32 size)186 int i40e_allocate_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem,
187 u32 size)
188 {
189 mem->size = size;
190 mem->va = kzalloc(size, GFP_KERNEL);
191
192 if (!mem->va)
193 return -ENOMEM;
194
195 return 0;
196 }
197
198 /**
199 * i40e_free_virt_mem - OS specific memory free for shared code
200 * @hw: pointer to the HW structure
201 * @mem: ptr to mem struct to free
202 **/
i40e_free_virt_mem(struct i40e_hw * hw,struct i40e_virt_mem * mem)203 int i40e_free_virt_mem(struct i40e_hw *hw, struct i40e_virt_mem *mem)
204 {
205 /* it's ok to kfree a NULL pointer */
206 kfree(mem->va);
207 mem->va = NULL;
208 mem->size = 0;
209
210 return 0;
211 }
212
213 /**
214 * i40e_get_lump - find a lump of free generic resource
215 * @pf: board private structure
216 * @pile: the pile of resource to search
217 * @needed: the number of items needed
218 * @id: an owner id to stick on the items assigned
219 *
220 * Returns the base item index of the lump, or negative for error
221 **/
i40e_get_lump(struct i40e_pf * pf,struct i40e_lump_tracking * pile,u16 needed,u16 id)222 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
223 u16 needed, u16 id)
224 {
225 int ret = -ENOMEM;
226 int i, j;
227
228 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
229 dev_info(&pf->pdev->dev,
230 "param err: pile=%s needed=%d id=0x%04x\n",
231 pile ? "<valid>" : "<null>", needed, id);
232 return -EINVAL;
233 }
234
235 /* Allocate last queue in the pile for FDIR VSI queue
236 * so it doesn't fragment the qp_pile
237 */
238 if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
239 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
240 dev_err(&pf->pdev->dev,
241 "Cannot allocate queue %d for I40E_VSI_FDIR\n",
242 pile->num_entries - 1);
243 return -ENOMEM;
244 }
245 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
246 return pile->num_entries - 1;
247 }
248
249 i = 0;
250 while (i < pile->num_entries) {
251 /* skip already allocated entries */
252 if (pile->list[i] & I40E_PILE_VALID_BIT) {
253 i++;
254 continue;
255 }
256
257 /* do we have enough in this lump? */
258 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
259 if (pile->list[i+j] & I40E_PILE_VALID_BIT)
260 break;
261 }
262
263 if (j == needed) {
264 /* there was enough, so assign it to the requestor */
265 for (j = 0; j < needed; j++)
266 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
267 ret = i;
268 break;
269 }
270
271 /* not enough, so skip over it and continue looking */
272 i += j;
273 }
274
275 return ret;
276 }
277
278 /**
279 * i40e_put_lump - return a lump of generic resource
280 * @pile: the pile of resource to search
281 * @index: the base item index
282 * @id: the owner id of the items assigned
283 *
284 * Returns the count of items in the lump
285 **/
i40e_put_lump(struct i40e_lump_tracking * pile,u16 index,u16 id)286 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
287 {
288 int valid_id = (id | I40E_PILE_VALID_BIT);
289 int count = 0;
290 u16 i;
291
292 if (!pile || index >= pile->num_entries)
293 return -EINVAL;
294
295 for (i = index;
296 i < pile->num_entries && pile->list[i] == valid_id;
297 i++) {
298 pile->list[i] = 0;
299 count++;
300 }
301
302
303 return count;
304 }
305
306 /**
307 * i40e_find_vsi_from_id - searches for the vsi with the given id
308 * @pf: the pf structure to search for the vsi
309 * @id: id of the vsi it is searching for
310 **/
i40e_find_vsi_from_id(struct i40e_pf * pf,u16 id)311 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
312 {
313 struct i40e_vsi *vsi;
314 int i;
315
316 i40e_pf_for_each_vsi(pf, i, vsi)
317 if (vsi->id == id)
318 return vsi;
319
320 return NULL;
321 }
322
323 /**
324 * i40e_service_event_schedule - Schedule the service task to wake up
325 * @pf: board private structure
326 *
327 * If not already scheduled, this puts the task into the work queue
328 **/
i40e_service_event_schedule(struct i40e_pf * pf)329 void i40e_service_event_schedule(struct i40e_pf *pf)
330 {
331 if ((!test_bit(__I40E_DOWN, pf->state) &&
332 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
333 test_bit(__I40E_RECOVERY_MODE, pf->state))
334 queue_work(i40e_wq, &pf->service_task);
335 }
336
337 /**
338 * i40e_tx_timeout - Respond to a Tx Hang
339 * @netdev: network interface device structure
340 * @txqueue: queue number timing out
341 *
342 * If any port has noticed a Tx timeout, it is likely that the whole
343 * device is munged, not just the one netdev port, so go for the full
344 * reset.
345 **/
i40e_tx_timeout(struct net_device * netdev,unsigned int txqueue)346 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
347 {
348 struct i40e_netdev_priv *np = netdev_priv(netdev);
349 struct i40e_vsi *vsi = np->vsi;
350 struct i40e_pf *pf = vsi->back;
351 struct i40e_ring *tx_ring = NULL;
352 unsigned int i;
353 u32 head, val;
354
355 pf->tx_timeout_count++;
356
357 /* with txqueue index, find the tx_ring struct */
358 for (i = 0; i < vsi->num_queue_pairs; i++) {
359 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
360 if (txqueue ==
361 vsi->tx_rings[i]->queue_index) {
362 tx_ring = vsi->tx_rings[i];
363 break;
364 }
365 }
366 }
367
368 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
369 pf->tx_timeout_recovery_level = 1; /* reset after some time */
370 else if (time_before(jiffies,
371 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
372 return; /* don't do any new action before the next timeout */
373
374 /* don't kick off another recovery if one is already pending */
375 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
376 return;
377
378 if (tx_ring) {
379 head = i40e_get_head(tx_ring);
380 /* Read interrupt register */
381 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
382 val = rd32(&pf->hw,
383 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
384 tx_ring->vsi->base_vector - 1));
385 else
386 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
387
388 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
389 vsi->seid, txqueue, tx_ring->next_to_clean,
390 head, tx_ring->next_to_use,
391 readl(tx_ring->tail), val);
392 }
393
394 pf->tx_timeout_last_recovery = jiffies;
395 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
396 pf->tx_timeout_recovery_level, txqueue);
397
398 switch (pf->tx_timeout_recovery_level) {
399 case 1:
400 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
401 break;
402 case 2:
403 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
404 break;
405 case 3:
406 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
407 break;
408 default:
409 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");
410 set_bit(__I40E_DOWN_REQUESTED, pf->state);
411 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);
412 break;
413 }
414
415 i40e_service_event_schedule(pf);
416 pf->tx_timeout_recovery_level++;
417 }
418
419 /**
420 * i40e_get_vsi_stats_struct - Get System Network Statistics
421 * @vsi: the VSI we care about
422 *
423 * Returns the address of the device statistics structure.
424 * The statistics are actually updated from the service task.
425 **/
i40e_get_vsi_stats_struct(struct i40e_vsi * vsi)426 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
427 {
428 return &vsi->net_stats;
429 }
430
431 /**
432 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
433 * @ring: Tx ring to get statistics from
434 * @stats: statistics entry to be updated
435 **/
i40e_get_netdev_stats_struct_tx(struct i40e_ring * ring,struct rtnl_link_stats64 * stats)436 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
437 struct rtnl_link_stats64 *stats)
438 {
439 u64 bytes, packets;
440 unsigned int start;
441
442 do {
443 start = u64_stats_fetch_begin(&ring->syncp);
444 packets = ring->stats.packets;
445 bytes = ring->stats.bytes;
446 } while (u64_stats_fetch_retry(&ring->syncp, start));
447
448 stats->tx_packets += packets;
449 stats->tx_bytes += bytes;
450 }
451
452 /**
453 * i40e_get_netdev_stats_struct - Get statistics for netdev interface
454 * @netdev: network interface device structure
455 * @stats: data structure to store statistics
456 *
457 * Returns the address of the device statistics structure.
458 * The statistics are actually updated from the service task.
459 **/
i40e_get_netdev_stats_struct(struct net_device * netdev,struct rtnl_link_stats64 * stats)460 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
461 struct rtnl_link_stats64 *stats)
462 {
463 struct i40e_netdev_priv *np = netdev_priv(netdev);
464 struct i40e_vsi *vsi = np->vsi;
465 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
466 struct i40e_ring *ring;
467 int i;
468
469 if (test_bit(__I40E_VSI_DOWN, vsi->state))
470 return;
471
472 if (!vsi->tx_rings)
473 return;
474
475 rcu_read_lock();
476 for (i = 0; i < vsi->num_queue_pairs; i++) {
477 u64 bytes, packets;
478 unsigned int start;
479
480 ring = READ_ONCE(vsi->tx_rings[i]);
481 if (!ring)
482 continue;
483 i40e_get_netdev_stats_struct_tx(ring, stats);
484
485 if (i40e_enabled_xdp_vsi(vsi)) {
486 ring = READ_ONCE(vsi->xdp_rings[i]);
487 if (!ring)
488 continue;
489 i40e_get_netdev_stats_struct_tx(ring, stats);
490 }
491
492 ring = READ_ONCE(vsi->rx_rings[i]);
493 if (!ring)
494 continue;
495 do {
496 start = u64_stats_fetch_begin(&ring->syncp);
497 packets = ring->stats.packets;
498 bytes = ring->stats.bytes;
499 } while (u64_stats_fetch_retry(&ring->syncp, start));
500
501 stats->rx_packets += packets;
502 stats->rx_bytes += bytes;
503
504 }
505 rcu_read_unlock();
506
507 /* following stats updated by i40e_watchdog_subtask() */
508 stats->multicast = vsi_stats->multicast;
509 stats->tx_errors = vsi_stats->tx_errors;
510 stats->tx_dropped = vsi_stats->tx_dropped;
511 stats->rx_errors = vsi_stats->rx_errors;
512 stats->rx_dropped = vsi_stats->rx_dropped;
513 stats->rx_missed_errors = vsi_stats->rx_missed_errors;
514 stats->rx_crc_errors = vsi_stats->rx_crc_errors;
515 stats->rx_length_errors = vsi_stats->rx_length_errors;
516 }
517
518 /**
519 * i40e_vsi_reset_stats - Resets all stats of the given vsi
520 * @vsi: the VSI to have its stats reset
521 **/
i40e_vsi_reset_stats(struct i40e_vsi * vsi)522 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
523 {
524 struct rtnl_link_stats64 *ns;
525 int i;
526
527 if (!vsi)
528 return;
529
530 ns = i40e_get_vsi_stats_struct(vsi);
531 memset(ns, 0, sizeof(*ns));
532 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
533 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
534 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
535 if (vsi->rx_rings && vsi->rx_rings[0]) {
536 for (i = 0; i < vsi->num_queue_pairs; i++) {
537 memset(&vsi->rx_rings[i]->stats, 0,
538 sizeof(vsi->rx_rings[i]->stats));
539 memset(&vsi->rx_rings[i]->rx_stats, 0,
540 sizeof(vsi->rx_rings[i]->rx_stats));
541 memset(&vsi->tx_rings[i]->stats, 0,
542 sizeof(vsi->tx_rings[i]->stats));
543 memset(&vsi->tx_rings[i]->tx_stats, 0,
544 sizeof(vsi->tx_rings[i]->tx_stats));
545 }
546 }
547 vsi->stat_offsets_loaded = false;
548 }
549
550 /**
551 * i40e_pf_reset_stats - Reset all of the stats for the given PF
552 * @pf: the PF to be reset
553 **/
i40e_pf_reset_stats(struct i40e_pf * pf)554 void i40e_pf_reset_stats(struct i40e_pf *pf)
555 {
556 struct i40e_veb *veb;
557 int i;
558
559 memset(&pf->stats, 0, sizeof(pf->stats));
560 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
561 pf->stat_offsets_loaded = false;
562
563 i40e_pf_for_each_veb(pf, i, veb) {
564 memset(&veb->stats, 0, sizeof(veb->stats));
565 memset(&veb->stats_offsets, 0, sizeof(veb->stats_offsets));
566 memset(&veb->tc_stats, 0, sizeof(veb->tc_stats));
567 memset(&veb->tc_stats_offsets, 0, sizeof(veb->tc_stats_offsets));
568 veb->stat_offsets_loaded = false;
569 }
570 pf->hw_csum_rx_error = 0;
571 }
572
573 /**
574 * i40e_compute_pci_to_hw_id - compute index form PCI function.
575 * @vsi: ptr to the VSI to read from.
576 * @hw: ptr to the hardware info.
577 **/
i40e_compute_pci_to_hw_id(struct i40e_vsi * vsi,struct i40e_hw * hw)578 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw)
579 {
580 int pf_count = i40e_get_pf_count(hw);
581
582 if (vsi->type == I40E_VSI_SRIOV)
583 return (hw->port * BIT(7)) / pf_count + vsi->vf_id;
584
585 return hw->port + BIT(7);
586 }
587
588 /**
589 * i40e_stat_update64 - read and update a 64 bit stat from the chip.
590 * @hw: ptr to the hardware info.
591 * @hireg: the high 32 bit reg to read.
592 * @loreg: the low 32 bit reg to read.
593 * @offset_loaded: has the initial offset been loaded yet.
594 * @offset: ptr to current offset value.
595 * @stat: ptr to the stat.
596 *
597 * Since the device stats are not reset at PFReset, they will not
598 * be zeroed when the driver starts. We'll save the first values read
599 * and use them as offsets to be subtracted from the raw values in order
600 * to report stats that count from zero.
601 **/
i40e_stat_update64(struct i40e_hw * hw,u32 hireg,u32 loreg,bool offset_loaded,u64 * offset,u64 * stat)602 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg,
603 bool offset_loaded, u64 *offset, u64 *stat)
604 {
605 u64 new_data;
606
607 new_data = rd64(hw, loreg);
608
609 if (!offset_loaded || new_data < *offset)
610 *offset = new_data;
611 *stat = new_data - *offset;
612 }
613
614 /**
615 * i40e_stat_update48 - read and update a 48 bit stat from the chip
616 * @hw: ptr to the hardware info
617 * @hireg: the high 32 bit reg to read
618 * @loreg: the low 32 bit reg to read
619 * @offset_loaded: has the initial offset been loaded yet
620 * @offset: ptr to current offset value
621 * @stat: ptr to the stat
622 *
623 * Since the device stats are not reset at PFReset, they likely will not
624 * be zeroed when the driver starts. We'll save the first values read
625 * and use them as offsets to be subtracted from the raw values in order
626 * to report stats that count from zero. In the process, we also manage
627 * the potential roll-over.
628 **/
i40e_stat_update48(struct i40e_hw * hw,u32 hireg,u32 loreg,bool offset_loaded,u64 * offset,u64 * stat)629 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
630 bool offset_loaded, u64 *offset, u64 *stat)
631 {
632 u64 new_data;
633
634 if (hw->device_id == I40E_DEV_ID_QEMU) {
635 new_data = rd32(hw, loreg);
636 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
637 } else {
638 new_data = rd64(hw, loreg);
639 }
640 if (!offset_loaded)
641 *offset = new_data;
642 if (likely(new_data >= *offset))
643 *stat = new_data - *offset;
644 else
645 *stat = (new_data + BIT_ULL(48)) - *offset;
646 *stat &= 0xFFFFFFFFFFFFULL;
647 }
648
649 /**
650 * i40e_stat_update32 - read and update a 32 bit stat from the chip
651 * @hw: ptr to the hardware info
652 * @reg: the hw reg to read
653 * @offset_loaded: has the initial offset been loaded yet
654 * @offset: ptr to current offset value
655 * @stat: ptr to the stat
656 **/
i40e_stat_update32(struct i40e_hw * hw,u32 reg,bool offset_loaded,u64 * offset,u64 * stat)657 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
658 bool offset_loaded, u64 *offset, u64 *stat)
659 {
660 u32 new_data;
661
662 new_data = rd32(hw, reg);
663 if (!offset_loaded)
664 *offset = new_data;
665 if (likely(new_data >= *offset))
666 *stat = (u32)(new_data - *offset);
667 else
668 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
669 }
670
671 /**
672 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
673 * @hw: ptr to the hardware info
674 * @reg: the hw reg to read and clear
675 * @stat: ptr to the stat
676 **/
i40e_stat_update_and_clear32(struct i40e_hw * hw,u32 reg,u64 * stat)677 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
678 {
679 u32 new_data = rd32(hw, reg);
680
681 wr32(hw, reg, 1); /* must write a nonzero value to clear register */
682 *stat += new_data;
683 }
684
685 /**
686 * i40e_stats_update_rx_discards - update rx_discards.
687 * @vsi: ptr to the VSI to be updated.
688 * @hw: ptr to the hardware info.
689 * @stat_idx: VSI's stat_counter_idx.
690 * @offset_loaded: ptr to the VSI's stat_offsets_loaded.
691 * @stat_offset: ptr to stat_offset to store first read of specific register.
692 * @stat: ptr to VSI's stat to be updated.
693 **/
694 static void
i40e_stats_update_rx_discards(struct i40e_vsi * vsi,struct i40e_hw * hw,int stat_idx,bool offset_loaded,struct i40e_eth_stats * stat_offset,struct i40e_eth_stats * stat)695 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw,
696 int stat_idx, bool offset_loaded,
697 struct i40e_eth_stats *stat_offset,
698 struct i40e_eth_stats *stat)
699 {
700 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded,
701 &stat_offset->rx_discards, &stat->rx_discards);
702 i40e_stat_update64(hw,
703 I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)),
704 I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)),
705 offset_loaded, &stat_offset->rx_discards_other,
706 &stat->rx_discards_other);
707 }
708
709 /**
710 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
711 * @vsi: the VSI to be updated
712 **/
i40e_update_eth_stats(struct i40e_vsi * vsi)713 void i40e_update_eth_stats(struct i40e_vsi *vsi)
714 {
715 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
716 struct i40e_pf *pf = vsi->back;
717 struct i40e_hw *hw = &pf->hw;
718 struct i40e_eth_stats *oes;
719 struct i40e_eth_stats *es; /* device's eth stats */
720
721 es = &vsi->eth_stats;
722 oes = &vsi->eth_stats_offsets;
723
724 /* Gather up the stats that the hw collects */
725 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
726 vsi->stat_offsets_loaded,
727 &oes->tx_errors, &es->tx_errors);
728 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
729 vsi->stat_offsets_loaded,
730 &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
731
732 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
733 I40E_GLV_GORCL(stat_idx),
734 vsi->stat_offsets_loaded,
735 &oes->rx_bytes, &es->rx_bytes);
736 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
737 I40E_GLV_UPRCL(stat_idx),
738 vsi->stat_offsets_loaded,
739 &oes->rx_unicast, &es->rx_unicast);
740 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
741 I40E_GLV_MPRCL(stat_idx),
742 vsi->stat_offsets_loaded,
743 &oes->rx_multicast, &es->rx_multicast);
744 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
745 I40E_GLV_BPRCL(stat_idx),
746 vsi->stat_offsets_loaded,
747 &oes->rx_broadcast, &es->rx_broadcast);
748
749 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
750 I40E_GLV_GOTCL(stat_idx),
751 vsi->stat_offsets_loaded,
752 &oes->tx_bytes, &es->tx_bytes);
753 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
754 I40E_GLV_UPTCL(stat_idx),
755 vsi->stat_offsets_loaded,
756 &oes->tx_unicast, &es->tx_unicast);
757 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
758 I40E_GLV_MPTCL(stat_idx),
759 vsi->stat_offsets_loaded,
760 &oes->tx_multicast, &es->tx_multicast);
761 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
762 I40E_GLV_BPTCL(stat_idx),
763 vsi->stat_offsets_loaded,
764 &oes->tx_broadcast, &es->tx_broadcast);
765
766 i40e_stats_update_rx_discards(vsi, hw, stat_idx,
767 vsi->stat_offsets_loaded, oes, es);
768
769 vsi->stat_offsets_loaded = true;
770 }
771
772 /**
773 * i40e_update_veb_stats - Update Switch component statistics
774 * @veb: the VEB being updated
775 **/
i40e_update_veb_stats(struct i40e_veb * veb)776 void i40e_update_veb_stats(struct i40e_veb *veb)
777 {
778 struct i40e_pf *pf = veb->pf;
779 struct i40e_hw *hw = &pf->hw;
780 struct i40e_eth_stats *oes;
781 struct i40e_eth_stats *es; /* device's eth stats */
782 struct i40e_veb_tc_stats *veb_oes;
783 struct i40e_veb_tc_stats *veb_es;
784 int i, idx = 0;
785
786 idx = veb->stats_idx;
787 es = &veb->stats;
788 oes = &veb->stats_offsets;
789 veb_es = &veb->tc_stats;
790 veb_oes = &veb->tc_stats_offsets;
791
792 /* Gather up the stats that the hw collects */
793 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
794 veb->stat_offsets_loaded,
795 &oes->tx_discards, &es->tx_discards);
796 if (hw->revision_id > 0)
797 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
798 veb->stat_offsets_loaded,
799 &oes->rx_unknown_protocol,
800 &es->rx_unknown_protocol);
801 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
802 veb->stat_offsets_loaded,
803 &oes->rx_bytes, &es->rx_bytes);
804 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
805 veb->stat_offsets_loaded,
806 &oes->rx_unicast, &es->rx_unicast);
807 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
808 veb->stat_offsets_loaded,
809 &oes->rx_multicast, &es->rx_multicast);
810 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
811 veb->stat_offsets_loaded,
812 &oes->rx_broadcast, &es->rx_broadcast);
813
814 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
815 veb->stat_offsets_loaded,
816 &oes->tx_bytes, &es->tx_bytes);
817 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
818 veb->stat_offsets_loaded,
819 &oes->tx_unicast, &es->tx_unicast);
820 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
821 veb->stat_offsets_loaded,
822 &oes->tx_multicast, &es->tx_multicast);
823 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
824 veb->stat_offsets_loaded,
825 &oes->tx_broadcast, &es->tx_broadcast);
826 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
827 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
828 I40E_GLVEBTC_RPCL(i, idx),
829 veb->stat_offsets_loaded,
830 &veb_oes->tc_rx_packets[i],
831 &veb_es->tc_rx_packets[i]);
832 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
833 I40E_GLVEBTC_RBCL(i, idx),
834 veb->stat_offsets_loaded,
835 &veb_oes->tc_rx_bytes[i],
836 &veb_es->tc_rx_bytes[i]);
837 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
838 I40E_GLVEBTC_TPCL(i, idx),
839 veb->stat_offsets_loaded,
840 &veb_oes->tc_tx_packets[i],
841 &veb_es->tc_tx_packets[i]);
842 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
843 I40E_GLVEBTC_TBCL(i, idx),
844 veb->stat_offsets_loaded,
845 &veb_oes->tc_tx_bytes[i],
846 &veb_es->tc_tx_bytes[i]);
847 }
848 veb->stat_offsets_loaded = true;
849 }
850
851 /**
852 * i40e_update_vsi_stats - Update the vsi statistics counters.
853 * @vsi: the VSI to be updated
854 *
855 * There are a few instances where we store the same stat in a
856 * couple of different structs. This is partly because we have
857 * the netdev stats that need to be filled out, which is slightly
858 * different from the "eth_stats" defined by the chip and used in
859 * VF communications. We sort it out here.
860 **/
i40e_update_vsi_stats(struct i40e_vsi * vsi)861 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
862 {
863 u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy;
864 struct i40e_pf *pf = vsi->back;
865 struct rtnl_link_stats64 *ons;
866 struct rtnl_link_stats64 *ns; /* netdev stats */
867 struct i40e_eth_stats *oes;
868 struct i40e_eth_stats *es; /* device's eth stats */
869 u64 tx_restart, tx_busy;
870 struct i40e_ring *p;
871 u64 bytes, packets;
872 unsigned int start;
873 u64 tx_linearize;
874 u64 tx_force_wb;
875 u64 tx_stopped;
876 u64 rx_p, rx_b;
877 u64 tx_p, tx_b;
878 u16 q;
879
880 if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
881 test_bit(__I40E_CONFIG_BUSY, pf->state))
882 return;
883
884 ns = i40e_get_vsi_stats_struct(vsi);
885 ons = &vsi->net_stats_offsets;
886 es = &vsi->eth_stats;
887 oes = &vsi->eth_stats_offsets;
888
889 /* Gather up the netdev and vsi stats that the driver collects
890 * on the fly during packet processing
891 */
892 rx_b = rx_p = 0;
893 tx_b = tx_p = 0;
894 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
895 tx_stopped = 0;
896 rx_page = 0;
897 rx_buf = 0;
898 rx_reuse = 0;
899 rx_alloc = 0;
900 rx_waive = 0;
901 rx_busy = 0;
902 rcu_read_lock();
903 for (q = 0; q < vsi->num_queue_pairs; q++) {
904 /* locate Tx ring */
905 p = READ_ONCE(vsi->tx_rings[q]);
906 if (!p)
907 continue;
908
909 do {
910 start = u64_stats_fetch_begin(&p->syncp);
911 packets = p->stats.packets;
912 bytes = p->stats.bytes;
913 } while (u64_stats_fetch_retry(&p->syncp, start));
914 tx_b += bytes;
915 tx_p += packets;
916 tx_restart += p->tx_stats.restart_queue;
917 tx_busy += p->tx_stats.tx_busy;
918 tx_linearize += p->tx_stats.tx_linearize;
919 tx_force_wb += p->tx_stats.tx_force_wb;
920 tx_stopped += p->tx_stats.tx_stopped;
921
922 /* locate Rx ring */
923 p = READ_ONCE(vsi->rx_rings[q]);
924 if (!p)
925 continue;
926
927 do {
928 start = u64_stats_fetch_begin(&p->syncp);
929 packets = p->stats.packets;
930 bytes = p->stats.bytes;
931 } while (u64_stats_fetch_retry(&p->syncp, start));
932 rx_b += bytes;
933 rx_p += packets;
934 rx_buf += p->rx_stats.alloc_buff_failed;
935 rx_page += p->rx_stats.alloc_page_failed;
936 rx_reuse += p->rx_stats.page_reuse_count;
937 rx_alloc += p->rx_stats.page_alloc_count;
938 rx_waive += p->rx_stats.page_waive_count;
939 rx_busy += p->rx_stats.page_busy_count;
940
941 if (i40e_enabled_xdp_vsi(vsi)) {
942 /* locate XDP ring */
943 p = READ_ONCE(vsi->xdp_rings[q]);
944 if (!p)
945 continue;
946
947 do {
948 start = u64_stats_fetch_begin(&p->syncp);
949 packets = p->stats.packets;
950 bytes = p->stats.bytes;
951 } while (u64_stats_fetch_retry(&p->syncp, start));
952 tx_b += bytes;
953 tx_p += packets;
954 tx_restart += p->tx_stats.restart_queue;
955 tx_busy += p->tx_stats.tx_busy;
956 tx_linearize += p->tx_stats.tx_linearize;
957 tx_force_wb += p->tx_stats.tx_force_wb;
958 }
959 }
960 rcu_read_unlock();
961 vsi->tx_restart = tx_restart;
962 vsi->tx_busy = tx_busy;
963 vsi->tx_linearize = tx_linearize;
964 vsi->tx_force_wb = tx_force_wb;
965 vsi->tx_stopped = tx_stopped;
966 vsi->rx_page_failed = rx_page;
967 vsi->rx_buf_failed = rx_buf;
968 vsi->rx_page_reuse = rx_reuse;
969 vsi->rx_page_alloc = rx_alloc;
970 vsi->rx_page_waive = rx_waive;
971 vsi->rx_page_busy = rx_busy;
972
973 ns->rx_packets = rx_p;
974 ns->rx_bytes = rx_b;
975 ns->tx_packets = tx_p;
976 ns->tx_bytes = tx_b;
977
978 /* update netdev stats from eth stats */
979 i40e_update_eth_stats(vsi);
980 ons->tx_errors = oes->tx_errors;
981 ns->tx_errors = es->tx_errors;
982 ons->multicast = oes->rx_multicast;
983 ns->multicast = es->rx_multicast;
984 ons->rx_dropped = oes->rx_discards_other;
985 ns->rx_dropped = es->rx_discards_other;
986 ons->rx_missed_errors = oes->rx_discards;
987 ns->rx_missed_errors = es->rx_discards;
988 ons->tx_dropped = oes->tx_discards;
989 ns->tx_dropped = es->tx_discards;
990
991 /* pull in a couple PF stats if this is the main vsi */
992 if (vsi->type == I40E_VSI_MAIN) {
993 ns->rx_crc_errors = pf->stats.crc_errors;
994 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
995 ns->rx_length_errors = pf->stats.rx_length_errors;
996 }
997 }
998
999 /**
1000 * i40e_update_pf_stats - Update the PF statistics counters.
1001 * @pf: the PF to be updated
1002 **/
i40e_update_pf_stats(struct i40e_pf * pf)1003 static void i40e_update_pf_stats(struct i40e_pf *pf)
1004 {
1005 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
1006 struct i40e_hw_port_stats *nsd = &pf->stats;
1007 struct i40e_hw *hw = &pf->hw;
1008 u32 val;
1009 int i;
1010
1011 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
1012 I40E_GLPRT_GORCL(hw->port),
1013 pf->stat_offsets_loaded,
1014 &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
1015 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
1016 I40E_GLPRT_GOTCL(hw->port),
1017 pf->stat_offsets_loaded,
1018 &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
1019 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
1020 pf->stat_offsets_loaded,
1021 &osd->eth.rx_discards,
1022 &nsd->eth.rx_discards);
1023 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
1024 I40E_GLPRT_UPRCL(hw->port),
1025 pf->stat_offsets_loaded,
1026 &osd->eth.rx_unicast,
1027 &nsd->eth.rx_unicast);
1028 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
1029 I40E_GLPRT_MPRCL(hw->port),
1030 pf->stat_offsets_loaded,
1031 &osd->eth.rx_multicast,
1032 &nsd->eth.rx_multicast);
1033 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
1034 I40E_GLPRT_BPRCL(hw->port),
1035 pf->stat_offsets_loaded,
1036 &osd->eth.rx_broadcast,
1037 &nsd->eth.rx_broadcast);
1038 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
1039 I40E_GLPRT_UPTCL(hw->port),
1040 pf->stat_offsets_loaded,
1041 &osd->eth.tx_unicast,
1042 &nsd->eth.tx_unicast);
1043 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
1044 I40E_GLPRT_MPTCL(hw->port),
1045 pf->stat_offsets_loaded,
1046 &osd->eth.tx_multicast,
1047 &nsd->eth.tx_multicast);
1048 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
1049 I40E_GLPRT_BPTCL(hw->port),
1050 pf->stat_offsets_loaded,
1051 &osd->eth.tx_broadcast,
1052 &nsd->eth.tx_broadcast);
1053
1054 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
1055 pf->stat_offsets_loaded,
1056 &osd->tx_dropped_link_down,
1057 &nsd->tx_dropped_link_down);
1058
1059 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
1060 pf->stat_offsets_loaded,
1061 &osd->crc_errors, &nsd->crc_errors);
1062
1063 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
1064 pf->stat_offsets_loaded,
1065 &osd->illegal_bytes, &nsd->illegal_bytes);
1066
1067 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
1068 pf->stat_offsets_loaded,
1069 &osd->mac_local_faults,
1070 &nsd->mac_local_faults);
1071 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
1072 pf->stat_offsets_loaded,
1073 &osd->mac_remote_faults,
1074 &nsd->mac_remote_faults);
1075
1076 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
1077 pf->stat_offsets_loaded,
1078 &osd->rx_length_errors,
1079 &nsd->rx_length_errors);
1080
1081 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
1082 pf->stat_offsets_loaded,
1083 &osd->link_xon_rx, &nsd->link_xon_rx);
1084 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
1085 pf->stat_offsets_loaded,
1086 &osd->link_xon_tx, &nsd->link_xon_tx);
1087 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1088 pf->stat_offsets_loaded,
1089 &osd->link_xoff_rx, &nsd->link_xoff_rx);
1090 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1091 pf->stat_offsets_loaded,
1092 &osd->link_xoff_tx, &nsd->link_xoff_tx);
1093
1094 for (i = 0; i < 8; i++) {
1095 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1096 pf->stat_offsets_loaded,
1097 &osd->priority_xoff_rx[i],
1098 &nsd->priority_xoff_rx[i]);
1099 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1100 pf->stat_offsets_loaded,
1101 &osd->priority_xon_rx[i],
1102 &nsd->priority_xon_rx[i]);
1103 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1104 pf->stat_offsets_loaded,
1105 &osd->priority_xon_tx[i],
1106 &nsd->priority_xon_tx[i]);
1107 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1108 pf->stat_offsets_loaded,
1109 &osd->priority_xoff_tx[i],
1110 &nsd->priority_xoff_tx[i]);
1111 i40e_stat_update32(hw,
1112 I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1113 pf->stat_offsets_loaded,
1114 &osd->priority_xon_2_xoff[i],
1115 &nsd->priority_xon_2_xoff[i]);
1116 }
1117
1118 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1119 I40E_GLPRT_PRC64L(hw->port),
1120 pf->stat_offsets_loaded,
1121 &osd->rx_size_64, &nsd->rx_size_64);
1122 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1123 I40E_GLPRT_PRC127L(hw->port),
1124 pf->stat_offsets_loaded,
1125 &osd->rx_size_127, &nsd->rx_size_127);
1126 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1127 I40E_GLPRT_PRC255L(hw->port),
1128 pf->stat_offsets_loaded,
1129 &osd->rx_size_255, &nsd->rx_size_255);
1130 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1131 I40E_GLPRT_PRC511L(hw->port),
1132 pf->stat_offsets_loaded,
1133 &osd->rx_size_511, &nsd->rx_size_511);
1134 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1135 I40E_GLPRT_PRC1023L(hw->port),
1136 pf->stat_offsets_loaded,
1137 &osd->rx_size_1023, &nsd->rx_size_1023);
1138 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1139 I40E_GLPRT_PRC1522L(hw->port),
1140 pf->stat_offsets_loaded,
1141 &osd->rx_size_1522, &nsd->rx_size_1522);
1142 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1143 I40E_GLPRT_PRC9522L(hw->port),
1144 pf->stat_offsets_loaded,
1145 &osd->rx_size_big, &nsd->rx_size_big);
1146
1147 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1148 I40E_GLPRT_PTC64L(hw->port),
1149 pf->stat_offsets_loaded,
1150 &osd->tx_size_64, &nsd->tx_size_64);
1151 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1152 I40E_GLPRT_PTC127L(hw->port),
1153 pf->stat_offsets_loaded,
1154 &osd->tx_size_127, &nsd->tx_size_127);
1155 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1156 I40E_GLPRT_PTC255L(hw->port),
1157 pf->stat_offsets_loaded,
1158 &osd->tx_size_255, &nsd->tx_size_255);
1159 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1160 I40E_GLPRT_PTC511L(hw->port),
1161 pf->stat_offsets_loaded,
1162 &osd->tx_size_511, &nsd->tx_size_511);
1163 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1164 I40E_GLPRT_PTC1023L(hw->port),
1165 pf->stat_offsets_loaded,
1166 &osd->tx_size_1023, &nsd->tx_size_1023);
1167 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1168 I40E_GLPRT_PTC1522L(hw->port),
1169 pf->stat_offsets_loaded,
1170 &osd->tx_size_1522, &nsd->tx_size_1522);
1171 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1172 I40E_GLPRT_PTC9522L(hw->port),
1173 pf->stat_offsets_loaded,
1174 &osd->tx_size_big, &nsd->tx_size_big);
1175
1176 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1177 pf->stat_offsets_loaded,
1178 &osd->rx_undersize, &nsd->rx_undersize);
1179 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1180 pf->stat_offsets_loaded,
1181 &osd->rx_fragments, &nsd->rx_fragments);
1182 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1183 pf->stat_offsets_loaded,
1184 &osd->rx_oversize, &nsd->rx_oversize);
1185 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1186 pf->stat_offsets_loaded,
1187 &osd->rx_jabber, &nsd->rx_jabber);
1188
1189 /* FDIR stats */
1190 i40e_stat_update_and_clear32(hw,
1191 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1192 &nsd->fd_atr_match);
1193 i40e_stat_update_and_clear32(hw,
1194 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1195 &nsd->fd_sb_match);
1196 i40e_stat_update_and_clear32(hw,
1197 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1198 &nsd->fd_atr_tunnel_match);
1199
1200 val = rd32(hw, I40E_PRTPM_EEE_STAT);
1201 nsd->tx_lpi_status =
1202 FIELD_GET(I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK, val);
1203 nsd->rx_lpi_status =
1204 FIELD_GET(I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK, val);
1205 i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1206 pf->stat_offsets_loaded,
1207 &osd->tx_lpi_count, &nsd->tx_lpi_count);
1208 i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1209 pf->stat_offsets_loaded,
1210 &osd->rx_lpi_count, &nsd->rx_lpi_count);
1211
1212 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
1213 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1214 nsd->fd_sb_status = true;
1215 else
1216 nsd->fd_sb_status = false;
1217
1218 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
1219 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1220 nsd->fd_atr_status = true;
1221 else
1222 nsd->fd_atr_status = false;
1223
1224 pf->stat_offsets_loaded = true;
1225 }
1226
1227 /**
1228 * i40e_update_stats - Update the various statistics counters.
1229 * @vsi: the VSI to be updated
1230 *
1231 * Update the various stats for this VSI and its related entities.
1232 **/
i40e_update_stats(struct i40e_vsi * vsi)1233 void i40e_update_stats(struct i40e_vsi *vsi)
1234 {
1235 struct i40e_pf *pf = vsi->back;
1236
1237 if (vsi->type == I40E_VSI_MAIN)
1238 i40e_update_pf_stats(pf);
1239
1240 i40e_update_vsi_stats(vsi);
1241 }
1242
1243 /**
1244 * i40e_count_filters - counts VSI mac filters
1245 * @vsi: the VSI to be searched
1246 *
1247 * Returns count of mac filters
1248 **/
i40e_count_filters(struct i40e_vsi * vsi)1249 int i40e_count_filters(struct i40e_vsi *vsi)
1250 {
1251 struct i40e_mac_filter *f;
1252 struct hlist_node *h;
1253 int bkt;
1254 int cnt = 0;
1255
1256 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1257 if (f->state == I40E_FILTER_NEW ||
1258 f->state == I40E_FILTER_ACTIVE)
1259 ++cnt;
1260 }
1261
1262 return cnt;
1263 }
1264
1265 /**
1266 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1267 * @vsi: the VSI to be searched
1268 * @macaddr: the MAC address
1269 * @vlan: the vlan
1270 *
1271 * Returns ptr to the filter object or NULL
1272 **/
i40e_find_filter(struct i40e_vsi * vsi,const u8 * macaddr,s16 vlan)1273 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1274 const u8 *macaddr, s16 vlan)
1275 {
1276 struct i40e_mac_filter *f;
1277 u64 key;
1278
1279 if (!vsi || !macaddr)
1280 return NULL;
1281
1282 key = i40e_addr_to_hkey(macaddr);
1283 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1284 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1285 (vlan == f->vlan))
1286 return f;
1287 }
1288 return NULL;
1289 }
1290
1291 /**
1292 * i40e_find_mac - Find a mac addr in the macvlan filters list
1293 * @vsi: the VSI to be searched
1294 * @macaddr: the MAC address we are searching for
1295 *
1296 * Returns the first filter with the provided MAC address or NULL if
1297 * MAC address was not found
1298 **/
i40e_find_mac(struct i40e_vsi * vsi,const u8 * macaddr)1299 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1300 {
1301 struct i40e_mac_filter *f;
1302 u64 key;
1303
1304 if (!vsi || !macaddr)
1305 return NULL;
1306
1307 key = i40e_addr_to_hkey(macaddr);
1308 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1309 if ((ether_addr_equal(macaddr, f->macaddr)))
1310 return f;
1311 }
1312 return NULL;
1313 }
1314
1315 /**
1316 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1317 * @vsi: the VSI to be searched
1318 *
1319 * Returns true if VSI is in vlan mode or false otherwise
1320 **/
i40e_is_vsi_in_vlan(struct i40e_vsi * vsi)1321 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1322 {
1323 /* If we have a PVID, always operate in VLAN mode */
1324 if (vsi->info.pvid)
1325 return true;
1326
1327 /* We need to operate in VLAN mode whenever we have any filters with
1328 * a VLAN other than I40E_VLAN_ALL. We could check the table each
1329 * time, incurring search cost repeatedly. However, we can notice two
1330 * things:
1331 *
1332 * 1) the only place where we can gain a VLAN filter is in
1333 * i40e_add_filter.
1334 *
1335 * 2) the only place where filters are actually removed is in
1336 * i40e_sync_filters_subtask.
1337 *
1338 * Thus, we can simply use a boolean value, has_vlan_filters which we
1339 * will set to true when we add a VLAN filter in i40e_add_filter. Then
1340 * we have to perform the full search after deleting filters in
1341 * i40e_sync_filters_subtask, but we already have to search
1342 * filters here and can perform the check at the same time. This
1343 * results in avoiding embedding a loop for VLAN mode inside another
1344 * loop over all the filters, and should maintain correctness as noted
1345 * above.
1346 */
1347 return vsi->has_vlan_filter;
1348 }
1349
1350 /**
1351 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1352 * @vsi: the VSI to configure
1353 * @tmp_add_list: list of filters ready to be added
1354 * @tmp_del_list: list of filters ready to be deleted
1355 * @vlan_filters: the number of active VLAN filters
1356 *
1357 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1358 * behave as expected. If we have any active VLAN filters remaining or about
1359 * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1360 * so that they only match against untagged traffic. If we no longer have any
1361 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1362 * so that they match against both tagged and untagged traffic. In this way,
1363 * we ensure that we correctly receive the desired traffic. This ensures that
1364 * when we have an active VLAN we will receive only untagged traffic and
1365 * traffic matching active VLANs. If we have no active VLANs then we will
1366 * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1367 *
1368 * Finally, in a similar fashion, this function also corrects filters when
1369 * there is an active PVID assigned to this VSI.
1370 *
1371 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1372 *
1373 * This function is only expected to be called from within
1374 * i40e_sync_vsi_filters.
1375 *
1376 * NOTE: This function expects to be called while under the
1377 * mac_filter_hash_lock
1378 */
i40e_correct_mac_vlan_filters(struct i40e_vsi * vsi,struct hlist_head * tmp_add_list,struct hlist_head * tmp_del_list,int vlan_filters)1379 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1380 struct hlist_head *tmp_add_list,
1381 struct hlist_head *tmp_del_list,
1382 int vlan_filters)
1383 {
1384 s16 pvid = le16_to_cpu(vsi->info.pvid);
1385 struct i40e_mac_filter *f, *add_head;
1386 struct i40e_new_mac_filter *new;
1387 struct hlist_node *h;
1388 int bkt, new_vlan;
1389
1390 /* To determine if a particular filter needs to be replaced we
1391 * have the three following conditions:
1392 *
1393 * a) if we have a PVID assigned, then all filters which are
1394 * not marked as VLAN=PVID must be replaced with filters that
1395 * are.
1396 * b) otherwise, if we have any active VLANS, all filters
1397 * which are marked as VLAN=-1 must be replaced with
1398 * filters marked as VLAN=0
1399 * c) finally, if we do not have any active VLANS, all filters
1400 * which are marked as VLAN=0 must be replaced with filters
1401 * marked as VLAN=-1
1402 */
1403
1404 /* Update the filters about to be added in place */
1405 hlist_for_each_entry(new, tmp_add_list, hlist) {
1406 if (pvid && new->f->vlan != pvid)
1407 new->f->vlan = pvid;
1408 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1409 new->f->vlan = 0;
1410 else if (!vlan_filters && new->f->vlan == 0)
1411 new->f->vlan = I40E_VLAN_ANY;
1412 }
1413
1414 /* Update the remaining active filters */
1415 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1416 /* Combine the checks for whether a filter needs to be changed
1417 * and then determine the new VLAN inside the if block, in
1418 * order to avoid duplicating code for adding the new filter
1419 * then deleting the old filter.
1420 */
1421 if ((pvid && f->vlan != pvid) ||
1422 (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1423 (!vlan_filters && f->vlan == 0)) {
1424 /* Determine the new vlan we will be adding */
1425 if (pvid)
1426 new_vlan = pvid;
1427 else if (vlan_filters)
1428 new_vlan = 0;
1429 else
1430 new_vlan = I40E_VLAN_ANY;
1431
1432 /* Create the new filter */
1433 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1434 if (!add_head)
1435 return -ENOMEM;
1436
1437 /* Create a temporary i40e_new_mac_filter */
1438 new = kzalloc(sizeof(*new), GFP_ATOMIC);
1439 if (!new)
1440 return -ENOMEM;
1441
1442 new->f = add_head;
1443 new->state = add_head->state;
1444
1445 /* Add the new filter to the tmp list */
1446 hlist_add_head(&new->hlist, tmp_add_list);
1447
1448 /* Put the original filter into the delete list */
1449 f->state = I40E_FILTER_REMOVE;
1450 hash_del(&f->hlist);
1451 hlist_add_head(&f->hlist, tmp_del_list);
1452 }
1453 }
1454
1455 vsi->has_vlan_filter = !!vlan_filters;
1456
1457 return 0;
1458 }
1459
1460 /**
1461 * i40e_get_vf_new_vlan - Get new vlan id on a vf
1462 * @vsi: the vsi to configure
1463 * @new_mac: new mac filter to be added
1464 * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL
1465 * @vlan_filters: the number of active VLAN filters
1466 * @trusted: flag if the VF is trusted
1467 *
1468 * Get new VLAN id based on current VLAN filters, trust, PVID
1469 * and vf-vlan-prune-disable flag.
1470 *
1471 * Returns the value of the new vlan filter or
1472 * the old value if no new filter is needed.
1473 */
i40e_get_vf_new_vlan(struct i40e_vsi * vsi,struct i40e_new_mac_filter * new_mac,struct i40e_mac_filter * f,int vlan_filters,bool trusted)1474 static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi,
1475 struct i40e_new_mac_filter *new_mac,
1476 struct i40e_mac_filter *f,
1477 int vlan_filters,
1478 bool trusted)
1479 {
1480 s16 pvid = le16_to_cpu(vsi->info.pvid);
1481 struct i40e_pf *pf = vsi->back;
1482 bool is_any;
1483
1484 if (new_mac)
1485 f = new_mac->f;
1486
1487 if (pvid && f->vlan != pvid)
1488 return pvid;
1489
1490 is_any = (trusted ||
1491 !test_bit(I40E_FLAG_VF_VLAN_PRUNING_ENA, pf->flags));
1492
1493 if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1494 (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1495 (is_any && !vlan_filters && f->vlan == 0)) {
1496 if (is_any)
1497 return I40E_VLAN_ANY;
1498 else
1499 return 0;
1500 }
1501
1502 return f->vlan;
1503 }
1504
1505 /**
1506 * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary
1507 * @vsi: the vsi to configure
1508 * @tmp_add_list: list of filters ready to be added
1509 * @tmp_del_list: list of filters ready to be deleted
1510 * @vlan_filters: the number of active VLAN filters
1511 * @trusted: flag if the VF is trusted
1512 *
1513 * Correct VF VLAN filters based on current VLAN filters, trust, PVID
1514 * and vf-vlan-prune-disable flag.
1515 *
1516 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1517 *
1518 * This function is only expected to be called from within
1519 * i40e_sync_vsi_filters.
1520 *
1521 * NOTE: This function expects to be called while under the
1522 * mac_filter_hash_lock
1523 */
i40e_correct_vf_mac_vlan_filters(struct i40e_vsi * vsi,struct hlist_head * tmp_add_list,struct hlist_head * tmp_del_list,int vlan_filters,bool trusted)1524 static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi,
1525 struct hlist_head *tmp_add_list,
1526 struct hlist_head *tmp_del_list,
1527 int vlan_filters,
1528 bool trusted)
1529 {
1530 struct i40e_mac_filter *f, *add_head;
1531 struct i40e_new_mac_filter *new_mac;
1532 struct hlist_node *h;
1533 int bkt, new_vlan;
1534
1535 hlist_for_each_entry(new_mac, tmp_add_list, hlist) {
1536 new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL,
1537 vlan_filters, trusted);
1538 }
1539
1540 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1541 new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters,
1542 trusted);
1543 if (new_vlan != f->vlan) {
1544 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1545 if (!add_head)
1546 return -ENOMEM;
1547 /* Create a temporary i40e_new_mac_filter */
1548 new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC);
1549 if (!new_mac)
1550 return -ENOMEM;
1551 new_mac->f = add_head;
1552 new_mac->state = add_head->state;
1553
1554 /* Add the new filter to the tmp list */
1555 hlist_add_head(&new_mac->hlist, tmp_add_list);
1556
1557 /* Put the original filter into the delete list */
1558 f->state = I40E_FILTER_REMOVE;
1559 hash_del(&f->hlist);
1560 hlist_add_head(&f->hlist, tmp_del_list);
1561 }
1562 }
1563
1564 vsi->has_vlan_filter = !!vlan_filters;
1565 return 0;
1566 }
1567
1568 /**
1569 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1570 * @vsi: the PF Main VSI - inappropriate for any other VSI
1571 * @macaddr: the MAC address
1572 *
1573 * Remove whatever filter the firmware set up so the driver can manage
1574 * its own filtering intelligently.
1575 **/
i40e_rm_default_mac_filter(struct i40e_vsi * vsi,u8 * macaddr)1576 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1577 {
1578 struct i40e_aqc_remove_macvlan_element_data element;
1579 struct i40e_pf *pf = vsi->back;
1580
1581 /* Only appropriate for the PF main VSI */
1582 if (vsi->type != I40E_VSI_MAIN)
1583 return;
1584
1585 memset(&element, 0, sizeof(element));
1586 ether_addr_copy(element.mac_addr, macaddr);
1587 element.vlan_tag = 0;
1588 /* Ignore error returns, some firmware does it this way... */
1589 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1590 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1591
1592 memset(&element, 0, sizeof(element));
1593 ether_addr_copy(element.mac_addr, macaddr);
1594 element.vlan_tag = 0;
1595 /* ...and some firmware does it this way. */
1596 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1597 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1598 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1599 }
1600
1601 /**
1602 * i40e_add_filter - Add a mac/vlan filter to the VSI
1603 * @vsi: the VSI to be searched
1604 * @macaddr: the MAC address
1605 * @vlan: the vlan
1606 *
1607 * Returns ptr to the filter object or NULL when no memory available.
1608 *
1609 * NOTE: This function is expected to be called with mac_filter_hash_lock
1610 * being held.
1611 **/
i40e_add_filter(struct i40e_vsi * vsi,const u8 * macaddr,s16 vlan)1612 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1613 const u8 *macaddr, s16 vlan)
1614 {
1615 struct i40e_mac_filter *f;
1616 u64 key;
1617
1618 if (!vsi || !macaddr)
1619 return NULL;
1620
1621 f = i40e_find_filter(vsi, macaddr, vlan);
1622 if (!f) {
1623 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1624 if (!f)
1625 return NULL;
1626
1627 /* Update the boolean indicating if we need to function in
1628 * VLAN mode.
1629 */
1630 if (vlan >= 0)
1631 vsi->has_vlan_filter = true;
1632
1633 ether_addr_copy(f->macaddr, macaddr);
1634 f->vlan = vlan;
1635 f->state = I40E_FILTER_NEW;
1636 INIT_HLIST_NODE(&f->hlist);
1637
1638 key = i40e_addr_to_hkey(macaddr);
1639 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1640
1641 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1642 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1643 }
1644
1645 /* If we're asked to add a filter that has been marked for removal, it
1646 * is safe to simply restore it to active state. __i40e_del_filter
1647 * will have simply deleted any filters which were previously marked
1648 * NEW or FAILED, so if it is currently marked REMOVE it must have
1649 * previously been ACTIVE. Since we haven't yet run the sync filters
1650 * task, just restore this filter to the ACTIVE state so that the
1651 * sync task leaves it in place
1652 */
1653 if (f->state == I40E_FILTER_REMOVE)
1654 f->state = I40E_FILTER_ACTIVE;
1655
1656 return f;
1657 }
1658
1659 /**
1660 * __i40e_del_filter - Remove a specific filter from the VSI
1661 * @vsi: VSI to remove from
1662 * @f: the filter to remove from the list
1663 *
1664 * This function should be called instead of i40e_del_filter only if you know
1665 * the exact filter you will remove already, such as via i40e_find_filter or
1666 * i40e_find_mac.
1667 *
1668 * NOTE: This function is expected to be called with mac_filter_hash_lock
1669 * being held.
1670 * ANOTHER NOTE: This function MUST be called from within the context of
1671 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1672 * instead of list_for_each_entry().
1673 **/
__i40e_del_filter(struct i40e_vsi * vsi,struct i40e_mac_filter * f)1674 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1675 {
1676 if (!f)
1677 return;
1678
1679 /* If the filter was never added to firmware then we can just delete it
1680 * directly and we don't want to set the status to remove or else an
1681 * admin queue command will unnecessarily fire.
1682 */
1683 if ((f->state == I40E_FILTER_FAILED) ||
1684 (f->state == I40E_FILTER_NEW)) {
1685 hash_del(&f->hlist);
1686 kfree(f);
1687 } else {
1688 f->state = I40E_FILTER_REMOVE;
1689 }
1690
1691 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1692 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1693 }
1694
1695 /**
1696 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1697 * @vsi: the VSI to be searched
1698 * @macaddr: the MAC address
1699 * @vlan: the VLAN
1700 *
1701 * NOTE: This function is expected to be called with mac_filter_hash_lock
1702 * being held.
1703 * ANOTHER NOTE: This function MUST be called from within the context of
1704 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1705 * instead of list_for_each_entry().
1706 **/
i40e_del_filter(struct i40e_vsi * vsi,const u8 * macaddr,s16 vlan)1707 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1708 {
1709 struct i40e_mac_filter *f;
1710
1711 if (!vsi || !macaddr)
1712 return;
1713
1714 f = i40e_find_filter(vsi, macaddr, vlan);
1715 __i40e_del_filter(vsi, f);
1716 }
1717
1718 /**
1719 * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1720 * @vsi: the VSI to be searched
1721 * @macaddr: the mac address to be filtered
1722 *
1723 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1724 * go through all the macvlan filters and add a macvlan filter for each
1725 * unique vlan that already exists. If a PVID has been assigned, instead only
1726 * add the macaddr to that VLAN.
1727 *
1728 * Returns last filter added on success, else NULL
1729 **/
i40e_add_mac_filter(struct i40e_vsi * vsi,const u8 * macaddr)1730 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1731 const u8 *macaddr)
1732 {
1733 struct i40e_mac_filter *f, *add = NULL;
1734 struct hlist_node *h;
1735 int bkt;
1736
1737 if (vsi->info.pvid)
1738 return i40e_add_filter(vsi, macaddr,
1739 le16_to_cpu(vsi->info.pvid));
1740
1741 if (!i40e_is_vsi_in_vlan(vsi))
1742 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1743
1744 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1745 if (f->state == I40E_FILTER_REMOVE)
1746 continue;
1747 add = i40e_add_filter(vsi, macaddr, f->vlan);
1748 if (!add)
1749 return NULL;
1750 }
1751
1752 return add;
1753 }
1754
1755 /**
1756 * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1757 * @vsi: the VSI to be searched
1758 * @macaddr: the mac address to be removed
1759 *
1760 * Removes a given MAC address from a VSI regardless of what VLAN it has been
1761 * associated with.
1762 *
1763 * Returns 0 for success, or error
1764 **/
i40e_del_mac_filter(struct i40e_vsi * vsi,const u8 * macaddr)1765 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1766 {
1767 struct i40e_mac_filter *f;
1768 struct hlist_node *h;
1769 bool found = false;
1770 int bkt;
1771
1772 lockdep_assert_held(&vsi->mac_filter_hash_lock);
1773 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1774 if (ether_addr_equal(macaddr, f->macaddr)) {
1775 __i40e_del_filter(vsi, f);
1776 found = true;
1777 }
1778 }
1779
1780 if (found)
1781 return 0;
1782 else
1783 return -ENOENT;
1784 }
1785
1786 /**
1787 * i40e_set_mac - NDO callback to set mac address
1788 * @netdev: network interface device structure
1789 * @p: pointer to an address structure
1790 *
1791 * Returns 0 on success, negative on failure
1792 **/
i40e_set_mac(struct net_device * netdev,void * p)1793 static int i40e_set_mac(struct net_device *netdev, void *p)
1794 {
1795 struct i40e_netdev_priv *np = netdev_priv(netdev);
1796 struct i40e_vsi *vsi = np->vsi;
1797 struct i40e_pf *pf = vsi->back;
1798 struct i40e_hw *hw = &pf->hw;
1799 struct sockaddr *addr = p;
1800
1801 if (!is_valid_ether_addr(addr->sa_data))
1802 return -EADDRNOTAVAIL;
1803
1804 if (test_bit(__I40E_DOWN, pf->state) ||
1805 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1806 return -EADDRNOTAVAIL;
1807
1808 if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1809 netdev_info(netdev, "returning to hw mac address %pM\n",
1810 hw->mac.addr);
1811 else
1812 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1813
1814 /* Copy the address first, so that we avoid a possible race with
1815 * .set_rx_mode().
1816 * - Remove old address from MAC filter
1817 * - Copy new address
1818 * - Add new address to MAC filter
1819 */
1820 spin_lock_bh(&vsi->mac_filter_hash_lock);
1821 i40e_del_mac_filter(vsi, netdev->dev_addr);
1822 eth_hw_addr_set(netdev, addr->sa_data);
1823 i40e_add_mac_filter(vsi, netdev->dev_addr);
1824 spin_unlock_bh(&vsi->mac_filter_hash_lock);
1825
1826 if (vsi->type == I40E_VSI_MAIN) {
1827 int ret;
1828
1829 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1830 addr->sa_data, NULL);
1831 if (ret)
1832 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n",
1833 ERR_PTR(ret),
1834 i40e_aq_str(hw, hw->aq.asq_last_status));
1835 }
1836
1837 /* schedule our worker thread which will take care of
1838 * applying the new filter changes
1839 */
1840 i40e_service_event_schedule(pf);
1841 return 0;
1842 }
1843
1844 /**
1845 * i40e_config_rss_aq - Prepare for RSS using AQ commands
1846 * @vsi: vsi structure
1847 * @seed: RSS hash seed
1848 * @lut: pointer to lookup table of lut_size
1849 * @lut_size: size of the lookup table
1850 **/
i40e_config_rss_aq(struct i40e_vsi * vsi,const u8 * seed,u8 * lut,u16 lut_size)1851 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1852 u8 *lut, u16 lut_size)
1853 {
1854 struct i40e_pf *pf = vsi->back;
1855 struct i40e_hw *hw = &pf->hw;
1856 int ret = 0;
1857
1858 if (seed) {
1859 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1860 (struct i40e_aqc_get_set_rss_key_data *)seed;
1861 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1862 if (ret) {
1863 dev_info(&pf->pdev->dev,
1864 "Cannot set RSS key, err %pe aq_err %s\n",
1865 ERR_PTR(ret),
1866 i40e_aq_str(hw, hw->aq.asq_last_status));
1867 return ret;
1868 }
1869 }
1870 if (lut) {
1871 bool pf_lut = vsi->type == I40E_VSI_MAIN;
1872
1873 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1874 if (ret) {
1875 dev_info(&pf->pdev->dev,
1876 "Cannot set RSS lut, err %pe aq_err %s\n",
1877 ERR_PTR(ret),
1878 i40e_aq_str(hw, hw->aq.asq_last_status));
1879 return ret;
1880 }
1881 }
1882 return ret;
1883 }
1884
1885 /**
1886 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1887 * @vsi: VSI structure
1888 **/
i40e_vsi_config_rss(struct i40e_vsi * vsi)1889 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1890 {
1891 struct i40e_pf *pf = vsi->back;
1892 u8 seed[I40E_HKEY_ARRAY_SIZE];
1893 u8 *lut;
1894 int ret;
1895
1896 if (!test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
1897 return 0;
1898 if (!vsi->rss_size)
1899 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1900 vsi->num_queue_pairs);
1901 if (!vsi->rss_size)
1902 return -EINVAL;
1903 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1904 if (!lut)
1905 return -ENOMEM;
1906
1907 /* Use the user configured hash keys and lookup table if there is one,
1908 * otherwise use default
1909 */
1910 if (vsi->rss_lut_user)
1911 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1912 else
1913 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1914 if (vsi->rss_hkey_user)
1915 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1916 else
1917 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1918 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1919 kfree(lut);
1920 return ret;
1921 }
1922
1923 /**
1924 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1925 * @vsi: the VSI being configured,
1926 * @ctxt: VSI context structure
1927 * @enabled_tc: number of traffic classes to enable
1928 *
1929 * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1930 **/
i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi * vsi,struct i40e_vsi_context * ctxt,u8 enabled_tc)1931 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1932 struct i40e_vsi_context *ctxt,
1933 u8 enabled_tc)
1934 {
1935 u16 qcount = 0, max_qcount, qmap, sections = 0;
1936 int i, override_q, pow, num_qps, ret;
1937 u8 netdev_tc = 0, offset = 0;
1938
1939 if (vsi->type != I40E_VSI_MAIN)
1940 return -EINVAL;
1941 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1942 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1943 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1944 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1945 num_qps = vsi->mqprio_qopt.qopt.count[0];
1946
1947 /* find the next higher power-of-2 of num queue pairs */
1948 pow = ilog2(num_qps);
1949 if (!is_power_of_2(num_qps))
1950 pow++;
1951 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1952 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1953
1954 /* Setup queue offset/count for all TCs for given VSI */
1955 max_qcount = vsi->mqprio_qopt.qopt.count[0];
1956 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1957 /* See if the given TC is enabled for the given VSI */
1958 if (vsi->tc_config.enabled_tc & BIT(i)) {
1959 offset = vsi->mqprio_qopt.qopt.offset[i];
1960 qcount = vsi->mqprio_qopt.qopt.count[i];
1961 if (qcount > max_qcount)
1962 max_qcount = qcount;
1963 vsi->tc_config.tc_info[i].qoffset = offset;
1964 vsi->tc_config.tc_info[i].qcount = qcount;
1965 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1966 } else {
1967 /* TC is not enabled so set the offset to
1968 * default queue and allocate one queue
1969 * for the given TC.
1970 */
1971 vsi->tc_config.tc_info[i].qoffset = 0;
1972 vsi->tc_config.tc_info[i].qcount = 1;
1973 vsi->tc_config.tc_info[i].netdev_tc = 0;
1974 }
1975 }
1976
1977 /* Set actual Tx/Rx queue pairs */
1978 vsi->num_queue_pairs = offset + qcount;
1979
1980 /* Setup queue TC[0].qmap for given VSI context */
1981 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1982 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1983 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1984 ctxt->info.valid_sections |= cpu_to_le16(sections);
1985
1986 /* Reconfigure RSS for main VSI with max queue count */
1987 vsi->rss_size = max_qcount;
1988 ret = i40e_vsi_config_rss(vsi);
1989 if (ret) {
1990 dev_info(&vsi->back->pdev->dev,
1991 "Failed to reconfig rss for num_queues (%u)\n",
1992 max_qcount);
1993 return ret;
1994 }
1995 vsi->reconfig_rss = true;
1996 dev_dbg(&vsi->back->pdev->dev,
1997 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1998
1999 /* Find queue count available for channel VSIs and starting offset
2000 * for channel VSIs
2001 */
2002 override_q = vsi->mqprio_qopt.qopt.count[0];
2003 if (override_q && override_q < vsi->num_queue_pairs) {
2004 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
2005 vsi->next_base_queue = override_q;
2006 }
2007 return 0;
2008 }
2009
2010 /**
2011 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
2012 * @vsi: the VSI being setup
2013 * @ctxt: VSI context structure
2014 * @enabled_tc: Enabled TCs bitmap
2015 * @is_add: True if called before Add VSI
2016 *
2017 * Setup VSI queue mapping for enabled traffic classes.
2018 **/
i40e_vsi_setup_queue_map(struct i40e_vsi * vsi,struct i40e_vsi_context * ctxt,u8 enabled_tc,bool is_add)2019 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
2020 struct i40e_vsi_context *ctxt,
2021 u8 enabled_tc,
2022 bool is_add)
2023 {
2024 struct i40e_pf *pf = vsi->back;
2025 u16 num_tc_qps = 0;
2026 u16 sections = 0;
2027 u8 netdev_tc = 0;
2028 u16 numtc = 1;
2029 u16 qcount;
2030 u8 offset;
2031 u16 qmap;
2032 int i;
2033
2034 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2035 offset = 0;
2036 /* zero out queue mapping, it will get updated on the end of the function */
2037 memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
2038
2039 if (vsi->type == I40E_VSI_MAIN) {
2040 /* This code helps add more queue to the VSI if we have
2041 * more cores than RSS can support, the higher cores will
2042 * be served by ATR or other filters. Furthermore, the
2043 * non-zero req_queue_pairs says that user requested a new
2044 * queue count via ethtool's set_channels, so use this
2045 * value for queues distribution across traffic classes
2046 * We need at least one queue pair for the interface
2047 * to be usable as we see in else statement.
2048 */
2049 if (vsi->req_queue_pairs > 0)
2050 vsi->num_queue_pairs = vsi->req_queue_pairs;
2051 else if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2052 vsi->num_queue_pairs = pf->num_lan_msix;
2053 else
2054 vsi->num_queue_pairs = 1;
2055 }
2056
2057 /* Number of queues per enabled TC */
2058 if (vsi->type == I40E_VSI_MAIN ||
2059 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
2060 num_tc_qps = vsi->num_queue_pairs;
2061 else
2062 num_tc_qps = vsi->alloc_queue_pairs;
2063
2064 if (enabled_tc && test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
2065 /* Find numtc from enabled TC bitmap */
2066 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2067 if (enabled_tc & BIT(i)) /* TC is enabled */
2068 numtc++;
2069 }
2070 if (!numtc) {
2071 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
2072 numtc = 1;
2073 }
2074 num_tc_qps = num_tc_qps / numtc;
2075 num_tc_qps = min_t(int, num_tc_qps,
2076 i40e_pf_get_max_q_per_tc(pf));
2077 }
2078
2079 vsi->tc_config.numtc = numtc;
2080 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
2081
2082 /* Do not allow use more TC queue pairs than MSI-X vectors exist */
2083 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
2084 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
2085
2086 /* Setup queue offset/count for all TCs for given VSI */
2087 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2088 /* See if the given TC is enabled for the given VSI */
2089 if (vsi->tc_config.enabled_tc & BIT(i)) {
2090 /* TC is enabled */
2091 int pow, num_qps;
2092
2093 switch (vsi->type) {
2094 case I40E_VSI_MAIN:
2095 if ((!test_bit(I40E_FLAG_FD_SB_ENA,
2096 pf->flags) &&
2097 !test_bit(I40E_FLAG_FD_ATR_ENA,
2098 pf->flags)) ||
2099 vsi->tc_config.enabled_tc != 1) {
2100 qcount = min_t(int, pf->alloc_rss_size,
2101 num_tc_qps);
2102 break;
2103 }
2104 fallthrough;
2105 case I40E_VSI_FDIR:
2106 case I40E_VSI_SRIOV:
2107 case I40E_VSI_VMDQ2:
2108 default:
2109 qcount = num_tc_qps;
2110 WARN_ON(i != 0);
2111 break;
2112 }
2113 vsi->tc_config.tc_info[i].qoffset = offset;
2114 vsi->tc_config.tc_info[i].qcount = qcount;
2115
2116 /* find the next higher power-of-2 of num queue pairs */
2117 num_qps = qcount;
2118 pow = 0;
2119 while (num_qps && (BIT_ULL(pow) < qcount)) {
2120 pow++;
2121 num_qps >>= 1;
2122 }
2123
2124 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
2125 qmap =
2126 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2127 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
2128
2129 offset += qcount;
2130 } else {
2131 /* TC is not enabled so set the offset to
2132 * default queue and allocate one queue
2133 * for the given TC.
2134 */
2135 vsi->tc_config.tc_info[i].qoffset = 0;
2136 vsi->tc_config.tc_info[i].qcount = 1;
2137 vsi->tc_config.tc_info[i].netdev_tc = 0;
2138
2139 qmap = 0;
2140 }
2141 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
2142 }
2143 /* Do not change previously set num_queue_pairs for PFs and VFs*/
2144 if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
2145 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
2146 (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
2147 vsi->num_queue_pairs = offset;
2148
2149 /* Scheduler section valid can only be set for ADD VSI */
2150 if (is_add) {
2151 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
2152
2153 ctxt->info.up_enable_bits = enabled_tc;
2154 }
2155 if (vsi->type == I40E_VSI_SRIOV) {
2156 ctxt->info.mapping_flags |=
2157 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2158 for (i = 0; i < vsi->num_queue_pairs; i++)
2159 ctxt->info.queue_mapping[i] =
2160 cpu_to_le16(vsi->base_queue + i);
2161 } else {
2162 ctxt->info.mapping_flags |=
2163 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2164 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
2165 }
2166 ctxt->info.valid_sections |= cpu_to_le16(sections);
2167 }
2168
2169 /**
2170 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
2171 * @netdev: the netdevice
2172 * @addr: address to add
2173 *
2174 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2175 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2176 */
i40e_addr_sync(struct net_device * netdev,const u8 * addr)2177 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
2178 {
2179 struct i40e_netdev_priv *np = netdev_priv(netdev);
2180 struct i40e_vsi *vsi = np->vsi;
2181
2182 if (i40e_add_mac_filter(vsi, addr))
2183 return 0;
2184 else
2185 return -ENOMEM;
2186 }
2187
2188 /**
2189 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2190 * @netdev: the netdevice
2191 * @addr: address to add
2192 *
2193 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
2194 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2195 */
i40e_addr_unsync(struct net_device * netdev,const u8 * addr)2196 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
2197 {
2198 struct i40e_netdev_priv *np = netdev_priv(netdev);
2199 struct i40e_vsi *vsi = np->vsi;
2200
2201 /* Under some circumstances, we might receive a request to delete
2202 * our own device address from our uc list. Because we store the
2203 * device address in the VSI's MAC/VLAN filter list, we need to ignore
2204 * such requests and not delete our device address from this list.
2205 */
2206 if (ether_addr_equal(addr, netdev->dev_addr))
2207 return 0;
2208
2209 i40e_del_mac_filter(vsi, addr);
2210
2211 return 0;
2212 }
2213
2214 /**
2215 * i40e_set_rx_mode - NDO callback to set the netdev filters
2216 * @netdev: network interface device structure
2217 **/
i40e_set_rx_mode(struct net_device * netdev)2218 static void i40e_set_rx_mode(struct net_device *netdev)
2219 {
2220 struct i40e_netdev_priv *np = netdev_priv(netdev);
2221 struct i40e_vsi *vsi = np->vsi;
2222
2223 spin_lock_bh(&vsi->mac_filter_hash_lock);
2224
2225 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2226 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2227
2228 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2229
2230 /* check for other flag changes */
2231 if (vsi->current_netdev_flags != vsi->netdev->flags) {
2232 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2233 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
2234 }
2235 }
2236
2237 /**
2238 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2239 * @vsi: Pointer to VSI struct
2240 * @from: Pointer to list which contains MAC filter entries - changes to
2241 * those entries needs to be undone.
2242 *
2243 * MAC filter entries from this list were slated for deletion.
2244 **/
i40e_undo_del_filter_entries(struct i40e_vsi * vsi,struct hlist_head * from)2245 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2246 struct hlist_head *from)
2247 {
2248 struct i40e_mac_filter *f;
2249 struct hlist_node *h;
2250
2251 hlist_for_each_entry_safe(f, h, from, hlist) {
2252 u64 key = i40e_addr_to_hkey(f->macaddr);
2253
2254 /* Move the element back into MAC filter list*/
2255 hlist_del(&f->hlist);
2256 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2257 }
2258 }
2259
2260 /**
2261 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2262 * @vsi: Pointer to vsi struct
2263 * @from: Pointer to list which contains MAC filter entries - changes to
2264 * those entries needs to be undone.
2265 *
2266 * MAC filter entries from this list were slated for addition.
2267 **/
i40e_undo_add_filter_entries(struct i40e_vsi * vsi,struct hlist_head * from)2268 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2269 struct hlist_head *from)
2270 {
2271 struct i40e_new_mac_filter *new;
2272 struct hlist_node *h;
2273
2274 hlist_for_each_entry_safe(new, h, from, hlist) {
2275 /* We can simply free the wrapper structure */
2276 hlist_del(&new->hlist);
2277 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2278 kfree(new);
2279 }
2280 }
2281
2282 /**
2283 * i40e_next_filter - Get the next non-broadcast filter from a list
2284 * @next: pointer to filter in list
2285 *
2286 * Returns the next non-broadcast filter in the list. Required so that we
2287 * ignore broadcast filters within the list, since these are not handled via
2288 * the normal firmware update path.
2289 */
2290 static
i40e_next_filter(struct i40e_new_mac_filter * next)2291 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2292 {
2293 hlist_for_each_entry_continue(next, hlist) {
2294 if (!is_broadcast_ether_addr(next->f->macaddr))
2295 return next;
2296 }
2297
2298 return NULL;
2299 }
2300
2301 /**
2302 * i40e_update_filter_state - Update filter state based on return data
2303 * from firmware
2304 * @count: Number of filters added
2305 * @add_list: return data from fw
2306 * @add_head: pointer to first filter in current batch
2307 *
2308 * MAC filter entries from list were slated to be added to device. Returns
2309 * number of successful filters. Note that 0 does NOT mean success!
2310 **/
2311 static int
i40e_update_filter_state(int count,struct i40e_aqc_add_macvlan_element_data * add_list,struct i40e_new_mac_filter * add_head)2312 i40e_update_filter_state(int count,
2313 struct i40e_aqc_add_macvlan_element_data *add_list,
2314 struct i40e_new_mac_filter *add_head)
2315 {
2316 int retval = 0;
2317 int i;
2318
2319 for (i = 0; i < count; i++) {
2320 /* Always check status of each filter. We don't need to check
2321 * the firmware return status because we pre-set the filter
2322 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2323 * request to the adminq. Thus, if it no longer matches then
2324 * we know the filter is active.
2325 */
2326 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2327 add_head->state = I40E_FILTER_FAILED;
2328 } else {
2329 add_head->state = I40E_FILTER_ACTIVE;
2330 retval++;
2331 }
2332
2333 add_head = i40e_next_filter(add_head);
2334 if (!add_head)
2335 break;
2336 }
2337
2338 return retval;
2339 }
2340
2341 /**
2342 * i40e_aqc_del_filters - Request firmware to delete a set of filters
2343 * @vsi: ptr to the VSI
2344 * @vsi_name: name to display in messages
2345 * @list: the list of filters to send to firmware
2346 * @num_del: the number of filters to delete
2347 * @retval: Set to -EIO on failure to delete
2348 *
2349 * Send a request to firmware via AdminQ to delete a set of filters. Uses
2350 * *retval instead of a return value so that success does not force ret_val to
2351 * be set to 0. This ensures that a sequence of calls to this function
2352 * preserve the previous value of *retval on successful delete.
2353 */
2354 static
i40e_aqc_del_filters(struct i40e_vsi * vsi,const char * vsi_name,struct i40e_aqc_remove_macvlan_element_data * list,int num_del,int * retval)2355 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2356 struct i40e_aqc_remove_macvlan_element_data *list,
2357 int num_del, int *retval)
2358 {
2359 struct i40e_hw *hw = &vsi->back->hw;
2360 enum i40e_admin_queue_err aq_status;
2361 int aq_ret;
2362
2363 aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL,
2364 &aq_status);
2365
2366 /* Explicitly ignore and do not report when firmware returns ENOENT */
2367 if (aq_ret && !(aq_status == I40E_AQ_RC_ENOENT)) {
2368 *retval = -EIO;
2369 dev_info(&vsi->back->pdev->dev,
2370 "ignoring delete macvlan error on %s, err %pe, aq_err %s\n",
2371 vsi_name, ERR_PTR(aq_ret),
2372 i40e_aq_str(hw, aq_status));
2373 }
2374 }
2375
2376 /**
2377 * i40e_aqc_add_filters - Request firmware to add a set of filters
2378 * @vsi: ptr to the VSI
2379 * @vsi_name: name to display in messages
2380 * @list: the list of filters to send to firmware
2381 * @add_head: Position in the add hlist
2382 * @num_add: the number of filters to add
2383 *
2384 * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2385 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2386 * space for more filters.
2387 */
2388 static
i40e_aqc_add_filters(struct i40e_vsi * vsi,const char * vsi_name,struct i40e_aqc_add_macvlan_element_data * list,struct i40e_new_mac_filter * add_head,int num_add)2389 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2390 struct i40e_aqc_add_macvlan_element_data *list,
2391 struct i40e_new_mac_filter *add_head,
2392 int num_add)
2393 {
2394 struct i40e_hw *hw = &vsi->back->hw;
2395 enum i40e_admin_queue_err aq_status;
2396 int fcnt;
2397
2398 i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status);
2399 fcnt = i40e_update_filter_state(num_add, list, add_head);
2400
2401 if (fcnt != num_add) {
2402 if (vsi->type == I40E_VSI_MAIN) {
2403 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2404 dev_warn(&vsi->back->pdev->dev,
2405 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2406 i40e_aq_str(hw, aq_status), vsi_name);
2407 } else if (vsi->type == I40E_VSI_SRIOV ||
2408 vsi->type == I40E_VSI_VMDQ1 ||
2409 vsi->type == I40E_VSI_VMDQ2) {
2410 dev_warn(&vsi->back->pdev->dev,
2411 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2412 i40e_aq_str(hw, aq_status), vsi_name,
2413 vsi_name);
2414 } else {
2415 dev_warn(&vsi->back->pdev->dev,
2416 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2417 i40e_aq_str(hw, aq_status), vsi_name,
2418 vsi->type);
2419 }
2420 }
2421 }
2422
2423 /**
2424 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2425 * @vsi: pointer to the VSI
2426 * @vsi_name: the VSI name
2427 * @f: filter data
2428 *
2429 * This function sets or clears the promiscuous broadcast flags for VLAN
2430 * filters in order to properly receive broadcast frames. Assumes that only
2431 * broadcast filters are passed.
2432 *
2433 * Returns status indicating success or failure;
2434 **/
2435 static int
i40e_aqc_broadcast_filter(struct i40e_vsi * vsi,const char * vsi_name,struct i40e_mac_filter * f)2436 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2437 struct i40e_mac_filter *f)
2438 {
2439 bool enable = f->state == I40E_FILTER_NEW;
2440 struct i40e_hw *hw = &vsi->back->hw;
2441 int aq_ret;
2442
2443 if (f->vlan == I40E_VLAN_ANY) {
2444 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2445 vsi->seid,
2446 enable,
2447 NULL);
2448 } else {
2449 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2450 vsi->seid,
2451 enable,
2452 f->vlan,
2453 NULL);
2454 }
2455
2456 if (aq_ret) {
2457 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2458 dev_warn(&vsi->back->pdev->dev,
2459 "Error %s, forcing overflow promiscuous on %s\n",
2460 i40e_aq_str(hw, hw->aq.asq_last_status),
2461 vsi_name);
2462 }
2463
2464 return aq_ret;
2465 }
2466
2467 /**
2468 * i40e_set_promiscuous - set promiscuous mode
2469 * @pf: board private structure
2470 * @promisc: promisc on or off
2471 *
2472 * There are different ways of setting promiscuous mode on a PF depending on
2473 * what state/environment we're in. This identifies and sets it appropriately.
2474 * Returns 0 on success.
2475 **/
i40e_set_promiscuous(struct i40e_pf * pf,bool promisc)2476 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2477 {
2478 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
2479 struct i40e_hw *hw = &pf->hw;
2480 int aq_ret;
2481
2482 if (vsi->type == I40E_VSI_MAIN &&
2483 i40e_pf_get_main_veb(pf) &&
2484 !test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
2485 /* set defport ON for Main VSI instead of true promisc
2486 * this way we will get all unicast/multicast and VLAN
2487 * promisc behavior but will not get VF or VMDq traffic
2488 * replicated on the Main VSI.
2489 */
2490 if (promisc)
2491 aq_ret = i40e_aq_set_default_vsi(hw,
2492 vsi->seid,
2493 NULL);
2494 else
2495 aq_ret = i40e_aq_clear_default_vsi(hw,
2496 vsi->seid,
2497 NULL);
2498 if (aq_ret) {
2499 dev_info(&pf->pdev->dev,
2500 "Set default VSI failed, err %pe, aq_err %s\n",
2501 ERR_PTR(aq_ret),
2502 i40e_aq_str(hw, hw->aq.asq_last_status));
2503 }
2504 } else {
2505 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2506 hw,
2507 vsi->seid,
2508 promisc, NULL,
2509 true);
2510 if (aq_ret) {
2511 dev_info(&pf->pdev->dev,
2512 "set unicast promisc failed, err %pe, aq_err %s\n",
2513 ERR_PTR(aq_ret),
2514 i40e_aq_str(hw, hw->aq.asq_last_status));
2515 }
2516 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2517 hw,
2518 vsi->seid,
2519 promisc, NULL);
2520 if (aq_ret) {
2521 dev_info(&pf->pdev->dev,
2522 "set multicast promisc failed, err %pe, aq_err %s\n",
2523 ERR_PTR(aq_ret),
2524 i40e_aq_str(hw, hw->aq.asq_last_status));
2525 }
2526 }
2527
2528 if (!aq_ret)
2529 pf->cur_promisc = promisc;
2530
2531 return aq_ret;
2532 }
2533
2534 /**
2535 * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2536 * @vsi: ptr to the VSI
2537 *
2538 * Push any outstanding VSI filter changes through the AdminQ.
2539 *
2540 * Returns 0 or error value
2541 **/
i40e_sync_vsi_filters(struct i40e_vsi * vsi)2542 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2543 {
2544 struct hlist_head tmp_add_list, tmp_del_list;
2545 struct i40e_mac_filter *f;
2546 struct i40e_new_mac_filter *new, *add_head = NULL;
2547 struct i40e_hw *hw = &vsi->back->hw;
2548 bool old_overflow, new_overflow;
2549 unsigned int failed_filters = 0;
2550 unsigned int vlan_filters = 0;
2551 char vsi_name[16] = "PF";
2552 int filter_list_len = 0;
2553 u32 changed_flags = 0;
2554 struct hlist_node *h;
2555 struct i40e_pf *pf;
2556 int num_add = 0;
2557 int num_del = 0;
2558 int aq_ret = 0;
2559 int retval = 0;
2560 u16 cmd_flags;
2561 int list_size;
2562 int bkt;
2563
2564 /* empty array typed pointers, kcalloc later */
2565 struct i40e_aqc_add_macvlan_element_data *add_list;
2566 struct i40e_aqc_remove_macvlan_element_data *del_list;
2567
2568 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2569 usleep_range(1000, 2000);
2570 pf = vsi->back;
2571
2572 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2573
2574 if (vsi->netdev) {
2575 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2576 vsi->current_netdev_flags = vsi->netdev->flags;
2577 }
2578
2579 INIT_HLIST_HEAD(&tmp_add_list);
2580 INIT_HLIST_HEAD(&tmp_del_list);
2581
2582 if (vsi->type == I40E_VSI_SRIOV)
2583 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2584 else if (vsi->type != I40E_VSI_MAIN)
2585 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2586
2587 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2588 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2589
2590 spin_lock_bh(&vsi->mac_filter_hash_lock);
2591 /* Create a list of filters to delete. */
2592 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2593 if (f->state == I40E_FILTER_REMOVE) {
2594 /* Move the element into temporary del_list */
2595 hash_del(&f->hlist);
2596 hlist_add_head(&f->hlist, &tmp_del_list);
2597
2598 /* Avoid counting removed filters */
2599 continue;
2600 }
2601 if (f->state == I40E_FILTER_NEW) {
2602 /* Create a temporary i40e_new_mac_filter */
2603 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2604 if (!new)
2605 goto err_no_memory_locked;
2606
2607 /* Store pointer to the real filter */
2608 new->f = f;
2609 new->state = f->state;
2610
2611 /* Add it to the hash list */
2612 hlist_add_head(&new->hlist, &tmp_add_list);
2613 }
2614
2615 /* Count the number of active (current and new) VLAN
2616 * filters we have now. Does not count filters which
2617 * are marked for deletion.
2618 */
2619 if (f->vlan > 0)
2620 vlan_filters++;
2621 }
2622
2623 if (vsi->type != I40E_VSI_SRIOV)
2624 retval = i40e_correct_mac_vlan_filters
2625 (vsi, &tmp_add_list, &tmp_del_list,
2626 vlan_filters);
2627 else if (pf->vf)
2628 retval = i40e_correct_vf_mac_vlan_filters
2629 (vsi, &tmp_add_list, &tmp_del_list,
2630 vlan_filters, pf->vf[vsi->vf_id].trusted);
2631
2632 hlist_for_each_entry(new, &tmp_add_list, hlist)
2633 netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
2634
2635 if (retval)
2636 goto err_no_memory_locked;
2637
2638 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2639 }
2640
2641 /* Now process 'del_list' outside the lock */
2642 if (!hlist_empty(&tmp_del_list)) {
2643 filter_list_len = hw->aq.asq_buf_size /
2644 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2645 list_size = filter_list_len *
2646 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2647 del_list = kzalloc(list_size, GFP_ATOMIC);
2648 if (!del_list)
2649 goto err_no_memory;
2650
2651 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2652 cmd_flags = 0;
2653
2654 /* handle broadcast filters by updating the broadcast
2655 * promiscuous flag and release filter list.
2656 */
2657 if (is_broadcast_ether_addr(f->macaddr)) {
2658 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2659
2660 hlist_del(&f->hlist);
2661 kfree(f);
2662 continue;
2663 }
2664
2665 /* add to delete list */
2666 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2667 if (f->vlan == I40E_VLAN_ANY) {
2668 del_list[num_del].vlan_tag = 0;
2669 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2670 } else {
2671 del_list[num_del].vlan_tag =
2672 cpu_to_le16((u16)(f->vlan));
2673 }
2674
2675 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2676 del_list[num_del].flags = cmd_flags;
2677 num_del++;
2678
2679 /* flush a full buffer */
2680 if (num_del == filter_list_len) {
2681 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2682 num_del, &retval);
2683 memset(del_list, 0, list_size);
2684 num_del = 0;
2685 }
2686 /* Release memory for MAC filter entries which were
2687 * synced up with HW.
2688 */
2689 hlist_del(&f->hlist);
2690 kfree(f);
2691 }
2692
2693 if (num_del) {
2694 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2695 num_del, &retval);
2696 }
2697
2698 kfree(del_list);
2699 del_list = NULL;
2700 }
2701
2702 if (!hlist_empty(&tmp_add_list)) {
2703 /* Do all the adds now. */
2704 filter_list_len = hw->aq.asq_buf_size /
2705 sizeof(struct i40e_aqc_add_macvlan_element_data);
2706 list_size = filter_list_len *
2707 sizeof(struct i40e_aqc_add_macvlan_element_data);
2708 add_list = kzalloc(list_size, GFP_ATOMIC);
2709 if (!add_list)
2710 goto err_no_memory;
2711
2712 num_add = 0;
2713 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2714 /* handle broadcast filters by updating the broadcast
2715 * promiscuous flag instead of adding a MAC filter.
2716 */
2717 if (is_broadcast_ether_addr(new->f->macaddr)) {
2718 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2719 new->f))
2720 new->state = I40E_FILTER_FAILED;
2721 else
2722 new->state = I40E_FILTER_ACTIVE;
2723 continue;
2724 }
2725
2726 /* add to add array */
2727 if (num_add == 0)
2728 add_head = new;
2729 cmd_flags = 0;
2730 ether_addr_copy(add_list[num_add].mac_addr,
2731 new->f->macaddr);
2732 if (new->f->vlan == I40E_VLAN_ANY) {
2733 add_list[num_add].vlan_tag = 0;
2734 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2735 } else {
2736 add_list[num_add].vlan_tag =
2737 cpu_to_le16((u16)(new->f->vlan));
2738 }
2739 add_list[num_add].queue_number = 0;
2740 /* set invalid match method for later detection */
2741 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2742 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2743 add_list[num_add].flags = cpu_to_le16(cmd_flags);
2744 num_add++;
2745
2746 /* flush a full buffer */
2747 if (num_add == filter_list_len) {
2748 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2749 add_head, num_add);
2750 memset(add_list, 0, list_size);
2751 num_add = 0;
2752 }
2753 }
2754 if (num_add) {
2755 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2756 num_add);
2757 }
2758 /* Now move all of the filters from the temp add list back to
2759 * the VSI's list.
2760 */
2761 spin_lock_bh(&vsi->mac_filter_hash_lock);
2762 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2763 /* Only update the state if we're still NEW */
2764 if (new->f->state == I40E_FILTER_NEW)
2765 new->f->state = new->state;
2766 hlist_del(&new->hlist);
2767 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2768 kfree(new);
2769 }
2770 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2771 kfree(add_list);
2772 add_list = NULL;
2773 }
2774
2775 /* Determine the number of active and failed filters. */
2776 spin_lock_bh(&vsi->mac_filter_hash_lock);
2777 vsi->active_filters = 0;
2778 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2779 if (f->state == I40E_FILTER_ACTIVE)
2780 vsi->active_filters++;
2781 else if (f->state == I40E_FILTER_FAILED)
2782 failed_filters++;
2783 }
2784 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2785
2786 /* Check if we are able to exit overflow promiscuous mode. We can
2787 * safely exit if we didn't just enter, we no longer have any failed
2788 * filters, and we have reduced filters below the threshold value.
2789 */
2790 if (old_overflow && !failed_filters &&
2791 vsi->active_filters < vsi->promisc_threshold) {
2792 dev_info(&pf->pdev->dev,
2793 "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2794 vsi_name);
2795 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2796 vsi->promisc_threshold = 0;
2797 }
2798
2799 /* if the VF is not trusted do not do promisc */
2800 if (vsi->type == I40E_VSI_SRIOV && pf->vf &&
2801 !pf->vf[vsi->vf_id].trusted) {
2802 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2803 goto out;
2804 }
2805
2806 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2807
2808 /* If we are entering overflow promiscuous, we need to calculate a new
2809 * threshold for when we are safe to exit
2810 */
2811 if (!old_overflow && new_overflow)
2812 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2813
2814 /* check for changes in promiscuous modes */
2815 if (changed_flags & IFF_ALLMULTI) {
2816 bool cur_multipromisc;
2817
2818 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2819 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2820 vsi->seid,
2821 cur_multipromisc,
2822 NULL);
2823 if (aq_ret) {
2824 retval = i40e_aq_rc_to_posix(aq_ret,
2825 hw->aq.asq_last_status);
2826 dev_info(&pf->pdev->dev,
2827 "set multi promisc failed on %s, err %pe aq_err %s\n",
2828 vsi_name,
2829 ERR_PTR(aq_ret),
2830 i40e_aq_str(hw, hw->aq.asq_last_status));
2831 } else {
2832 dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2833 cur_multipromisc ? "entering" : "leaving");
2834 }
2835 }
2836
2837 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2838 bool cur_promisc;
2839
2840 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2841 new_overflow);
2842 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2843 if (aq_ret) {
2844 retval = i40e_aq_rc_to_posix(aq_ret,
2845 hw->aq.asq_last_status);
2846 dev_info(&pf->pdev->dev,
2847 "Setting promiscuous %s failed on %s, err %pe aq_err %s\n",
2848 cur_promisc ? "on" : "off",
2849 vsi_name,
2850 ERR_PTR(aq_ret),
2851 i40e_aq_str(hw, hw->aq.asq_last_status));
2852 }
2853 }
2854 out:
2855 /* if something went wrong then set the changed flag so we try again */
2856 if (retval)
2857 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2858
2859 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2860 return retval;
2861
2862 err_no_memory:
2863 /* Restore elements on the temporary add and delete lists */
2864 spin_lock_bh(&vsi->mac_filter_hash_lock);
2865 err_no_memory_locked:
2866 i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2867 i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2868 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2869
2870 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2871 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2872 return -ENOMEM;
2873 }
2874
2875 /**
2876 * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2877 * @pf: board private structure
2878 **/
i40e_sync_filters_subtask(struct i40e_pf * pf)2879 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2880 {
2881 struct i40e_vsi *vsi;
2882 int v;
2883
2884 if (!pf)
2885 return;
2886 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2887 return;
2888 if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2889 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2890 return;
2891 }
2892
2893 i40e_pf_for_each_vsi(pf, v, vsi) {
2894 if ((vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2895 !test_bit(__I40E_VSI_RELEASING, vsi->state)) {
2896 int ret = i40e_sync_vsi_filters(vsi);
2897
2898 if (ret) {
2899 /* come back and try again later */
2900 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2901 pf->state);
2902 break;
2903 }
2904 }
2905 }
2906 }
2907
2908 /**
2909 * i40e_calculate_vsi_rx_buf_len - Calculates buffer length
2910 *
2911 * @vsi: VSI to calculate rx_buf_len from
2912 */
i40e_calculate_vsi_rx_buf_len(struct i40e_vsi * vsi)2913 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
2914 {
2915 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags))
2916 return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048);
2917
2918 return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
2919 }
2920
2921 /**
2922 * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI
2923 * @vsi: the vsi
2924 * @xdp_prog: XDP program
2925 **/
i40e_max_vsi_frame_size(struct i40e_vsi * vsi,struct bpf_prog * xdp_prog)2926 static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi,
2927 struct bpf_prog *xdp_prog)
2928 {
2929 u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
2930 u16 chain_len;
2931
2932 if (xdp_prog && !xdp_prog->aux->xdp_has_frags)
2933 chain_len = 1;
2934 else
2935 chain_len = I40E_MAX_CHAINED_RX_BUFFERS;
2936
2937 return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER);
2938 }
2939
2940 /**
2941 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2942 * @netdev: network interface device structure
2943 * @new_mtu: new value for maximum frame size
2944 *
2945 * Returns 0 on success, negative on failure
2946 **/
i40e_change_mtu(struct net_device * netdev,int new_mtu)2947 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2948 {
2949 struct i40e_netdev_priv *np = netdev_priv(netdev);
2950 struct i40e_vsi *vsi = np->vsi;
2951 struct i40e_pf *pf = vsi->back;
2952 int frame_size;
2953
2954 frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
2955 if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) {
2956 netdev_err(netdev, "Error changing mtu to %d, Max is %d\n",
2957 new_mtu, frame_size - I40E_PACKET_HDR_PAD);
2958 return -EINVAL;
2959 }
2960
2961 netdev_dbg(netdev, "changing MTU from %d to %d\n",
2962 netdev->mtu, new_mtu);
2963 WRITE_ONCE(netdev->mtu, new_mtu);
2964 if (netif_running(netdev))
2965 i40e_vsi_reinit_locked(vsi);
2966 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2967 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2968 return 0;
2969 }
2970
2971 /**
2972 * i40e_ioctl - Access the hwtstamp interface
2973 * @netdev: network interface device structure
2974 * @ifr: interface request data
2975 * @cmd: ioctl command
2976 **/
i40e_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)2977 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2978 {
2979 struct i40e_netdev_priv *np = netdev_priv(netdev);
2980 struct i40e_pf *pf = np->vsi->back;
2981
2982 switch (cmd) {
2983 case SIOCGHWTSTAMP:
2984 return i40e_ptp_get_ts_config(pf, ifr);
2985 case SIOCSHWTSTAMP:
2986 return i40e_ptp_set_ts_config(pf, ifr);
2987 default:
2988 return -EOPNOTSUPP;
2989 }
2990 }
2991
2992 /**
2993 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2994 * @vsi: the vsi being adjusted
2995 **/
i40e_vlan_stripping_enable(struct i40e_vsi * vsi)2996 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2997 {
2998 struct i40e_vsi_context ctxt;
2999 int ret;
3000
3001 /* Don't modify stripping options if a port VLAN is active */
3002 if (vsi->info.pvid)
3003 return;
3004
3005 if ((vsi->info.valid_sections &
3006 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3007 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
3008 return; /* already enabled */
3009
3010 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3011 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3012 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3013
3014 ctxt.seid = vsi->seid;
3015 ctxt.info = vsi->info;
3016 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3017 if (ret) {
3018 dev_info(&vsi->back->pdev->dev,
3019 "update vlan stripping failed, err %pe aq_err %s\n",
3020 ERR_PTR(ret),
3021 i40e_aq_str(&vsi->back->hw,
3022 vsi->back->hw.aq.asq_last_status));
3023 }
3024 }
3025
3026 /**
3027 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
3028 * @vsi: the vsi being adjusted
3029 **/
i40e_vlan_stripping_disable(struct i40e_vsi * vsi)3030 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
3031 {
3032 struct i40e_vsi_context ctxt;
3033 int ret;
3034
3035 /* Don't modify stripping options if a port VLAN is active */
3036 if (vsi->info.pvid)
3037 return;
3038
3039 if ((vsi->info.valid_sections &
3040 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3041 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
3042 I40E_AQ_VSI_PVLAN_EMOD_MASK))
3043 return; /* already disabled */
3044
3045 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3046 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3047 I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3048
3049 ctxt.seid = vsi->seid;
3050 ctxt.info = vsi->info;
3051 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3052 if (ret) {
3053 dev_info(&vsi->back->pdev->dev,
3054 "update vlan stripping failed, err %pe aq_err %s\n",
3055 ERR_PTR(ret),
3056 i40e_aq_str(&vsi->back->hw,
3057 vsi->back->hw.aq.asq_last_status));
3058 }
3059 }
3060
3061 /**
3062 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
3063 * @vsi: the vsi being configured
3064 * @vid: vlan id to be added (0 = untagged only , -1 = any)
3065 *
3066 * This is a helper function for adding a new MAC/VLAN filter with the
3067 * specified VLAN for each existing MAC address already in the hash table.
3068 * This function does *not* perform any accounting to update filters based on
3069 * VLAN mode.
3070 *
3071 * NOTE: this function expects to be called while under the
3072 * mac_filter_hash_lock
3073 **/
i40e_add_vlan_all_mac(struct i40e_vsi * vsi,s16 vid)3074 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3075 {
3076 struct i40e_mac_filter *f, *add_f;
3077 struct hlist_node *h;
3078 int bkt;
3079
3080 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3081 /* If we're asked to add a filter that has been marked for
3082 * removal, it is safe to simply restore it to active state.
3083 * __i40e_del_filter will have simply deleted any filters which
3084 * were previously marked NEW or FAILED, so if it is currently
3085 * marked REMOVE it must have previously been ACTIVE. Since we
3086 * haven't yet run the sync filters task, just restore this
3087 * filter to the ACTIVE state so that the sync task leaves it
3088 * in place.
3089 */
3090 if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) {
3091 f->state = I40E_FILTER_ACTIVE;
3092 continue;
3093 } else if (f->state == I40E_FILTER_REMOVE) {
3094 continue;
3095 }
3096 add_f = i40e_add_filter(vsi, f->macaddr, vid);
3097 if (!add_f) {
3098 dev_info(&vsi->back->pdev->dev,
3099 "Could not add vlan filter %d for %pM\n",
3100 vid, f->macaddr);
3101 return -ENOMEM;
3102 }
3103 }
3104
3105 return 0;
3106 }
3107
3108 /**
3109 * i40e_vsi_add_vlan - Add VSI membership for given VLAN
3110 * @vsi: the VSI being configured
3111 * @vid: VLAN id to be added
3112 **/
i40e_vsi_add_vlan(struct i40e_vsi * vsi,u16 vid)3113 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
3114 {
3115 int err;
3116
3117 if (vsi->info.pvid)
3118 return -EINVAL;
3119
3120 /* The network stack will attempt to add VID=0, with the intention to
3121 * receive priority tagged packets with a VLAN of 0. Our HW receives
3122 * these packets by default when configured to receive untagged
3123 * packets, so we don't need to add a filter for this case.
3124 * Additionally, HW interprets adding a VID=0 filter as meaning to
3125 * receive *only* tagged traffic and stops receiving untagged traffic.
3126 * Thus, we do not want to actually add a filter for VID=0
3127 */
3128 if (!vid)
3129 return 0;
3130
3131 /* Locked once because all functions invoked below iterates list*/
3132 spin_lock_bh(&vsi->mac_filter_hash_lock);
3133 err = i40e_add_vlan_all_mac(vsi, vid);
3134 spin_unlock_bh(&vsi->mac_filter_hash_lock);
3135 if (err)
3136 return err;
3137
3138 /* schedule our worker thread which will take care of
3139 * applying the new filter changes
3140 */
3141 i40e_service_event_schedule(vsi->back);
3142 return 0;
3143 }
3144
3145 /**
3146 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
3147 * @vsi: the vsi being configured
3148 * @vid: vlan id to be removed (0 = untagged only , -1 = any)
3149 *
3150 * This function should be used to remove all VLAN filters which match the
3151 * given VID. It does not schedule the service event and does not take the
3152 * mac_filter_hash_lock so it may be combined with other operations under
3153 * a single invocation of the mac_filter_hash_lock.
3154 *
3155 * NOTE: this function expects to be called while under the
3156 * mac_filter_hash_lock
3157 */
i40e_rm_vlan_all_mac(struct i40e_vsi * vsi,s16 vid)3158 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3159 {
3160 struct i40e_mac_filter *f;
3161 struct hlist_node *h;
3162 int bkt;
3163
3164 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3165 if (f->vlan == vid)
3166 __i40e_del_filter(vsi, f);
3167 }
3168 }
3169
3170 /**
3171 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
3172 * @vsi: the VSI being configured
3173 * @vid: VLAN id to be removed
3174 **/
i40e_vsi_kill_vlan(struct i40e_vsi * vsi,u16 vid)3175 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
3176 {
3177 if (!vid || vsi->info.pvid)
3178 return;
3179
3180 spin_lock_bh(&vsi->mac_filter_hash_lock);
3181 i40e_rm_vlan_all_mac(vsi, vid);
3182 spin_unlock_bh(&vsi->mac_filter_hash_lock);
3183
3184 /* schedule our worker thread which will take care of
3185 * applying the new filter changes
3186 */
3187 i40e_service_event_schedule(vsi->back);
3188 }
3189
3190 /**
3191 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
3192 * @netdev: network interface to be adjusted
3193 * @proto: unused protocol value
3194 * @vid: vlan id to be added
3195 *
3196 * net_device_ops implementation for adding vlan ids
3197 **/
i40e_vlan_rx_add_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)3198 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
3199 __always_unused __be16 proto, u16 vid)
3200 {
3201 struct i40e_netdev_priv *np = netdev_priv(netdev);
3202 struct i40e_vsi *vsi = np->vsi;
3203 int ret = 0;
3204
3205 if (vid >= VLAN_N_VID)
3206 return -EINVAL;
3207
3208 ret = i40e_vsi_add_vlan(vsi, vid);
3209 if (!ret)
3210 set_bit(vid, vsi->active_vlans);
3211
3212 return ret;
3213 }
3214
3215 /**
3216 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
3217 * @netdev: network interface to be adjusted
3218 * @proto: unused protocol value
3219 * @vid: vlan id to be added
3220 **/
i40e_vlan_rx_add_vid_up(struct net_device * netdev,__always_unused __be16 proto,u16 vid)3221 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
3222 __always_unused __be16 proto, u16 vid)
3223 {
3224 struct i40e_netdev_priv *np = netdev_priv(netdev);
3225 struct i40e_vsi *vsi = np->vsi;
3226
3227 if (vid >= VLAN_N_VID)
3228 return;
3229 set_bit(vid, vsi->active_vlans);
3230 }
3231
3232 /**
3233 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
3234 * @netdev: network interface to be adjusted
3235 * @proto: unused protocol value
3236 * @vid: vlan id to be removed
3237 *
3238 * net_device_ops implementation for removing vlan ids
3239 **/
i40e_vlan_rx_kill_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)3240 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
3241 __always_unused __be16 proto, u16 vid)
3242 {
3243 struct i40e_netdev_priv *np = netdev_priv(netdev);
3244 struct i40e_vsi *vsi = np->vsi;
3245
3246 /* return code is ignored as there is nothing a user
3247 * can do about failure to remove and a log message was
3248 * already printed from the other function
3249 */
3250 i40e_vsi_kill_vlan(vsi, vid);
3251
3252 clear_bit(vid, vsi->active_vlans);
3253
3254 return 0;
3255 }
3256
3257 /**
3258 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
3259 * @vsi: the vsi being brought back up
3260 **/
i40e_restore_vlan(struct i40e_vsi * vsi)3261 static void i40e_restore_vlan(struct i40e_vsi *vsi)
3262 {
3263 u16 vid;
3264
3265 if (!vsi->netdev)
3266 return;
3267
3268 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3269 i40e_vlan_stripping_enable(vsi);
3270 else
3271 i40e_vlan_stripping_disable(vsi);
3272
3273 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
3274 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
3275 vid);
3276 }
3277
3278 /**
3279 * i40e_vsi_add_pvid - Add pvid for the VSI
3280 * @vsi: the vsi being adjusted
3281 * @vid: the vlan id to set as a PVID
3282 **/
i40e_vsi_add_pvid(struct i40e_vsi * vsi,u16 vid)3283 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
3284 {
3285 struct i40e_vsi_context ctxt;
3286 int ret;
3287
3288 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3289 vsi->info.pvid = cpu_to_le16(vid);
3290 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3291 I40E_AQ_VSI_PVLAN_INSERT_PVID |
3292 I40E_AQ_VSI_PVLAN_EMOD_STR;
3293
3294 ctxt.seid = vsi->seid;
3295 ctxt.info = vsi->info;
3296 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3297 if (ret) {
3298 dev_info(&vsi->back->pdev->dev,
3299 "add pvid failed, err %pe aq_err %s\n",
3300 ERR_PTR(ret),
3301 i40e_aq_str(&vsi->back->hw,
3302 vsi->back->hw.aq.asq_last_status));
3303 return -ENOENT;
3304 }
3305
3306 return 0;
3307 }
3308
3309 /**
3310 * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3311 * @vsi: the vsi being adjusted
3312 *
3313 * Just use the vlan_rx_register() service to put it back to normal
3314 **/
i40e_vsi_remove_pvid(struct i40e_vsi * vsi)3315 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3316 {
3317 vsi->info.pvid = 0;
3318
3319 i40e_vlan_stripping_disable(vsi);
3320 }
3321
3322 /**
3323 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3324 * @vsi: ptr to the VSI
3325 *
3326 * If this function returns with an error, then it's possible one or
3327 * more of the rings is populated (while the rest are not). It is the
3328 * callers duty to clean those orphaned rings.
3329 *
3330 * Return 0 on success, negative on failure
3331 **/
i40e_vsi_setup_tx_resources(struct i40e_vsi * vsi)3332 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3333 {
3334 int i, err = 0;
3335
3336 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3337 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3338
3339 if (!i40e_enabled_xdp_vsi(vsi))
3340 return err;
3341
3342 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3343 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3344
3345 return err;
3346 }
3347
3348 /**
3349 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3350 * @vsi: ptr to the VSI
3351 *
3352 * Free VSI's transmit software resources
3353 **/
i40e_vsi_free_tx_resources(struct i40e_vsi * vsi)3354 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3355 {
3356 int i;
3357
3358 if (vsi->tx_rings) {
3359 for (i = 0; i < vsi->num_queue_pairs; i++)
3360 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3361 i40e_free_tx_resources(vsi->tx_rings[i]);
3362 }
3363
3364 if (vsi->xdp_rings) {
3365 for (i = 0; i < vsi->num_queue_pairs; i++)
3366 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3367 i40e_free_tx_resources(vsi->xdp_rings[i]);
3368 }
3369 }
3370
3371 /**
3372 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3373 * @vsi: ptr to the VSI
3374 *
3375 * If this function returns with an error, then it's possible one or
3376 * more of the rings is populated (while the rest are not). It is the
3377 * callers duty to clean those orphaned rings.
3378 *
3379 * Return 0 on success, negative on failure
3380 **/
i40e_vsi_setup_rx_resources(struct i40e_vsi * vsi)3381 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3382 {
3383 int i, err = 0;
3384
3385 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3386 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3387 return err;
3388 }
3389
3390 /**
3391 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3392 * @vsi: ptr to the VSI
3393 *
3394 * Free all receive software resources
3395 **/
i40e_vsi_free_rx_resources(struct i40e_vsi * vsi)3396 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3397 {
3398 int i;
3399
3400 if (!vsi->rx_rings)
3401 return;
3402
3403 for (i = 0; i < vsi->num_queue_pairs; i++)
3404 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3405 i40e_free_rx_resources(vsi->rx_rings[i]);
3406 }
3407
3408 /**
3409 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3410 * @ring: The Tx ring to configure
3411 *
3412 * This enables/disables XPS for a given Tx descriptor ring
3413 * based on the TCs enabled for the VSI that ring belongs to.
3414 **/
i40e_config_xps_tx_ring(struct i40e_ring * ring)3415 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3416 {
3417 int cpu;
3418
3419 if (!ring->q_vector || !ring->netdev || ring->ch)
3420 return;
3421
3422 /* We only initialize XPS once, so as not to overwrite user settings */
3423 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3424 return;
3425
3426 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3427 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3428 ring->queue_index);
3429 }
3430
3431 /**
3432 * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3433 * @ring: The Tx or Rx ring
3434 *
3435 * Returns the AF_XDP buffer pool or NULL.
3436 **/
i40e_xsk_pool(struct i40e_ring * ring)3437 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3438 {
3439 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3440 int qid = ring->queue_index;
3441
3442 if (ring_is_xdp(ring))
3443 qid -= ring->vsi->alloc_queue_pairs;
3444
3445 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3446 return NULL;
3447
3448 return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3449 }
3450
3451 /**
3452 * i40e_configure_tx_ring - Configure a transmit ring context and rest
3453 * @ring: The Tx ring to configure
3454 *
3455 * Configure the Tx descriptor ring in the HMC context.
3456 **/
i40e_configure_tx_ring(struct i40e_ring * ring)3457 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3458 {
3459 struct i40e_vsi *vsi = ring->vsi;
3460 u16 pf_q = vsi->base_queue + ring->queue_index;
3461 struct i40e_hw *hw = &vsi->back->hw;
3462 struct i40e_hmc_obj_txq tx_ctx;
3463 u32 qtx_ctl = 0;
3464 int err = 0;
3465
3466 if (ring_is_xdp(ring))
3467 ring->xsk_pool = i40e_xsk_pool(ring);
3468
3469 /* some ATR related tx ring init */
3470 if (test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags)) {
3471 ring->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
3472 ring->atr_count = 0;
3473 } else {
3474 ring->atr_sample_rate = 0;
3475 }
3476
3477 /* configure XPS */
3478 i40e_config_xps_tx_ring(ring);
3479
3480 /* clear the context structure first */
3481 memset(&tx_ctx, 0, sizeof(tx_ctx));
3482
3483 tx_ctx.new_context = 1;
3484 tx_ctx.base = (ring->dma / 128);
3485 tx_ctx.qlen = ring->count;
3486 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags) ||
3487 test_bit(I40E_FLAG_FD_ATR_ENA, vsi->back->flags))
3488 tx_ctx.fd_ena = 1;
3489 if (test_bit(I40E_FLAG_PTP_ENA, vsi->back->flags))
3490 tx_ctx.timesync_ena = 1;
3491 /* FDIR VSI tx ring can still use RS bit and writebacks */
3492 if (vsi->type != I40E_VSI_FDIR)
3493 tx_ctx.head_wb_ena = 1;
3494 tx_ctx.head_wb_addr = ring->dma +
3495 (ring->count * sizeof(struct i40e_tx_desc));
3496
3497 /* As part of VSI creation/update, FW allocates certain
3498 * Tx arbitration queue sets for each TC enabled for
3499 * the VSI. The FW returns the handles to these queue
3500 * sets as part of the response buffer to Add VSI,
3501 * Update VSI, etc. AQ commands. It is expected that
3502 * these queue set handles be associated with the Tx
3503 * queues by the driver as part of the TX queue context
3504 * initialization. This has to be done regardless of
3505 * DCB as by default everything is mapped to TC0.
3506 */
3507
3508 if (ring->ch)
3509 tx_ctx.rdylist =
3510 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3511
3512 else
3513 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3514
3515 tx_ctx.rdylist_act = 0;
3516
3517 /* clear the context in the HMC */
3518 err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3519 if (err) {
3520 dev_info(&vsi->back->pdev->dev,
3521 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3522 ring->queue_index, pf_q, err);
3523 return -ENOMEM;
3524 }
3525
3526 /* set the context in the HMC */
3527 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3528 if (err) {
3529 dev_info(&vsi->back->pdev->dev,
3530 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3531 ring->queue_index, pf_q, err);
3532 return -ENOMEM;
3533 }
3534
3535 /* Now associate this queue with this PCI function */
3536 if (ring->ch) {
3537 if (ring->ch->type == I40E_VSI_VMDQ2)
3538 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3539 else
3540 return -EINVAL;
3541
3542 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3543 ring->ch->vsi_number);
3544 } else {
3545 if (vsi->type == I40E_VSI_VMDQ2) {
3546 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3547 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_VFVM_INDX_MASK,
3548 vsi->id);
3549 } else {
3550 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3551 }
3552 }
3553
3554 qtx_ctl |= FIELD_PREP(I40E_QTX_CTL_PF_INDX_MASK, hw->pf_id);
3555 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3556 i40e_flush(hw);
3557
3558 /* cache tail off for easier writes later */
3559 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3560
3561 return 0;
3562 }
3563
3564 /**
3565 * i40e_rx_offset - Return expected offset into page to access data
3566 * @rx_ring: Ring we are requesting offset of
3567 *
3568 * Returns the offset value for ring into the data buffer.
3569 */
i40e_rx_offset(struct i40e_ring * rx_ring)3570 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
3571 {
3572 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
3573 }
3574
3575 /**
3576 * i40e_configure_rx_ring - Configure a receive ring context
3577 * @ring: The Rx ring to configure
3578 *
3579 * Configure the Rx descriptor ring in the HMC context.
3580 **/
i40e_configure_rx_ring(struct i40e_ring * ring)3581 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3582 {
3583 struct i40e_vsi *vsi = ring->vsi;
3584 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3585 u16 pf_q = vsi->base_queue + ring->queue_index;
3586 struct i40e_hw *hw = &vsi->back->hw;
3587 struct i40e_hmc_obj_rxq rx_ctx;
3588 int err = 0;
3589 bool ok;
3590
3591 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3592
3593 /* clear the context structure first */
3594 memset(&rx_ctx, 0, sizeof(rx_ctx));
3595
3596 ring->rx_buf_len = vsi->rx_buf_len;
3597
3598 /* XDP RX-queue info only needed for RX rings exposed to XDP */
3599 if (ring->vsi->type != I40E_VSI_MAIN)
3600 goto skip;
3601
3602 if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) {
3603 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3604 ring->queue_index,
3605 ring->q_vector->napi.napi_id,
3606 ring->rx_buf_len);
3607 if (err)
3608 return err;
3609 }
3610
3611 ring->xsk_pool = i40e_xsk_pool(ring);
3612 if (ring->xsk_pool) {
3613 xdp_rxq_info_unreg(&ring->xdp_rxq);
3614 ring->rx_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool);
3615 err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev,
3616 ring->queue_index,
3617 ring->q_vector->napi.napi_id,
3618 ring->rx_buf_len);
3619 if (err)
3620 return err;
3621 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3622 MEM_TYPE_XSK_BUFF_POOL,
3623 NULL);
3624 if (err)
3625 return err;
3626 dev_info(&vsi->back->pdev->dev,
3627 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3628 ring->queue_index);
3629
3630 } else {
3631 err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3632 MEM_TYPE_PAGE_SHARED,
3633 NULL);
3634 if (err)
3635 return err;
3636 }
3637
3638 skip:
3639 xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq);
3640
3641 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3642 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3643
3644 rx_ctx.base = (ring->dma / 128);
3645 rx_ctx.qlen = ring->count;
3646
3647 /* use 16 byte descriptors */
3648 rx_ctx.dsize = 0;
3649
3650 /* descriptor type is always zero
3651 * rx_ctx.dtype = 0;
3652 */
3653 rx_ctx.hsplit_0 = 0;
3654
3655 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3656 if (hw->revision_id == 0)
3657 rx_ctx.lrxqthresh = 0;
3658 else
3659 rx_ctx.lrxqthresh = 1;
3660 rx_ctx.crcstrip = 1;
3661 rx_ctx.l2tsel = 1;
3662 /* this controls whether VLAN is stripped from inner headers */
3663 rx_ctx.showiv = 0;
3664 /* set the prefena field to 1 because the manual says to */
3665 rx_ctx.prefena = 1;
3666
3667 /* clear the context in the HMC */
3668 err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3669 if (err) {
3670 dev_info(&vsi->back->pdev->dev,
3671 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3672 ring->queue_index, pf_q, err);
3673 return -ENOMEM;
3674 }
3675
3676 /* set the context in the HMC */
3677 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3678 if (err) {
3679 dev_info(&vsi->back->pdev->dev,
3680 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3681 ring->queue_index, pf_q, err);
3682 return -ENOMEM;
3683 }
3684
3685 /* configure Rx buffer alignment */
3686 if (!vsi->netdev || test_bit(I40E_FLAG_LEGACY_RX_ENA, vsi->back->flags)) {
3687 if (I40E_2K_TOO_SMALL_WITH_PADDING) {
3688 dev_info(&vsi->back->pdev->dev,
3689 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
3690 return -EOPNOTSUPP;
3691 }
3692 clear_ring_build_skb_enabled(ring);
3693 } else {
3694 set_ring_build_skb_enabled(ring);
3695 }
3696
3697 ring->rx_offset = i40e_rx_offset(ring);
3698
3699 /* cache tail for quicker writes, and clear the reg before use */
3700 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3701 writel(0, ring->tail);
3702
3703 if (ring->xsk_pool) {
3704 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3705 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3706 } else {
3707 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3708 }
3709 if (!ok) {
3710 /* Log this in case the user has forgotten to give the kernel
3711 * any buffers, even later in the application.
3712 */
3713 dev_info(&vsi->back->pdev->dev,
3714 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3715 ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3716 ring->queue_index, pf_q);
3717 }
3718
3719 return 0;
3720 }
3721
3722 /**
3723 * i40e_vsi_configure_tx - Configure the VSI for Tx
3724 * @vsi: VSI structure describing this set of rings and resources
3725 *
3726 * Configure the Tx VSI for operation.
3727 **/
i40e_vsi_configure_tx(struct i40e_vsi * vsi)3728 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3729 {
3730 int err = 0;
3731 u16 i;
3732
3733 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3734 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3735
3736 if (err || !i40e_enabled_xdp_vsi(vsi))
3737 return err;
3738
3739 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3740 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3741
3742 return err;
3743 }
3744
3745 /**
3746 * i40e_vsi_configure_rx - Configure the VSI for Rx
3747 * @vsi: the VSI being configured
3748 *
3749 * Configure the Rx VSI for operation.
3750 **/
i40e_vsi_configure_rx(struct i40e_vsi * vsi)3751 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3752 {
3753 int err = 0;
3754 u16 i;
3755
3756 vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
3757 vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
3758
3759 #if (PAGE_SIZE < 8192)
3760 if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&
3761 vsi->netdev->mtu <= ETH_DATA_LEN) {
3762 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3763 vsi->max_frame = vsi->rx_buf_len;
3764 }
3765 #endif
3766
3767 /* set up individual rings */
3768 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3769 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3770
3771 return err;
3772 }
3773
3774 /**
3775 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3776 * @vsi: ptr to the VSI
3777 **/
i40e_vsi_config_dcb_rings(struct i40e_vsi * vsi)3778 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3779 {
3780 struct i40e_ring *tx_ring, *rx_ring;
3781 u16 qoffset, qcount;
3782 int i, n;
3783
3784 if (!test_bit(I40E_FLAG_DCB_ENA, vsi->back->flags)) {
3785 /* Reset the TC information */
3786 for (i = 0; i < vsi->num_queue_pairs; i++) {
3787 rx_ring = vsi->rx_rings[i];
3788 tx_ring = vsi->tx_rings[i];
3789 rx_ring->dcb_tc = 0;
3790 tx_ring->dcb_tc = 0;
3791 }
3792 return;
3793 }
3794
3795 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3796 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3797 continue;
3798
3799 qoffset = vsi->tc_config.tc_info[n].qoffset;
3800 qcount = vsi->tc_config.tc_info[n].qcount;
3801 for (i = qoffset; i < (qoffset + qcount); i++) {
3802 rx_ring = vsi->rx_rings[i];
3803 tx_ring = vsi->tx_rings[i];
3804 rx_ring->dcb_tc = n;
3805 tx_ring->dcb_tc = n;
3806 }
3807 }
3808 }
3809
3810 /**
3811 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3812 * @vsi: ptr to the VSI
3813 **/
i40e_set_vsi_rx_mode(struct i40e_vsi * vsi)3814 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3815 {
3816 if (vsi->netdev)
3817 i40e_set_rx_mode(vsi->netdev);
3818 }
3819
3820 /**
3821 * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3822 * @pf: Pointer to the targeted PF
3823 *
3824 * Set all flow director counters to 0.
3825 */
i40e_reset_fdir_filter_cnt(struct i40e_pf * pf)3826 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3827 {
3828 pf->fd_tcp4_filter_cnt = 0;
3829 pf->fd_udp4_filter_cnt = 0;
3830 pf->fd_sctp4_filter_cnt = 0;
3831 pf->fd_ip4_filter_cnt = 0;
3832 pf->fd_tcp6_filter_cnt = 0;
3833 pf->fd_udp6_filter_cnt = 0;
3834 pf->fd_sctp6_filter_cnt = 0;
3835 pf->fd_ip6_filter_cnt = 0;
3836 }
3837
3838 /**
3839 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3840 * @vsi: Pointer to the targeted VSI
3841 *
3842 * This function replays the hlist on the hw where all the SB Flow Director
3843 * filters were saved.
3844 **/
i40e_fdir_filter_restore(struct i40e_vsi * vsi)3845 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3846 {
3847 struct i40e_fdir_filter *filter;
3848 struct i40e_pf *pf = vsi->back;
3849 struct hlist_node *node;
3850
3851 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
3852 return;
3853
3854 /* Reset FDir counters as we're replaying all existing filters */
3855 i40e_reset_fdir_filter_cnt(pf);
3856
3857 hlist_for_each_entry_safe(filter, node,
3858 &pf->fdir_filter_list, fdir_node) {
3859 i40e_add_del_fdir(vsi, filter, true);
3860 }
3861 }
3862
3863 /**
3864 * i40e_vsi_configure - Set up the VSI for action
3865 * @vsi: the VSI being configured
3866 **/
i40e_vsi_configure(struct i40e_vsi * vsi)3867 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3868 {
3869 int err;
3870
3871 i40e_set_vsi_rx_mode(vsi);
3872 i40e_restore_vlan(vsi);
3873 i40e_vsi_config_dcb_rings(vsi);
3874 err = i40e_vsi_configure_tx(vsi);
3875 if (!err)
3876 err = i40e_vsi_configure_rx(vsi);
3877
3878 return err;
3879 }
3880
3881 /**
3882 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3883 * @vsi: the VSI being configured
3884 **/
i40e_vsi_configure_msix(struct i40e_vsi * vsi)3885 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3886 {
3887 bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3888 struct i40e_pf *pf = vsi->back;
3889 struct i40e_hw *hw = &pf->hw;
3890 u16 vector;
3891 int i, q;
3892 u32 qp;
3893
3894 /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3895 * and PFINT_LNKLSTn registers, e.g.:
3896 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts)
3897 */
3898 qp = vsi->base_queue;
3899 vector = vsi->base_vector;
3900 for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3901 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3902
3903 q_vector->rx.next_update = jiffies + 1;
3904 q_vector->rx.target_itr =
3905 ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3906 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3907 q_vector->rx.target_itr >> 1);
3908 q_vector->rx.current_itr = q_vector->rx.target_itr;
3909
3910 q_vector->tx.next_update = jiffies + 1;
3911 q_vector->tx.target_itr =
3912 ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3913 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3914 q_vector->tx.target_itr >> 1);
3915 q_vector->tx.current_itr = q_vector->tx.target_itr;
3916
3917 /* Set ITR for software interrupts triggered after exiting
3918 * busy-loop polling.
3919 */
3920 wr32(hw, I40E_PFINT_ITRN(I40E_SW_ITR, vector - 1),
3921 I40E_ITR_20K);
3922
3923 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3924 i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3925
3926 /* begin of linked list for RX queue assigned to this vector */
3927 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3928 for (q = 0; q < q_vector->num_ringpairs; q++) {
3929 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3930 u32 val;
3931
3932 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3933 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3934 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3935 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3936 (I40E_QUEUE_TYPE_TX <<
3937 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3938
3939 wr32(hw, I40E_QINT_RQCTL(qp), val);
3940
3941 if (has_xdp) {
3942 /* TX queue with next queue set to TX */
3943 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3944 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3945 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3946 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3947 (I40E_QUEUE_TYPE_TX <<
3948 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3949
3950 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3951 }
3952 /* TX queue with next RX or end of linked list */
3953 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3954 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3955 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3956 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3957 (I40E_QUEUE_TYPE_RX <<
3958 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3959
3960 /* Terminate the linked list */
3961 if (q == (q_vector->num_ringpairs - 1))
3962 val |= (I40E_QUEUE_END_OF_LIST <<
3963 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3964
3965 wr32(hw, I40E_QINT_TQCTL(qp), val);
3966 qp++;
3967 }
3968 }
3969
3970 i40e_flush(hw);
3971 }
3972
3973 /**
3974 * i40e_enable_misc_int_causes - enable the non-queue interrupts
3975 * @pf: pointer to private device data structure
3976 **/
i40e_enable_misc_int_causes(struct i40e_pf * pf)3977 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3978 {
3979 struct i40e_hw *hw = &pf->hw;
3980 u32 val;
3981
3982 /* clear things first */
3983 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */
3984 rd32(hw, I40E_PFINT_ICR0); /* read to clear */
3985
3986 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
3987 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
3988 I40E_PFINT_ICR0_ENA_GRST_MASK |
3989 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3990 I40E_PFINT_ICR0_ENA_GPIO_MASK |
3991 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
3992 I40E_PFINT_ICR0_ENA_VFLR_MASK |
3993 I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3994
3995 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
3996 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3997
3998 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
3999 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4000
4001 wr32(hw, I40E_PFINT_ICR0_ENA, val);
4002
4003 /* SW_ITR_IDX = 0, but don't change INTENA */
4004 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
4005 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
4006
4007 /* OTHER_ITR_IDX = 0 */
4008 wr32(hw, I40E_PFINT_STAT_CTL0, 0);
4009 }
4010
4011 /**
4012 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
4013 * @vsi: the VSI being configured
4014 **/
i40e_configure_msi_and_legacy(struct i40e_vsi * vsi)4015 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
4016 {
4017 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
4018 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4019 struct i40e_pf *pf = vsi->back;
4020 struct i40e_hw *hw = &pf->hw;
4021
4022 /* set the ITR configuration */
4023 q_vector->rx.next_update = jiffies + 1;
4024 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
4025 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
4026 q_vector->rx.current_itr = q_vector->rx.target_itr;
4027 q_vector->tx.next_update = jiffies + 1;
4028 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
4029 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
4030 q_vector->tx.current_itr = q_vector->tx.target_itr;
4031
4032 i40e_enable_misc_int_causes(pf);
4033
4034 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
4035 wr32(hw, I40E_PFINT_LNKLST0, 0);
4036
4037 /* Associate the queue pair to the vector and enable the queue
4038 * interrupt RX queue in linked list with next queue set to TX
4039 */
4040 wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX));
4041
4042 if (i40e_enabled_xdp_vsi(vsi)) {
4043 /* TX queue in linked list with next queue set to TX */
4044 wr32(hw, I40E_QINT_TQCTL(nextqp),
4045 I40E_QINT_TQCTL_VAL(nextqp, 0, TX));
4046 }
4047
4048 /* last TX queue so the next RX queue doesn't matter */
4049 wr32(hw, I40E_QINT_TQCTL(0),
4050 I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX));
4051 i40e_flush(hw);
4052 }
4053
4054 /**
4055 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
4056 * @pf: board private structure
4057 **/
i40e_irq_dynamic_disable_icr0(struct i40e_pf * pf)4058 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
4059 {
4060 struct i40e_hw *hw = &pf->hw;
4061
4062 wr32(hw, I40E_PFINT_DYN_CTL0,
4063 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4064 i40e_flush(hw);
4065 }
4066
4067 /**
4068 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
4069 * @pf: board private structure
4070 **/
i40e_irq_dynamic_enable_icr0(struct i40e_pf * pf)4071 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
4072 {
4073 struct i40e_hw *hw = &pf->hw;
4074 u32 val;
4075
4076 val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
4077 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4078 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4079
4080 wr32(hw, I40E_PFINT_DYN_CTL0, val);
4081 i40e_flush(hw);
4082 }
4083
4084 /**
4085 * i40e_msix_clean_rings - MSIX mode Interrupt Handler
4086 * @irq: interrupt number
4087 * @data: pointer to a q_vector
4088 **/
i40e_msix_clean_rings(int irq,void * data)4089 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
4090 {
4091 struct i40e_q_vector *q_vector = data;
4092
4093 if (!q_vector->tx.ring && !q_vector->rx.ring)
4094 return IRQ_HANDLED;
4095
4096 napi_schedule_irqoff(&q_vector->napi);
4097
4098 return IRQ_HANDLED;
4099 }
4100
4101 /**
4102 * i40e_irq_affinity_notify - Callback for affinity changes
4103 * @notify: context as to what irq was changed
4104 * @mask: the new affinity mask
4105 *
4106 * This is a callback function used by the irq_set_affinity_notifier function
4107 * so that we may register to receive changes to the irq affinity masks.
4108 **/
i40e_irq_affinity_notify(struct irq_affinity_notify * notify,const cpumask_t * mask)4109 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
4110 const cpumask_t *mask)
4111 {
4112 struct i40e_q_vector *q_vector =
4113 container_of(notify, struct i40e_q_vector, affinity_notify);
4114
4115 cpumask_copy(&q_vector->affinity_mask, mask);
4116 }
4117
4118 /**
4119 * i40e_irq_affinity_release - Callback for affinity notifier release
4120 * @ref: internal core kernel usage
4121 *
4122 * This is a callback function used by the irq_set_affinity_notifier function
4123 * to inform the current notification subscriber that they will no longer
4124 * receive notifications.
4125 **/
i40e_irq_affinity_release(struct kref * ref)4126 static void i40e_irq_affinity_release(struct kref *ref) {}
4127
4128 /**
4129 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
4130 * @vsi: the VSI being configured
4131 * @basename: name for the vector
4132 *
4133 * Allocates MSI-X vectors and requests interrupts from the kernel.
4134 **/
i40e_vsi_request_irq_msix(struct i40e_vsi * vsi,char * basename)4135 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
4136 {
4137 int q_vectors = vsi->num_q_vectors;
4138 struct i40e_pf *pf = vsi->back;
4139 int base = vsi->base_vector;
4140 int rx_int_idx = 0;
4141 int tx_int_idx = 0;
4142 int vector, err;
4143 int irq_num;
4144 int cpu;
4145
4146 for (vector = 0; vector < q_vectors; vector++) {
4147 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
4148
4149 irq_num = pf->msix_entries[base + vector].vector;
4150
4151 if (q_vector->tx.ring && q_vector->rx.ring) {
4152 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4153 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
4154 tx_int_idx++;
4155 } else if (q_vector->rx.ring) {
4156 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4157 "%s-%s-%d", basename, "rx", rx_int_idx++);
4158 } else if (q_vector->tx.ring) {
4159 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4160 "%s-%s-%d", basename, "tx", tx_int_idx++);
4161 } else {
4162 /* skip this unused q_vector */
4163 continue;
4164 }
4165 err = request_irq(irq_num,
4166 vsi->irq_handler,
4167 0,
4168 q_vector->name,
4169 q_vector);
4170 if (err) {
4171 dev_info(&pf->pdev->dev,
4172 "MSIX request_irq failed, error: %d\n", err);
4173 goto free_queue_irqs;
4174 }
4175
4176 /* register for affinity change notifications */
4177 q_vector->irq_num = irq_num;
4178 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
4179 q_vector->affinity_notify.release = i40e_irq_affinity_release;
4180 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
4181 /* Spread affinity hints out across online CPUs.
4182 *
4183 * get_cpu_mask returns a static constant mask with
4184 * a permanent lifetime so it's ok to pass to
4185 * irq_update_affinity_hint without making a copy.
4186 */
4187 cpu = cpumask_local_spread(q_vector->v_idx, -1);
4188 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
4189 }
4190
4191 vsi->irqs_ready = true;
4192 return 0;
4193
4194 free_queue_irqs:
4195 while (vector) {
4196 vector--;
4197 irq_num = pf->msix_entries[base + vector].vector;
4198 irq_set_affinity_notifier(irq_num, NULL);
4199 irq_update_affinity_hint(irq_num, NULL);
4200 free_irq(irq_num, &vsi->q_vectors[vector]);
4201 }
4202 return err;
4203 }
4204
4205 /**
4206 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
4207 * @vsi: the VSI being un-configured
4208 **/
i40e_vsi_disable_irq(struct i40e_vsi * vsi)4209 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
4210 {
4211 struct i40e_pf *pf = vsi->back;
4212 struct i40e_hw *hw = &pf->hw;
4213 int base = vsi->base_vector;
4214 int i;
4215
4216 /* disable interrupt causation from each queue */
4217 for (i = 0; i < vsi->num_queue_pairs; i++) {
4218 u32 val;
4219
4220 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4221 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
4222 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4223
4224 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
4225 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
4226 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
4227
4228 if (!i40e_enabled_xdp_vsi(vsi))
4229 continue;
4230 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
4231 }
4232
4233 /* disable each interrupt */
4234 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4235 for (i = vsi->base_vector;
4236 i < (vsi->num_q_vectors + vsi->base_vector); i++)
4237 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
4238
4239 i40e_flush(hw);
4240 for (i = 0; i < vsi->num_q_vectors; i++)
4241 synchronize_irq(pf->msix_entries[i + base].vector);
4242 } else {
4243 /* Legacy and MSI mode - this stops all interrupt handling */
4244 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
4245 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
4246 i40e_flush(hw);
4247 synchronize_irq(pf->pdev->irq);
4248 }
4249 }
4250
4251 /**
4252 * i40e_vsi_enable_irq - Enable IRQ for the given VSI
4253 * @vsi: the VSI being configured
4254 **/
i40e_vsi_enable_irq(struct i40e_vsi * vsi)4255 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
4256 {
4257 struct i40e_pf *pf = vsi->back;
4258 int i;
4259
4260 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4261 for (i = 0; i < vsi->num_q_vectors; i++)
4262 i40e_irq_dynamic_enable(vsi, i);
4263 } else {
4264 i40e_irq_dynamic_enable_icr0(pf);
4265 }
4266
4267 i40e_flush(&pf->hw);
4268 return 0;
4269 }
4270
4271 /**
4272 * i40e_free_misc_vector - Free the vector that handles non-queue events
4273 * @pf: board private structure
4274 **/
i40e_free_misc_vector(struct i40e_pf * pf)4275 static void i40e_free_misc_vector(struct i40e_pf *pf)
4276 {
4277 /* Disable ICR 0 */
4278 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
4279 i40e_flush(&pf->hw);
4280
4281 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
4282 free_irq(pf->msix_entries[0].vector, pf);
4283 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
4284 }
4285 }
4286
4287 /**
4288 * i40e_intr - MSI/Legacy and non-queue interrupt handler
4289 * @irq: interrupt number
4290 * @data: pointer to a q_vector
4291 *
4292 * This is the handler used for all MSI/Legacy interrupts, and deals
4293 * with both queue and non-queue interrupts. This is also used in
4294 * MSIX mode to handle the non-queue interrupts.
4295 **/
i40e_intr(int irq,void * data)4296 static irqreturn_t i40e_intr(int irq, void *data)
4297 {
4298 struct i40e_pf *pf = (struct i40e_pf *)data;
4299 struct i40e_hw *hw = &pf->hw;
4300 irqreturn_t ret = IRQ_NONE;
4301 u32 icr0, icr0_remaining;
4302 u32 val, ena_mask;
4303
4304 icr0 = rd32(hw, I40E_PFINT_ICR0);
4305 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4306
4307 /* if sharing a legacy IRQ, we might get called w/o an intr pending */
4308 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4309 goto enable_intr;
4310
4311 /* if interrupt but no bits showing, must be SWINT */
4312 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4313 (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4314 pf->sw_int_count++;
4315
4316 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
4317 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4318 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4319 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4320 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4321 }
4322
4323 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4324 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4325 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
4326 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4327
4328 /* We do not have a way to disarm Queue causes while leaving
4329 * interrupt enabled for all other causes, ideally
4330 * interrupt should be disabled while we are in NAPI but
4331 * this is not a performance path and napi_schedule()
4332 * can deal with rescheduling.
4333 */
4334 if (!test_bit(__I40E_DOWN, pf->state))
4335 napi_schedule_irqoff(&q_vector->napi);
4336 }
4337
4338 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4339 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4340 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4341 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4342 }
4343
4344 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4345 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4346 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4347 }
4348
4349 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4350 /* disable any further VFLR event notifications */
4351 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4352 u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4353
4354 reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4355 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4356 } else {
4357 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4358 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4359 }
4360 }
4361
4362 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4363 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4364 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4365 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4366 val = rd32(hw, I40E_GLGEN_RSTAT);
4367 val = FIELD_GET(I40E_GLGEN_RSTAT_RESET_TYPE_MASK, val);
4368 if (val == I40E_RESET_CORER) {
4369 pf->corer_count++;
4370 } else if (val == I40E_RESET_GLOBR) {
4371 pf->globr_count++;
4372 } else if (val == I40E_RESET_EMPR) {
4373 pf->empr_count++;
4374 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4375 }
4376 }
4377
4378 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4379 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4380 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4381 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4382 rd32(hw, I40E_PFHMC_ERRORINFO),
4383 rd32(hw, I40E_PFHMC_ERRORDATA));
4384 }
4385
4386 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4387 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4388
4389 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)
4390 schedule_work(&pf->ptp_extts0_work);
4391
4392 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)
4393 i40e_ptp_tx_hwtstamp(pf);
4394
4395 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4396 }
4397
4398 /* If a critical error is pending we have no choice but to reset the
4399 * device.
4400 * Report and mask out any remaining unexpected interrupts.
4401 */
4402 icr0_remaining = icr0 & ena_mask;
4403 if (icr0_remaining) {
4404 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4405 icr0_remaining);
4406 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4407 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4408 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4409 dev_info(&pf->pdev->dev, "device will be reset\n");
4410 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4411 i40e_service_event_schedule(pf);
4412 }
4413 ena_mask &= ~icr0_remaining;
4414 }
4415 ret = IRQ_HANDLED;
4416
4417 enable_intr:
4418 /* re-enable interrupt causes */
4419 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4420 if (!test_bit(__I40E_DOWN, pf->state) ||
4421 test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4422 i40e_service_event_schedule(pf);
4423 i40e_irq_dynamic_enable_icr0(pf);
4424 }
4425
4426 return ret;
4427 }
4428
4429 /**
4430 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4431 * @tx_ring: tx ring to clean
4432 * @budget: how many cleans we're allowed
4433 *
4434 * Returns true if there's any budget left (e.g. the clean is finished)
4435 **/
i40e_clean_fdir_tx_irq(struct i40e_ring * tx_ring,int budget)4436 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4437 {
4438 struct i40e_vsi *vsi = tx_ring->vsi;
4439 u16 i = tx_ring->next_to_clean;
4440 struct i40e_tx_buffer *tx_buf;
4441 struct i40e_tx_desc *tx_desc;
4442
4443 tx_buf = &tx_ring->tx_bi[i];
4444 tx_desc = I40E_TX_DESC(tx_ring, i);
4445 i -= tx_ring->count;
4446
4447 do {
4448 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4449
4450 /* if next_to_watch is not set then there is no work pending */
4451 if (!eop_desc)
4452 break;
4453
4454 /* prevent any other reads prior to eop_desc */
4455 smp_rmb();
4456
4457 /* if the descriptor isn't done, no work yet to do */
4458 if (!(eop_desc->cmd_type_offset_bsz &
4459 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4460 break;
4461
4462 /* clear next_to_watch to prevent false hangs */
4463 tx_buf->next_to_watch = NULL;
4464
4465 tx_desc->buffer_addr = 0;
4466 tx_desc->cmd_type_offset_bsz = 0;
4467 /* move past filter desc */
4468 tx_buf++;
4469 tx_desc++;
4470 i++;
4471 if (unlikely(!i)) {
4472 i -= tx_ring->count;
4473 tx_buf = tx_ring->tx_bi;
4474 tx_desc = I40E_TX_DESC(tx_ring, 0);
4475 }
4476 /* unmap skb header data */
4477 dma_unmap_single(tx_ring->dev,
4478 dma_unmap_addr(tx_buf, dma),
4479 dma_unmap_len(tx_buf, len),
4480 DMA_TO_DEVICE);
4481 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4482 kfree(tx_buf->raw_buf);
4483
4484 tx_buf->raw_buf = NULL;
4485 tx_buf->tx_flags = 0;
4486 tx_buf->next_to_watch = NULL;
4487 dma_unmap_len_set(tx_buf, len, 0);
4488 tx_desc->buffer_addr = 0;
4489 tx_desc->cmd_type_offset_bsz = 0;
4490
4491 /* move us past the eop_desc for start of next FD desc */
4492 tx_buf++;
4493 tx_desc++;
4494 i++;
4495 if (unlikely(!i)) {
4496 i -= tx_ring->count;
4497 tx_buf = tx_ring->tx_bi;
4498 tx_desc = I40E_TX_DESC(tx_ring, 0);
4499 }
4500
4501 /* update budget accounting */
4502 budget--;
4503 } while (likely(budget));
4504
4505 i += tx_ring->count;
4506 tx_ring->next_to_clean = i;
4507
4508 if (test_bit(I40E_FLAG_MSIX_ENA, vsi->back->flags))
4509 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4510
4511 return budget > 0;
4512 }
4513
4514 /**
4515 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4516 * @irq: interrupt number
4517 * @data: pointer to a q_vector
4518 **/
i40e_fdir_clean_ring(int irq,void * data)4519 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4520 {
4521 struct i40e_q_vector *q_vector = data;
4522 struct i40e_vsi *vsi;
4523
4524 if (!q_vector->tx.ring)
4525 return IRQ_HANDLED;
4526
4527 vsi = q_vector->tx.ring->vsi;
4528 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4529
4530 return IRQ_HANDLED;
4531 }
4532
4533 /**
4534 * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4535 * @vsi: the VSI being configured
4536 * @v_idx: vector index
4537 * @qp_idx: queue pair index
4538 **/
i40e_map_vector_to_qp(struct i40e_vsi * vsi,int v_idx,int qp_idx)4539 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4540 {
4541 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4542 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4543 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4544
4545 tx_ring->q_vector = q_vector;
4546 tx_ring->next = q_vector->tx.ring;
4547 q_vector->tx.ring = tx_ring;
4548 q_vector->tx.count++;
4549
4550 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4551 if (i40e_enabled_xdp_vsi(vsi)) {
4552 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4553
4554 xdp_ring->q_vector = q_vector;
4555 xdp_ring->next = q_vector->tx.ring;
4556 q_vector->tx.ring = xdp_ring;
4557 q_vector->tx.count++;
4558 }
4559
4560 rx_ring->q_vector = q_vector;
4561 rx_ring->next = q_vector->rx.ring;
4562 q_vector->rx.ring = rx_ring;
4563 q_vector->rx.count++;
4564 }
4565
4566 /**
4567 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4568 * @vsi: the VSI being configured
4569 *
4570 * This function maps descriptor rings to the queue-specific vectors
4571 * we were allotted through the MSI-X enabling code. Ideally, we'd have
4572 * one vector per queue pair, but on a constrained vector budget, we
4573 * group the queue pairs as "efficiently" as possible.
4574 **/
i40e_vsi_map_rings_to_vectors(struct i40e_vsi * vsi)4575 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4576 {
4577 int qp_remaining = vsi->num_queue_pairs;
4578 int q_vectors = vsi->num_q_vectors;
4579 int num_ringpairs;
4580 int v_start = 0;
4581 int qp_idx = 0;
4582
4583 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4584 * group them so there are multiple queues per vector.
4585 * It is also important to go through all the vectors available to be
4586 * sure that if we don't use all the vectors, that the remaining vectors
4587 * are cleared. This is especially important when decreasing the
4588 * number of queues in use.
4589 */
4590 for (; v_start < q_vectors; v_start++) {
4591 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4592
4593 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4594
4595 q_vector->num_ringpairs = num_ringpairs;
4596 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4597
4598 q_vector->rx.count = 0;
4599 q_vector->tx.count = 0;
4600 q_vector->rx.ring = NULL;
4601 q_vector->tx.ring = NULL;
4602
4603 while (num_ringpairs--) {
4604 i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4605 qp_idx++;
4606 qp_remaining--;
4607 }
4608 }
4609 }
4610
4611 /**
4612 * i40e_vsi_request_irq - Request IRQ from the OS
4613 * @vsi: the VSI being configured
4614 * @basename: name for the vector
4615 **/
i40e_vsi_request_irq(struct i40e_vsi * vsi,char * basename)4616 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4617 {
4618 struct i40e_pf *pf = vsi->back;
4619 int err;
4620
4621 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
4622 err = i40e_vsi_request_irq_msix(vsi, basename);
4623 else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags))
4624 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4625 pf->int_name, pf);
4626 else
4627 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4628 pf->int_name, pf);
4629
4630 if (err)
4631 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4632
4633 return err;
4634 }
4635
4636 #ifdef CONFIG_NET_POLL_CONTROLLER
4637 /**
4638 * i40e_netpoll - A Polling 'interrupt' handler
4639 * @netdev: network interface device structure
4640 *
4641 * This is used by netconsole to send skbs without having to re-enable
4642 * interrupts. It's not called while the normal interrupt routine is executing.
4643 **/
i40e_netpoll(struct net_device * netdev)4644 static void i40e_netpoll(struct net_device *netdev)
4645 {
4646 struct i40e_netdev_priv *np = netdev_priv(netdev);
4647 struct i40e_vsi *vsi = np->vsi;
4648 struct i40e_pf *pf = vsi->back;
4649 int i;
4650
4651 /* if interface is down do nothing */
4652 if (test_bit(__I40E_VSI_DOWN, vsi->state))
4653 return;
4654
4655 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4656 for (i = 0; i < vsi->num_q_vectors; i++)
4657 i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4658 } else {
4659 i40e_intr(pf->pdev->irq, netdev);
4660 }
4661 }
4662 #endif
4663
4664 #define I40E_QTX_ENA_WAIT_COUNT 50
4665
4666 /**
4667 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4668 * @pf: the PF being configured
4669 * @pf_q: the PF queue
4670 * @enable: enable or disable state of the queue
4671 *
4672 * This routine will wait for the given Tx queue of the PF to reach the
4673 * enabled or disabled state.
4674 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4675 * multiple retries; else will return 0 in case of success.
4676 **/
i40e_pf_txq_wait(struct i40e_pf * pf,int pf_q,bool enable)4677 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4678 {
4679 int i;
4680 u32 tx_reg;
4681
4682 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4683 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4684 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4685 break;
4686
4687 usleep_range(10, 20);
4688 }
4689 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4690 return -ETIMEDOUT;
4691
4692 return 0;
4693 }
4694
4695 /**
4696 * i40e_control_tx_q - Start or stop a particular Tx queue
4697 * @pf: the PF structure
4698 * @pf_q: the PF queue to configure
4699 * @enable: start or stop the queue
4700 *
4701 * This function enables or disables a single queue. Note that any delay
4702 * required after the operation is expected to be handled by the caller of
4703 * this function.
4704 **/
i40e_control_tx_q(struct i40e_pf * pf,int pf_q,bool enable)4705 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4706 {
4707 struct i40e_hw *hw = &pf->hw;
4708 u32 tx_reg;
4709 int i;
4710
4711 /* warn the TX unit of coming changes */
4712 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4713 if (!enable)
4714 usleep_range(10, 20);
4715
4716 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4717 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4718 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4719 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4720 break;
4721 usleep_range(1000, 2000);
4722 }
4723
4724 /* Skip if the queue is already in the requested state */
4725 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4726 return;
4727
4728 /* turn on/off the queue */
4729 if (enable) {
4730 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4731 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4732 } else {
4733 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4734 }
4735
4736 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4737 }
4738
4739 /**
4740 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4741 * @seid: VSI SEID
4742 * @pf: the PF structure
4743 * @pf_q: the PF queue to configure
4744 * @is_xdp: true if the queue is used for XDP
4745 * @enable: start or stop the queue
4746 **/
i40e_control_wait_tx_q(int seid,struct i40e_pf * pf,int pf_q,bool is_xdp,bool enable)4747 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4748 bool is_xdp, bool enable)
4749 {
4750 int ret;
4751
4752 i40e_control_tx_q(pf, pf_q, enable);
4753
4754 /* wait for the change to finish */
4755 ret = i40e_pf_txq_wait(pf, pf_q, enable);
4756 if (ret) {
4757 dev_info(&pf->pdev->dev,
4758 "VSI seid %d %sTx ring %d %sable timeout\n",
4759 seid, (is_xdp ? "XDP " : ""), pf_q,
4760 (enable ? "en" : "dis"));
4761 }
4762
4763 return ret;
4764 }
4765
4766 /**
4767 * i40e_vsi_enable_tx - Start a VSI's rings
4768 * @vsi: the VSI being configured
4769 **/
i40e_vsi_enable_tx(struct i40e_vsi * vsi)4770 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4771 {
4772 struct i40e_pf *pf = vsi->back;
4773 int i, pf_q, ret = 0;
4774
4775 pf_q = vsi->base_queue;
4776 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4777 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4778 pf_q,
4779 false /*is xdp*/, true);
4780 if (ret)
4781 break;
4782
4783 if (!i40e_enabled_xdp_vsi(vsi))
4784 continue;
4785
4786 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4787 pf_q + vsi->alloc_queue_pairs,
4788 true /*is xdp*/, true);
4789 if (ret)
4790 break;
4791 }
4792 return ret;
4793 }
4794
4795 /**
4796 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4797 * @pf: the PF being configured
4798 * @pf_q: the PF queue
4799 * @enable: enable or disable state of the queue
4800 *
4801 * This routine will wait for the given Rx queue of the PF to reach the
4802 * enabled or disabled state.
4803 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4804 * multiple retries; else will return 0 in case of success.
4805 **/
i40e_pf_rxq_wait(struct i40e_pf * pf,int pf_q,bool enable)4806 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4807 {
4808 int i;
4809 u32 rx_reg;
4810
4811 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4812 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4813 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4814 break;
4815
4816 usleep_range(10, 20);
4817 }
4818 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4819 return -ETIMEDOUT;
4820
4821 return 0;
4822 }
4823
4824 /**
4825 * i40e_control_rx_q - Start or stop a particular Rx queue
4826 * @pf: the PF structure
4827 * @pf_q: the PF queue to configure
4828 * @enable: start or stop the queue
4829 *
4830 * This function enables or disables a single queue. Note that
4831 * any delay required after the operation is expected to be
4832 * handled by the caller of this function.
4833 **/
i40e_control_rx_q(struct i40e_pf * pf,int pf_q,bool enable)4834 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4835 {
4836 struct i40e_hw *hw = &pf->hw;
4837 u32 rx_reg;
4838 int i;
4839
4840 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4841 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4842 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4843 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4844 break;
4845 usleep_range(1000, 2000);
4846 }
4847
4848 /* Skip if the queue is already in the requested state */
4849 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4850 return;
4851
4852 /* turn on/off the queue */
4853 if (enable)
4854 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4855 else
4856 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4857
4858 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4859 }
4860
4861 /**
4862 * i40e_control_wait_rx_q
4863 * @pf: the PF structure
4864 * @pf_q: queue being configured
4865 * @enable: start or stop the rings
4866 *
4867 * This function enables or disables a single queue along with waiting
4868 * for the change to finish. The caller of this function should handle
4869 * the delays needed in the case of disabling queues.
4870 **/
i40e_control_wait_rx_q(struct i40e_pf * pf,int pf_q,bool enable)4871 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4872 {
4873 int ret = 0;
4874
4875 i40e_control_rx_q(pf, pf_q, enable);
4876
4877 /* wait for the change to finish */
4878 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4879 if (ret)
4880 return ret;
4881
4882 return ret;
4883 }
4884
4885 /**
4886 * i40e_vsi_enable_rx - Start a VSI's rings
4887 * @vsi: the VSI being configured
4888 **/
i40e_vsi_enable_rx(struct i40e_vsi * vsi)4889 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4890 {
4891 struct i40e_pf *pf = vsi->back;
4892 int i, pf_q, ret = 0;
4893
4894 pf_q = vsi->base_queue;
4895 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4896 ret = i40e_control_wait_rx_q(pf, pf_q, true);
4897 if (ret) {
4898 dev_info(&pf->pdev->dev,
4899 "VSI seid %d Rx ring %d enable timeout\n",
4900 vsi->seid, pf_q);
4901 break;
4902 }
4903 }
4904
4905 return ret;
4906 }
4907
4908 /**
4909 * i40e_vsi_start_rings - Start a VSI's rings
4910 * @vsi: the VSI being configured
4911 **/
i40e_vsi_start_rings(struct i40e_vsi * vsi)4912 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4913 {
4914 int ret = 0;
4915
4916 /* do rx first for enable and last for disable */
4917 ret = i40e_vsi_enable_rx(vsi);
4918 if (ret)
4919 return ret;
4920 ret = i40e_vsi_enable_tx(vsi);
4921
4922 return ret;
4923 }
4924
4925 #define I40E_DISABLE_TX_GAP_MSEC 50
4926
4927 /**
4928 * i40e_vsi_stop_rings - Stop a VSI's rings
4929 * @vsi: the VSI being configured
4930 **/
i40e_vsi_stop_rings(struct i40e_vsi * vsi)4931 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4932 {
4933 struct i40e_pf *pf = vsi->back;
4934 u32 pf_q, tx_q_end, rx_q_end;
4935
4936 /* When port TX is suspended, don't wait */
4937 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4938 return i40e_vsi_stop_rings_no_wait(vsi);
4939
4940 tx_q_end = vsi->base_queue +
4941 vsi->alloc_queue_pairs * (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
4942 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4943 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, false);
4944
4945 rx_q_end = vsi->base_queue + vsi->num_queue_pairs;
4946 for (pf_q = vsi->base_queue; pf_q < rx_q_end; pf_q++)
4947 i40e_control_rx_q(pf, pf_q, false);
4948
4949 msleep(I40E_DISABLE_TX_GAP_MSEC);
4950 for (pf_q = vsi->base_queue; pf_q < tx_q_end; pf_q++)
4951 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4952
4953 i40e_vsi_wait_queues_disabled(vsi);
4954 }
4955
4956 /**
4957 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4958 * @vsi: the VSI being shutdown
4959 *
4960 * This function stops all the rings for a VSI but does not delay to verify
4961 * that rings have been disabled. It is expected that the caller is shutting
4962 * down multiple VSIs at once and will delay together for all the VSIs after
4963 * initiating the shutdown. This is particularly useful for shutting down lots
4964 * of VFs together. Otherwise, a large delay can be incurred while configuring
4965 * each VSI in serial.
4966 **/
i40e_vsi_stop_rings_no_wait(struct i40e_vsi * vsi)4967 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4968 {
4969 struct i40e_pf *pf = vsi->back;
4970 int i, pf_q;
4971
4972 pf_q = vsi->base_queue;
4973 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4974 i40e_control_tx_q(pf, pf_q, false);
4975 i40e_control_rx_q(pf, pf_q, false);
4976 }
4977 }
4978
4979 /**
4980 * i40e_vsi_free_irq - Free the irq association with the OS
4981 * @vsi: the VSI being configured
4982 **/
i40e_vsi_free_irq(struct i40e_vsi * vsi)4983 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4984 {
4985 struct i40e_pf *pf = vsi->back;
4986 struct i40e_hw *hw = &pf->hw;
4987 int base = vsi->base_vector;
4988 u32 val, qp;
4989 int i;
4990
4991 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
4992 if (!vsi->q_vectors)
4993 return;
4994
4995 if (!vsi->irqs_ready)
4996 return;
4997
4998 vsi->irqs_ready = false;
4999 for (i = 0; i < vsi->num_q_vectors; i++) {
5000 int irq_num;
5001 u16 vector;
5002
5003 vector = i + base;
5004 irq_num = pf->msix_entries[vector].vector;
5005
5006 /* free only the irqs that were actually requested */
5007 if (!vsi->q_vectors[i] ||
5008 !vsi->q_vectors[i]->num_ringpairs)
5009 continue;
5010
5011 /* clear the affinity notifier in the IRQ descriptor */
5012 irq_set_affinity_notifier(irq_num, NULL);
5013 /* remove our suggested affinity mask for this IRQ */
5014 irq_update_affinity_hint(irq_num, NULL);
5015 free_irq(irq_num, vsi->q_vectors[i]);
5016
5017 /* Tear down the interrupt queue link list
5018 *
5019 * We know that they come in pairs and always
5020 * the Rx first, then the Tx. To clear the
5021 * link list, stick the EOL value into the
5022 * next_q field of the registers.
5023 */
5024 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
5025 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK,
5026 val);
5027 val |= I40E_QUEUE_END_OF_LIST
5028 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
5029 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
5030
5031 while (qp != I40E_QUEUE_END_OF_LIST) {
5032 u32 next;
5033
5034 val = rd32(hw, I40E_QINT_RQCTL(qp));
5035
5036 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
5037 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5038 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5039 I40E_QINT_RQCTL_INTEVENT_MASK);
5040
5041 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5042 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5043
5044 wr32(hw, I40E_QINT_RQCTL(qp), val);
5045
5046 val = rd32(hw, I40E_QINT_TQCTL(qp));
5047
5048 next = FIELD_GET(I40E_QINT_TQCTL_NEXTQ_INDX_MASK,
5049 val);
5050
5051 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
5052 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5053 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
5054 I40E_QINT_TQCTL_INTEVENT_MASK);
5055
5056 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5057 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5058
5059 wr32(hw, I40E_QINT_TQCTL(qp), val);
5060 qp = next;
5061 }
5062 }
5063 } else {
5064 free_irq(pf->pdev->irq, pf);
5065
5066 val = rd32(hw, I40E_PFINT_LNKLST0);
5067 qp = FIELD_GET(I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK, val);
5068 val |= I40E_QUEUE_END_OF_LIST
5069 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
5070 wr32(hw, I40E_PFINT_LNKLST0, val);
5071
5072 val = rd32(hw, I40E_QINT_RQCTL(qp));
5073 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
5074 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5075 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
5076 I40E_QINT_RQCTL_INTEVENT_MASK);
5077
5078 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5079 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5080
5081 wr32(hw, I40E_QINT_RQCTL(qp), val);
5082
5083 val = rd32(hw, I40E_QINT_TQCTL(qp));
5084
5085 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
5086 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5087 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
5088 I40E_QINT_TQCTL_INTEVENT_MASK);
5089
5090 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5091 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5092
5093 wr32(hw, I40E_QINT_TQCTL(qp), val);
5094 }
5095 }
5096
5097 /**
5098 * i40e_free_q_vector - Free memory allocated for specific interrupt vector
5099 * @vsi: the VSI being configured
5100 * @v_idx: Index of vector to be freed
5101 *
5102 * This function frees the memory allocated to the q_vector. In addition if
5103 * NAPI is enabled it will delete any references to the NAPI struct prior
5104 * to freeing the q_vector.
5105 **/
i40e_free_q_vector(struct i40e_vsi * vsi,int v_idx)5106 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
5107 {
5108 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
5109 struct i40e_ring *ring;
5110
5111 if (!q_vector)
5112 return;
5113
5114 /* disassociate q_vector from rings */
5115 i40e_for_each_ring(ring, q_vector->tx)
5116 ring->q_vector = NULL;
5117
5118 i40e_for_each_ring(ring, q_vector->rx)
5119 ring->q_vector = NULL;
5120
5121 /* only VSI w/ an associated netdev is set up w/ NAPI */
5122 if (vsi->netdev)
5123 netif_napi_del(&q_vector->napi);
5124
5125 vsi->q_vectors[v_idx] = NULL;
5126
5127 kfree_rcu(q_vector, rcu);
5128 }
5129
5130 /**
5131 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
5132 * @vsi: the VSI being un-configured
5133 *
5134 * This frees the memory allocated to the q_vectors and
5135 * deletes references to the NAPI struct.
5136 **/
i40e_vsi_free_q_vectors(struct i40e_vsi * vsi)5137 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
5138 {
5139 int v_idx;
5140
5141 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
5142 i40e_free_q_vector(vsi, v_idx);
5143 }
5144
5145 /**
5146 * i40e_reset_interrupt_capability - Disable interrupt setup in OS
5147 * @pf: board private structure
5148 **/
i40e_reset_interrupt_capability(struct i40e_pf * pf)5149 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
5150 {
5151 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
5152 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
5153 pci_disable_msix(pf->pdev);
5154 kfree(pf->msix_entries);
5155 pf->msix_entries = NULL;
5156 kfree(pf->irq_pile);
5157 pf->irq_pile = NULL;
5158 } else if (test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
5159 pci_disable_msi(pf->pdev);
5160 }
5161 clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
5162 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
5163 }
5164
5165 /**
5166 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
5167 * @pf: board private structure
5168 *
5169 * We go through and clear interrupt specific resources and reset the structure
5170 * to pre-load conditions
5171 **/
i40e_clear_interrupt_scheme(struct i40e_pf * pf)5172 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
5173 {
5174 struct i40e_vsi *vsi;
5175 int i;
5176
5177 if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
5178 i40e_free_misc_vector(pf);
5179
5180 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
5181 I40E_IWARP_IRQ_PILE_ID);
5182
5183 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
5184
5185 i40e_pf_for_each_vsi(pf, i, vsi)
5186 i40e_vsi_free_q_vectors(vsi);
5187
5188 i40e_reset_interrupt_capability(pf);
5189 }
5190
5191 /**
5192 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
5193 * @vsi: the VSI being configured
5194 **/
i40e_napi_enable_all(struct i40e_vsi * vsi)5195 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
5196 {
5197 int q_idx;
5198
5199 if (!vsi->netdev)
5200 return;
5201
5202 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5203 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5204
5205 if (q_vector->rx.ring || q_vector->tx.ring)
5206 napi_enable(&q_vector->napi);
5207 }
5208 }
5209
5210 /**
5211 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
5212 * @vsi: the VSI being configured
5213 **/
i40e_napi_disable_all(struct i40e_vsi * vsi)5214 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
5215 {
5216 int q_idx;
5217
5218 if (!vsi->netdev)
5219 return;
5220
5221 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5222 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5223
5224 if (q_vector->rx.ring || q_vector->tx.ring)
5225 napi_disable(&q_vector->napi);
5226 }
5227 }
5228
5229 /**
5230 * i40e_vsi_close - Shut down a VSI
5231 * @vsi: the vsi to be quelled
5232 **/
i40e_vsi_close(struct i40e_vsi * vsi)5233 static void i40e_vsi_close(struct i40e_vsi *vsi)
5234 {
5235 struct i40e_pf *pf = vsi->back;
5236 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
5237 i40e_down(vsi);
5238 i40e_vsi_free_irq(vsi);
5239 i40e_vsi_free_tx_resources(vsi);
5240 i40e_vsi_free_rx_resources(vsi);
5241 vsi->current_netdev_flags = 0;
5242 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
5243 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
5244 set_bit(__I40E_CLIENT_RESET, pf->state);
5245 }
5246
5247 /**
5248 * i40e_quiesce_vsi - Pause a given VSI
5249 * @vsi: the VSI being paused
5250 **/
i40e_quiesce_vsi(struct i40e_vsi * vsi)5251 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
5252 {
5253 if (test_bit(__I40E_VSI_DOWN, vsi->state))
5254 return;
5255
5256 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
5257 if (vsi->netdev && netif_running(vsi->netdev))
5258 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5259 else
5260 i40e_vsi_close(vsi);
5261 }
5262
5263 /**
5264 * i40e_unquiesce_vsi - Resume a given VSI
5265 * @vsi: the VSI being resumed
5266 **/
i40e_unquiesce_vsi(struct i40e_vsi * vsi)5267 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
5268 {
5269 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
5270 return;
5271
5272 if (vsi->netdev && netif_running(vsi->netdev))
5273 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5274 else
5275 i40e_vsi_open(vsi); /* this clears the DOWN bit */
5276 }
5277
5278 /**
5279 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5280 * @pf: the PF
5281 **/
i40e_pf_quiesce_all_vsi(struct i40e_pf * pf)5282 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
5283 {
5284 struct i40e_vsi *vsi;
5285 int v;
5286
5287 i40e_pf_for_each_vsi(pf, v, vsi)
5288 i40e_quiesce_vsi(vsi);
5289 }
5290
5291 /**
5292 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5293 * @pf: the PF
5294 **/
i40e_pf_unquiesce_all_vsi(struct i40e_pf * pf)5295 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5296 {
5297 struct i40e_vsi *vsi;
5298 int v;
5299
5300 i40e_pf_for_each_vsi(pf, v, vsi)
5301 i40e_unquiesce_vsi(vsi);
5302 }
5303
5304 /**
5305 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5306 * @vsi: the VSI being configured
5307 *
5308 * Wait until all queues on a given VSI have been disabled.
5309 **/
i40e_vsi_wait_queues_disabled(struct i40e_vsi * vsi)5310 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5311 {
5312 struct i40e_pf *pf = vsi->back;
5313 int i, pf_q, ret;
5314
5315 pf_q = vsi->base_queue;
5316 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5317 /* Check and wait for the Tx queue */
5318 ret = i40e_pf_txq_wait(pf, pf_q, false);
5319 if (ret) {
5320 dev_info(&pf->pdev->dev,
5321 "VSI seid %d Tx ring %d disable timeout\n",
5322 vsi->seid, pf_q);
5323 return ret;
5324 }
5325
5326 if (!i40e_enabled_xdp_vsi(vsi))
5327 goto wait_rx;
5328
5329 /* Check and wait for the XDP Tx queue */
5330 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5331 false);
5332 if (ret) {
5333 dev_info(&pf->pdev->dev,
5334 "VSI seid %d XDP Tx ring %d disable timeout\n",
5335 vsi->seid, pf_q);
5336 return ret;
5337 }
5338 wait_rx:
5339 /* Check and wait for the Rx queue */
5340 ret = i40e_pf_rxq_wait(pf, pf_q, false);
5341 if (ret) {
5342 dev_info(&pf->pdev->dev,
5343 "VSI seid %d Rx ring %d disable timeout\n",
5344 vsi->seid, pf_q);
5345 return ret;
5346 }
5347 }
5348
5349 return 0;
5350 }
5351
5352 #ifdef CONFIG_I40E_DCB
5353 /**
5354 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5355 * @pf: the PF
5356 *
5357 * This function waits for the queues to be in disabled state for all the
5358 * VSIs that are managed by this PF.
5359 **/
i40e_pf_wait_queues_disabled(struct i40e_pf * pf)5360 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5361 {
5362 struct i40e_vsi *vsi;
5363 int v, ret = 0;
5364
5365 i40e_pf_for_each_vsi(pf, v, vsi) {
5366 ret = i40e_vsi_wait_queues_disabled(vsi);
5367 if (ret)
5368 break;
5369 }
5370
5371 return ret;
5372 }
5373
5374 #endif
5375
5376 /**
5377 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5378 * @pf: pointer to PF
5379 *
5380 * Get TC map for ISCSI PF type that will include iSCSI TC
5381 * and LAN TC.
5382 **/
i40e_get_iscsi_tc_map(struct i40e_pf * pf)5383 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5384 {
5385 struct i40e_dcb_app_priority_table app;
5386 struct i40e_hw *hw = &pf->hw;
5387 u8 enabled_tc = 1; /* TC0 is always enabled */
5388 u8 tc, i;
5389 /* Get the iSCSI APP TLV */
5390 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5391
5392 for (i = 0; i < dcbcfg->numapps; i++) {
5393 app = dcbcfg->app[i];
5394 if (app.selector == I40E_APP_SEL_TCPIP &&
5395 app.protocolid == I40E_APP_PROTOID_ISCSI) {
5396 tc = dcbcfg->etscfg.prioritytable[app.priority];
5397 enabled_tc |= BIT(tc);
5398 break;
5399 }
5400 }
5401
5402 return enabled_tc;
5403 }
5404
5405 /**
5406 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config
5407 * @dcbcfg: the corresponding DCBx configuration structure
5408 *
5409 * Return the number of TCs from given DCBx configuration
5410 **/
i40e_dcb_get_num_tc(struct i40e_dcbx_config * dcbcfg)5411 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5412 {
5413 int i, tc_unused = 0;
5414 u8 num_tc = 0;
5415 u8 ret = 0;
5416
5417 /* Scan the ETS Config Priority Table to find
5418 * traffic class enabled for a given priority
5419 * and create a bitmask of enabled TCs
5420 */
5421 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5422 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5423
5424 /* Now scan the bitmask to check for
5425 * contiguous TCs starting with TC0
5426 */
5427 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5428 if (num_tc & BIT(i)) {
5429 if (!tc_unused) {
5430 ret++;
5431 } else {
5432 pr_err("Non-contiguous TC - Disabling DCB\n");
5433 return 1;
5434 }
5435 } else {
5436 tc_unused = 1;
5437 }
5438 }
5439
5440 /* There is always at least TC0 */
5441 if (!ret)
5442 ret = 1;
5443
5444 return ret;
5445 }
5446
5447 /**
5448 * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5449 * @dcbcfg: the corresponding DCBx configuration structure
5450 *
5451 * Query the current DCB configuration and return the number of
5452 * traffic classes enabled from the given DCBX config
5453 **/
i40e_dcb_get_enabled_tc(struct i40e_dcbx_config * dcbcfg)5454 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5455 {
5456 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5457 u8 enabled_tc = 1;
5458 u8 i;
5459
5460 for (i = 0; i < num_tc; i++)
5461 enabled_tc |= BIT(i);
5462
5463 return enabled_tc;
5464 }
5465
5466 /**
5467 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5468 * @pf: PF being queried
5469 *
5470 * Query the current MQPRIO configuration and return the number of
5471 * traffic classes enabled.
5472 **/
i40e_mqprio_get_enabled_tc(struct i40e_pf * pf)5473 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5474 {
5475 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
5476 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5477 u8 enabled_tc = 1, i;
5478
5479 for (i = 1; i < num_tc; i++)
5480 enabled_tc |= BIT(i);
5481 return enabled_tc;
5482 }
5483
5484 /**
5485 * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5486 * @pf: PF being queried
5487 *
5488 * Return number of traffic classes enabled for the given PF
5489 **/
i40e_pf_get_num_tc(struct i40e_pf * pf)5490 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5491 {
5492 u8 i, enabled_tc = 1;
5493 u8 num_tc = 0;
5494
5495 if (i40e_is_tc_mqprio_enabled(pf)) {
5496 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
5497
5498 return vsi->mqprio_qopt.qopt.num_tc;
5499 }
5500
5501 /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5502 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5503 return 1;
5504
5505 /* SFP mode will be enabled for all TCs on port */
5506 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5507 return i40e_dcb_get_num_tc(&pf->hw.local_dcbx_config);
5508
5509 /* MFP mode return count of enabled TCs for this PF */
5510 if (pf->hw.func_caps.iscsi)
5511 enabled_tc = i40e_get_iscsi_tc_map(pf);
5512 else
5513 return 1; /* Only TC0 */
5514
5515 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5516 if (enabled_tc & BIT(i))
5517 num_tc++;
5518 }
5519 return num_tc;
5520 }
5521
5522 /**
5523 * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5524 * @pf: PF being queried
5525 *
5526 * Return a bitmap for enabled traffic classes for this PF.
5527 **/
i40e_pf_get_tc_map(struct i40e_pf * pf)5528 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5529 {
5530 if (i40e_is_tc_mqprio_enabled(pf))
5531 return i40e_mqprio_get_enabled_tc(pf);
5532
5533 /* If neither MQPRIO nor DCB is enabled for this PF then just return
5534 * default TC
5535 */
5536 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags))
5537 return I40E_DEFAULT_TRAFFIC_CLASS;
5538
5539 /* SFP mode we want PF to be enabled for all TCs */
5540 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
5541 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5542
5543 /* MFP enabled and iSCSI PF type */
5544 if (pf->hw.func_caps.iscsi)
5545 return i40e_get_iscsi_tc_map(pf);
5546 else
5547 return I40E_DEFAULT_TRAFFIC_CLASS;
5548 }
5549
5550 /**
5551 * i40e_vsi_get_bw_info - Query VSI BW Information
5552 * @vsi: the VSI being queried
5553 *
5554 * Returns 0 on success, negative value on failure
5555 **/
i40e_vsi_get_bw_info(struct i40e_vsi * vsi)5556 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5557 {
5558 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5559 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5560 struct i40e_pf *pf = vsi->back;
5561 struct i40e_hw *hw = &pf->hw;
5562 u32 tc_bw_max;
5563 int ret;
5564 int i;
5565
5566 /* Get the VSI level BW configuration */
5567 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5568 if (ret) {
5569 dev_info(&pf->pdev->dev,
5570 "couldn't get PF vsi bw config, err %pe aq_err %s\n",
5571 ERR_PTR(ret),
5572 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5573 return -EINVAL;
5574 }
5575
5576 /* Get the VSI level BW configuration per TC */
5577 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5578 NULL);
5579 if (ret) {
5580 dev_info(&pf->pdev->dev,
5581 "couldn't get PF vsi ets bw config, err %pe aq_err %s\n",
5582 ERR_PTR(ret),
5583 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5584 return -EINVAL;
5585 }
5586
5587 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5588 dev_info(&pf->pdev->dev,
5589 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5590 bw_config.tc_valid_bits,
5591 bw_ets_config.tc_valid_bits);
5592 /* Still continuing */
5593 }
5594
5595 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5596 vsi->bw_max_quanta = bw_config.max_bw;
5597 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5598 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5599 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5600 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5601 vsi->bw_ets_limit_credits[i] =
5602 le16_to_cpu(bw_ets_config.credits[i]);
5603 /* 3 bits out of 4 for each TC */
5604 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5605 }
5606
5607 return 0;
5608 }
5609
5610 /**
5611 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5612 * @vsi: the VSI being configured
5613 * @enabled_tc: TC bitmap
5614 * @bw_share: BW shared credits per TC
5615 *
5616 * Returns 0 on success, negative value on failure
5617 **/
i40e_vsi_configure_bw_alloc(struct i40e_vsi * vsi,u8 enabled_tc,u8 * bw_share)5618 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5619 u8 *bw_share)
5620 {
5621 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5622 struct i40e_pf *pf = vsi->back;
5623 int ret;
5624 int i;
5625
5626 /* There is no need to reset BW when mqprio mode is on. */
5627 if (i40e_is_tc_mqprio_enabled(pf))
5628 return 0;
5629 if (!vsi->mqprio_qopt.qopt.hw && !test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
5630 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5631 if (ret)
5632 dev_info(&pf->pdev->dev,
5633 "Failed to reset tx rate for vsi->seid %u\n",
5634 vsi->seid);
5635 return ret;
5636 }
5637 memset(&bw_data, 0, sizeof(bw_data));
5638 bw_data.tc_valid_bits = enabled_tc;
5639 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5640 bw_data.tc_bw_credits[i] = bw_share[i];
5641
5642 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5643 if (ret) {
5644 dev_info(&pf->pdev->dev,
5645 "AQ command Config VSI BW allocation per TC failed = %d\n",
5646 pf->hw.aq.asq_last_status);
5647 return -EINVAL;
5648 }
5649
5650 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5651 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5652
5653 return 0;
5654 }
5655
5656 /**
5657 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5658 * @vsi: the VSI being configured
5659 * @enabled_tc: TC map to be enabled
5660 *
5661 **/
i40e_vsi_config_netdev_tc(struct i40e_vsi * vsi,u8 enabled_tc)5662 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5663 {
5664 struct net_device *netdev = vsi->netdev;
5665 struct i40e_pf *pf = vsi->back;
5666 struct i40e_hw *hw = &pf->hw;
5667 u8 netdev_tc = 0;
5668 int i;
5669 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5670
5671 if (!netdev)
5672 return;
5673
5674 if (!enabled_tc) {
5675 netdev_reset_tc(netdev);
5676 return;
5677 }
5678
5679 /* Set up actual enabled TCs on the VSI */
5680 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5681 return;
5682
5683 /* set per TC queues for the VSI */
5684 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5685 /* Only set TC queues for enabled tcs
5686 *
5687 * e.g. For a VSI that has TC0 and TC3 enabled the
5688 * enabled_tc bitmap would be 0x00001001; the driver
5689 * will set the numtc for netdev as 2 that will be
5690 * referenced by the netdev layer as TC 0 and 1.
5691 */
5692 if (vsi->tc_config.enabled_tc & BIT(i))
5693 netdev_set_tc_queue(netdev,
5694 vsi->tc_config.tc_info[i].netdev_tc,
5695 vsi->tc_config.tc_info[i].qcount,
5696 vsi->tc_config.tc_info[i].qoffset);
5697 }
5698
5699 if (i40e_is_tc_mqprio_enabled(pf))
5700 return;
5701
5702 /* Assign UP2TC map for the VSI */
5703 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5704 /* Get the actual TC# for the UP */
5705 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5706 /* Get the mapped netdev TC# for the UP */
5707 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc;
5708 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5709 }
5710 }
5711
5712 /**
5713 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5714 * @vsi: the VSI being configured
5715 * @ctxt: the ctxt buffer returned from AQ VSI update param command
5716 **/
i40e_vsi_update_queue_map(struct i40e_vsi * vsi,struct i40e_vsi_context * ctxt)5717 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5718 struct i40e_vsi_context *ctxt)
5719 {
5720 /* copy just the sections touched not the entire info
5721 * since not all sections are valid as returned by
5722 * update vsi params
5723 */
5724 vsi->info.mapping_flags = ctxt->info.mapping_flags;
5725 memcpy(&vsi->info.queue_mapping,
5726 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5727 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5728 sizeof(vsi->info.tc_mapping));
5729 }
5730
5731 /**
5732 * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5733 * @vsi: the VSI being reconfigured
5734 * @vsi_offset: offset from main VF VSI
5735 */
i40e_update_adq_vsi_queues(struct i40e_vsi * vsi,int vsi_offset)5736 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5737 {
5738 struct i40e_vsi_context ctxt = {};
5739 struct i40e_pf *pf;
5740 struct i40e_hw *hw;
5741 int ret;
5742
5743 if (!vsi)
5744 return -EINVAL;
5745 pf = vsi->back;
5746 hw = &pf->hw;
5747
5748 ctxt.seid = vsi->seid;
5749 ctxt.pf_num = hw->pf_id;
5750 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5751 ctxt.uplink_seid = vsi->uplink_seid;
5752 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5753 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5754 ctxt.info = vsi->info;
5755
5756 i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
5757 false);
5758 if (vsi->reconfig_rss) {
5759 vsi->rss_size = min_t(int, pf->alloc_rss_size,
5760 vsi->num_queue_pairs);
5761 ret = i40e_vsi_config_rss(vsi);
5762 if (ret) {
5763 dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5764 return ret;
5765 }
5766 vsi->reconfig_rss = false;
5767 }
5768
5769 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5770 if (ret) {
5771 dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n",
5772 ERR_PTR(ret),
5773 i40e_aq_str(hw, hw->aq.asq_last_status));
5774 return ret;
5775 }
5776 /* update the local VSI info with updated queue map */
5777 i40e_vsi_update_queue_map(vsi, &ctxt);
5778 vsi->info.valid_sections = 0;
5779
5780 return ret;
5781 }
5782
5783 /**
5784 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5785 * @vsi: VSI to be configured
5786 * @enabled_tc: TC bitmap
5787 *
5788 * This configures a particular VSI for TCs that are mapped to the
5789 * given TC bitmap. It uses default bandwidth share for TCs across
5790 * VSIs to configure TC for a particular VSI.
5791 *
5792 * NOTE:
5793 * It is expected that the VSI queues have been quisced before calling
5794 * this function.
5795 **/
i40e_vsi_config_tc(struct i40e_vsi * vsi,u8 enabled_tc)5796 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5797 {
5798 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5799 struct i40e_pf *pf = vsi->back;
5800 struct i40e_hw *hw = &pf->hw;
5801 struct i40e_vsi_context ctxt;
5802 int ret = 0;
5803 int i;
5804
5805 /* Check if enabled_tc is same as existing or new TCs */
5806 if (vsi->tc_config.enabled_tc == enabled_tc &&
5807 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5808 return ret;
5809
5810 /* Enable ETS TCs with equal BW Share for now across all VSIs */
5811 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5812 if (enabled_tc & BIT(i))
5813 bw_share[i] = 1;
5814 }
5815
5816 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5817 if (ret) {
5818 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5819
5820 dev_info(&pf->pdev->dev,
5821 "Failed configuring TC map %d for VSI %d\n",
5822 enabled_tc, vsi->seid);
5823 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5824 &bw_config, NULL);
5825 if (ret) {
5826 dev_info(&pf->pdev->dev,
5827 "Failed querying vsi bw info, err %pe aq_err %s\n",
5828 ERR_PTR(ret),
5829 i40e_aq_str(hw, hw->aq.asq_last_status));
5830 goto out;
5831 }
5832 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5833 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5834
5835 if (!valid_tc)
5836 valid_tc = bw_config.tc_valid_bits;
5837 /* Always enable TC0, no matter what */
5838 valid_tc |= 1;
5839 dev_info(&pf->pdev->dev,
5840 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5841 enabled_tc, bw_config.tc_valid_bits, valid_tc);
5842 enabled_tc = valid_tc;
5843 }
5844
5845 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5846 if (ret) {
5847 dev_err(&pf->pdev->dev,
5848 "Unable to configure TC map %d for VSI %d\n",
5849 enabled_tc, vsi->seid);
5850 goto out;
5851 }
5852 }
5853
5854 /* Update Queue Pairs Mapping for currently enabled UPs */
5855 ctxt.seid = vsi->seid;
5856 ctxt.pf_num = vsi->back->hw.pf_id;
5857 ctxt.vf_num = 0;
5858 ctxt.uplink_seid = vsi->uplink_seid;
5859 ctxt.info = vsi->info;
5860 if (i40e_is_tc_mqprio_enabled(pf)) {
5861 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5862 if (ret)
5863 goto out;
5864 } else {
5865 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5866 }
5867
5868 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5869 * queues changed.
5870 */
5871 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5872 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5873 vsi->num_queue_pairs);
5874 ret = i40e_vsi_config_rss(vsi);
5875 if (ret) {
5876 dev_info(&vsi->back->pdev->dev,
5877 "Failed to reconfig rss for num_queues\n");
5878 return ret;
5879 }
5880 vsi->reconfig_rss = false;
5881 }
5882 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
5883 ctxt.info.valid_sections |=
5884 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5885 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5886 }
5887
5888 /* Update the VSI after updating the VSI queue-mapping
5889 * information
5890 */
5891 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5892 if (ret) {
5893 dev_info(&pf->pdev->dev,
5894 "Update vsi tc config failed, err %pe aq_err %s\n",
5895 ERR_PTR(ret),
5896 i40e_aq_str(hw, hw->aq.asq_last_status));
5897 goto out;
5898 }
5899 /* update the local VSI info with updated queue map */
5900 i40e_vsi_update_queue_map(vsi, &ctxt);
5901 vsi->info.valid_sections = 0;
5902
5903 /* Update current VSI BW information */
5904 ret = i40e_vsi_get_bw_info(vsi);
5905 if (ret) {
5906 dev_info(&pf->pdev->dev,
5907 "Failed updating vsi bw info, err %pe aq_err %s\n",
5908 ERR_PTR(ret),
5909 i40e_aq_str(hw, hw->aq.asq_last_status));
5910 goto out;
5911 }
5912
5913 /* Update the netdev TC setup */
5914 i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5915 out:
5916 return ret;
5917 }
5918
5919 /**
5920 * i40e_vsi_reconfig_tc - Reconfigure VSI Tx Scheduler for stored TC map
5921 * @vsi: VSI to be reconfigured
5922 *
5923 * This reconfigures a particular VSI for TCs that are mapped to the
5924 * TC bitmap stored previously for the VSI.
5925 *
5926 * Context: It is expected that the VSI queues have been quisced before
5927 * calling this function.
5928 *
5929 * Return: 0 on success, negative value on failure
5930 **/
i40e_vsi_reconfig_tc(struct i40e_vsi * vsi)5931 static int i40e_vsi_reconfig_tc(struct i40e_vsi *vsi)
5932 {
5933 u8 enabled_tc;
5934
5935 enabled_tc = vsi->tc_config.enabled_tc;
5936 vsi->tc_config.enabled_tc = 0;
5937
5938 return i40e_vsi_config_tc(vsi, enabled_tc);
5939 }
5940
5941 /**
5942 * i40e_get_link_speed - Returns link speed for the interface
5943 * @vsi: VSI to be configured
5944 *
5945 **/
i40e_get_link_speed(struct i40e_vsi * vsi)5946 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5947 {
5948 struct i40e_pf *pf = vsi->back;
5949
5950 switch (pf->hw.phy.link_info.link_speed) {
5951 case I40E_LINK_SPEED_40GB:
5952 return 40000;
5953 case I40E_LINK_SPEED_25GB:
5954 return 25000;
5955 case I40E_LINK_SPEED_20GB:
5956 return 20000;
5957 case I40E_LINK_SPEED_10GB:
5958 return 10000;
5959 case I40E_LINK_SPEED_1GB:
5960 return 1000;
5961 default:
5962 return -EINVAL;
5963 }
5964 }
5965
5966 /**
5967 * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5968 * @vsi: Pointer to vsi structure
5969 * @max_tx_rate: max TX rate in bytes to be converted into Mbits
5970 *
5971 * Helper function to convert units before send to set BW limit
5972 **/
i40e_bw_bytes_to_mbits(struct i40e_vsi * vsi,u64 max_tx_rate)5973 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)
5974 {
5975 if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {
5976 dev_warn(&vsi->back->pdev->dev,
5977 "Setting max tx rate to minimum usable value of 50Mbps.\n");
5978 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5979 } else {
5980 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
5981 }
5982
5983 return max_tx_rate;
5984 }
5985
5986 /**
5987 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5988 * @vsi: VSI to be configured
5989 * @seid: seid of the channel/VSI
5990 * @max_tx_rate: max TX rate to be configured as BW limit
5991 *
5992 * Helper function to set BW limit for a given VSI
5993 **/
i40e_set_bw_limit(struct i40e_vsi * vsi,u16 seid,u64 max_tx_rate)5994 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5995 {
5996 struct i40e_pf *pf = vsi->back;
5997 u64 credits = 0;
5998 int speed = 0;
5999 int ret = 0;
6000
6001 speed = i40e_get_link_speed(vsi);
6002 if (max_tx_rate > speed) {
6003 dev_err(&pf->pdev->dev,
6004 "Invalid max tx rate %llu specified for VSI seid %d.",
6005 max_tx_rate, seid);
6006 return -EINVAL;
6007 }
6008 if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {
6009 dev_warn(&pf->pdev->dev,
6010 "Setting max tx rate to minimum usable value of 50Mbps.\n");
6011 max_tx_rate = I40E_BW_CREDIT_DIVISOR;
6012 }
6013
6014 /* Tx rate credits are in values of 50Mbps, 0 is disabled */
6015 credits = max_tx_rate;
6016 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6017 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
6018 I40E_MAX_BW_INACTIVE_ACCUM, NULL);
6019 if (ret)
6020 dev_err(&pf->pdev->dev,
6021 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n",
6022 max_tx_rate, seid, ERR_PTR(ret),
6023 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6024 return ret;
6025 }
6026
6027 /**
6028 * i40e_remove_queue_channels - Remove queue channels for the TCs
6029 * @vsi: VSI to be configured
6030 *
6031 * Remove queue channels for the TCs
6032 **/
i40e_remove_queue_channels(struct i40e_vsi * vsi)6033 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
6034 {
6035 enum i40e_admin_queue_err last_aq_status;
6036 struct i40e_cloud_filter *cfilter;
6037 struct i40e_channel *ch, *ch_tmp;
6038 struct i40e_pf *pf = vsi->back;
6039 struct hlist_node *node;
6040 int ret, i;
6041
6042 /* Reset rss size that was stored when reconfiguring rss for
6043 * channel VSIs with non-power-of-2 queue count.
6044 */
6045 vsi->current_rss_size = 0;
6046
6047 /* perform cleanup for channels if they exist */
6048 if (list_empty(&vsi->ch_list))
6049 return;
6050
6051 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6052 struct i40e_vsi *p_vsi;
6053
6054 list_del(&ch->list);
6055 p_vsi = ch->parent_vsi;
6056 if (!p_vsi || !ch->initialized) {
6057 kfree(ch);
6058 continue;
6059 }
6060 /* Reset queue contexts */
6061 for (i = 0; i < ch->num_queue_pairs; i++) {
6062 struct i40e_ring *tx_ring, *rx_ring;
6063 u16 pf_q;
6064
6065 pf_q = ch->base_queue + i;
6066 tx_ring = vsi->tx_rings[pf_q];
6067 tx_ring->ch = NULL;
6068
6069 rx_ring = vsi->rx_rings[pf_q];
6070 rx_ring->ch = NULL;
6071 }
6072
6073 /* Reset BW configured for this VSI via mqprio */
6074 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
6075 if (ret)
6076 dev_info(&vsi->back->pdev->dev,
6077 "Failed to reset tx rate for ch->seid %u\n",
6078 ch->seid);
6079
6080 /* delete cloud filters associated with this channel */
6081 hlist_for_each_entry_safe(cfilter, node,
6082 &pf->cloud_filter_list, cloud_node) {
6083 if (cfilter->seid != ch->seid)
6084 continue;
6085
6086 hash_del(&cfilter->cloud_node);
6087 if (cfilter->dst_port)
6088 ret = i40e_add_del_cloud_filter_big_buf(vsi,
6089 cfilter,
6090 false);
6091 else
6092 ret = i40e_add_del_cloud_filter(vsi, cfilter,
6093 false);
6094 last_aq_status = pf->hw.aq.asq_last_status;
6095 if (ret)
6096 dev_info(&pf->pdev->dev,
6097 "Failed to delete cloud filter, err %pe aq_err %s\n",
6098 ERR_PTR(ret),
6099 i40e_aq_str(&pf->hw, last_aq_status));
6100 kfree(cfilter);
6101 }
6102
6103 /* delete VSI from FW */
6104 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
6105 NULL);
6106 if (ret)
6107 dev_err(&vsi->back->pdev->dev,
6108 "unable to remove channel (%d) for parent VSI(%d)\n",
6109 ch->seid, p_vsi->seid);
6110 kfree(ch);
6111 }
6112 INIT_LIST_HEAD(&vsi->ch_list);
6113 }
6114
6115 /**
6116 * i40e_get_max_queues_for_channel
6117 * @vsi: ptr to VSI to which channels are associated with
6118 *
6119 * Helper function which returns max value among the queue counts set on the
6120 * channels/TCs created.
6121 **/
i40e_get_max_queues_for_channel(struct i40e_vsi * vsi)6122 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
6123 {
6124 struct i40e_channel *ch, *ch_tmp;
6125 int max = 0;
6126
6127 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6128 if (!ch->initialized)
6129 continue;
6130 if (ch->num_queue_pairs > max)
6131 max = ch->num_queue_pairs;
6132 }
6133
6134 return max;
6135 }
6136
6137 /**
6138 * i40e_validate_num_queues - validate num_queues w.r.t channel
6139 * @pf: ptr to PF device
6140 * @num_queues: number of queues
6141 * @vsi: the parent VSI
6142 * @reconfig_rss: indicates should the RSS be reconfigured or not
6143 *
6144 * This function validates number of queues in the context of new channel
6145 * which is being established and determines if RSS should be reconfigured
6146 * or not for parent VSI.
6147 **/
i40e_validate_num_queues(struct i40e_pf * pf,int num_queues,struct i40e_vsi * vsi,bool * reconfig_rss)6148 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
6149 struct i40e_vsi *vsi, bool *reconfig_rss)
6150 {
6151 int max_ch_queues;
6152
6153 if (!reconfig_rss)
6154 return -EINVAL;
6155
6156 *reconfig_rss = false;
6157 if (vsi->current_rss_size) {
6158 if (num_queues > vsi->current_rss_size) {
6159 dev_dbg(&pf->pdev->dev,
6160 "Error: num_queues (%d) > vsi's current_size(%d)\n",
6161 num_queues, vsi->current_rss_size);
6162 return -EINVAL;
6163 } else if ((num_queues < vsi->current_rss_size) &&
6164 (!is_power_of_2(num_queues))) {
6165 dev_dbg(&pf->pdev->dev,
6166 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
6167 num_queues, vsi->current_rss_size);
6168 return -EINVAL;
6169 }
6170 }
6171
6172 if (!is_power_of_2(num_queues)) {
6173 /* Find the max num_queues configured for channel if channel
6174 * exist.
6175 * if channel exist, then enforce 'num_queues' to be more than
6176 * max ever queues configured for channel.
6177 */
6178 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
6179 if (num_queues < max_ch_queues) {
6180 dev_dbg(&pf->pdev->dev,
6181 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
6182 num_queues, max_ch_queues);
6183 return -EINVAL;
6184 }
6185 *reconfig_rss = true;
6186 }
6187
6188 return 0;
6189 }
6190
6191 /**
6192 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
6193 * @vsi: the VSI being setup
6194 * @rss_size: size of RSS, accordingly LUT gets reprogrammed
6195 *
6196 * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
6197 **/
i40e_vsi_reconfig_rss(struct i40e_vsi * vsi,u16 rss_size)6198 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
6199 {
6200 struct i40e_pf *pf = vsi->back;
6201 u8 seed[I40E_HKEY_ARRAY_SIZE];
6202 struct i40e_hw *hw = &pf->hw;
6203 int local_rss_size;
6204 u8 *lut;
6205 int ret;
6206
6207 if (!vsi->rss_size)
6208 return -EINVAL;
6209
6210 if (rss_size > vsi->rss_size)
6211 return -EINVAL;
6212
6213 local_rss_size = min_t(int, vsi->rss_size, rss_size);
6214 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
6215 if (!lut)
6216 return -ENOMEM;
6217
6218 /* Ignoring user configured lut if there is one */
6219 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
6220
6221 /* Use user configured hash key if there is one, otherwise
6222 * use default.
6223 */
6224 if (vsi->rss_hkey_user)
6225 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
6226 else
6227 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
6228
6229 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
6230 if (ret) {
6231 dev_info(&pf->pdev->dev,
6232 "Cannot set RSS lut, err %pe aq_err %s\n",
6233 ERR_PTR(ret),
6234 i40e_aq_str(hw, hw->aq.asq_last_status));
6235 kfree(lut);
6236 return ret;
6237 }
6238 kfree(lut);
6239
6240 /* Do the update w.r.t. storing rss_size */
6241 if (!vsi->orig_rss_size)
6242 vsi->orig_rss_size = vsi->rss_size;
6243 vsi->current_rss_size = local_rss_size;
6244
6245 return ret;
6246 }
6247
6248 /**
6249 * i40e_channel_setup_queue_map - Setup a channel queue map
6250 * @pf: ptr to PF device
6251 * @ctxt: VSI context structure
6252 * @ch: ptr to channel structure
6253 *
6254 * Setup queue map for a specific channel
6255 **/
i40e_channel_setup_queue_map(struct i40e_pf * pf,struct i40e_vsi_context * ctxt,struct i40e_channel * ch)6256 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
6257 struct i40e_vsi_context *ctxt,
6258 struct i40e_channel *ch)
6259 {
6260 u16 qcount, qmap, sections = 0;
6261 u8 offset = 0;
6262 int pow;
6263
6264 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
6265 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
6266
6267 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
6268 ch->num_queue_pairs = qcount;
6269
6270 /* find the next higher power-of-2 of num queue pairs */
6271 pow = ilog2(qcount);
6272 if (!is_power_of_2(qcount))
6273 pow++;
6274
6275 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6276 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
6277
6278 /* Setup queue TC[0].qmap for given VSI context */
6279 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
6280
6281 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
6282 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
6283 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
6284 ctxt->info.valid_sections |= cpu_to_le16(sections);
6285 }
6286
6287 /**
6288 * i40e_add_channel - add a channel by adding VSI
6289 * @pf: ptr to PF device
6290 * @uplink_seid: underlying HW switching element (VEB) ID
6291 * @ch: ptr to channel structure
6292 *
6293 * Add a channel (VSI) using add_vsi and queue_map
6294 **/
i40e_add_channel(struct i40e_pf * pf,u16 uplink_seid,struct i40e_channel * ch)6295 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
6296 struct i40e_channel *ch)
6297 {
6298 struct i40e_hw *hw = &pf->hw;
6299 struct i40e_vsi_context ctxt;
6300 u8 enabled_tc = 0x1; /* TC0 enabled */
6301 int ret;
6302
6303 if (ch->type != I40E_VSI_VMDQ2) {
6304 dev_info(&pf->pdev->dev,
6305 "add new vsi failed, ch->type %d\n", ch->type);
6306 return -EINVAL;
6307 }
6308
6309 memset(&ctxt, 0, sizeof(ctxt));
6310 ctxt.pf_num = hw->pf_id;
6311 ctxt.vf_num = 0;
6312 ctxt.uplink_seid = uplink_seid;
6313 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
6314 if (ch->type == I40E_VSI_VMDQ2)
6315 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6316
6317 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6318 ctxt.info.valid_sections |=
6319 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6320 ctxt.info.switch_id =
6321 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6322 }
6323
6324 /* Set queue map for a given VSI context */
6325 i40e_channel_setup_queue_map(pf, &ctxt, ch);
6326
6327 /* Now time to create VSI */
6328 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6329 if (ret) {
6330 dev_info(&pf->pdev->dev,
6331 "add new vsi failed, err %pe aq_err %s\n",
6332 ERR_PTR(ret),
6333 i40e_aq_str(&pf->hw,
6334 pf->hw.aq.asq_last_status));
6335 return -ENOENT;
6336 }
6337
6338 /* Success, update channel, set enabled_tc only if the channel
6339 * is not a macvlan
6340 */
6341 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6342 ch->seid = ctxt.seid;
6343 ch->vsi_number = ctxt.vsi_number;
6344 ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6345
6346 /* copy just the sections touched not the entire info
6347 * since not all sections are valid as returned by
6348 * update vsi params
6349 */
6350 ch->info.mapping_flags = ctxt.info.mapping_flags;
6351 memcpy(&ch->info.queue_mapping,
6352 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6353 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6354 sizeof(ctxt.info.tc_mapping));
6355
6356 return 0;
6357 }
6358
i40e_channel_config_bw(struct i40e_vsi * vsi,struct i40e_channel * ch,u8 * bw_share)6359 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6360 u8 *bw_share)
6361 {
6362 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6363 int ret;
6364 int i;
6365
6366 memset(&bw_data, 0, sizeof(bw_data));
6367 bw_data.tc_valid_bits = ch->enabled_tc;
6368 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6369 bw_data.tc_bw_credits[i] = bw_share[i];
6370
6371 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
6372 &bw_data, NULL);
6373 if (ret) {
6374 dev_info(&vsi->back->pdev->dev,
6375 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6376 vsi->back->hw.aq.asq_last_status, ch->seid);
6377 return -EINVAL;
6378 }
6379
6380 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6381 ch->info.qs_handle[i] = bw_data.qs_handles[i];
6382
6383 return 0;
6384 }
6385
6386 /**
6387 * i40e_channel_config_tx_ring - config TX ring associated with new channel
6388 * @pf: ptr to PF device
6389 * @vsi: the VSI being setup
6390 * @ch: ptr to channel structure
6391 *
6392 * Configure TX rings associated with channel (VSI) since queues are being
6393 * from parent VSI.
6394 **/
i40e_channel_config_tx_ring(struct i40e_pf * pf,struct i40e_vsi * vsi,struct i40e_channel * ch)6395 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6396 struct i40e_vsi *vsi,
6397 struct i40e_channel *ch)
6398 {
6399 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6400 int ret;
6401 int i;
6402
6403 /* Enable ETS TCs with equal BW Share for now across all VSIs */
6404 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6405 if (ch->enabled_tc & BIT(i))
6406 bw_share[i] = 1;
6407 }
6408
6409 /* configure BW for new VSI */
6410 ret = i40e_channel_config_bw(vsi, ch, bw_share);
6411 if (ret) {
6412 dev_info(&vsi->back->pdev->dev,
6413 "Failed configuring TC map %d for channel (seid %u)\n",
6414 ch->enabled_tc, ch->seid);
6415 return ret;
6416 }
6417
6418 for (i = 0; i < ch->num_queue_pairs; i++) {
6419 struct i40e_ring *tx_ring, *rx_ring;
6420 u16 pf_q;
6421
6422 pf_q = ch->base_queue + i;
6423
6424 /* Get to TX ring ptr of main VSI, for re-setup TX queue
6425 * context
6426 */
6427 tx_ring = vsi->tx_rings[pf_q];
6428 tx_ring->ch = ch;
6429
6430 /* Get the RX ring ptr */
6431 rx_ring = vsi->rx_rings[pf_q];
6432 rx_ring->ch = ch;
6433 }
6434
6435 return 0;
6436 }
6437
6438 /**
6439 * i40e_setup_hw_channel - setup new channel
6440 * @pf: ptr to PF device
6441 * @vsi: the VSI being setup
6442 * @ch: ptr to channel structure
6443 * @uplink_seid: underlying HW switching element (VEB) ID
6444 * @type: type of channel to be created (VMDq2/VF)
6445 *
6446 * Setup new channel (VSI) based on specified type (VMDq2/VF)
6447 * and configures TX rings accordingly
6448 **/
i40e_setup_hw_channel(struct i40e_pf * pf,struct i40e_vsi * vsi,struct i40e_channel * ch,u16 uplink_seid,u8 type)6449 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6450 struct i40e_vsi *vsi,
6451 struct i40e_channel *ch,
6452 u16 uplink_seid, u8 type)
6453 {
6454 int ret;
6455
6456 ch->initialized = false;
6457 ch->base_queue = vsi->next_base_queue;
6458 ch->type = type;
6459
6460 /* Proceed with creation of channel (VMDq2) VSI */
6461 ret = i40e_add_channel(pf, uplink_seid, ch);
6462 if (ret) {
6463 dev_info(&pf->pdev->dev,
6464 "failed to add_channel using uplink_seid %u\n",
6465 uplink_seid);
6466 return ret;
6467 }
6468
6469 /* Mark the successful creation of channel */
6470 ch->initialized = true;
6471
6472 /* Reconfigure TX queues using QTX_CTL register */
6473 ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6474 if (ret) {
6475 dev_info(&pf->pdev->dev,
6476 "failed to configure TX rings for channel %u\n",
6477 ch->seid);
6478 return ret;
6479 }
6480
6481 /* update 'next_base_queue' */
6482 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6483 dev_dbg(&pf->pdev->dev,
6484 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6485 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6486 ch->num_queue_pairs,
6487 vsi->next_base_queue);
6488 return ret;
6489 }
6490
6491 /**
6492 * i40e_setup_channel - setup new channel using uplink element
6493 * @pf: ptr to PF device
6494 * @vsi: pointer to the VSI to set up the channel within
6495 * @ch: ptr to channel structure
6496 *
6497 * Setup new channel (VSI) based on specified type (VMDq2/VF)
6498 * and uplink switching element (uplink_seid)
6499 **/
i40e_setup_channel(struct i40e_pf * pf,struct i40e_vsi * vsi,struct i40e_channel * ch)6500 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6501 struct i40e_channel *ch)
6502 {
6503 struct i40e_vsi *main_vsi;
6504 u8 vsi_type;
6505 u16 seid;
6506 int ret;
6507
6508 if (vsi->type == I40E_VSI_MAIN) {
6509 vsi_type = I40E_VSI_VMDQ2;
6510 } else {
6511 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6512 vsi->type);
6513 return false;
6514 }
6515
6516 /* underlying switching element */
6517 main_vsi = i40e_pf_get_main_vsi(pf);
6518 seid = main_vsi->uplink_seid;
6519
6520 /* create channel (VSI), configure TX rings */
6521 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6522 if (ret) {
6523 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6524 return false;
6525 }
6526
6527 return ch->initialized ? true : false;
6528 }
6529
6530 /**
6531 * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6532 * @vsi: ptr to VSI which has PF backing
6533 *
6534 * Sets up switch mode correctly if it needs to be changed and perform
6535 * what are allowed modes.
6536 **/
i40e_validate_and_set_switch_mode(struct i40e_vsi * vsi)6537 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6538 {
6539 u8 mode;
6540 struct i40e_pf *pf = vsi->back;
6541 struct i40e_hw *hw = &pf->hw;
6542 int ret;
6543
6544 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6545 if (ret)
6546 return -EINVAL;
6547
6548 if (hw->dev_caps.switch_mode) {
6549 /* if switch mode is set, support mode2 (non-tunneled for
6550 * cloud filter) for now
6551 */
6552 u32 switch_mode = hw->dev_caps.switch_mode &
6553 I40E_SWITCH_MODE_MASK;
6554 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6555 if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6556 return 0;
6557 dev_err(&pf->pdev->dev,
6558 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6559 hw->dev_caps.switch_mode);
6560 return -EINVAL;
6561 }
6562 }
6563
6564 /* Set Bit 7 to be valid */
6565 mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6566
6567 /* Set L4type for TCP support */
6568 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6569
6570 /* Set cloud filter mode */
6571 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6572
6573 /* Prep mode field for set_switch_config */
6574 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6575 pf->last_sw_conf_valid_flags,
6576 mode, NULL);
6577 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6578 dev_err(&pf->pdev->dev,
6579 "couldn't set switch config bits, err %pe aq_err %s\n",
6580 ERR_PTR(ret),
6581 i40e_aq_str(hw,
6582 hw->aq.asq_last_status));
6583
6584 return ret;
6585 }
6586
6587 /**
6588 * i40e_create_queue_channel - function to create channel
6589 * @vsi: VSI to be configured
6590 * @ch: ptr to channel (it contains channel specific params)
6591 *
6592 * This function creates channel (VSI) using num_queues specified by user,
6593 * reconfigs RSS if needed.
6594 **/
i40e_create_queue_channel(struct i40e_vsi * vsi,struct i40e_channel * ch)6595 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6596 struct i40e_channel *ch)
6597 {
6598 struct i40e_pf *pf = vsi->back;
6599 bool reconfig_rss;
6600 int err;
6601
6602 if (!ch)
6603 return -EINVAL;
6604
6605 if (!ch->num_queue_pairs) {
6606 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6607 ch->num_queue_pairs);
6608 return -EINVAL;
6609 }
6610
6611 /* validate user requested num_queues for channel */
6612 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6613 &reconfig_rss);
6614 if (err) {
6615 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6616 ch->num_queue_pairs);
6617 return -EINVAL;
6618 }
6619
6620 /* By default we are in VEPA mode, if this is the first VF/VMDq
6621 * VSI to be added switch to VEB mode.
6622 */
6623
6624 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
6625 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
6626
6627 if (vsi->type == I40E_VSI_MAIN) {
6628 if (i40e_is_tc_mqprio_enabled(pf))
6629 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
6630 else
6631 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6632 }
6633 /* now onwards for main VSI, number of queues will be value
6634 * of TC0's queue count
6635 */
6636 }
6637
6638 /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6639 * it should be more than num_queues
6640 */
6641 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6642 dev_dbg(&pf->pdev->dev,
6643 "Error: cnt_q_avail (%u) less than num_queues %d\n",
6644 vsi->cnt_q_avail, ch->num_queue_pairs);
6645 return -EINVAL;
6646 }
6647
6648 /* reconfig_rss only if vsi type is MAIN_VSI */
6649 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6650 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6651 if (err) {
6652 dev_info(&pf->pdev->dev,
6653 "Error: unable to reconfig rss for num_queues (%u)\n",
6654 ch->num_queue_pairs);
6655 return -EINVAL;
6656 }
6657 }
6658
6659 if (!i40e_setup_channel(pf, vsi, ch)) {
6660 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6661 return -EINVAL;
6662 }
6663
6664 dev_info(&pf->pdev->dev,
6665 "Setup channel (id:%u) utilizing num_queues %d\n",
6666 ch->seid, ch->num_queue_pairs);
6667
6668 /* configure VSI for BW limit */
6669 if (ch->max_tx_rate) {
6670 u64 credits = ch->max_tx_rate;
6671
6672 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6673 return -EINVAL;
6674
6675 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6676 dev_dbg(&pf->pdev->dev,
6677 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6678 ch->max_tx_rate,
6679 credits,
6680 ch->seid);
6681 }
6682
6683 /* in case of VF, this will be main SRIOV VSI */
6684 ch->parent_vsi = vsi;
6685
6686 /* and update main_vsi's count for queue_available to use */
6687 vsi->cnt_q_avail -= ch->num_queue_pairs;
6688
6689 return 0;
6690 }
6691
6692 /**
6693 * i40e_configure_queue_channels - Add queue channel for the given TCs
6694 * @vsi: VSI to be configured
6695 *
6696 * Configures queue channel mapping to the given TCs
6697 **/
i40e_configure_queue_channels(struct i40e_vsi * vsi)6698 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6699 {
6700 struct i40e_channel *ch;
6701 u64 max_rate = 0;
6702 int ret = 0, i;
6703
6704 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6705 vsi->tc_seid_map[0] = vsi->seid;
6706 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6707 if (vsi->tc_config.enabled_tc & BIT(i)) {
6708 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6709 if (!ch) {
6710 ret = -ENOMEM;
6711 goto err_free;
6712 }
6713
6714 INIT_LIST_HEAD(&ch->list);
6715 ch->num_queue_pairs =
6716 vsi->tc_config.tc_info[i].qcount;
6717 ch->base_queue =
6718 vsi->tc_config.tc_info[i].qoffset;
6719
6720 /* Bandwidth limit through tc interface is in bytes/s,
6721 * change to Mbit/s
6722 */
6723 max_rate = vsi->mqprio_qopt.max_rate[i];
6724 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6725 ch->max_tx_rate = max_rate;
6726
6727 list_add_tail(&ch->list, &vsi->ch_list);
6728
6729 ret = i40e_create_queue_channel(vsi, ch);
6730 if (ret) {
6731 dev_err(&vsi->back->pdev->dev,
6732 "Failed creating queue channel with TC%d: queues %d\n",
6733 i, ch->num_queue_pairs);
6734 goto err_free;
6735 }
6736 vsi->tc_seid_map[i] = ch->seid;
6737 }
6738 }
6739
6740 /* reset to reconfigure TX queue contexts */
6741 i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true);
6742 return ret;
6743
6744 err_free:
6745 i40e_remove_queue_channels(vsi);
6746 return ret;
6747 }
6748
6749 /**
6750 * i40e_veb_config_tc - Configure TCs for given VEB
6751 * @veb: given VEB
6752 * @enabled_tc: TC bitmap
6753 *
6754 * Configures given TC bitmap for VEB (switching) element
6755 **/
i40e_veb_config_tc(struct i40e_veb * veb,u8 enabled_tc)6756 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6757 {
6758 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6759 struct i40e_pf *pf = veb->pf;
6760 int ret = 0;
6761 int i;
6762
6763 /* No TCs or already enabled TCs just return */
6764 if (!enabled_tc || veb->enabled_tc == enabled_tc)
6765 return ret;
6766
6767 bw_data.tc_valid_bits = enabled_tc;
6768 /* bw_data.absolute_credits is not set (relative) */
6769
6770 /* Enable ETS TCs with equal BW Share for now */
6771 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6772 if (enabled_tc & BIT(i))
6773 bw_data.tc_bw_share_credits[i] = 1;
6774 }
6775
6776 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6777 &bw_data, NULL);
6778 if (ret) {
6779 dev_info(&pf->pdev->dev,
6780 "VEB bw config failed, err %pe aq_err %s\n",
6781 ERR_PTR(ret),
6782 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6783 goto out;
6784 }
6785
6786 /* Update the BW information */
6787 ret = i40e_veb_get_bw_info(veb);
6788 if (ret) {
6789 dev_info(&pf->pdev->dev,
6790 "Failed getting veb bw config, err %pe aq_err %s\n",
6791 ERR_PTR(ret),
6792 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6793 }
6794
6795 out:
6796 return ret;
6797 }
6798
6799 #ifdef CONFIG_I40E_DCB
6800 /**
6801 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6802 * @pf: PF struct
6803 *
6804 * Reconfigure VEB/VSIs on a given PF; it is assumed that
6805 * the caller would've quiesce all the VSIs before calling
6806 * this function
6807 **/
i40e_dcb_reconfigure(struct i40e_pf * pf)6808 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6809 {
6810 struct i40e_vsi *vsi;
6811 struct i40e_veb *veb;
6812 u8 tc_map = 0;
6813 int ret;
6814 int v;
6815
6816 /* Enable the TCs available on PF to all VEBs */
6817 tc_map = i40e_pf_get_tc_map(pf);
6818 if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6819 return;
6820
6821 i40e_pf_for_each_veb(pf, v, veb) {
6822 ret = i40e_veb_config_tc(veb, tc_map);
6823 if (ret) {
6824 dev_info(&pf->pdev->dev,
6825 "Failed configuring TC for VEB seid=%d\n",
6826 veb->seid);
6827 /* Will try to configure as many components */
6828 }
6829 }
6830
6831 /* Update each VSI */
6832 i40e_pf_for_each_vsi(pf, v, vsi) {
6833 /* - Enable all TCs for the LAN VSI
6834 * - For all others keep them at TC0 for now
6835 */
6836 if (vsi->type == I40E_VSI_MAIN)
6837 tc_map = i40e_pf_get_tc_map(pf);
6838 else
6839 tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6840
6841 ret = i40e_vsi_config_tc(vsi, tc_map);
6842 if (ret) {
6843 dev_info(&pf->pdev->dev,
6844 "Failed configuring TC for VSI seid=%d\n",
6845 vsi->seid);
6846 /* Will try to configure as many components */
6847 } else {
6848 /* Re-configure VSI vectors based on updated TC map */
6849 i40e_vsi_map_rings_to_vectors(vsi);
6850 if (vsi->netdev)
6851 i40e_dcbnl_set_all(vsi);
6852 }
6853 }
6854 }
6855
6856 /**
6857 * i40e_resume_port_tx - Resume port Tx
6858 * @pf: PF struct
6859 *
6860 * Resume a port's Tx and issue a PF reset in case of failure to
6861 * resume.
6862 **/
i40e_resume_port_tx(struct i40e_pf * pf)6863 static int i40e_resume_port_tx(struct i40e_pf *pf)
6864 {
6865 struct i40e_hw *hw = &pf->hw;
6866 int ret;
6867
6868 ret = i40e_aq_resume_port_tx(hw, NULL);
6869 if (ret) {
6870 dev_info(&pf->pdev->dev,
6871 "Resume Port Tx failed, err %pe aq_err %s\n",
6872 ERR_PTR(ret),
6873 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6874 /* Schedule PF reset to recover */
6875 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6876 i40e_service_event_schedule(pf);
6877 }
6878
6879 return ret;
6880 }
6881
6882 /**
6883 * i40e_suspend_port_tx - Suspend port Tx
6884 * @pf: PF struct
6885 *
6886 * Suspend a port's Tx and issue a PF reset in case of failure.
6887 **/
i40e_suspend_port_tx(struct i40e_pf * pf)6888 static int i40e_suspend_port_tx(struct i40e_pf *pf)
6889 {
6890 struct i40e_hw *hw = &pf->hw;
6891 int ret;
6892
6893 ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);
6894 if (ret) {
6895 dev_info(&pf->pdev->dev,
6896 "Suspend Port Tx failed, err %pe aq_err %s\n",
6897 ERR_PTR(ret),
6898 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6899 /* Schedule PF reset to recover */
6900 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6901 i40e_service_event_schedule(pf);
6902 }
6903
6904 return ret;
6905 }
6906
6907 /**
6908 * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6909 * @pf: PF being configured
6910 * @new_cfg: New DCBX configuration
6911 *
6912 * Program DCB settings into HW and reconfigure VEB/VSIs on
6913 * given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6914 **/
i40e_hw_set_dcb_config(struct i40e_pf * pf,struct i40e_dcbx_config * new_cfg)6915 static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6916 struct i40e_dcbx_config *new_cfg)
6917 {
6918 struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6919 int ret;
6920
6921 /* Check if need reconfiguration */
6922 if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {
6923 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6924 return 0;
6925 }
6926
6927 /* Config change disable all VSIs */
6928 i40e_pf_quiesce_all_vsi(pf);
6929
6930 /* Copy the new config to the current config */
6931 *old_cfg = *new_cfg;
6932 old_cfg->etsrec = old_cfg->etscfg;
6933 ret = i40e_set_dcb_config(&pf->hw);
6934 if (ret) {
6935 dev_info(&pf->pdev->dev,
6936 "Set DCB Config failed, err %pe aq_err %s\n",
6937 ERR_PTR(ret),
6938 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6939 goto out;
6940 }
6941
6942 /* Changes in configuration update VEB/VSI */
6943 i40e_dcb_reconfigure(pf);
6944 out:
6945 /* In case of reset do not try to resume anything */
6946 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6947 /* Re-start the VSIs if disabled */
6948 ret = i40e_resume_port_tx(pf);
6949 /* In case of error no point in resuming VSIs */
6950 if (ret)
6951 goto err;
6952 i40e_pf_unquiesce_all_vsi(pf);
6953 }
6954 err:
6955 return ret;
6956 }
6957
6958 /**
6959 * i40e_hw_dcb_config - Program new DCBX settings into HW
6960 * @pf: PF being configured
6961 * @new_cfg: New DCBX configuration
6962 *
6963 * Program DCB settings into HW and reconfigure VEB/VSIs on
6964 * given PF
6965 **/
i40e_hw_dcb_config(struct i40e_pf * pf,struct i40e_dcbx_config * new_cfg)6966 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6967 {
6968 struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6969 u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6970 u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6971 struct i40e_dcbx_config *old_cfg;
6972 u8 mode[I40E_MAX_TRAFFIC_CLASS];
6973 struct i40e_rx_pb_config pb_cfg;
6974 struct i40e_hw *hw = &pf->hw;
6975 u8 num_ports = hw->num_ports;
6976 bool need_reconfig;
6977 int ret = -EINVAL;
6978 u8 lltc_map = 0;
6979 u8 tc_map = 0;
6980 u8 new_numtc;
6981 u8 i;
6982
6983 dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6984 /* Un-pack information to Program ETS HW via shared API
6985 * numtc, tcmap
6986 * LLTC map
6987 * ETS/NON-ETS arbiter mode
6988 * max exponent (credit refills)
6989 * Total number of ports
6990 * PFC priority bit-map
6991 * Priority Table
6992 * BW % per TC
6993 * Arbiter mode between UPs sharing same TC
6994 * TSA table (ETS or non-ETS)
6995 * EEE enabled or not
6996 * MFS TC table
6997 */
6998
6999 new_numtc = i40e_dcb_get_num_tc(new_cfg);
7000
7001 memset(&ets_data, 0, sizeof(ets_data));
7002 for (i = 0; i < new_numtc; i++) {
7003 tc_map |= BIT(i);
7004 switch (new_cfg->etscfg.tsatable[i]) {
7005 case I40E_IEEE_TSA_ETS:
7006 prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
7007 ets_data.tc_bw_share_credits[i] =
7008 new_cfg->etscfg.tcbwtable[i];
7009 break;
7010 case I40E_IEEE_TSA_STRICT:
7011 prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
7012 lltc_map |= BIT(i);
7013 ets_data.tc_bw_share_credits[i] =
7014 I40E_DCB_STRICT_PRIO_CREDITS;
7015 break;
7016 default:
7017 /* Invalid TSA type */
7018 need_reconfig = false;
7019 goto out;
7020 }
7021 }
7022
7023 old_cfg = &hw->local_dcbx_config;
7024 /* Check if need reconfiguration */
7025 need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
7026
7027 /* If needed, enable/disable frame tagging, disable all VSIs
7028 * and suspend port tx
7029 */
7030 if (need_reconfig) {
7031 /* Enable DCB tagging only when more than one TC */
7032 if (new_numtc > 1)
7033 set_bit(I40E_FLAG_DCB_ENA, pf->flags);
7034 else
7035 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7036
7037 set_bit(__I40E_PORT_SUSPENDED, pf->state);
7038 /* Reconfiguration needed quiesce all VSIs */
7039 i40e_pf_quiesce_all_vsi(pf);
7040 ret = i40e_suspend_port_tx(pf);
7041 if (ret)
7042 goto err;
7043 }
7044
7045 /* Configure Port ETS Tx Scheduler */
7046 ets_data.tc_valid_bits = tc_map;
7047 ets_data.tc_strict_priority_flags = lltc_map;
7048 ret = i40e_aq_config_switch_comp_ets
7049 (hw, pf->mac_seid, &ets_data,
7050 i40e_aqc_opc_modify_switching_comp_ets, NULL);
7051 if (ret) {
7052 dev_info(&pf->pdev->dev,
7053 "Modify Port ETS failed, err %pe aq_err %s\n",
7054 ERR_PTR(ret),
7055 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7056 goto out;
7057 }
7058
7059 /* Configure Rx ETS HW */
7060 memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
7061 i40e_dcb_hw_set_num_tc(hw, new_numtc);
7062 i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,
7063 I40E_DCB_ARB_MODE_STRICT_PRIORITY,
7064 I40E_DCB_DEFAULT_MAX_EXPONENT,
7065 lltc_map);
7066 i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);
7067 i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,
7068 prio_type);
7069 i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,
7070 new_cfg->etscfg.prioritytable);
7071 i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);
7072
7073 /* Configure Rx Packet Buffers in HW */
7074 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7075 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
7076
7077 mfs_tc[i] = main_vsi->netdev->mtu;
7078 mfs_tc[i] += I40E_PACKET_HDR_PAD;
7079 }
7080
7081 i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
7082 false, new_cfg->pfc.pfcenable,
7083 mfs_tc, &pb_cfg);
7084 i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);
7085
7086 /* Update the local Rx Packet buffer config */
7087 pf->pb_cfg = pb_cfg;
7088
7089 /* Inform the FW about changes to DCB configuration */
7090 ret = i40e_aq_dcb_updated(&pf->hw, NULL);
7091 if (ret) {
7092 dev_info(&pf->pdev->dev,
7093 "DCB Updated failed, err %pe aq_err %s\n",
7094 ERR_PTR(ret),
7095 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7096 goto out;
7097 }
7098
7099 /* Update the port DCBx configuration */
7100 *old_cfg = *new_cfg;
7101
7102 /* Changes in configuration update VEB/VSI */
7103 i40e_dcb_reconfigure(pf);
7104 out:
7105 /* Re-start the VSIs if disabled */
7106 if (need_reconfig) {
7107 ret = i40e_resume_port_tx(pf);
7108
7109 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
7110 /* In case of error no point in resuming VSIs */
7111 if (ret)
7112 goto err;
7113
7114 /* Wait for the PF's queues to be disabled */
7115 ret = i40e_pf_wait_queues_disabled(pf);
7116 if (ret) {
7117 /* Schedule PF reset to recover */
7118 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7119 i40e_service_event_schedule(pf);
7120 goto err;
7121 } else {
7122 i40e_pf_unquiesce_all_vsi(pf);
7123 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7124 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
7125 }
7126 /* registers are set, lets apply */
7127 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps))
7128 ret = i40e_hw_set_dcb_config(pf, new_cfg);
7129 }
7130
7131 err:
7132 return ret;
7133 }
7134
7135 /**
7136 * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
7137 * @pf: PF being queried
7138 *
7139 * Set default DCB configuration in case DCB is to be done in SW.
7140 **/
i40e_dcb_sw_default_config(struct i40e_pf * pf)7141 int i40e_dcb_sw_default_config(struct i40e_pf *pf)
7142 {
7143 struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
7144 struct i40e_aqc_configure_switching_comp_ets_data ets_data;
7145 struct i40e_hw *hw = &pf->hw;
7146 int err;
7147
7148 if (test_bit(I40E_HW_CAP_USE_SET_LLDP_MIB, pf->hw.caps)) {
7149 /* Update the local cached instance with TC0 ETS */
7150 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
7151 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7152 pf->tmp_cfg.etscfg.maxtcs = 0;
7153 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7154 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
7155 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
7156 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
7157 /* FW needs one App to configure HW */
7158 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
7159 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
7160 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
7161 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
7162
7163 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);
7164 }
7165
7166 memset(&ets_data, 0, sizeof(ets_data));
7167 ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
7168 ets_data.tc_strict_priority_flags = 0; /* ETS */
7169 ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
7170
7171 /* Enable ETS on the Physical port */
7172 err = i40e_aq_config_switch_comp_ets
7173 (hw, pf->mac_seid, &ets_data,
7174 i40e_aqc_opc_enable_switching_comp_ets, NULL);
7175 if (err) {
7176 dev_info(&pf->pdev->dev,
7177 "Enable Port ETS failed, err %pe aq_err %s\n",
7178 ERR_PTR(err),
7179 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7180 err = -ENOENT;
7181 goto out;
7182 }
7183
7184 /* Update the local cached instance with TC0 ETS */
7185 dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7186 dcb_cfg->etscfg.cbs = 0;
7187 dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
7188 dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7189
7190 out:
7191 return err;
7192 }
7193
7194 /**
7195 * i40e_init_pf_dcb - Initialize DCB configuration
7196 * @pf: PF being configured
7197 *
7198 * Query the current DCB configuration and cache it
7199 * in the hardware structure
7200 **/
i40e_init_pf_dcb(struct i40e_pf * pf)7201 static int i40e_init_pf_dcb(struct i40e_pf *pf)
7202 {
7203 struct i40e_hw *hw = &pf->hw;
7204 int err;
7205
7206 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
7207 * Also do not enable DCBx if FW LLDP agent is disabled
7208 */
7209 if (test_bit(I40E_HW_CAP_NO_DCB_SUPPORT, pf->hw.caps)) {
7210 dev_info(&pf->pdev->dev, "DCB is not supported.\n");
7211 err = -EOPNOTSUPP;
7212 goto out;
7213 }
7214 if (test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) {
7215 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
7216 err = i40e_dcb_sw_default_config(pf);
7217 if (err) {
7218 dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
7219 goto out;
7220 }
7221 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
7222 pf->dcbx_cap = DCB_CAP_DCBX_HOST |
7223 DCB_CAP_DCBX_VER_IEEE;
7224 /* at init capable but disabled */
7225 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
7226 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7227 goto out;
7228 }
7229 err = i40e_init_dcb(hw, true);
7230 if (!err) {
7231 /* Device/Function is not DCBX capable */
7232 if ((!hw->func_caps.dcb) ||
7233 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
7234 dev_info(&pf->pdev->dev,
7235 "DCBX offload is not supported or is disabled for this PF.\n");
7236 } else {
7237 /* When status is not DISABLED then DCBX in FW */
7238 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
7239 DCB_CAP_DCBX_VER_IEEE;
7240
7241 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
7242 /* Enable DCB tagging only when more than one TC
7243 * or explicitly disable if only one TC
7244 */
7245 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
7246 set_bit(I40E_FLAG_DCB_ENA, pf->flags);
7247 else
7248 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
7249 dev_dbg(&pf->pdev->dev,
7250 "DCBX offload is supported for this PF.\n");
7251 }
7252 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
7253 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
7254 set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags);
7255 } else {
7256 dev_info(&pf->pdev->dev,
7257 "Query for DCB configuration failed, err %pe aq_err %s\n",
7258 ERR_PTR(err),
7259 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7260 }
7261
7262 out:
7263 return err;
7264 }
7265 #endif /* CONFIG_I40E_DCB */
7266
i40e_print_link_message_eee(struct i40e_vsi * vsi,const char * speed,const char * fc)7267 static void i40e_print_link_message_eee(struct i40e_vsi *vsi,
7268 const char *speed, const char *fc)
7269 {
7270 struct ethtool_keee kedata;
7271
7272 memzero_explicit(&kedata, sizeof(kedata));
7273 if (vsi->netdev->ethtool_ops->get_eee)
7274 vsi->netdev->ethtool_ops->get_eee(vsi->netdev, &kedata);
7275
7276 if (!linkmode_empty(kedata.supported))
7277 netdev_info(vsi->netdev,
7278 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s, EEE: %s\n",
7279 speed, fc,
7280 kedata.eee_enabled ? "Enabled" : "Disabled");
7281 else
7282 netdev_info(vsi->netdev,
7283 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7284 speed, fc);
7285 }
7286
7287 /**
7288 * i40e_print_link_message - print link up or down
7289 * @vsi: the VSI for which link needs a message
7290 * @isup: true of link is up, false otherwise
7291 */
i40e_print_link_message(struct i40e_vsi * vsi,bool isup)7292 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
7293 {
7294 enum i40e_aq_link_speed new_speed;
7295 struct i40e_pf *pf = vsi->back;
7296 char *speed = "Unknown";
7297 char *fc = "Unknown";
7298 char *fec = "";
7299 char *req_fec = "";
7300 char *an = "";
7301
7302 if (isup)
7303 new_speed = pf->hw.phy.link_info.link_speed;
7304 else
7305 new_speed = I40E_LINK_SPEED_UNKNOWN;
7306
7307 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
7308 return;
7309 vsi->current_isup = isup;
7310 vsi->current_speed = new_speed;
7311 if (!isup) {
7312 netdev_info(vsi->netdev, "NIC Link is Down\n");
7313 return;
7314 }
7315
7316 /* Warn user if link speed on NPAR enabled partition is not at
7317 * least 10GB
7318 */
7319 if (pf->hw.func_caps.npar_enable &&
7320 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
7321 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
7322 netdev_warn(vsi->netdev,
7323 "The partition detected link speed that is less than 10Gbps\n");
7324
7325 switch (pf->hw.phy.link_info.link_speed) {
7326 case I40E_LINK_SPEED_40GB:
7327 speed = "40 G";
7328 break;
7329 case I40E_LINK_SPEED_20GB:
7330 speed = "20 G";
7331 break;
7332 case I40E_LINK_SPEED_25GB:
7333 speed = "25 G";
7334 break;
7335 case I40E_LINK_SPEED_10GB:
7336 speed = "10 G";
7337 break;
7338 case I40E_LINK_SPEED_5GB:
7339 speed = "5 G";
7340 break;
7341 case I40E_LINK_SPEED_2_5GB:
7342 speed = "2.5 G";
7343 break;
7344 case I40E_LINK_SPEED_1GB:
7345 speed = "1000 M";
7346 break;
7347 case I40E_LINK_SPEED_100MB:
7348 speed = "100 M";
7349 break;
7350 default:
7351 break;
7352 }
7353
7354 switch (pf->hw.fc.current_mode) {
7355 case I40E_FC_FULL:
7356 fc = "RX/TX";
7357 break;
7358 case I40E_FC_TX_PAUSE:
7359 fc = "TX";
7360 break;
7361 case I40E_FC_RX_PAUSE:
7362 fc = "RX";
7363 break;
7364 default:
7365 fc = "None";
7366 break;
7367 }
7368
7369 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
7370 req_fec = "None";
7371 fec = "None";
7372 an = "False";
7373
7374 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7375 an = "True";
7376
7377 if (pf->hw.phy.link_info.fec_info &
7378 I40E_AQ_CONFIG_FEC_KR_ENA)
7379 fec = "CL74 FC-FEC/BASE-R";
7380 else if (pf->hw.phy.link_info.fec_info &
7381 I40E_AQ_CONFIG_FEC_RS_ENA)
7382 fec = "CL108 RS-FEC";
7383
7384 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
7385 * both RS and FC are requested
7386 */
7387 if (vsi->back->hw.phy.link_info.req_fec_info &
7388 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7389 if (vsi->back->hw.phy.link_info.req_fec_info &
7390 I40E_AQ_REQUEST_FEC_RS)
7391 req_fec = "CL108 RS-FEC";
7392 else
7393 req_fec = "CL74 FC-FEC/BASE-R";
7394 }
7395 netdev_info(vsi->netdev,
7396 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7397 speed, req_fec, fec, an, fc);
7398 } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7399 req_fec = "None";
7400 fec = "None";
7401 an = "False";
7402
7403 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7404 an = "True";
7405
7406 if (pf->hw.phy.link_info.fec_info &
7407 I40E_AQ_CONFIG_FEC_KR_ENA)
7408 fec = "CL74 FC-FEC/BASE-R";
7409
7410 if (pf->hw.phy.link_info.req_fec_info &
7411 I40E_AQ_REQUEST_FEC_KR)
7412 req_fec = "CL74 FC-FEC/BASE-R";
7413
7414 netdev_info(vsi->netdev,
7415 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7416 speed, req_fec, fec, an, fc);
7417 } else {
7418 i40e_print_link_message_eee(vsi, speed, fc);
7419 }
7420
7421 }
7422
7423 /**
7424 * i40e_up_complete - Finish the last steps of bringing up a connection
7425 * @vsi: the VSI being configured
7426 **/
i40e_up_complete(struct i40e_vsi * vsi)7427 static int i40e_up_complete(struct i40e_vsi *vsi)
7428 {
7429 struct i40e_pf *pf = vsi->back;
7430 int err;
7431
7432 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
7433 i40e_vsi_configure_msix(vsi);
7434 else
7435 i40e_configure_msi_and_legacy(vsi);
7436
7437 /* start rings */
7438 err = i40e_vsi_start_rings(vsi);
7439 if (err)
7440 return err;
7441
7442 clear_bit(__I40E_VSI_DOWN, vsi->state);
7443 i40e_napi_enable_all(vsi);
7444 i40e_vsi_enable_irq(vsi);
7445
7446 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7447 (vsi->netdev)) {
7448 i40e_print_link_message(vsi, true);
7449 netif_tx_start_all_queues(vsi->netdev);
7450 netif_carrier_on(vsi->netdev);
7451 }
7452
7453 /* replay FDIR SB filters */
7454 if (vsi->type == I40E_VSI_FDIR) {
7455 /* reset fd counters */
7456 pf->fd_add_err = 0;
7457 pf->fd_atr_cnt = 0;
7458 i40e_fdir_filter_restore(vsi);
7459 }
7460
7461 /* On the next run of the service_task, notify any clients of the new
7462 * opened netdev
7463 */
7464 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7465 i40e_service_event_schedule(pf);
7466
7467 return 0;
7468 }
7469
7470 /**
7471 * i40e_vsi_reinit_locked - Reset the VSI
7472 * @vsi: the VSI being configured
7473 *
7474 * Rebuild the ring structs after some configuration
7475 * has changed, e.g. MTU size.
7476 **/
i40e_vsi_reinit_locked(struct i40e_vsi * vsi)7477 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7478 {
7479 struct i40e_pf *pf = vsi->back;
7480
7481 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
7482 usleep_range(1000, 2000);
7483 i40e_down(vsi);
7484
7485 i40e_up(vsi);
7486 clear_bit(__I40E_CONFIG_BUSY, pf->state);
7487 }
7488
7489 /**
7490 * i40e_force_link_state - Force the link status
7491 * @pf: board private structure
7492 * @is_up: whether the link state should be forced up or down
7493 **/
i40e_force_link_state(struct i40e_pf * pf,bool is_up)7494 static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7495 {
7496 struct i40e_aq_get_phy_abilities_resp abilities;
7497 struct i40e_aq_set_phy_config config = {0};
7498 bool non_zero_phy_type = is_up;
7499 struct i40e_hw *hw = &pf->hw;
7500 u64 mask;
7501 u8 speed;
7502 int err;
7503
7504 /* Card might've been put in an unstable state by other drivers
7505 * and applications, which causes incorrect speed values being
7506 * set on startup. In order to clear speed registers, we call
7507 * get_phy_capabilities twice, once to get initial state of
7508 * available speeds, and once to get current PHY config.
7509 */
7510 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
7511 NULL);
7512 if (err) {
7513 dev_err(&pf->pdev->dev,
7514 "failed to get phy cap., ret = %pe last_status = %s\n",
7515 ERR_PTR(err),
7516 i40e_aq_str(hw, hw->aq.asq_last_status));
7517 return err;
7518 }
7519 speed = abilities.link_speed;
7520
7521 /* Get the current phy config */
7522 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
7523 NULL);
7524 if (err) {
7525 dev_err(&pf->pdev->dev,
7526 "failed to get phy cap., ret = %pe last_status = %s\n",
7527 ERR_PTR(err),
7528 i40e_aq_str(hw, hw->aq.asq_last_status));
7529 return err;
7530 }
7531
7532 /* If link needs to go up, but was not forced to go down,
7533 * and its speed values are OK, no need for a flap
7534 * if non_zero_phy_type was set, still need to force up
7535 */
7536 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags))
7537 non_zero_phy_type = true;
7538 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7539 return 0;
7540
7541 /* To force link we need to set bits for all supported PHY types,
7542 * but there are now more than 32, so we need to split the bitmap
7543 * across two fields.
7544 */
7545 mask = I40E_PHY_TYPES_BITMASK;
7546 config.phy_type =
7547 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7548 config.phy_type_ext =
7549 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7550 /* Copy the old settings, except of phy_type */
7551 config.abilities = abilities.abilities;
7552 if (test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags)) {
7553 if (is_up)
7554 config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7555 else
7556 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7557 }
7558 if (abilities.link_speed != 0)
7559 config.link_speed = abilities.link_speed;
7560 else
7561 config.link_speed = speed;
7562 config.eee_capability = abilities.eee_capability;
7563 config.eeer = abilities.eeer_val;
7564 config.low_power_ctrl = abilities.d3_lpan;
7565 config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7566 I40E_AQ_PHY_FEC_CONFIG_MASK;
7567 err = i40e_aq_set_phy_config(hw, &config, NULL);
7568
7569 if (err) {
7570 dev_err(&pf->pdev->dev,
7571 "set phy config ret = %pe last_status = %s\n",
7572 ERR_PTR(err),
7573 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7574 return err;
7575 }
7576
7577 /* Update the link info */
7578 err = i40e_update_link_info(hw);
7579 if (err) {
7580 /* Wait a little bit (on 40G cards it sometimes takes a really
7581 * long time for link to come back from the atomic reset)
7582 * and try once more
7583 */
7584 msleep(1000);
7585 i40e_update_link_info(hw);
7586 }
7587
7588 i40e_aq_set_link_restart_an(hw, is_up, NULL);
7589
7590 return 0;
7591 }
7592
7593 /**
7594 * i40e_up - Bring the connection back up after being down
7595 * @vsi: the VSI being configured
7596 **/
i40e_up(struct i40e_vsi * vsi)7597 int i40e_up(struct i40e_vsi *vsi)
7598 {
7599 int err;
7600
7601 if (vsi->type == I40E_VSI_MAIN &&
7602 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7603 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7604 i40e_force_link_state(vsi->back, true);
7605
7606 err = i40e_vsi_configure(vsi);
7607 if (!err)
7608 err = i40e_up_complete(vsi);
7609
7610 return err;
7611 }
7612
7613 /**
7614 * i40e_down - Shutdown the connection processing
7615 * @vsi: the VSI being stopped
7616 **/
i40e_down(struct i40e_vsi * vsi)7617 void i40e_down(struct i40e_vsi *vsi)
7618 {
7619 int i;
7620
7621 /* It is assumed that the caller of this function
7622 * sets the vsi->state __I40E_VSI_DOWN bit.
7623 */
7624 if (vsi->netdev) {
7625 netif_carrier_off(vsi->netdev);
7626 netif_tx_disable(vsi->netdev);
7627 }
7628 i40e_vsi_disable_irq(vsi);
7629 i40e_vsi_stop_rings(vsi);
7630 if (vsi->type == I40E_VSI_MAIN &&
7631 (test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags) ||
7632 test_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, vsi->back->flags)))
7633 i40e_force_link_state(vsi->back, false);
7634 i40e_napi_disable_all(vsi);
7635
7636 for (i = 0; i < vsi->num_queue_pairs; i++) {
7637 i40e_clean_tx_ring(vsi->tx_rings[i]);
7638 if (i40e_enabled_xdp_vsi(vsi)) {
7639 /* Make sure that in-progress ndo_xdp_xmit and
7640 * ndo_xsk_wakeup calls are completed.
7641 */
7642 synchronize_rcu();
7643 i40e_clean_tx_ring(vsi->xdp_rings[i]);
7644 }
7645 i40e_clean_rx_ring(vsi->rx_rings[i]);
7646 }
7647
7648 }
7649
7650 /**
7651 * i40e_validate_mqprio_qopt- validate queue mapping info
7652 * @vsi: the VSI being configured
7653 * @mqprio_qopt: queue parametrs
7654 **/
i40e_validate_mqprio_qopt(struct i40e_vsi * vsi,struct tc_mqprio_qopt_offload * mqprio_qopt)7655 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7656 struct tc_mqprio_qopt_offload *mqprio_qopt)
7657 {
7658 u64 sum_max_rate = 0;
7659 u64 max_rate = 0;
7660 int i;
7661
7662 if (mqprio_qopt->qopt.offset[0] != 0 ||
7663 mqprio_qopt->qopt.num_tc < 1 ||
7664 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7665 return -EINVAL;
7666 for (i = 0; ; i++) {
7667 if (!mqprio_qopt->qopt.count[i])
7668 return -EINVAL;
7669 if (mqprio_qopt->min_rate[i]) {
7670 dev_err(&vsi->back->pdev->dev,
7671 "Invalid min tx rate (greater than 0) specified\n");
7672 return -EINVAL;
7673 }
7674 max_rate = mqprio_qopt->max_rate[i];
7675 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7676 sum_max_rate += max_rate;
7677
7678 if (i >= mqprio_qopt->qopt.num_tc - 1)
7679 break;
7680 if (mqprio_qopt->qopt.offset[i + 1] !=
7681 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7682 return -EINVAL;
7683 }
7684 if (vsi->num_queue_pairs <
7685 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7686 dev_err(&vsi->back->pdev->dev,
7687 "Failed to create traffic channel, insufficient number of queues.\n");
7688 return -EINVAL;
7689 }
7690 if (sum_max_rate > i40e_get_link_speed(vsi)) {
7691 dev_err(&vsi->back->pdev->dev,
7692 "Invalid max tx rate specified\n");
7693 return -EINVAL;
7694 }
7695 return 0;
7696 }
7697
7698 /**
7699 * i40e_vsi_set_default_tc_config - set default values for tc configuration
7700 * @vsi: the VSI being configured
7701 **/
i40e_vsi_set_default_tc_config(struct i40e_vsi * vsi)7702 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7703 {
7704 u16 qcount;
7705 int i;
7706
7707 /* Only TC0 is enabled */
7708 vsi->tc_config.numtc = 1;
7709 vsi->tc_config.enabled_tc = 1;
7710 qcount = min_t(int, vsi->alloc_queue_pairs,
7711 i40e_pf_get_max_q_per_tc(vsi->back));
7712 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7713 /* For the TC that is not enabled set the offset to default
7714 * queue and allocate one queue for the given TC.
7715 */
7716 vsi->tc_config.tc_info[i].qoffset = 0;
7717 if (i == 0)
7718 vsi->tc_config.tc_info[i].qcount = qcount;
7719 else
7720 vsi->tc_config.tc_info[i].qcount = 1;
7721 vsi->tc_config.tc_info[i].netdev_tc = 0;
7722 }
7723 }
7724
7725 /**
7726 * i40e_del_macvlan_filter
7727 * @hw: pointer to the HW structure
7728 * @seid: seid of the channel VSI
7729 * @macaddr: the mac address to apply as a filter
7730 * @aq_err: store the admin Q error
7731 *
7732 * This function deletes a mac filter on the channel VSI which serves as the
7733 * macvlan. Returns 0 on success.
7734 **/
i40e_del_macvlan_filter(struct i40e_hw * hw,u16 seid,const u8 * macaddr,int * aq_err)7735 static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7736 const u8 *macaddr, int *aq_err)
7737 {
7738 struct i40e_aqc_remove_macvlan_element_data element;
7739 int status;
7740
7741 memset(&element, 0, sizeof(element));
7742 ether_addr_copy(element.mac_addr, macaddr);
7743 element.vlan_tag = 0;
7744 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7745 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7746 *aq_err = hw->aq.asq_last_status;
7747
7748 return status;
7749 }
7750
7751 /**
7752 * i40e_add_macvlan_filter
7753 * @hw: pointer to the HW structure
7754 * @seid: seid of the channel VSI
7755 * @macaddr: the mac address to apply as a filter
7756 * @aq_err: store the admin Q error
7757 *
7758 * This function adds a mac filter on the channel VSI which serves as the
7759 * macvlan. Returns 0 on success.
7760 **/
i40e_add_macvlan_filter(struct i40e_hw * hw,u16 seid,const u8 * macaddr,int * aq_err)7761 static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7762 const u8 *macaddr, int *aq_err)
7763 {
7764 struct i40e_aqc_add_macvlan_element_data element;
7765 u16 cmd_flags = 0;
7766 int status;
7767
7768 ether_addr_copy(element.mac_addr, macaddr);
7769 element.vlan_tag = 0;
7770 element.queue_number = 0;
7771 element.match_method = I40E_AQC_MM_ERR_NO_RES;
7772 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7773 element.flags = cpu_to_le16(cmd_flags);
7774 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7775 *aq_err = hw->aq.asq_last_status;
7776
7777 return status;
7778 }
7779
7780 /**
7781 * i40e_reset_ch_rings - Reset the queue contexts in a channel
7782 * @vsi: the VSI we want to access
7783 * @ch: the channel we want to access
7784 */
i40e_reset_ch_rings(struct i40e_vsi * vsi,struct i40e_channel * ch)7785 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7786 {
7787 struct i40e_ring *tx_ring, *rx_ring;
7788 u16 pf_q;
7789 int i;
7790
7791 for (i = 0; i < ch->num_queue_pairs; i++) {
7792 pf_q = ch->base_queue + i;
7793 tx_ring = vsi->tx_rings[pf_q];
7794 tx_ring->ch = NULL;
7795 rx_ring = vsi->rx_rings[pf_q];
7796 rx_ring->ch = NULL;
7797 }
7798 }
7799
7800 /**
7801 * i40e_free_macvlan_channels
7802 * @vsi: the VSI we want to access
7803 *
7804 * This function frees the Qs of the channel VSI from
7805 * the stack and also deletes the channel VSIs which
7806 * serve as macvlans.
7807 */
i40e_free_macvlan_channels(struct i40e_vsi * vsi)7808 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7809 {
7810 struct i40e_channel *ch, *ch_tmp;
7811 int ret;
7812
7813 if (list_empty(&vsi->macvlan_list))
7814 return;
7815
7816 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7817 struct i40e_vsi *parent_vsi;
7818
7819 if (i40e_is_channel_macvlan(ch)) {
7820 i40e_reset_ch_rings(vsi, ch);
7821 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7822 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7823 netdev_set_sb_channel(ch->fwd->netdev, 0);
7824 kfree(ch->fwd);
7825 ch->fwd = NULL;
7826 }
7827
7828 list_del(&ch->list);
7829 parent_vsi = ch->parent_vsi;
7830 if (!parent_vsi || !ch->initialized) {
7831 kfree(ch);
7832 continue;
7833 }
7834
7835 /* remove the VSI */
7836 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7837 NULL);
7838 if (ret)
7839 dev_err(&vsi->back->pdev->dev,
7840 "unable to remove channel (%d) for parent VSI(%d)\n",
7841 ch->seid, parent_vsi->seid);
7842 kfree(ch);
7843 }
7844 vsi->macvlan_cnt = 0;
7845 }
7846
7847 /**
7848 * i40e_fwd_ring_up - bring the macvlan device up
7849 * @vsi: the VSI we want to access
7850 * @vdev: macvlan netdevice
7851 * @fwd: the private fwd structure
7852 */
i40e_fwd_ring_up(struct i40e_vsi * vsi,struct net_device * vdev,struct i40e_fwd_adapter * fwd)7853 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7854 struct i40e_fwd_adapter *fwd)
7855 {
7856 struct i40e_channel *ch = NULL, *ch_tmp, *iter;
7857 int ret = 0, num_tc = 1, i, aq_err;
7858 struct i40e_pf *pf = vsi->back;
7859 struct i40e_hw *hw = &pf->hw;
7860
7861 /* Go through the list and find an available channel */
7862 list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {
7863 if (!i40e_is_channel_macvlan(iter)) {
7864 iter->fwd = fwd;
7865 /* record configuration for macvlan interface in vdev */
7866 for (i = 0; i < num_tc; i++)
7867 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7868 i,
7869 iter->num_queue_pairs,
7870 iter->base_queue);
7871 for (i = 0; i < iter->num_queue_pairs; i++) {
7872 struct i40e_ring *tx_ring, *rx_ring;
7873 u16 pf_q;
7874
7875 pf_q = iter->base_queue + i;
7876
7877 /* Get to TX ring ptr */
7878 tx_ring = vsi->tx_rings[pf_q];
7879 tx_ring->ch = iter;
7880
7881 /* Get the RX ring ptr */
7882 rx_ring = vsi->rx_rings[pf_q];
7883 rx_ring->ch = iter;
7884 }
7885 ch = iter;
7886 break;
7887 }
7888 }
7889
7890 if (!ch)
7891 return -EINVAL;
7892
7893 /* Guarantee all rings are updated before we update the
7894 * MAC address filter.
7895 */
7896 wmb();
7897
7898 /* Add a mac filter */
7899 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7900 if (ret) {
7901 /* if we cannot add the MAC rule then disable the offload */
7902 macvlan_release_l2fw_offload(vdev);
7903 for (i = 0; i < ch->num_queue_pairs; i++) {
7904 struct i40e_ring *rx_ring;
7905 u16 pf_q;
7906
7907 pf_q = ch->base_queue + i;
7908 rx_ring = vsi->rx_rings[pf_q];
7909 rx_ring->netdev = NULL;
7910 }
7911 dev_info(&pf->pdev->dev,
7912 "Error adding mac filter on macvlan err %pe, aq_err %s\n",
7913 ERR_PTR(ret),
7914 i40e_aq_str(hw, aq_err));
7915 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7916 }
7917
7918 return ret;
7919 }
7920
7921 /**
7922 * i40e_setup_macvlans - create the channels which will be macvlans
7923 * @vsi: the VSI we want to access
7924 * @macvlan_cnt: no. of macvlans to be setup
7925 * @qcnt: no. of Qs per macvlan
7926 * @vdev: macvlan netdevice
7927 */
i40e_setup_macvlans(struct i40e_vsi * vsi,u16 macvlan_cnt,u16 qcnt,struct net_device * vdev)7928 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7929 struct net_device *vdev)
7930 {
7931 struct i40e_pf *pf = vsi->back;
7932 struct i40e_hw *hw = &pf->hw;
7933 struct i40e_vsi_context ctxt;
7934 u16 sections, qmap, num_qps;
7935 struct i40e_channel *ch;
7936 int i, pow, ret = 0;
7937 u8 offset = 0;
7938
7939 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7940 return -EINVAL;
7941
7942 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7943
7944 /* find the next higher power-of-2 of num queue pairs */
7945 pow = fls(roundup_pow_of_two(num_qps) - 1);
7946
7947 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7948 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7949
7950 /* Setup context bits for the main VSI */
7951 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7952 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7953 memset(&ctxt, 0, sizeof(ctxt));
7954 ctxt.seid = vsi->seid;
7955 ctxt.pf_num = vsi->back->hw.pf_id;
7956 ctxt.vf_num = 0;
7957 ctxt.uplink_seid = vsi->uplink_seid;
7958 ctxt.info = vsi->info;
7959 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7960 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7961 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7962 ctxt.info.valid_sections |= cpu_to_le16(sections);
7963
7964 /* Reconfigure RSS for main VSI with new max queue count */
7965 vsi->rss_size = max_t(u16, num_qps, qcnt);
7966 ret = i40e_vsi_config_rss(vsi);
7967 if (ret) {
7968 dev_info(&pf->pdev->dev,
7969 "Failed to reconfig RSS for num_queues (%u)\n",
7970 vsi->rss_size);
7971 return ret;
7972 }
7973 vsi->reconfig_rss = true;
7974 dev_dbg(&vsi->back->pdev->dev,
7975 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7976 vsi->next_base_queue = num_qps;
7977 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7978
7979 /* Update the VSI after updating the VSI queue-mapping
7980 * information
7981 */
7982 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7983 if (ret) {
7984 dev_info(&pf->pdev->dev,
7985 "Update vsi tc config failed, err %pe aq_err %s\n",
7986 ERR_PTR(ret),
7987 i40e_aq_str(hw, hw->aq.asq_last_status));
7988 return ret;
7989 }
7990 /* update the local VSI info with updated queue map */
7991 i40e_vsi_update_queue_map(vsi, &ctxt);
7992 vsi->info.valid_sections = 0;
7993
7994 /* Create channels for macvlans */
7995 INIT_LIST_HEAD(&vsi->macvlan_list);
7996 for (i = 0; i < macvlan_cnt; i++) {
7997 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7998 if (!ch) {
7999 ret = -ENOMEM;
8000 goto err_free;
8001 }
8002 INIT_LIST_HEAD(&ch->list);
8003 ch->num_queue_pairs = qcnt;
8004 if (!i40e_setup_channel(pf, vsi, ch)) {
8005 ret = -EINVAL;
8006 kfree(ch);
8007 goto err_free;
8008 }
8009 ch->parent_vsi = vsi;
8010 vsi->cnt_q_avail -= ch->num_queue_pairs;
8011 vsi->macvlan_cnt++;
8012 list_add_tail(&ch->list, &vsi->macvlan_list);
8013 }
8014
8015 return ret;
8016
8017 err_free:
8018 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
8019 i40e_free_macvlan_channels(vsi);
8020
8021 return ret;
8022 }
8023
8024 /**
8025 * i40e_fwd_add - configure macvlans
8026 * @netdev: net device to configure
8027 * @vdev: macvlan netdevice
8028 **/
i40e_fwd_add(struct net_device * netdev,struct net_device * vdev)8029 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
8030 {
8031 struct i40e_netdev_priv *np = netdev_priv(netdev);
8032 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
8033 struct i40e_vsi *vsi = np->vsi;
8034 struct i40e_pf *pf = vsi->back;
8035 struct i40e_fwd_adapter *fwd;
8036 int avail_macvlan, ret;
8037
8038 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8039 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
8040 return ERR_PTR(-EINVAL);
8041 }
8042 if (i40e_is_tc_mqprio_enabled(pf)) {
8043 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
8044 return ERR_PTR(-EINVAL);
8045 }
8046 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
8047 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
8048 return ERR_PTR(-EINVAL);
8049 }
8050
8051 /* The macvlan device has to be a single Q device so that the
8052 * tc_to_txq field can be reused to pick the tx queue.
8053 */
8054 if (netif_is_multiqueue(vdev))
8055 return ERR_PTR(-ERANGE);
8056
8057 if (!vsi->macvlan_cnt) {
8058 /* reserve bit 0 for the pf device */
8059 set_bit(0, vsi->fwd_bitmask);
8060
8061 /* Try to reserve as many queues as possible for macvlans. First
8062 * reserve 3/4th of max vectors, then half, then quarter and
8063 * calculate Qs per macvlan as you go
8064 */
8065 vectors = pf->num_lan_msix;
8066 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
8067 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
8068 q_per_macvlan = 4;
8069 macvlan_cnt = (vectors - 32) / 4;
8070 } else if (vectors <= 64 && vectors > 32) {
8071 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
8072 q_per_macvlan = 2;
8073 macvlan_cnt = (vectors - 16) / 2;
8074 } else if (vectors <= 32 && vectors > 16) {
8075 /* allocate 1 Q per macvlan and 16 Qs to the PF*/
8076 q_per_macvlan = 1;
8077 macvlan_cnt = vectors - 16;
8078 } else if (vectors <= 16 && vectors > 8) {
8079 /* allocate 1 Q per macvlan and 8 Qs to the PF */
8080 q_per_macvlan = 1;
8081 macvlan_cnt = vectors - 8;
8082 } else {
8083 /* allocate 1 Q per macvlan and 1 Q to the PF */
8084 q_per_macvlan = 1;
8085 macvlan_cnt = vectors - 1;
8086 }
8087
8088 if (macvlan_cnt == 0)
8089 return ERR_PTR(-EBUSY);
8090
8091 /* Quiesce VSI queues */
8092 i40e_quiesce_vsi(vsi);
8093
8094 /* sets up the macvlans but does not "enable" them */
8095 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
8096 vdev);
8097 if (ret)
8098 return ERR_PTR(ret);
8099
8100 /* Unquiesce VSI */
8101 i40e_unquiesce_vsi(vsi);
8102 }
8103 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
8104 vsi->macvlan_cnt);
8105 if (avail_macvlan >= I40E_MAX_MACVLANS)
8106 return ERR_PTR(-EBUSY);
8107
8108 /* create the fwd struct */
8109 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
8110 if (!fwd)
8111 return ERR_PTR(-ENOMEM);
8112
8113 set_bit(avail_macvlan, vsi->fwd_bitmask);
8114 fwd->bit_no = avail_macvlan;
8115 netdev_set_sb_channel(vdev, avail_macvlan);
8116 fwd->netdev = vdev;
8117
8118 if (!netif_running(netdev))
8119 return fwd;
8120
8121 /* Set fwd ring up */
8122 ret = i40e_fwd_ring_up(vsi, vdev, fwd);
8123 if (ret) {
8124 /* unbind the queues and drop the subordinate channel config */
8125 netdev_unbind_sb_channel(netdev, vdev);
8126 netdev_set_sb_channel(vdev, 0);
8127
8128 kfree(fwd);
8129 return ERR_PTR(-EINVAL);
8130 }
8131
8132 return fwd;
8133 }
8134
8135 /**
8136 * i40e_del_all_macvlans - Delete all the mac filters on the channels
8137 * @vsi: the VSI we want to access
8138 */
i40e_del_all_macvlans(struct i40e_vsi * vsi)8139 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
8140 {
8141 struct i40e_channel *ch, *ch_tmp;
8142 struct i40e_pf *pf = vsi->back;
8143 struct i40e_hw *hw = &pf->hw;
8144 int aq_err, ret = 0;
8145
8146 if (list_empty(&vsi->macvlan_list))
8147 return;
8148
8149 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8150 if (i40e_is_channel_macvlan(ch)) {
8151 ret = i40e_del_macvlan_filter(hw, ch->seid,
8152 i40e_channel_mac(ch),
8153 &aq_err);
8154 if (!ret) {
8155 /* Reset queue contexts */
8156 i40e_reset_ch_rings(vsi, ch);
8157 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8158 netdev_unbind_sb_channel(vsi->netdev,
8159 ch->fwd->netdev);
8160 netdev_set_sb_channel(ch->fwd->netdev, 0);
8161 kfree(ch->fwd);
8162 ch->fwd = NULL;
8163 }
8164 }
8165 }
8166 }
8167
8168 /**
8169 * i40e_fwd_del - delete macvlan interfaces
8170 * @netdev: net device to configure
8171 * @vdev: macvlan netdevice
8172 */
i40e_fwd_del(struct net_device * netdev,void * vdev)8173 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
8174 {
8175 struct i40e_netdev_priv *np = netdev_priv(netdev);
8176 struct i40e_fwd_adapter *fwd = vdev;
8177 struct i40e_channel *ch, *ch_tmp;
8178 struct i40e_vsi *vsi = np->vsi;
8179 struct i40e_pf *pf = vsi->back;
8180 struct i40e_hw *hw = &pf->hw;
8181 int aq_err, ret = 0;
8182
8183 /* Find the channel associated with the macvlan and del mac filter */
8184 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8185 if (i40e_is_channel_macvlan(ch) &&
8186 ether_addr_equal(i40e_channel_mac(ch),
8187 fwd->netdev->dev_addr)) {
8188 ret = i40e_del_macvlan_filter(hw, ch->seid,
8189 i40e_channel_mac(ch),
8190 &aq_err);
8191 if (!ret) {
8192 /* Reset queue contexts */
8193 i40e_reset_ch_rings(vsi, ch);
8194 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8195 netdev_unbind_sb_channel(netdev, fwd->netdev);
8196 netdev_set_sb_channel(fwd->netdev, 0);
8197 kfree(ch->fwd);
8198 ch->fwd = NULL;
8199 } else {
8200 dev_info(&pf->pdev->dev,
8201 "Error deleting mac filter on macvlan err %pe, aq_err %s\n",
8202 ERR_PTR(ret),
8203 i40e_aq_str(hw, aq_err));
8204 }
8205 break;
8206 }
8207 }
8208 }
8209
8210 /**
8211 * i40e_setup_tc - configure multiple traffic classes
8212 * @netdev: net device to configure
8213 * @type_data: tc offload data
8214 **/
i40e_setup_tc(struct net_device * netdev,void * type_data)8215 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
8216 {
8217 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8218 struct i40e_netdev_priv *np = netdev_priv(netdev);
8219 struct i40e_vsi *vsi = np->vsi;
8220 struct i40e_pf *pf = vsi->back;
8221 u8 enabled_tc = 0, num_tc, hw;
8222 bool need_reset = false;
8223 int old_queue_pairs;
8224 int ret = -EINVAL;
8225 u16 mode;
8226 int i;
8227
8228 old_queue_pairs = vsi->num_queue_pairs;
8229 num_tc = mqprio_qopt->qopt.num_tc;
8230 hw = mqprio_qopt->qopt.hw;
8231 mode = mqprio_qopt->mode;
8232 if (!hw) {
8233 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8234 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8235 goto config_tc;
8236 }
8237
8238 /* Check if MFP enabled */
8239 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags)) {
8240 netdev_info(netdev,
8241 "Configuring TC not supported in MFP mode\n");
8242 return ret;
8243 }
8244 switch (mode) {
8245 case TC_MQPRIO_MODE_DCB:
8246 clear_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8247
8248 /* Check if DCB enabled to continue */
8249 if (!test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8250 netdev_info(netdev,
8251 "DCB is not enabled for adapter\n");
8252 return ret;
8253 }
8254
8255 /* Check whether tc count is within enabled limit */
8256 if (num_tc > i40e_pf_get_num_tc(pf)) {
8257 netdev_info(netdev,
8258 "TC count greater than enabled on link for adapter\n");
8259 return ret;
8260 }
8261 break;
8262 case TC_MQPRIO_MODE_CHANNEL:
8263 if (test_bit(I40E_FLAG_DCB_ENA, pf->flags)) {
8264 netdev_info(netdev,
8265 "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
8266 return ret;
8267 }
8268 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
8269 return ret;
8270 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
8271 if (ret)
8272 return ret;
8273 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
8274 sizeof(*mqprio_qopt));
8275 set_bit(I40E_FLAG_TC_MQPRIO_ENA, pf->flags);
8276 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
8277 break;
8278 default:
8279 return -EINVAL;
8280 }
8281
8282 config_tc:
8283 /* Generate TC map for number of tc requested */
8284 for (i = 0; i < num_tc; i++)
8285 enabled_tc |= BIT(i);
8286
8287 /* Requesting same TC configuration as already enabled */
8288 if (enabled_tc == vsi->tc_config.enabled_tc &&
8289 mode != TC_MQPRIO_MODE_CHANNEL)
8290 return 0;
8291
8292 /* Quiesce VSI queues */
8293 i40e_quiesce_vsi(vsi);
8294
8295 if (!hw && !i40e_is_tc_mqprio_enabled(pf))
8296 i40e_remove_queue_channels(vsi);
8297
8298 /* Configure VSI for enabled TCs */
8299 ret = i40e_vsi_config_tc(vsi, enabled_tc);
8300 if (ret) {
8301 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
8302 vsi->seid);
8303 need_reset = true;
8304 goto exit;
8305 } else if (enabled_tc &&
8306 (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
8307 netdev_info(netdev,
8308 "Failed to create channel. Override queues (%u) not power of 2\n",
8309 vsi->tc_config.tc_info[0].qcount);
8310 ret = -EINVAL;
8311 need_reset = true;
8312 goto exit;
8313 }
8314
8315 dev_info(&vsi->back->pdev->dev,
8316 "Setup channel (id:%u) utilizing num_queues %d\n",
8317 vsi->seid, vsi->tc_config.tc_info[0].qcount);
8318
8319 if (i40e_is_tc_mqprio_enabled(pf)) {
8320 if (vsi->mqprio_qopt.max_rate[0]) {
8321 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
8322 vsi->mqprio_qopt.max_rate[0]);
8323
8324 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
8325 if (!ret) {
8326 u64 credits = max_tx_rate;
8327
8328 do_div(credits, I40E_BW_CREDIT_DIVISOR);
8329 dev_dbg(&vsi->back->pdev->dev,
8330 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
8331 max_tx_rate,
8332 credits,
8333 vsi->seid);
8334 } else {
8335 need_reset = true;
8336 goto exit;
8337 }
8338 }
8339 ret = i40e_configure_queue_channels(vsi);
8340 if (ret) {
8341 vsi->num_queue_pairs = old_queue_pairs;
8342 netdev_info(netdev,
8343 "Failed configuring queue channels\n");
8344 need_reset = true;
8345 goto exit;
8346 }
8347 }
8348
8349 exit:
8350 /* Reset the configuration data to defaults, only TC0 is enabled */
8351 if (need_reset) {
8352 i40e_vsi_set_default_tc_config(vsi);
8353 need_reset = false;
8354 }
8355
8356 /* Unquiesce VSI */
8357 i40e_unquiesce_vsi(vsi);
8358 return ret;
8359 }
8360
8361 /**
8362 * i40e_set_cld_element - sets cloud filter element data
8363 * @filter: cloud filter rule
8364 * @cld: ptr to cloud filter element data
8365 *
8366 * This is helper function to copy data into cloud filter element
8367 **/
8368 static inline void
i40e_set_cld_element(struct i40e_cloud_filter * filter,struct i40e_aqc_cloud_filters_element_data * cld)8369 i40e_set_cld_element(struct i40e_cloud_filter *filter,
8370 struct i40e_aqc_cloud_filters_element_data *cld)
8371 {
8372 u32 ipa;
8373 int i;
8374
8375 memset(cld, 0, sizeof(*cld));
8376 ether_addr_copy(cld->outer_mac, filter->dst_mac);
8377 ether_addr_copy(cld->inner_mac, filter->src_mac);
8378
8379 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
8380 return;
8381
8382 if (filter->n_proto == ETH_P_IPV6) {
8383 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1)
8384 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
8385 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
8386
8387 *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
8388 }
8389 } else {
8390 ipa = be32_to_cpu(filter->dst_ipv4);
8391
8392 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8393 }
8394
8395 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8396
8397 /* tenant_id is not supported by FW now, once the support is enabled
8398 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8399 */
8400 if (filter->tenant_id)
8401 return;
8402 }
8403
8404 /**
8405 * i40e_add_del_cloud_filter - Add/del cloud filter
8406 * @vsi: pointer to VSI
8407 * @filter: cloud filter rule
8408 * @add: if true, add, if false, delete
8409 *
8410 * Add or delete a cloud filter for a specific flow spec.
8411 * Returns 0 if the filter were successfully added.
8412 **/
i40e_add_del_cloud_filter(struct i40e_vsi * vsi,struct i40e_cloud_filter * filter,bool add)8413 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8414 struct i40e_cloud_filter *filter, bool add)
8415 {
8416 struct i40e_aqc_cloud_filters_element_data cld_filter;
8417 struct i40e_pf *pf = vsi->back;
8418 int ret;
8419 static const u16 flag_table[128] = {
8420 [I40E_CLOUD_FILTER_FLAGS_OMAC] =
8421 I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8422 [I40E_CLOUD_FILTER_FLAGS_IMAC] =
8423 I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8424 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] =
8425 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8426 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8427 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8428 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8429 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8430 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8431 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8432 [I40E_CLOUD_FILTER_FLAGS_IIP] =
8433 I40E_AQC_ADD_CLOUD_FILTER_IIP,
8434 };
8435
8436 if (filter->flags >= ARRAY_SIZE(flag_table))
8437 return -EIO;
8438
8439 memset(&cld_filter, 0, sizeof(cld_filter));
8440
8441 /* copy element needed to add cloud filter from filter */
8442 i40e_set_cld_element(filter, &cld_filter);
8443
8444 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8445 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8446 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8447
8448 if (filter->n_proto == ETH_P_IPV6)
8449 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8450 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8451 else
8452 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8453 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8454
8455 if (add)
8456 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
8457 &cld_filter, 1);
8458 else
8459 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
8460 &cld_filter, 1);
8461 if (ret)
8462 dev_dbg(&pf->pdev->dev,
8463 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8464 add ? "add" : "delete", filter->dst_port, ret,
8465 pf->hw.aq.asq_last_status);
8466 else
8467 dev_info(&pf->pdev->dev,
8468 "%s cloud filter for VSI: %d\n",
8469 add ? "Added" : "Deleted", filter->seid);
8470 return ret;
8471 }
8472
8473 /**
8474 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8475 * @vsi: pointer to VSI
8476 * @filter: cloud filter rule
8477 * @add: if true, add, if false, delete
8478 *
8479 * Add or delete a cloud filter for a specific flow spec using big buffer.
8480 * Returns 0 if the filter were successfully added.
8481 **/
i40e_add_del_cloud_filter_big_buf(struct i40e_vsi * vsi,struct i40e_cloud_filter * filter,bool add)8482 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8483 struct i40e_cloud_filter *filter,
8484 bool add)
8485 {
8486 struct i40e_aqc_cloud_filters_element_bb cld_filter;
8487 struct i40e_pf *pf = vsi->back;
8488 int ret;
8489
8490 /* Both (src/dst) valid mac_addr are not supported */
8491 if ((is_valid_ether_addr(filter->dst_mac) &&
8492 is_valid_ether_addr(filter->src_mac)) ||
8493 (is_multicast_ether_addr(filter->dst_mac) &&
8494 is_multicast_ether_addr(filter->src_mac)))
8495 return -EOPNOTSUPP;
8496
8497 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8498 * ports are not supported via big buffer now.
8499 */
8500 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8501 return -EOPNOTSUPP;
8502
8503 /* adding filter using src_port/src_ip is not supported at this stage */
8504 if (filter->src_port ||
8505 (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8506 !ipv6_addr_any(&filter->ip.v6.src_ip6))
8507 return -EOPNOTSUPP;
8508
8509 memset(&cld_filter, 0, sizeof(cld_filter));
8510
8511 /* copy element needed to add cloud filter from filter */
8512 i40e_set_cld_element(filter, &cld_filter.element);
8513
8514 if (is_valid_ether_addr(filter->dst_mac) ||
8515 is_valid_ether_addr(filter->src_mac) ||
8516 is_multicast_ether_addr(filter->dst_mac) ||
8517 is_multicast_ether_addr(filter->src_mac)) {
8518 /* MAC + IP : unsupported mode */
8519 if (filter->dst_ipv4)
8520 return -EOPNOTSUPP;
8521
8522 /* since we validated that L4 port must be valid before
8523 * we get here, start with respective "flags" value
8524 * and update if vlan is present or not
8525 */
8526 cld_filter.element.flags =
8527 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8528
8529 if (filter->vlan_id) {
8530 cld_filter.element.flags =
8531 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8532 }
8533
8534 } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8535 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
8536 cld_filter.element.flags =
8537 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8538 if (filter->n_proto == ETH_P_IPV6)
8539 cld_filter.element.flags |=
8540 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8541 else
8542 cld_filter.element.flags |=
8543 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8544 } else {
8545 dev_err(&pf->pdev->dev,
8546 "either mac or ip has to be valid for cloud filter\n");
8547 return -EINVAL;
8548 }
8549
8550 /* Now copy L4 port in Byte 6..7 in general fields */
8551 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8552 be16_to_cpu(filter->dst_port);
8553
8554 if (add) {
8555 /* Validate current device switch mode, change if necessary */
8556 ret = i40e_validate_and_set_switch_mode(vsi);
8557 if (ret) {
8558 dev_err(&pf->pdev->dev,
8559 "failed to set switch mode, ret %d\n",
8560 ret);
8561 return ret;
8562 }
8563
8564 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
8565 &cld_filter, 1);
8566 } else {
8567 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
8568 &cld_filter, 1);
8569 }
8570
8571 if (ret)
8572 dev_dbg(&pf->pdev->dev,
8573 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8574 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8575 else
8576 dev_info(&pf->pdev->dev,
8577 "%s cloud filter for VSI: %d, L4 port: %d\n",
8578 add ? "add" : "delete", filter->seid,
8579 ntohs(filter->dst_port));
8580 return ret;
8581 }
8582
8583 /**
8584 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8585 * @vsi: Pointer to VSI
8586 * @f: Pointer to struct flow_cls_offload
8587 * @filter: Pointer to cloud filter structure
8588 *
8589 **/
i40e_parse_cls_flower(struct i40e_vsi * vsi,struct flow_cls_offload * f,struct i40e_cloud_filter * filter)8590 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8591 struct flow_cls_offload *f,
8592 struct i40e_cloud_filter *filter)
8593 {
8594 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8595 struct flow_dissector *dissector = rule->match.dissector;
8596 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8597 struct i40e_pf *pf = vsi->back;
8598 u8 field_flags = 0;
8599
8600 if (dissector->used_keys &
8601 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
8602 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
8603 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8604 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
8605 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8606 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8607 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
8608 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8609 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%llx\n",
8610 dissector->used_keys);
8611 return -EOPNOTSUPP;
8612 }
8613
8614 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8615 struct flow_match_enc_keyid match;
8616
8617 flow_rule_match_enc_keyid(rule, &match);
8618 if (match.mask->keyid != 0)
8619 field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8620
8621 filter->tenant_id = be32_to_cpu(match.key->keyid);
8622 }
8623
8624 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8625 struct flow_match_basic match;
8626
8627 flow_rule_match_basic(rule, &match);
8628 n_proto_key = ntohs(match.key->n_proto);
8629 n_proto_mask = ntohs(match.mask->n_proto);
8630
8631 if (n_proto_key == ETH_P_ALL) {
8632 n_proto_key = 0;
8633 n_proto_mask = 0;
8634 }
8635 filter->n_proto = n_proto_key & n_proto_mask;
8636 filter->ip_proto = match.key->ip_proto;
8637 }
8638
8639 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8640 struct flow_match_eth_addrs match;
8641
8642 flow_rule_match_eth_addrs(rule, &match);
8643
8644 /* use is_broadcast and is_zero to check for all 0xf or 0 */
8645 if (!is_zero_ether_addr(match.mask->dst)) {
8646 if (is_broadcast_ether_addr(match.mask->dst)) {
8647 field_flags |= I40E_CLOUD_FIELD_OMAC;
8648 } else {
8649 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8650 match.mask->dst);
8651 return -EIO;
8652 }
8653 }
8654
8655 if (!is_zero_ether_addr(match.mask->src)) {
8656 if (is_broadcast_ether_addr(match.mask->src)) {
8657 field_flags |= I40E_CLOUD_FIELD_IMAC;
8658 } else {
8659 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8660 match.mask->src);
8661 return -EIO;
8662 }
8663 }
8664 ether_addr_copy(filter->dst_mac, match.key->dst);
8665 ether_addr_copy(filter->src_mac, match.key->src);
8666 }
8667
8668 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8669 struct flow_match_vlan match;
8670
8671 flow_rule_match_vlan(rule, &match);
8672 if (match.mask->vlan_id) {
8673 if (match.mask->vlan_id == VLAN_VID_MASK) {
8674 field_flags |= I40E_CLOUD_FIELD_IVLAN;
8675
8676 } else {
8677 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8678 match.mask->vlan_id);
8679 return -EIO;
8680 }
8681 }
8682
8683 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8684 }
8685
8686 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8687 struct flow_match_control match;
8688
8689 flow_rule_match_control(rule, &match);
8690 addr_type = match.key->addr_type;
8691
8692 if (flow_rule_has_control_flags(match.mask->flags,
8693 f->common.extack))
8694 return -EOPNOTSUPP;
8695 }
8696
8697 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8698 struct flow_match_ipv4_addrs match;
8699
8700 flow_rule_match_ipv4_addrs(rule, &match);
8701 if (match.mask->dst) {
8702 if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8703 field_flags |= I40E_CLOUD_FIELD_IIP;
8704 } else {
8705 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8706 &match.mask->dst);
8707 return -EIO;
8708 }
8709 }
8710
8711 if (match.mask->src) {
8712 if (match.mask->src == cpu_to_be32(0xffffffff)) {
8713 field_flags |= I40E_CLOUD_FIELD_IIP;
8714 } else {
8715 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8716 &match.mask->src);
8717 return -EIO;
8718 }
8719 }
8720
8721 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8722 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8723 return -EIO;
8724 }
8725 filter->dst_ipv4 = match.key->dst;
8726 filter->src_ipv4 = match.key->src;
8727 }
8728
8729 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8730 struct flow_match_ipv6_addrs match;
8731
8732 flow_rule_match_ipv6_addrs(rule, &match);
8733
8734 /* src and dest IPV6 address should not be LOOPBACK
8735 * (0:0:0:0:0:0:0:1), which can be represented as ::1
8736 */
8737 if (ipv6_addr_loopback(&match.key->dst) ||
8738 ipv6_addr_loopback(&match.key->src)) {
8739 dev_err(&pf->pdev->dev,
8740 "Bad ipv6, addr is LOOPBACK\n");
8741 return -EIO;
8742 }
8743 if (!ipv6_addr_any(&match.mask->dst) ||
8744 !ipv6_addr_any(&match.mask->src))
8745 field_flags |= I40E_CLOUD_FIELD_IIP;
8746
8747 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8748 sizeof(filter->src_ipv6));
8749 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8750 sizeof(filter->dst_ipv6));
8751 }
8752
8753 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8754 struct flow_match_ports match;
8755
8756 flow_rule_match_ports(rule, &match);
8757 if (match.mask->src) {
8758 if (match.mask->src == cpu_to_be16(0xffff)) {
8759 field_flags |= I40E_CLOUD_FIELD_IIP;
8760 } else {
8761 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8762 be16_to_cpu(match.mask->src));
8763 return -EIO;
8764 }
8765 }
8766
8767 if (match.mask->dst) {
8768 if (match.mask->dst == cpu_to_be16(0xffff)) {
8769 field_flags |= I40E_CLOUD_FIELD_IIP;
8770 } else {
8771 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8772 be16_to_cpu(match.mask->dst));
8773 return -EIO;
8774 }
8775 }
8776
8777 filter->dst_port = match.key->dst;
8778 filter->src_port = match.key->src;
8779
8780 switch (filter->ip_proto) {
8781 case IPPROTO_TCP:
8782 case IPPROTO_UDP:
8783 break;
8784 default:
8785 dev_err(&pf->pdev->dev,
8786 "Only UDP and TCP transport are supported\n");
8787 return -EINVAL;
8788 }
8789 }
8790 filter->flags = field_flags;
8791 return 0;
8792 }
8793
8794 /**
8795 * i40e_handle_tclass: Forward to a traffic class on the device
8796 * @vsi: Pointer to VSI
8797 * @tc: traffic class index on the device
8798 * @filter: Pointer to cloud filter structure
8799 *
8800 **/
i40e_handle_tclass(struct i40e_vsi * vsi,u32 tc,struct i40e_cloud_filter * filter)8801 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8802 struct i40e_cloud_filter *filter)
8803 {
8804 struct i40e_channel *ch, *ch_tmp;
8805
8806 /* direct to a traffic class on the same device */
8807 if (tc == 0) {
8808 filter->seid = vsi->seid;
8809 return 0;
8810 } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8811 if (!filter->dst_port) {
8812 dev_err(&vsi->back->pdev->dev,
8813 "Specify destination port to direct to traffic class that is not default\n");
8814 return -EINVAL;
8815 }
8816 if (list_empty(&vsi->ch_list))
8817 return -EINVAL;
8818 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8819 list) {
8820 if (ch->seid == vsi->tc_seid_map[tc])
8821 filter->seid = ch->seid;
8822 }
8823 return 0;
8824 }
8825 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8826 return -EINVAL;
8827 }
8828
8829 /**
8830 * i40e_configure_clsflower - Configure tc flower filters
8831 * @vsi: Pointer to VSI
8832 * @cls_flower: Pointer to struct flow_cls_offload
8833 *
8834 **/
i40e_configure_clsflower(struct i40e_vsi * vsi,struct flow_cls_offload * cls_flower)8835 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8836 struct flow_cls_offload *cls_flower)
8837 {
8838 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8839 struct i40e_cloud_filter *filter = NULL;
8840 struct i40e_pf *pf = vsi->back;
8841 int err = 0;
8842
8843 if (tc < 0) {
8844 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8845 return -EOPNOTSUPP;
8846 }
8847
8848 if (!tc) {
8849 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");
8850 return -EINVAL;
8851 }
8852
8853 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8854 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8855 return -EBUSY;
8856
8857 if (pf->fdir_pf_active_filters ||
8858 (!hlist_empty(&pf->fdir_filter_list))) {
8859 dev_err(&vsi->back->pdev->dev,
8860 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8861 return -EINVAL;
8862 }
8863
8864 if (test_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags)) {
8865 dev_err(&vsi->back->pdev->dev,
8866 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8867 clear_bit(I40E_FLAG_FD_SB_ENA, vsi->back->flags);
8868 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, vsi->back->flags);
8869 }
8870
8871 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8872 if (!filter)
8873 return -ENOMEM;
8874
8875 filter->cookie = cls_flower->cookie;
8876
8877 err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8878 if (err < 0)
8879 goto err;
8880
8881 err = i40e_handle_tclass(vsi, tc, filter);
8882 if (err < 0)
8883 goto err;
8884
8885 /* Add cloud filter */
8886 if (filter->dst_port)
8887 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8888 else
8889 err = i40e_add_del_cloud_filter(vsi, filter, true);
8890
8891 if (err) {
8892 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8893 err);
8894 goto err;
8895 }
8896
8897 /* add filter to the ordered list */
8898 INIT_HLIST_NODE(&filter->cloud_node);
8899
8900 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8901
8902 pf->num_cloud_filters++;
8903
8904 return err;
8905 err:
8906 kfree(filter);
8907 return err;
8908 }
8909
8910 /**
8911 * i40e_find_cloud_filter - Find the could filter in the list
8912 * @vsi: Pointer to VSI
8913 * @cookie: filter specific cookie
8914 *
8915 **/
i40e_find_cloud_filter(struct i40e_vsi * vsi,unsigned long * cookie)8916 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8917 unsigned long *cookie)
8918 {
8919 struct i40e_cloud_filter *filter = NULL;
8920 struct hlist_node *node2;
8921
8922 hlist_for_each_entry_safe(filter, node2,
8923 &vsi->back->cloud_filter_list, cloud_node)
8924 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8925 return filter;
8926 return NULL;
8927 }
8928
8929 /**
8930 * i40e_delete_clsflower - Remove tc flower filters
8931 * @vsi: Pointer to VSI
8932 * @cls_flower: Pointer to struct flow_cls_offload
8933 *
8934 **/
i40e_delete_clsflower(struct i40e_vsi * vsi,struct flow_cls_offload * cls_flower)8935 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8936 struct flow_cls_offload *cls_flower)
8937 {
8938 struct i40e_cloud_filter *filter = NULL;
8939 struct i40e_pf *pf = vsi->back;
8940 int err = 0;
8941
8942 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8943
8944 if (!filter)
8945 return -EINVAL;
8946
8947 hash_del(&filter->cloud_node);
8948
8949 if (filter->dst_port)
8950 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8951 else
8952 err = i40e_add_del_cloud_filter(vsi, filter, false);
8953
8954 kfree(filter);
8955 if (err) {
8956 dev_err(&pf->pdev->dev,
8957 "Failed to delete cloud filter, err %pe\n",
8958 ERR_PTR(err));
8959 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8960 }
8961
8962 pf->num_cloud_filters--;
8963 if (!pf->num_cloud_filters)
8964 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
8965 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
8966 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
8967 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
8968 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
8969 }
8970 return 0;
8971 }
8972
8973 /**
8974 * i40e_setup_tc_cls_flower - flower classifier offloads
8975 * @np: net device to configure
8976 * @cls_flower: offload data
8977 **/
i40e_setup_tc_cls_flower(struct i40e_netdev_priv * np,struct flow_cls_offload * cls_flower)8978 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8979 struct flow_cls_offload *cls_flower)
8980 {
8981 struct i40e_vsi *vsi = np->vsi;
8982
8983 switch (cls_flower->command) {
8984 case FLOW_CLS_REPLACE:
8985 return i40e_configure_clsflower(vsi, cls_flower);
8986 case FLOW_CLS_DESTROY:
8987 return i40e_delete_clsflower(vsi, cls_flower);
8988 case FLOW_CLS_STATS:
8989 return -EOPNOTSUPP;
8990 default:
8991 return -EOPNOTSUPP;
8992 }
8993 }
8994
i40e_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)8995 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8996 void *cb_priv)
8997 {
8998 struct i40e_netdev_priv *np = cb_priv;
8999
9000 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
9001 return -EOPNOTSUPP;
9002
9003 switch (type) {
9004 case TC_SETUP_CLSFLOWER:
9005 return i40e_setup_tc_cls_flower(np, type_data);
9006
9007 default:
9008 return -EOPNOTSUPP;
9009 }
9010 }
9011
9012 static LIST_HEAD(i40e_block_cb_list);
9013
__i40e_setup_tc(struct net_device * netdev,enum tc_setup_type type,void * type_data)9014 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
9015 void *type_data)
9016 {
9017 struct i40e_netdev_priv *np = netdev_priv(netdev);
9018
9019 switch (type) {
9020 case TC_SETUP_QDISC_MQPRIO:
9021 return i40e_setup_tc(netdev, type_data);
9022 case TC_SETUP_BLOCK:
9023 return flow_block_cb_setup_simple(type_data,
9024 &i40e_block_cb_list,
9025 i40e_setup_tc_block_cb,
9026 np, np, true);
9027 default:
9028 return -EOPNOTSUPP;
9029 }
9030 }
9031
9032 /**
9033 * i40e_open - Called when a network interface is made active
9034 * @netdev: network interface device structure
9035 *
9036 * The open entry point is called when a network interface is made
9037 * active by the system (IFF_UP). At this point all resources needed
9038 * for transmit and receive operations are allocated, the interrupt
9039 * handler is registered with the OS, the netdev watchdog subtask is
9040 * enabled, and the stack is notified that the interface is ready.
9041 *
9042 * Returns 0 on success, negative value on failure
9043 **/
i40e_open(struct net_device * netdev)9044 int i40e_open(struct net_device *netdev)
9045 {
9046 struct i40e_netdev_priv *np = netdev_priv(netdev);
9047 struct i40e_vsi *vsi = np->vsi;
9048 struct i40e_pf *pf = vsi->back;
9049 int err;
9050
9051 /* disallow open during test or if eeprom is broken */
9052 if (test_bit(__I40E_TESTING, pf->state) ||
9053 test_bit(__I40E_BAD_EEPROM, pf->state))
9054 return -EBUSY;
9055
9056 netif_carrier_off(netdev);
9057
9058 if (i40e_force_link_state(pf, true))
9059 return -EAGAIN;
9060
9061 err = i40e_vsi_open(vsi);
9062 if (err)
9063 return err;
9064
9065 /* configure global TSO hardware offload settings */
9066 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
9067 TCP_FLAG_FIN) >> 16);
9068 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
9069 TCP_FLAG_FIN |
9070 TCP_FLAG_CWR) >> 16);
9071 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
9072 udp_tunnel_get_rx_info(netdev);
9073
9074 return 0;
9075 }
9076
9077 /**
9078 * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
9079 * @vsi: vsi structure
9080 *
9081 * This updates netdev's number of tx/rx queues
9082 *
9083 * Returns status of setting tx/rx queues
9084 **/
i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi * vsi)9085 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
9086 {
9087 int ret;
9088
9089 ret = netif_set_real_num_rx_queues(vsi->netdev,
9090 vsi->num_queue_pairs);
9091 if (ret)
9092 return ret;
9093
9094 return netif_set_real_num_tx_queues(vsi->netdev,
9095 vsi->num_queue_pairs);
9096 }
9097
9098 /**
9099 * i40e_vsi_open -
9100 * @vsi: the VSI to open
9101 *
9102 * Finish initialization of the VSI.
9103 *
9104 * Returns 0 on success, negative value on failure
9105 *
9106 * Note: expects to be called while under rtnl_lock()
9107 **/
i40e_vsi_open(struct i40e_vsi * vsi)9108 int i40e_vsi_open(struct i40e_vsi *vsi)
9109 {
9110 struct i40e_pf *pf = vsi->back;
9111 char int_name[I40E_INT_NAME_STR_LEN];
9112 int err;
9113
9114 /* allocate descriptors */
9115 err = i40e_vsi_setup_tx_resources(vsi);
9116 if (err)
9117 goto err_setup_tx;
9118 err = i40e_vsi_setup_rx_resources(vsi);
9119 if (err)
9120 goto err_setup_rx;
9121
9122 err = i40e_vsi_configure(vsi);
9123 if (err)
9124 goto err_setup_rx;
9125
9126 if (vsi->netdev) {
9127 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
9128 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
9129 err = i40e_vsi_request_irq(vsi, int_name);
9130 if (err)
9131 goto err_setup_rx;
9132
9133 /* Notify the stack of the actual queue counts. */
9134 err = i40e_netif_set_realnum_tx_rx_queues(vsi);
9135 if (err)
9136 goto err_set_queues;
9137
9138 } else if (vsi->type == I40E_VSI_FDIR) {
9139 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
9140 dev_driver_string(&pf->pdev->dev),
9141 dev_name(&pf->pdev->dev));
9142 err = i40e_vsi_request_irq(vsi, int_name);
9143 if (err)
9144 goto err_setup_rx;
9145
9146 } else {
9147 err = -EINVAL;
9148 goto err_setup_rx;
9149 }
9150
9151 err = i40e_up_complete(vsi);
9152 if (err)
9153 goto err_up_complete;
9154
9155 return 0;
9156
9157 err_up_complete:
9158 i40e_down(vsi);
9159 err_set_queues:
9160 i40e_vsi_free_irq(vsi);
9161 err_setup_rx:
9162 i40e_vsi_free_rx_resources(vsi);
9163 err_setup_tx:
9164 i40e_vsi_free_tx_resources(vsi);
9165 if (vsi->type == I40E_VSI_MAIN)
9166 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
9167
9168 return err;
9169 }
9170
9171 /**
9172 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
9173 * @pf: Pointer to PF
9174 *
9175 * This function destroys the hlist where all the Flow Director
9176 * filters were saved.
9177 **/
i40e_fdir_filter_exit(struct i40e_pf * pf)9178 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
9179 {
9180 struct i40e_fdir_filter *filter;
9181 struct i40e_flex_pit *pit_entry, *tmp;
9182 struct hlist_node *node2;
9183
9184 hlist_for_each_entry_safe(filter, node2,
9185 &pf->fdir_filter_list, fdir_node) {
9186 hlist_del(&filter->fdir_node);
9187 kfree(filter);
9188 }
9189
9190 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
9191 list_del(&pit_entry->list);
9192 kfree(pit_entry);
9193 }
9194 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
9195
9196 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
9197 list_del(&pit_entry->list);
9198 kfree(pit_entry);
9199 }
9200 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
9201
9202 pf->fdir_pf_active_filters = 0;
9203 i40e_reset_fdir_filter_cnt(pf);
9204
9205 /* Reprogram the default input set for TCP/IPv4 */
9206 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9207 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9208 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9209
9210 /* Reprogram the default input set for TCP/IPv6 */
9211 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
9212 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9213 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9214
9215 /* Reprogram the default input set for UDP/IPv4 */
9216 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
9217 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9218 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9219
9220 /* Reprogram the default input set for UDP/IPv6 */
9221 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
9222 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9223 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9224
9225 /* Reprogram the default input set for SCTP/IPv4 */
9226 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
9227 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9228 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9229
9230 /* Reprogram the default input set for SCTP/IPv6 */
9231 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
9232 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9233 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9234
9235 /* Reprogram the default input set for Other/IPv4 */
9236 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
9237 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9238
9239 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
9240 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9241
9242 /* Reprogram the default input set for Other/IPv6 */
9243 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
9244 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9245
9246 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,
9247 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9248 }
9249
9250 /**
9251 * i40e_cloud_filter_exit - Cleans up the cloud filters
9252 * @pf: Pointer to PF
9253 *
9254 * This function destroys the hlist where all the cloud filters
9255 * were saved.
9256 **/
i40e_cloud_filter_exit(struct i40e_pf * pf)9257 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
9258 {
9259 struct i40e_cloud_filter *cfilter;
9260 struct hlist_node *node;
9261
9262 hlist_for_each_entry_safe(cfilter, node,
9263 &pf->cloud_filter_list, cloud_node) {
9264 hlist_del(&cfilter->cloud_node);
9265 kfree(cfilter);
9266 }
9267 pf->num_cloud_filters = 0;
9268
9269 if (test_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags) &&
9270 !test_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags)) {
9271 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
9272 clear_bit(I40E_FLAG_FD_SB_TO_CLOUD_FILTER, pf->flags);
9273 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
9274 }
9275 }
9276
9277 /**
9278 * i40e_close - Disables a network interface
9279 * @netdev: network interface device structure
9280 *
9281 * The close entry point is called when an interface is de-activated
9282 * by the OS. The hardware is still under the driver's control, but
9283 * this netdev interface is disabled.
9284 *
9285 * Returns 0, this is not allowed to fail
9286 **/
i40e_close(struct net_device * netdev)9287 int i40e_close(struct net_device *netdev)
9288 {
9289 struct i40e_netdev_priv *np = netdev_priv(netdev);
9290 struct i40e_vsi *vsi = np->vsi;
9291
9292 i40e_vsi_close(vsi);
9293
9294 return 0;
9295 }
9296
9297 /**
9298 * i40e_do_reset - Start a PF or Core Reset sequence
9299 * @pf: board private structure
9300 * @reset_flags: which reset is requested
9301 * @lock_acquired: indicates whether or not the lock has been acquired
9302 * before this function was called.
9303 *
9304 * The essential difference in resets is that the PF Reset
9305 * doesn't clear the packet buffers, doesn't reset the PE
9306 * firmware, and doesn't bother the other PFs on the chip.
9307 **/
i40e_do_reset(struct i40e_pf * pf,u32 reset_flags,bool lock_acquired)9308 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
9309 {
9310 struct i40e_vsi *vsi;
9311 u32 val;
9312 int i;
9313
9314 /* do the biggest reset indicated */
9315 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
9316
9317 /* Request a Global Reset
9318 *
9319 * This will start the chip's countdown to the actual full
9320 * chip reset event, and a warning interrupt to be sent
9321 * to all PFs, including the requestor. Our handler
9322 * for the warning interrupt will deal with the shutdown
9323 * and recovery of the switch setup.
9324 */
9325 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
9326 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9327 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
9328 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9329
9330 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
9331
9332 /* Request a Core Reset
9333 *
9334 * Same as Global Reset, except does *not* include the MAC/PHY
9335 */
9336 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
9337 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9338 val |= I40E_GLGEN_RTRIG_CORER_MASK;
9339 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9340 i40e_flush(&pf->hw);
9341
9342 } else if (reset_flags & I40E_PF_RESET_FLAG) {
9343
9344 /* Request a PF Reset
9345 *
9346 * Resets only the PF-specific registers
9347 *
9348 * This goes directly to the tear-down and rebuild of
9349 * the switch, since we need to do all the recovery as
9350 * for the Core Reset.
9351 */
9352 dev_dbg(&pf->pdev->dev, "PFR requested\n");
9353 i40e_handle_reset_warning(pf, lock_acquired);
9354
9355 } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
9356 /* Request a PF Reset
9357 *
9358 * Resets PF and reinitializes PFs VSI.
9359 */
9360 i40e_prep_for_reset(pf);
9361 i40e_reset_and_rebuild(pf, true, lock_acquired);
9362 dev_info(&pf->pdev->dev,
9363 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
9364 "FW LLDP is disabled\n" :
9365 "FW LLDP is enabled\n");
9366
9367 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
9368 /* Find the VSI(s) that requested a re-init */
9369 dev_info(&pf->pdev->dev, "VSI reinit requested\n");
9370
9371 i40e_pf_for_each_vsi(pf, i, vsi) {
9372 if (test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
9373 vsi->state))
9374 i40e_vsi_reinit_locked(vsi);
9375 }
9376 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
9377 /* Find the VSI(s) that needs to be brought down */
9378 dev_info(&pf->pdev->dev, "VSI down requested\n");
9379
9380 i40e_pf_for_each_vsi(pf, i, vsi) {
9381 if (test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
9382 vsi->state)) {
9383 set_bit(__I40E_VSI_DOWN, vsi->state);
9384 i40e_down(vsi);
9385 }
9386 }
9387 } else {
9388 dev_info(&pf->pdev->dev,
9389 "bad reset request 0x%08x\n", reset_flags);
9390 }
9391 }
9392
9393 #ifdef CONFIG_I40E_DCB
9394 /**
9395 * i40e_dcb_need_reconfig - Check if DCB needs reconfig
9396 * @pf: board private structure
9397 * @old_cfg: current DCB config
9398 * @new_cfg: new DCB config
9399 **/
i40e_dcb_need_reconfig(struct i40e_pf * pf,struct i40e_dcbx_config * old_cfg,struct i40e_dcbx_config * new_cfg)9400 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
9401 struct i40e_dcbx_config *old_cfg,
9402 struct i40e_dcbx_config *new_cfg)
9403 {
9404 bool need_reconfig = false;
9405
9406 /* Check if ETS configuration has changed */
9407 if (memcmp(&new_cfg->etscfg,
9408 &old_cfg->etscfg,
9409 sizeof(new_cfg->etscfg))) {
9410 /* If Priority Table has changed reconfig is needed */
9411 if (memcmp(&new_cfg->etscfg.prioritytable,
9412 &old_cfg->etscfg.prioritytable,
9413 sizeof(new_cfg->etscfg.prioritytable))) {
9414 need_reconfig = true;
9415 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9416 }
9417
9418 if (memcmp(&new_cfg->etscfg.tcbwtable,
9419 &old_cfg->etscfg.tcbwtable,
9420 sizeof(new_cfg->etscfg.tcbwtable)))
9421 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9422
9423 if (memcmp(&new_cfg->etscfg.tsatable,
9424 &old_cfg->etscfg.tsatable,
9425 sizeof(new_cfg->etscfg.tsatable)))
9426 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9427 }
9428
9429 /* Check if PFC configuration has changed */
9430 if (memcmp(&new_cfg->pfc,
9431 &old_cfg->pfc,
9432 sizeof(new_cfg->pfc))) {
9433 need_reconfig = true;
9434 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9435 }
9436
9437 /* Check if APP Table has changed */
9438 if (memcmp(&new_cfg->app,
9439 &old_cfg->app,
9440 sizeof(new_cfg->app))) {
9441 need_reconfig = true;
9442 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9443 }
9444
9445 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9446 return need_reconfig;
9447 }
9448
9449 /**
9450 * i40e_handle_lldp_event - Handle LLDP Change MIB event
9451 * @pf: board private structure
9452 * @e: event info posted on ARQ
9453 **/
i40e_handle_lldp_event(struct i40e_pf * pf,struct i40e_arq_event_info * e)9454 static int i40e_handle_lldp_event(struct i40e_pf *pf,
9455 struct i40e_arq_event_info *e)
9456 {
9457 struct i40e_aqc_lldp_get_mib *mib =
9458 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9459 struct i40e_hw *hw = &pf->hw;
9460 struct i40e_dcbx_config tmp_dcbx_cfg;
9461 bool need_reconfig = false;
9462 int ret = 0;
9463 u8 type;
9464
9465 /* X710-T*L 2.5G and 5G speeds don't support DCB */
9466 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9467 (hw->phy.link_info.link_speed &
9468 ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9469 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9470 /* let firmware decide if the DCB should be disabled */
9471 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9472
9473 /* Not DCB capable or capability disabled */
9474 if (!test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
9475 return ret;
9476
9477 /* Ignore if event is not for Nearest Bridge */
9478 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9479 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9480 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9481 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9482 return ret;
9483
9484 /* Check MIB Type and return if event for Remote MIB update */
9485 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9486 dev_dbg(&pf->pdev->dev,
9487 "LLDP event mib type %s\n", type ? "remote" : "local");
9488 if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9489 /* Update the remote cached instance and return */
9490 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9491 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9492 &hw->remote_dcbx_config);
9493 goto exit;
9494 }
9495
9496 /* Store the old configuration */
9497 tmp_dcbx_cfg = hw->local_dcbx_config;
9498
9499 /* Reset the old DCBx configuration data */
9500 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9501 /* Get updated DCBX data from firmware */
9502 ret = i40e_get_dcb_config(&pf->hw);
9503 if (ret) {
9504 /* X710-T*L 2.5G and 5G speeds don't support DCB */
9505 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9506 (hw->phy.link_info.link_speed &
9507 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9508 dev_warn(&pf->pdev->dev,
9509 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
9510 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
9511 } else {
9512 dev_info(&pf->pdev->dev,
9513 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n",
9514 ERR_PTR(ret),
9515 i40e_aq_str(&pf->hw,
9516 pf->hw.aq.asq_last_status));
9517 }
9518 goto exit;
9519 }
9520
9521 /* No change detected in DCBX configs */
9522 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
9523 sizeof(tmp_dcbx_cfg))) {
9524 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9525 goto exit;
9526 }
9527
9528 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
9529 &hw->local_dcbx_config);
9530
9531 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
9532
9533 if (!need_reconfig)
9534 goto exit;
9535
9536 /* Enable DCB tagging only when more than one TC */
9537 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
9538 set_bit(I40E_FLAG_DCB_ENA, pf->flags);
9539 else
9540 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
9541
9542 set_bit(__I40E_PORT_SUSPENDED, pf->state);
9543 /* Reconfiguration needed quiesce all VSIs */
9544 i40e_pf_quiesce_all_vsi(pf);
9545
9546 /* Changes in configuration update VEB/VSI */
9547 i40e_dcb_reconfigure(pf);
9548
9549 ret = i40e_resume_port_tx(pf);
9550
9551 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
9552 /* In case of error no point in resuming VSIs */
9553 if (ret)
9554 goto exit;
9555
9556 /* Wait for the PF's queues to be disabled */
9557 ret = i40e_pf_wait_queues_disabled(pf);
9558 if (ret) {
9559 /* Schedule PF reset to recover */
9560 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9561 i40e_service_event_schedule(pf);
9562 } else {
9563 i40e_pf_unquiesce_all_vsi(pf);
9564 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
9565 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
9566 }
9567
9568 exit:
9569 return ret;
9570 }
9571 #endif /* CONFIG_I40E_DCB */
9572
9573 /**
9574 * i40e_do_reset_safe - Protected reset path for userland calls.
9575 * @pf: board private structure
9576 * @reset_flags: which reset is requested
9577 *
9578 **/
i40e_do_reset_safe(struct i40e_pf * pf,u32 reset_flags)9579 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9580 {
9581 rtnl_lock();
9582 i40e_do_reset(pf, reset_flags, true);
9583 rtnl_unlock();
9584 }
9585
9586 /**
9587 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9588 * @pf: board private structure
9589 * @e: event info posted on ARQ
9590 *
9591 * Handler for LAN Queue Overflow Event generated by the firmware for PF
9592 * and VF queues
9593 **/
i40e_handle_lan_overflow_event(struct i40e_pf * pf,struct i40e_arq_event_info * e)9594 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9595 struct i40e_arq_event_info *e)
9596 {
9597 struct i40e_aqc_lan_overflow *data =
9598 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9599 u32 queue = le32_to_cpu(data->prtdcb_rupto);
9600 u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9601 struct i40e_hw *hw = &pf->hw;
9602 struct i40e_vf *vf;
9603 u16 vf_id;
9604
9605 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9606 queue, qtx_ctl);
9607
9608 if (FIELD_GET(I40E_QTX_CTL_PFVF_Q_MASK, qtx_ctl) !=
9609 I40E_QTX_CTL_VF_QUEUE)
9610 return;
9611
9612 /* Queue belongs to VF, find the VF and issue VF reset */
9613 vf_id = FIELD_GET(I40E_QTX_CTL_VFVM_INDX_MASK, qtx_ctl);
9614 vf_id -= hw->func_caps.vf_base_id;
9615 vf = &pf->vf[vf_id];
9616 i40e_vc_notify_vf_reset(vf);
9617 /* Allow VF to process pending reset notification */
9618 msleep(20);
9619 i40e_reset_vf(vf, false);
9620 }
9621
9622 /**
9623 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9624 * @pf: board private structure
9625 **/
i40e_get_cur_guaranteed_fd_count(struct i40e_pf * pf)9626 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9627 {
9628 u32 val, fcnt_prog;
9629
9630 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9631 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9632 return fcnt_prog;
9633 }
9634
9635 /**
9636 * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9637 * @pf: board private structure
9638 **/
i40e_get_current_fd_count(struct i40e_pf * pf)9639 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9640 {
9641 u32 val, fcnt_prog;
9642
9643 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9644 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9645 FIELD_GET(I40E_PFQF_FDSTAT_BEST_CNT_MASK, val);
9646 return fcnt_prog;
9647 }
9648
9649 /**
9650 * i40e_get_global_fd_count - Get total FD filters programmed on device
9651 * @pf: board private structure
9652 **/
i40e_get_global_fd_count(struct i40e_pf * pf)9653 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9654 {
9655 u32 val, fcnt_prog;
9656
9657 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9658 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9659 FIELD_GET(I40E_GLQF_FDCNT_0_BESTCNT_MASK, val);
9660 return fcnt_prog;
9661 }
9662
9663 /**
9664 * i40e_reenable_fdir_sb - Restore FDir SB capability
9665 * @pf: board private structure
9666 **/
i40e_reenable_fdir_sb(struct i40e_pf * pf)9667 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9668 {
9669 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9670 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
9671 (I40E_DEBUG_FD & pf->hw.debug_mask))
9672 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9673 }
9674
9675 /**
9676 * i40e_reenable_fdir_atr - Restore FDir ATR capability
9677 * @pf: board private structure
9678 **/
i40e_reenable_fdir_atr(struct i40e_pf * pf)9679 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9680 {
9681 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9682 /* ATR uses the same filtering logic as SB rules. It only
9683 * functions properly if the input set mask is at the default
9684 * settings. It is safe to restore the default input set
9685 * because there are no active TCPv4 filter rules.
9686 */
9687 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9688 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9689 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9690
9691 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
9692 (I40E_DEBUG_FD & pf->hw.debug_mask))
9693 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9694 }
9695 }
9696
9697 /**
9698 * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9699 * @pf: board private structure
9700 * @filter: FDir filter to remove
9701 */
i40e_delete_invalid_filter(struct i40e_pf * pf,struct i40e_fdir_filter * filter)9702 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9703 struct i40e_fdir_filter *filter)
9704 {
9705 /* Update counters */
9706 pf->fdir_pf_active_filters--;
9707 pf->fd_inv = 0;
9708
9709 switch (filter->flow_type) {
9710 case TCP_V4_FLOW:
9711 pf->fd_tcp4_filter_cnt--;
9712 break;
9713 case UDP_V4_FLOW:
9714 pf->fd_udp4_filter_cnt--;
9715 break;
9716 case SCTP_V4_FLOW:
9717 pf->fd_sctp4_filter_cnt--;
9718 break;
9719 case TCP_V6_FLOW:
9720 pf->fd_tcp6_filter_cnt--;
9721 break;
9722 case UDP_V6_FLOW:
9723 pf->fd_udp6_filter_cnt--;
9724 break;
9725 case SCTP_V6_FLOW:
9726 pf->fd_udp6_filter_cnt--;
9727 break;
9728 case IP_USER_FLOW:
9729 switch (filter->ipl4_proto) {
9730 case IPPROTO_TCP:
9731 pf->fd_tcp4_filter_cnt--;
9732 break;
9733 case IPPROTO_UDP:
9734 pf->fd_udp4_filter_cnt--;
9735 break;
9736 case IPPROTO_SCTP:
9737 pf->fd_sctp4_filter_cnt--;
9738 break;
9739 case IPPROTO_IP:
9740 pf->fd_ip4_filter_cnt--;
9741 break;
9742 }
9743 break;
9744 case IPV6_USER_FLOW:
9745 switch (filter->ipl4_proto) {
9746 case IPPROTO_TCP:
9747 pf->fd_tcp6_filter_cnt--;
9748 break;
9749 case IPPROTO_UDP:
9750 pf->fd_udp6_filter_cnt--;
9751 break;
9752 case IPPROTO_SCTP:
9753 pf->fd_sctp6_filter_cnt--;
9754 break;
9755 case IPPROTO_IP:
9756 pf->fd_ip6_filter_cnt--;
9757 break;
9758 }
9759 break;
9760 }
9761
9762 /* Remove the filter from the list and free memory */
9763 hlist_del(&filter->fdir_node);
9764 kfree(filter);
9765 }
9766
9767 /**
9768 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9769 * @pf: board private structure
9770 **/
i40e_fdir_check_and_reenable(struct i40e_pf * pf)9771 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9772 {
9773 struct i40e_fdir_filter *filter;
9774 u32 fcnt_prog, fcnt_avail;
9775 struct hlist_node *node;
9776
9777 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9778 return;
9779
9780 /* Check if we have enough room to re-enable FDir SB capability. */
9781 fcnt_prog = i40e_get_global_fd_count(pf);
9782 fcnt_avail = pf->fdir_pf_filter_count;
9783 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9784 (pf->fd_add_err == 0) ||
9785 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9786 i40e_reenable_fdir_sb(pf);
9787
9788 /* We should wait for even more space before re-enabling ATR.
9789 * Additionally, we cannot enable ATR as long as we still have TCP SB
9790 * rules active.
9791 */
9792 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9793 pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9794 i40e_reenable_fdir_atr(pf);
9795
9796 /* if hw had a problem adding a filter, delete it */
9797 if (pf->fd_inv > 0) {
9798 hlist_for_each_entry_safe(filter, node,
9799 &pf->fdir_filter_list, fdir_node)
9800 if (filter->fd_id == pf->fd_inv)
9801 i40e_delete_invalid_filter(pf, filter);
9802 }
9803 }
9804
9805 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9806 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9807 /**
9808 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9809 * @pf: board private structure
9810 **/
i40e_fdir_flush_and_replay(struct i40e_pf * pf)9811 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9812 {
9813 unsigned long min_flush_time;
9814 int flush_wait_retry = 50;
9815 bool disable_atr = false;
9816 int fd_room;
9817 int reg;
9818
9819 if (!time_after(jiffies, pf->fd_flush_timestamp +
9820 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9821 return;
9822
9823 /* If the flush is happening too quick and we have mostly SB rules we
9824 * should not re-enable ATR for some time.
9825 */
9826 min_flush_time = pf->fd_flush_timestamp +
9827 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9828 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9829
9830 if (!(time_after(jiffies, min_flush_time)) &&
9831 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9832 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9833 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9834 disable_atr = true;
9835 }
9836
9837 pf->fd_flush_timestamp = jiffies;
9838 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9839 /* flush all filters */
9840 wr32(&pf->hw, I40E_PFQF_CTL_1,
9841 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9842 i40e_flush(&pf->hw);
9843 pf->fd_flush_cnt++;
9844 pf->fd_add_err = 0;
9845 do {
9846 /* Check FD flush status every 5-6msec */
9847 usleep_range(5000, 6000);
9848 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9849 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9850 break;
9851 } while (flush_wait_retry--);
9852 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9853 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9854 } else {
9855 /* replay sideband filters */
9856 i40e_fdir_filter_restore(i40e_pf_get_main_vsi(pf));
9857 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9858 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9859 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9860 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9861 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9862 }
9863 }
9864
9865 /**
9866 * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9867 * @pf: board private structure
9868 **/
i40e_get_current_atr_cnt(struct i40e_pf * pf)9869 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9870 {
9871 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9872 }
9873
9874 /**
9875 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9876 * @pf: board private structure
9877 **/
i40e_fdir_reinit_subtask(struct i40e_pf * pf)9878 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9879 {
9880
9881 /* if interface is down do nothing */
9882 if (test_bit(__I40E_DOWN, pf->state))
9883 return;
9884
9885 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9886 i40e_fdir_flush_and_replay(pf);
9887
9888 i40e_fdir_check_and_reenable(pf);
9889
9890 }
9891
9892 /**
9893 * i40e_vsi_link_event - notify VSI of a link event
9894 * @vsi: vsi to be notified
9895 * @link_up: link up or down
9896 **/
i40e_vsi_link_event(struct i40e_vsi * vsi,bool link_up)9897 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9898 {
9899 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9900 return;
9901
9902 switch (vsi->type) {
9903 case I40E_VSI_MAIN:
9904 if (!vsi->netdev || !vsi->netdev_registered)
9905 break;
9906
9907 if (link_up) {
9908 netif_carrier_on(vsi->netdev);
9909 netif_tx_wake_all_queues(vsi->netdev);
9910 } else {
9911 netif_carrier_off(vsi->netdev);
9912 netif_tx_stop_all_queues(vsi->netdev);
9913 }
9914 break;
9915
9916 case I40E_VSI_SRIOV:
9917 case I40E_VSI_VMDQ2:
9918 case I40E_VSI_CTRL:
9919 case I40E_VSI_IWARP:
9920 case I40E_VSI_MIRROR:
9921 default:
9922 /* there is no notification for other VSIs */
9923 break;
9924 }
9925 }
9926
9927 /**
9928 * i40e_veb_link_event - notify elements on the veb of a link event
9929 * @veb: veb to be notified
9930 * @link_up: link up or down
9931 **/
i40e_veb_link_event(struct i40e_veb * veb,bool link_up)9932 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9933 {
9934 struct i40e_vsi *vsi;
9935 struct i40e_pf *pf;
9936 int i;
9937
9938 if (!veb || !veb->pf)
9939 return;
9940 pf = veb->pf;
9941
9942 /* Send link event to contained VSIs */
9943 i40e_pf_for_each_vsi(pf, i, vsi)
9944 if (vsi->uplink_seid == veb->seid)
9945 i40e_vsi_link_event(vsi, link_up);
9946 }
9947
9948 /**
9949 * i40e_link_event - Update netif_carrier status
9950 * @pf: board private structure
9951 **/
i40e_link_event(struct i40e_pf * pf)9952 static void i40e_link_event(struct i40e_pf *pf)
9953 {
9954 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
9955 struct i40e_veb *veb = i40e_pf_get_main_veb(pf);
9956 u8 new_link_speed, old_link_speed;
9957 bool new_link, old_link;
9958 int status;
9959 #ifdef CONFIG_I40E_DCB
9960 int err;
9961 #endif /* CONFIG_I40E_DCB */
9962
9963 /* set this to force the get_link_status call to refresh state */
9964 pf->hw.phy.get_link_info = true;
9965 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9966 status = i40e_get_link_status(&pf->hw, &new_link);
9967
9968 /* On success, disable temp link polling */
9969 if (status == 0) {
9970 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9971 } else {
9972 /* Enable link polling temporarily until i40e_get_link_status
9973 * returns 0
9974 */
9975 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9976 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9977 status);
9978 return;
9979 }
9980
9981 old_link_speed = pf->hw.phy.link_info_old.link_speed;
9982 new_link_speed = pf->hw.phy.link_info.link_speed;
9983
9984 if (new_link == old_link &&
9985 new_link_speed == old_link_speed &&
9986 (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9987 new_link == netif_carrier_ok(vsi->netdev)))
9988 return;
9989
9990 i40e_print_link_message(vsi, new_link);
9991
9992 /* Notify the base of the switch tree connected to
9993 * the link. Floating VEBs are not notified.
9994 */
9995 if (veb)
9996 i40e_veb_link_event(veb, new_link);
9997 else
9998 i40e_vsi_link_event(vsi, new_link);
9999
10000 if (pf->vf)
10001 i40e_vc_notify_link_state(pf);
10002
10003 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
10004 i40e_ptp_set_increment(pf);
10005 #ifdef CONFIG_I40E_DCB
10006 if (new_link == old_link)
10007 return;
10008 /* Not SW DCB so firmware will take care of default settings */
10009 if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
10010 return;
10011
10012 /* We cover here only link down, as after link up in case of SW DCB
10013 * SW LLDP agent will take care of setting it up
10014 */
10015 if (!new_link) {
10016 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
10017 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
10018 err = i40e_dcb_sw_default_config(pf);
10019 if (err) {
10020 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10021 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
10022 } else {
10023 pf->dcbx_cap = DCB_CAP_DCBX_HOST |
10024 DCB_CAP_DCBX_VER_IEEE;
10025 set_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10026 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
10027 }
10028 }
10029 #endif /* CONFIG_I40E_DCB */
10030 }
10031
10032 /**
10033 * i40e_watchdog_subtask - periodic checks not using event driven response
10034 * @pf: board private structure
10035 **/
i40e_watchdog_subtask(struct i40e_pf * pf)10036 static void i40e_watchdog_subtask(struct i40e_pf *pf)
10037 {
10038 struct i40e_vsi *vsi;
10039 struct i40e_veb *veb;
10040 int i;
10041
10042 /* if interface is down do nothing */
10043 if (test_bit(__I40E_DOWN, pf->state) ||
10044 test_bit(__I40E_CONFIG_BUSY, pf->state))
10045 return;
10046
10047 /* make sure we don't do these things too often */
10048 if (time_before(jiffies, (pf->service_timer_previous +
10049 pf->service_timer_period)))
10050 return;
10051 pf->service_timer_previous = jiffies;
10052
10053 if (test_bit(I40E_FLAG_LINK_POLLING_ENA, pf->flags) ||
10054 test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
10055 i40e_link_event(pf);
10056
10057 /* Update the stats for active netdevs so the network stack
10058 * can look at updated numbers whenever it cares to
10059 */
10060 i40e_pf_for_each_vsi(pf, i, vsi)
10061 if (vsi->netdev)
10062 i40e_update_stats(vsi);
10063
10064 if (test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags)) {
10065 /* Update the stats for the active switching components */
10066 i40e_pf_for_each_veb(pf, i, veb)
10067 i40e_update_veb_stats(veb);
10068 }
10069
10070 i40e_ptp_rx_hang(pf);
10071 i40e_ptp_tx_hang(pf);
10072 }
10073
10074 /**
10075 * i40e_reset_subtask - Set up for resetting the device and driver
10076 * @pf: board private structure
10077 **/
i40e_reset_subtask(struct i40e_pf * pf)10078 static void i40e_reset_subtask(struct i40e_pf *pf)
10079 {
10080 u32 reset_flags = 0;
10081
10082 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
10083 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
10084 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
10085 }
10086 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
10087 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
10088 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
10089 }
10090 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
10091 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
10092 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
10093 }
10094 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
10095 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
10096 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
10097 }
10098 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
10099 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
10100 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
10101 }
10102
10103 /* If there's a recovery already waiting, it takes
10104 * precedence before starting a new reset sequence.
10105 */
10106 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
10107 i40e_prep_for_reset(pf);
10108 i40e_reset(pf);
10109 i40e_rebuild(pf, false, false);
10110 }
10111
10112 /* If we're already down or resetting, just bail */
10113 if (reset_flags &&
10114 !test_bit(__I40E_DOWN, pf->state) &&
10115 !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
10116 i40e_do_reset(pf, reset_flags, false);
10117 }
10118 }
10119
10120 /**
10121 * i40e_handle_link_event - Handle link event
10122 * @pf: board private structure
10123 * @e: event info posted on ARQ
10124 **/
i40e_handle_link_event(struct i40e_pf * pf,struct i40e_arq_event_info * e)10125 static void i40e_handle_link_event(struct i40e_pf *pf,
10126 struct i40e_arq_event_info *e)
10127 {
10128 struct i40e_aqc_get_link_status *status =
10129 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
10130
10131 /* Do a new status request to re-enable LSE reporting
10132 * and load new status information into the hw struct
10133 * This completely ignores any state information
10134 * in the ARQ event info, instead choosing to always
10135 * issue the AQ update link status command.
10136 */
10137 i40e_link_event(pf);
10138
10139 /* Check if module meets thermal requirements */
10140 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
10141 dev_err(&pf->pdev->dev,
10142 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
10143 dev_err(&pf->pdev->dev,
10144 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10145 } else {
10146 /* check for unqualified module, if link is down, suppress
10147 * the message if link was forced to be down.
10148 */
10149 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
10150 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
10151 (!(status->link_info & I40E_AQ_LINK_UP)) &&
10152 (!test_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))) {
10153 dev_err(&pf->pdev->dev,
10154 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
10155 dev_err(&pf->pdev->dev,
10156 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10157 }
10158 }
10159 }
10160
10161 /**
10162 * i40e_clean_adminq_subtask - Clean the AdminQ rings
10163 * @pf: board private structure
10164 **/
i40e_clean_adminq_subtask(struct i40e_pf * pf)10165 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
10166 {
10167 struct i40e_arq_event_info event;
10168 struct i40e_hw *hw = &pf->hw;
10169 u16 pending, i = 0;
10170 u16 opcode;
10171 u32 oldval;
10172 int ret;
10173 u32 val;
10174
10175 /* Do not run clean AQ when PF reset fails */
10176 if (test_bit(__I40E_RESET_FAILED, pf->state))
10177 return;
10178
10179 /* check for error indications */
10180 val = rd32(&pf->hw, I40E_PF_ARQLEN);
10181 oldval = val;
10182 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
10183 if (hw->debug_mask & I40E_DEBUG_AQ)
10184 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
10185 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
10186 }
10187 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
10188 if (hw->debug_mask & I40E_DEBUG_AQ)
10189 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
10190 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
10191 pf->arq_overflows++;
10192 }
10193 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
10194 if (hw->debug_mask & I40E_DEBUG_AQ)
10195 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
10196 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
10197 }
10198 if (oldval != val)
10199 wr32(&pf->hw, I40E_PF_ARQLEN, val);
10200
10201 val = rd32(&pf->hw, I40E_PF_ATQLEN);
10202 oldval = val;
10203 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
10204 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10205 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
10206 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
10207 }
10208 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
10209 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10210 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
10211 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
10212 }
10213 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
10214 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10215 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
10216 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
10217 }
10218 if (oldval != val)
10219 wr32(&pf->hw, I40E_PF_ATQLEN, val);
10220
10221 event.buf_len = I40E_MAX_AQ_BUF_SIZE;
10222 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
10223 if (!event.msg_buf)
10224 return;
10225
10226 do {
10227 ret = i40e_clean_arq_element(hw, &event, &pending);
10228 if (ret == -EALREADY)
10229 break;
10230 else if (ret) {
10231 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
10232 break;
10233 }
10234
10235 opcode = le16_to_cpu(event.desc.opcode);
10236 switch (opcode) {
10237
10238 case i40e_aqc_opc_get_link_status:
10239 rtnl_lock();
10240 i40e_handle_link_event(pf, &event);
10241 rtnl_unlock();
10242 break;
10243 case i40e_aqc_opc_send_msg_to_pf:
10244 ret = i40e_vc_process_vf_msg(pf,
10245 le16_to_cpu(event.desc.retval),
10246 le32_to_cpu(event.desc.cookie_high),
10247 le32_to_cpu(event.desc.cookie_low),
10248 event.msg_buf,
10249 event.msg_len);
10250 break;
10251 case i40e_aqc_opc_lldp_update_mib:
10252 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
10253 #ifdef CONFIG_I40E_DCB
10254 rtnl_lock();
10255 i40e_handle_lldp_event(pf, &event);
10256 rtnl_unlock();
10257 #endif /* CONFIG_I40E_DCB */
10258 break;
10259 case i40e_aqc_opc_event_lan_overflow:
10260 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
10261 i40e_handle_lan_overflow_event(pf, &event);
10262 break;
10263 case i40e_aqc_opc_send_msg_to_peer:
10264 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
10265 break;
10266 case i40e_aqc_opc_nvm_erase:
10267 case i40e_aqc_opc_nvm_update:
10268 case i40e_aqc_opc_oem_post_update:
10269 i40e_debug(&pf->hw, I40E_DEBUG_NVM,
10270 "ARQ NVM operation 0x%04x completed\n",
10271 opcode);
10272 break;
10273 default:
10274 dev_info(&pf->pdev->dev,
10275 "ARQ: Unknown event 0x%04x ignored\n",
10276 opcode);
10277 break;
10278 }
10279 } while (i++ < I40E_AQ_WORK_LIMIT);
10280
10281 if (i < I40E_AQ_WORK_LIMIT)
10282 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
10283
10284 /* re-enable Admin queue interrupt cause */
10285 val = rd32(hw, I40E_PFINT_ICR0_ENA);
10286 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
10287 wr32(hw, I40E_PFINT_ICR0_ENA, val);
10288 i40e_flush(hw);
10289
10290 kfree(event.msg_buf);
10291 }
10292
10293 /**
10294 * i40e_verify_eeprom - make sure eeprom is good to use
10295 * @pf: board private structure
10296 **/
i40e_verify_eeprom(struct i40e_pf * pf)10297 static void i40e_verify_eeprom(struct i40e_pf *pf)
10298 {
10299 int err;
10300
10301 err = i40e_diag_eeprom_test(&pf->hw);
10302 if (err) {
10303 /* retry in case of garbage read */
10304 err = i40e_diag_eeprom_test(&pf->hw);
10305 if (err) {
10306 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
10307 err);
10308 set_bit(__I40E_BAD_EEPROM, pf->state);
10309 }
10310 }
10311
10312 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
10313 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
10314 clear_bit(__I40E_BAD_EEPROM, pf->state);
10315 }
10316 }
10317
10318 /**
10319 * i40e_enable_pf_switch_lb
10320 * @pf: pointer to the PF structure
10321 *
10322 * enable switch loop back or die - no point in a return value
10323 **/
i40e_enable_pf_switch_lb(struct i40e_pf * pf)10324 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
10325 {
10326 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
10327 struct i40e_vsi_context ctxt;
10328 int ret;
10329
10330 ctxt.seid = pf->main_vsi_seid;
10331 ctxt.pf_num = pf->hw.pf_id;
10332 ctxt.vf_num = 0;
10333 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10334 if (ret) {
10335 dev_info(&pf->pdev->dev,
10336 "couldn't get PF vsi config, err %pe aq_err %s\n",
10337 ERR_PTR(ret),
10338 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10339 return;
10340 }
10341 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10342 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10343 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10344
10345 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10346 if (ret) {
10347 dev_info(&pf->pdev->dev,
10348 "update vsi switch failed, err %pe aq_err %s\n",
10349 ERR_PTR(ret),
10350 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10351 }
10352 }
10353
10354 /**
10355 * i40e_disable_pf_switch_lb
10356 * @pf: pointer to the PF structure
10357 *
10358 * disable switch loop back or die - no point in a return value
10359 **/
i40e_disable_pf_switch_lb(struct i40e_pf * pf)10360 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
10361 {
10362 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
10363 struct i40e_vsi_context ctxt;
10364 int ret;
10365
10366 ctxt.seid = pf->main_vsi_seid;
10367 ctxt.pf_num = pf->hw.pf_id;
10368 ctxt.vf_num = 0;
10369 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10370 if (ret) {
10371 dev_info(&pf->pdev->dev,
10372 "couldn't get PF vsi config, err %pe aq_err %s\n",
10373 ERR_PTR(ret),
10374 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10375 return;
10376 }
10377 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10378 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10379 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10380
10381 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10382 if (ret) {
10383 dev_info(&pf->pdev->dev,
10384 "update vsi switch failed, err %pe aq_err %s\n",
10385 ERR_PTR(ret),
10386 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10387 }
10388 }
10389
10390 /**
10391 * i40e_config_bridge_mode - Configure the HW bridge mode
10392 * @veb: pointer to the bridge instance
10393 *
10394 * Configure the loop back mode for the LAN VSI that is downlink to the
10395 * specified HW bridge instance. It is expected this function is called
10396 * when a new HW bridge is instantiated.
10397 **/
i40e_config_bridge_mode(struct i40e_veb * veb)10398 static void i40e_config_bridge_mode(struct i40e_veb *veb)
10399 {
10400 struct i40e_pf *pf = veb->pf;
10401
10402 if (pf->hw.debug_mask & I40E_DEBUG_LAN)
10403 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
10404 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10405 if (veb->bridge_mode & BRIDGE_MODE_VEPA)
10406 i40e_disable_pf_switch_lb(pf);
10407 else
10408 i40e_enable_pf_switch_lb(pf);
10409 }
10410
10411 /**
10412 * i40e_reconstitute_veb - rebuild the VEB and VSIs connected to it
10413 * @veb: pointer to the VEB instance
10414 *
10415 * This is a function that builds the attached VSIs. We track the connections
10416 * through our own index numbers because the seid's from the HW could change
10417 * across the reset.
10418 **/
i40e_reconstitute_veb(struct i40e_veb * veb)10419 static int i40e_reconstitute_veb(struct i40e_veb *veb)
10420 {
10421 struct i40e_vsi *ctl_vsi = NULL;
10422 struct i40e_pf *pf = veb->pf;
10423 struct i40e_vsi *vsi;
10424 int v, ret;
10425
10426 /* As we do not maintain PV (port virtualizer) switch element then
10427 * there can be only one non-floating VEB that have uplink to MAC SEID
10428 * and its control VSI is the main one.
10429 */
10430 if (WARN_ON(veb->uplink_seid && veb->uplink_seid != pf->mac_seid)) {
10431 dev_err(&pf->pdev->dev,
10432 "Invalid uplink SEID for VEB %d\n", veb->idx);
10433 return -ENOENT;
10434 }
10435
10436 if (veb->uplink_seid == pf->mac_seid) {
10437 /* Check that the LAN VSI has VEB owning flag set */
10438 ctl_vsi = i40e_pf_get_main_vsi(pf);
10439
10440 if (WARN_ON(ctl_vsi->veb_idx != veb->idx ||
10441 !(ctl_vsi->flags & I40E_VSI_FLAG_VEB_OWNER))) {
10442 dev_err(&pf->pdev->dev,
10443 "Invalid control VSI for VEB %d\n", veb->idx);
10444 return -ENOENT;
10445 }
10446
10447 /* Add the control VSI to switch */
10448 ret = i40e_add_vsi(ctl_vsi);
10449 if (ret) {
10450 dev_err(&pf->pdev->dev,
10451 "Rebuild of owner VSI for VEB %d failed: %d\n",
10452 veb->idx, ret);
10453 return ret;
10454 }
10455
10456 i40e_vsi_reset_stats(ctl_vsi);
10457 }
10458
10459 /* create the VEB in the switch and move the VSI onto the VEB */
10460 ret = i40e_add_veb(veb, ctl_vsi);
10461 if (ret)
10462 return ret;
10463
10464 if (veb->uplink_seid) {
10465 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
10466 veb->bridge_mode = BRIDGE_MODE_VEB;
10467 else
10468 veb->bridge_mode = BRIDGE_MODE_VEPA;
10469 i40e_config_bridge_mode(veb);
10470 }
10471
10472 /* create the remaining VSIs attached to this VEB */
10473 i40e_pf_for_each_vsi(pf, v, vsi) {
10474 if (vsi == ctl_vsi)
10475 continue;
10476
10477 if (vsi->veb_idx == veb->idx) {
10478 vsi->uplink_seid = veb->seid;
10479 ret = i40e_add_vsi(vsi);
10480 if (ret) {
10481 dev_info(&pf->pdev->dev,
10482 "rebuild of vsi_idx %d failed: %d\n",
10483 v, ret);
10484 return ret;
10485 }
10486 i40e_vsi_reset_stats(vsi);
10487 }
10488 }
10489
10490 return ret;
10491 }
10492
10493 /**
10494 * i40e_get_capabilities - get info about the HW
10495 * @pf: the PF struct
10496 * @list_type: AQ capability to be queried
10497 **/
i40e_get_capabilities(struct i40e_pf * pf,enum i40e_admin_queue_opc list_type)10498 static int i40e_get_capabilities(struct i40e_pf *pf,
10499 enum i40e_admin_queue_opc list_type)
10500 {
10501 struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10502 u16 data_size;
10503 int buf_len;
10504 int err;
10505
10506 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10507 do {
10508 cap_buf = kzalloc(buf_len, GFP_KERNEL);
10509 if (!cap_buf)
10510 return -ENOMEM;
10511
10512 /* this loads the data into the hw struct for us */
10513 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
10514 &data_size, list_type,
10515 NULL);
10516 /* data loaded, buffer no longer needed */
10517 kfree(cap_buf);
10518
10519 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10520 /* retry with a larger buffer */
10521 buf_len = data_size;
10522 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
10523 dev_info(&pf->pdev->dev,
10524 "capability discovery failed, err %pe aq_err %s\n",
10525 ERR_PTR(err),
10526 i40e_aq_str(&pf->hw,
10527 pf->hw.aq.asq_last_status));
10528 return -ENODEV;
10529 }
10530 } while (err);
10531
10532 if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10533 if (list_type == i40e_aqc_opc_list_func_capabilities) {
10534 dev_info(&pf->pdev->dev,
10535 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10536 pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10537 pf->hw.func_caps.num_msix_vectors,
10538 pf->hw.func_caps.num_msix_vectors_vf,
10539 pf->hw.func_caps.fd_filters_guaranteed,
10540 pf->hw.func_caps.fd_filters_best_effort,
10541 pf->hw.func_caps.num_tx_qp,
10542 pf->hw.func_caps.num_vsis);
10543 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10544 dev_info(&pf->pdev->dev,
10545 "switch_mode=0x%04x, function_valid=0x%08x\n",
10546 pf->hw.dev_caps.switch_mode,
10547 pf->hw.dev_caps.valid_functions);
10548 dev_info(&pf->pdev->dev,
10549 "SR-IOV=%d, num_vfs for all function=%u\n",
10550 pf->hw.dev_caps.sr_iov_1_1,
10551 pf->hw.dev_caps.num_vfs);
10552 dev_info(&pf->pdev->dev,
10553 "num_vsis=%u, num_rx:%u, num_tx=%u\n",
10554 pf->hw.dev_caps.num_vsis,
10555 pf->hw.dev_caps.num_rx_qp,
10556 pf->hw.dev_caps.num_tx_qp);
10557 }
10558 }
10559 if (list_type == i40e_aqc_opc_list_func_capabilities) {
10560 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10561 + pf->hw.func_caps.num_vfs)
10562 if (pf->hw.revision_id == 0 &&
10563 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10564 dev_info(&pf->pdev->dev,
10565 "got num_vsis %d, setting num_vsis to %d\n",
10566 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10567 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10568 }
10569 }
10570 return 0;
10571 }
10572
10573 static int i40e_vsi_clear(struct i40e_vsi *vsi);
10574
10575 /**
10576 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10577 * @pf: board private structure
10578 **/
i40e_fdir_sb_setup(struct i40e_pf * pf)10579 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10580 {
10581 struct i40e_vsi *main_vsi, *vsi;
10582
10583 /* quick workaround for an NVM issue that leaves a critical register
10584 * uninitialized
10585 */
10586 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10587 static const u32 hkey[] = {
10588 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10589 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10590 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10591 0x95b3a76d};
10592 int i;
10593
10594 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10595 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10596 }
10597
10598 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
10599 return;
10600
10601 /* find existing VSI and see if it needs configuring */
10602 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10603
10604 /* create a new VSI if none exists */
10605 if (!vsi) {
10606 main_vsi = i40e_pf_get_main_vsi(pf);
10607 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, main_vsi->seid, 0);
10608 if (!vsi) {
10609 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10610 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
10611 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
10612 return;
10613 }
10614 }
10615
10616 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
10617 }
10618
10619 /**
10620 * i40e_fdir_teardown - release the Flow Director resources
10621 * @pf: board private structure
10622 **/
i40e_fdir_teardown(struct i40e_pf * pf)10623 static void i40e_fdir_teardown(struct i40e_pf *pf)
10624 {
10625 struct i40e_vsi *vsi;
10626
10627 i40e_fdir_filter_exit(pf);
10628 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10629 if (vsi)
10630 i40e_vsi_release(vsi);
10631 }
10632
10633 /**
10634 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10635 * @vsi: PF main vsi
10636 * @seid: seid of main or channel VSIs
10637 *
10638 * Rebuilds cloud filters associated with main VSI and channel VSIs if they
10639 * existed before reset
10640 **/
i40e_rebuild_cloud_filters(struct i40e_vsi * vsi,u16 seid)10641 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10642 {
10643 struct i40e_cloud_filter *cfilter;
10644 struct i40e_pf *pf = vsi->back;
10645 struct hlist_node *node;
10646 int ret;
10647
10648 /* Add cloud filters back if they exist */
10649 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10650 cloud_node) {
10651 if (cfilter->seid != seid)
10652 continue;
10653
10654 if (cfilter->dst_port)
10655 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10656 true);
10657 else
10658 ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10659
10660 if (ret) {
10661 dev_dbg(&pf->pdev->dev,
10662 "Failed to rebuild cloud filter, err %pe aq_err %s\n",
10663 ERR_PTR(ret),
10664 i40e_aq_str(&pf->hw,
10665 pf->hw.aq.asq_last_status));
10666 return ret;
10667 }
10668 }
10669 return 0;
10670 }
10671
10672 /**
10673 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10674 * @vsi: PF main vsi
10675 *
10676 * Rebuilds channel VSIs if they existed before reset
10677 **/
i40e_rebuild_channels(struct i40e_vsi * vsi)10678 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10679 {
10680 struct i40e_channel *ch, *ch_tmp;
10681 int ret;
10682
10683 if (list_empty(&vsi->ch_list))
10684 return 0;
10685
10686 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10687 if (!ch->initialized)
10688 break;
10689 /* Proceed with creation of channel (VMDq2) VSI */
10690 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10691 if (ret) {
10692 dev_info(&vsi->back->pdev->dev,
10693 "failed to rebuild channels using uplink_seid %u\n",
10694 vsi->uplink_seid);
10695 return ret;
10696 }
10697 /* Reconfigure TX queues using QTX_CTL register */
10698 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10699 if (ret) {
10700 dev_info(&vsi->back->pdev->dev,
10701 "failed to configure TX rings for channel %u\n",
10702 ch->seid);
10703 return ret;
10704 }
10705 /* update 'next_base_queue' */
10706 vsi->next_base_queue = vsi->next_base_queue +
10707 ch->num_queue_pairs;
10708 if (ch->max_tx_rate) {
10709 u64 credits = ch->max_tx_rate;
10710
10711 if (i40e_set_bw_limit(vsi, ch->seid,
10712 ch->max_tx_rate))
10713 return -EINVAL;
10714
10715 do_div(credits, I40E_BW_CREDIT_DIVISOR);
10716 dev_dbg(&vsi->back->pdev->dev,
10717 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10718 ch->max_tx_rate,
10719 credits,
10720 ch->seid);
10721 }
10722 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10723 if (ret) {
10724 dev_dbg(&vsi->back->pdev->dev,
10725 "Failed to rebuild cloud filters for channel VSI %u\n",
10726 ch->seid);
10727 return ret;
10728 }
10729 }
10730 return 0;
10731 }
10732
10733 /**
10734 * i40e_clean_xps_state - clean xps state for every tx_ring
10735 * @vsi: ptr to the VSI
10736 **/
i40e_clean_xps_state(struct i40e_vsi * vsi)10737 static void i40e_clean_xps_state(struct i40e_vsi *vsi)
10738 {
10739 int i;
10740
10741 if (vsi->tx_rings)
10742 for (i = 0; i < vsi->num_queue_pairs; i++)
10743 if (vsi->tx_rings[i])
10744 clear_bit(__I40E_TX_XPS_INIT_DONE,
10745 vsi->tx_rings[i]->state);
10746 }
10747
10748 /**
10749 * i40e_prep_for_reset - prep for the core to reset
10750 * @pf: board private structure
10751 *
10752 * Close up the VFs and other things in prep for PF Reset.
10753 **/
i40e_prep_for_reset(struct i40e_pf * pf)10754 static void i40e_prep_for_reset(struct i40e_pf *pf)
10755 {
10756 struct i40e_hw *hw = &pf->hw;
10757 struct i40e_vsi *vsi;
10758 int ret = 0;
10759 u32 v;
10760
10761 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10762 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10763 return;
10764 if (i40e_check_asq_alive(&pf->hw))
10765 i40e_vc_notify_reset(pf);
10766
10767 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10768
10769 /* quiesce the VSIs and their queues that are not already DOWN */
10770 i40e_pf_quiesce_all_vsi(pf);
10771
10772 i40e_pf_for_each_vsi(pf, v, vsi) {
10773 i40e_clean_xps_state(vsi);
10774 vsi->seid = 0;
10775 }
10776
10777 i40e_shutdown_adminq(&pf->hw);
10778
10779 /* call shutdown HMC */
10780 if (hw->hmc.hmc_obj) {
10781 ret = i40e_shutdown_lan_hmc(hw);
10782 if (ret)
10783 dev_warn(&pf->pdev->dev,
10784 "shutdown_lan_hmc failed: %d\n", ret);
10785 }
10786
10787 /* Save the current PTP time so that we can restore the time after the
10788 * reset completes.
10789 */
10790 i40e_ptp_save_hw_time(pf);
10791 }
10792
10793 /**
10794 * i40e_send_version - update firmware with driver version
10795 * @pf: PF struct
10796 */
i40e_send_version(struct i40e_pf * pf)10797 static void i40e_send_version(struct i40e_pf *pf)
10798 {
10799 struct i40e_driver_version dv;
10800
10801 dv.major_version = 0xff;
10802 dv.minor_version = 0xff;
10803 dv.build_version = 0xff;
10804 dv.subbuild_version = 0;
10805 strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10806 i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10807 }
10808
10809 /**
10810 * i40e_get_oem_version - get OEM specific version information
10811 * @hw: pointer to the hardware structure
10812 **/
i40e_get_oem_version(struct i40e_hw * hw)10813 static void i40e_get_oem_version(struct i40e_hw *hw)
10814 {
10815 u16 block_offset = 0xffff;
10816 u16 block_length = 0;
10817 u16 capabilities = 0;
10818 u16 gen_snap = 0;
10819 u16 release = 0;
10820
10821 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B
10822 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00
10823 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01
10824 #define I40E_NVM_OEM_GEN_OFFSET 0x02
10825 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03
10826 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F
10827 #define I40E_NVM_OEM_LENGTH 3
10828
10829 /* Check if pointer to OEM version block is valid. */
10830 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10831 if (block_offset == 0xffff)
10832 return;
10833
10834 /* Check if OEM version block has correct length. */
10835 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10836 &block_length);
10837 if (block_length < I40E_NVM_OEM_LENGTH)
10838 return;
10839
10840 /* Check if OEM version format is as expected. */
10841 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10842 &capabilities);
10843 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10844 return;
10845
10846 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10847 &gen_snap);
10848 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10849 &release);
10850 hw->nvm.oem_ver =
10851 FIELD_PREP(I40E_OEM_GEN_MASK | I40E_OEM_SNAP_MASK, gen_snap) |
10852 FIELD_PREP(I40E_OEM_RELEASE_MASK, release);
10853 hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10854 }
10855
10856 /**
10857 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10858 * @pf: board private structure
10859 **/
i40e_reset(struct i40e_pf * pf)10860 static int i40e_reset(struct i40e_pf *pf)
10861 {
10862 struct i40e_hw *hw = &pf->hw;
10863 int ret;
10864
10865 ret = i40e_pf_reset(hw);
10866 if (ret) {
10867 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10868 set_bit(__I40E_RESET_FAILED, pf->state);
10869 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10870 } else {
10871 pf->pfr_count++;
10872 }
10873 return ret;
10874 }
10875
10876 /**
10877 * i40e_rebuild - rebuild using a saved config
10878 * @pf: board private structure
10879 * @reinit: if the Main VSI needs to re-initialized.
10880 * @lock_acquired: indicates whether or not the lock has been acquired
10881 * before this function was called.
10882 **/
i40e_rebuild(struct i40e_pf * pf,bool reinit,bool lock_acquired)10883 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10884 {
10885 const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf);
10886 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
10887 struct i40e_hw *hw = &pf->hw;
10888 struct i40e_veb *veb;
10889 int ret;
10890 u32 val;
10891 int v;
10892
10893 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10894 is_recovery_mode_reported)
10895 i40e_set_ethtool_ops(vsi->netdev);
10896
10897 if (test_bit(__I40E_DOWN, pf->state) &&
10898 !test_bit(__I40E_RECOVERY_MODE, pf->state))
10899 goto clear_recovery;
10900 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10901
10902 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10903 ret = i40e_init_adminq(&pf->hw);
10904 if (ret) {
10905 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n",
10906 ERR_PTR(ret),
10907 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10908 goto clear_recovery;
10909 }
10910 i40e_get_oem_version(&pf->hw);
10911
10912 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {
10913 /* The following delay is necessary for firmware update. */
10914 mdelay(1000);
10915 }
10916
10917 /* re-verify the eeprom if we just had an EMP reset */
10918 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10919 i40e_verify_eeprom(pf);
10920
10921 /* if we are going out of or into recovery mode we have to act
10922 * accordingly with regard to resources initialization
10923 * and deinitialization
10924 */
10925 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10926 if (i40e_get_capabilities(pf,
10927 i40e_aqc_opc_list_func_capabilities))
10928 goto end_unlock;
10929
10930 if (is_recovery_mode_reported) {
10931 /* we're staying in recovery mode so we'll reinitialize
10932 * misc vector here
10933 */
10934 if (i40e_setup_misc_vector_for_recovery_mode(pf))
10935 goto end_unlock;
10936 } else {
10937 if (!lock_acquired)
10938 rtnl_lock();
10939 /* we're going out of recovery mode so we'll free
10940 * the IRQ allocated specifically for recovery mode
10941 * and restore the interrupt scheme
10942 */
10943 free_irq(pf->pdev->irq, pf);
10944 i40e_clear_interrupt_scheme(pf);
10945 if (i40e_restore_interrupt_scheme(pf))
10946 goto end_unlock;
10947 }
10948
10949 /* tell the firmware that we're starting */
10950 i40e_send_version(pf);
10951
10952 /* bail out in case recovery mode was detected, as there is
10953 * no need for further configuration.
10954 */
10955 goto end_unlock;
10956 }
10957
10958 i40e_clear_pxe_mode(hw);
10959 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10960 if (ret)
10961 goto end_core_reset;
10962
10963 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10964 hw->func_caps.num_rx_qp, 0, 0);
10965 if (ret) {
10966 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10967 goto end_core_reset;
10968 }
10969 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10970 if (ret) {
10971 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10972 goto end_core_reset;
10973 }
10974
10975 #ifdef CONFIG_I40E_DCB
10976 /* Enable FW to write a default DCB config on link-up
10977 * unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10978 * is not supported with new link speed
10979 */
10980 if (i40e_is_tc_mqprio_enabled(pf)) {
10981 i40e_aq_set_dcb_parameters(hw, false, NULL);
10982 } else {
10983 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10984 (hw->phy.link_info.link_speed &
10985 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10986 i40e_aq_set_dcb_parameters(hw, false, NULL);
10987 dev_warn(&pf->pdev->dev,
10988 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
10989 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10990 } else {
10991 i40e_aq_set_dcb_parameters(hw, true, NULL);
10992 ret = i40e_init_pf_dcb(pf);
10993 if (ret) {
10994 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10995 ret);
10996 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
10997 /* Continue without DCB enabled */
10998 }
10999 }
11000 }
11001
11002 #endif /* CONFIG_I40E_DCB */
11003 if (!lock_acquired)
11004 rtnl_lock();
11005 ret = i40e_setup_pf_switch(pf, reinit, true);
11006 if (ret)
11007 goto end_unlock;
11008
11009 /* The driver only wants link up/down and module qualification
11010 * reports from firmware. Note the negative logic.
11011 */
11012 ret = i40e_aq_set_phy_int_mask(&pf->hw,
11013 ~(I40E_AQ_EVENT_LINK_UPDOWN |
11014 I40E_AQ_EVENT_MEDIA_NA |
11015 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
11016 if (ret)
11017 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
11018 ERR_PTR(ret),
11019 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11020
11021 /* Rebuild the VSIs and VEBs that existed before reset.
11022 * They are still in our local switch element arrays, so only
11023 * need to rebuild the switch model in the HW.
11024 *
11025 * If there were VEBs but the reconstitution failed, we'll try
11026 * to recover minimal use by getting the basic PF VSI working.
11027 */
11028 if (vsi->uplink_seid != pf->mac_seid) {
11029 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
11030
11031 /* Rebuild VEBs */
11032 i40e_pf_for_each_veb(pf, v, veb) {
11033 ret = i40e_reconstitute_veb(veb);
11034 if (!ret)
11035 continue;
11036
11037 /* If Main VEB failed, we're in deep doodoo,
11038 * so give up rebuilding the switch and set up
11039 * for minimal rebuild of PF VSI.
11040 * If orphan failed, we'll report the error
11041 * but try to keep going.
11042 */
11043 if (veb->uplink_seid == pf->mac_seid) {
11044 dev_info(&pf->pdev->dev,
11045 "rebuild of switch failed: %d, will try to set up simple PF connection\n",
11046 ret);
11047 vsi->uplink_seid = pf->mac_seid;
11048 break;
11049 } else if (veb->uplink_seid == 0) {
11050 dev_info(&pf->pdev->dev,
11051 "rebuild of orphan VEB failed: %d\n",
11052 ret);
11053 }
11054 }
11055 }
11056
11057 if (vsi->uplink_seid == pf->mac_seid) {
11058 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
11059 /* no VEB, so rebuild only the Main VSI */
11060 ret = i40e_add_vsi(vsi);
11061 if (ret) {
11062 dev_info(&pf->pdev->dev,
11063 "rebuild of Main VSI failed: %d\n", ret);
11064 goto end_unlock;
11065 }
11066 }
11067
11068 if (vsi->mqprio_qopt.max_rate[0]) {
11069 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
11070 vsi->mqprio_qopt.max_rate[0]);
11071 u64 credits = 0;
11072
11073 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
11074 if (ret)
11075 goto end_unlock;
11076
11077 credits = max_tx_rate;
11078 do_div(credits, I40E_BW_CREDIT_DIVISOR);
11079 dev_dbg(&vsi->back->pdev->dev,
11080 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
11081 max_tx_rate,
11082 credits,
11083 vsi->seid);
11084 }
11085
11086 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
11087 if (ret)
11088 goto end_unlock;
11089
11090 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
11091 * for this main VSI if they exist
11092 */
11093 ret = i40e_rebuild_channels(vsi);
11094 if (ret)
11095 goto end_unlock;
11096
11097 /* Reconfigure hardware for allowing smaller MSS in the case
11098 * of TSO, so that we avoid the MDD being fired and causing
11099 * a reset in the case of small MSS+TSO.
11100 */
11101 #define I40E_REG_MSS 0x000E64DC
11102 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
11103 #define I40E_64BYTE_MSS 0x400000
11104 val = rd32(hw, I40E_REG_MSS);
11105 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11106 val &= ~I40E_REG_MSS_MIN_MASK;
11107 val |= I40E_64BYTE_MSS;
11108 wr32(hw, I40E_REG_MSS, val);
11109 }
11110
11111 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
11112 msleep(75);
11113 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11114 if (ret)
11115 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
11116 ERR_PTR(ret),
11117 i40e_aq_str(&pf->hw,
11118 pf->hw.aq.asq_last_status));
11119 }
11120 /* reinit the misc interrupt */
11121 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
11122 ret = i40e_setup_misc_vector(pf);
11123 if (ret)
11124 goto end_unlock;
11125 }
11126
11127 /* Add a filter to drop all Flow control frames from any VSI from being
11128 * transmitted. By doing so we stop a malicious VF from sending out
11129 * PAUSE or PFC frames and potentially controlling traffic for other
11130 * PF/VF VSIs.
11131 * The FW can still send Flow control frames if enabled.
11132 */
11133 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11134 pf->main_vsi_seid);
11135
11136 /* restart the VSIs that were rebuilt and running before the reset */
11137 i40e_pf_unquiesce_all_vsi(pf);
11138
11139 /* Release the RTNL lock before we start resetting VFs */
11140 if (!lock_acquired)
11141 rtnl_unlock();
11142
11143 /* Restore promiscuous settings */
11144 ret = i40e_set_promiscuous(pf, pf->cur_promisc);
11145 if (ret)
11146 dev_warn(&pf->pdev->dev,
11147 "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n",
11148 pf->cur_promisc ? "on" : "off",
11149 ERR_PTR(ret),
11150 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11151
11152 i40e_reset_all_vfs(pf, true);
11153
11154 /* tell the firmware that we're starting */
11155 i40e_send_version(pf);
11156
11157 /* We've already released the lock, so don't do it again */
11158 goto end_core_reset;
11159
11160 end_unlock:
11161 if (!lock_acquired)
11162 rtnl_unlock();
11163 end_core_reset:
11164 clear_bit(__I40E_RESET_FAILED, pf->state);
11165 clear_recovery:
11166 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
11167 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
11168 }
11169
11170 /**
11171 * i40e_reset_and_rebuild - reset and rebuild using a saved config
11172 * @pf: board private structure
11173 * @reinit: if the Main VSI needs to re-initialized.
11174 * @lock_acquired: indicates whether or not the lock has been acquired
11175 * before this function was called.
11176 **/
i40e_reset_and_rebuild(struct i40e_pf * pf,bool reinit,bool lock_acquired)11177 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
11178 bool lock_acquired)
11179 {
11180 int ret;
11181
11182 if (test_bit(__I40E_IN_REMOVE, pf->state))
11183 return;
11184 /* Now we wait for GRST to settle out.
11185 * We don't have to delete the VEBs or VSIs from the hw switch
11186 * because the reset will make them disappear.
11187 */
11188 ret = i40e_reset(pf);
11189 if (!ret)
11190 i40e_rebuild(pf, reinit, lock_acquired);
11191 else
11192 dev_err(&pf->pdev->dev, "%s: i40e_reset() FAILED", __func__);
11193 }
11194
11195 /**
11196 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
11197 * @pf: board private structure
11198 *
11199 * Close up the VFs and other things in prep for a Core Reset,
11200 * then get ready to rebuild the world.
11201 * @lock_acquired: indicates whether or not the lock has been acquired
11202 * before this function was called.
11203 **/
i40e_handle_reset_warning(struct i40e_pf * pf,bool lock_acquired)11204 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
11205 {
11206 i40e_prep_for_reset(pf);
11207 i40e_reset_and_rebuild(pf, false, lock_acquired);
11208 }
11209
11210 /**
11211 * i40e_handle_mdd_event
11212 * @pf: pointer to the PF structure
11213 *
11214 * Called from the MDD irq handler to identify possibly malicious vfs
11215 **/
i40e_handle_mdd_event(struct i40e_pf * pf)11216 static void i40e_handle_mdd_event(struct i40e_pf *pf)
11217 {
11218 struct i40e_hw *hw = &pf->hw;
11219 bool mdd_detected = false;
11220 struct i40e_vf *vf;
11221 u32 reg;
11222 int i;
11223
11224 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
11225 return;
11226
11227 /* find what triggered the MDD event */
11228 reg = rd32(hw, I40E_GL_MDET_TX);
11229 if (reg & I40E_GL_MDET_TX_VALID_MASK) {
11230 u8 pf_num = FIELD_GET(I40E_GL_MDET_TX_PF_NUM_MASK, reg);
11231 u16 vf_num = FIELD_GET(I40E_GL_MDET_TX_VF_NUM_MASK, reg);
11232 u8 event = FIELD_GET(I40E_GL_MDET_TX_EVENT_MASK, reg);
11233 u16 queue = FIELD_GET(I40E_GL_MDET_TX_QUEUE_MASK, reg) -
11234 pf->hw.func_caps.base_queue;
11235 if (netif_msg_tx_err(pf))
11236 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
11237 event, queue, pf_num, vf_num);
11238 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
11239 mdd_detected = true;
11240 }
11241 reg = rd32(hw, I40E_GL_MDET_RX);
11242 if (reg & I40E_GL_MDET_RX_VALID_MASK) {
11243 u8 func = FIELD_GET(I40E_GL_MDET_RX_FUNCTION_MASK, reg);
11244 u8 event = FIELD_GET(I40E_GL_MDET_RX_EVENT_MASK, reg);
11245 u16 queue = FIELD_GET(I40E_GL_MDET_RX_QUEUE_MASK, reg) -
11246 pf->hw.func_caps.base_queue;
11247 if (netif_msg_rx_err(pf))
11248 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
11249 event, queue, func);
11250 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
11251 mdd_detected = true;
11252 }
11253
11254 if (mdd_detected) {
11255 reg = rd32(hw, I40E_PF_MDET_TX);
11256 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
11257 wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
11258 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
11259 }
11260 reg = rd32(hw, I40E_PF_MDET_RX);
11261 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
11262 wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
11263 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
11264 }
11265 }
11266
11267 /* see if one of the VFs needs its hand slapped */
11268 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
11269 vf = &(pf->vf[i]);
11270 reg = rd32(hw, I40E_VP_MDET_TX(i));
11271 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
11272 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
11273 vf->num_mdd_events++;
11274 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
11275 i);
11276 dev_info(&pf->pdev->dev,
11277 "Use PF Control I/F to re-enable the VF\n");
11278 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11279 }
11280
11281 reg = rd32(hw, I40E_VP_MDET_RX(i));
11282 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
11283 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
11284 vf->num_mdd_events++;
11285 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
11286 i);
11287 dev_info(&pf->pdev->dev,
11288 "Use PF Control I/F to re-enable the VF\n");
11289 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11290 }
11291 }
11292
11293 /* re-enable mdd interrupt cause */
11294 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
11295 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
11296 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
11297 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
11298 i40e_flush(hw);
11299 }
11300
11301 /**
11302 * i40e_service_task - Run the driver's async subtasks
11303 * @work: pointer to work_struct containing our data
11304 **/
i40e_service_task(struct work_struct * work)11305 static void i40e_service_task(struct work_struct *work)
11306 {
11307 struct i40e_pf *pf = container_of(work,
11308 struct i40e_pf,
11309 service_task);
11310 unsigned long start_time = jiffies;
11311
11312 /* don't bother with service tasks if a reset is in progress */
11313 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
11314 test_bit(__I40E_SUSPENDED, pf->state))
11315 return;
11316
11317 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
11318 return;
11319
11320 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
11321 i40e_detect_recover_hung(pf);
11322 i40e_sync_filters_subtask(pf);
11323 i40e_reset_subtask(pf);
11324 i40e_handle_mdd_event(pf);
11325 i40e_vc_process_vflr_event(pf);
11326 i40e_watchdog_subtask(pf);
11327 i40e_fdir_reinit_subtask(pf);
11328 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
11329 /* Client subtask will reopen next time through. */
11330 i40e_notify_client_of_netdev_close(pf, true);
11331 } else {
11332 i40e_client_subtask(pf);
11333 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
11334 pf->state))
11335 i40e_notify_client_of_l2_param_changes(pf);
11336 }
11337 i40e_sync_filters_subtask(pf);
11338 } else {
11339 i40e_reset_subtask(pf);
11340 }
11341
11342 i40e_clean_adminq_subtask(pf);
11343
11344 /* flush memory to make sure state is correct before next watchdog */
11345 smp_mb__before_atomic();
11346 clear_bit(__I40E_SERVICE_SCHED, pf->state);
11347
11348 /* If the tasks have taken longer than one timer cycle or there
11349 * is more work to be done, reschedule the service task now
11350 * rather than wait for the timer to tick again.
11351 */
11352 if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
11353 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) ||
11354 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) ||
11355 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
11356 i40e_service_event_schedule(pf);
11357 }
11358
11359 /**
11360 * i40e_service_timer - timer callback
11361 * @t: timer list pointer
11362 **/
i40e_service_timer(struct timer_list * t)11363 static void i40e_service_timer(struct timer_list *t)
11364 {
11365 struct i40e_pf *pf = from_timer(pf, t, service_timer);
11366
11367 mod_timer(&pf->service_timer,
11368 round_jiffies(jiffies + pf->service_timer_period));
11369 i40e_service_event_schedule(pf);
11370 }
11371
11372 /**
11373 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11374 * @vsi: the VSI being configured
11375 **/
i40e_set_num_rings_in_vsi(struct i40e_vsi * vsi)11376 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
11377 {
11378 struct i40e_pf *pf = vsi->back;
11379
11380 switch (vsi->type) {
11381 case I40E_VSI_MAIN:
11382 vsi->alloc_queue_pairs = pf->num_lan_qps;
11383 if (!vsi->num_tx_desc)
11384 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11385 I40E_REQ_DESCRIPTOR_MULTIPLE);
11386 if (!vsi->num_rx_desc)
11387 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11388 I40E_REQ_DESCRIPTOR_MULTIPLE);
11389 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11390 vsi->num_q_vectors = pf->num_lan_msix;
11391 else
11392 vsi->num_q_vectors = 1;
11393
11394 break;
11395
11396 case I40E_VSI_FDIR:
11397 vsi->alloc_queue_pairs = 1;
11398 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11399 I40E_REQ_DESCRIPTOR_MULTIPLE);
11400 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11401 I40E_REQ_DESCRIPTOR_MULTIPLE);
11402 vsi->num_q_vectors = pf->num_fdsb_msix;
11403 break;
11404
11405 case I40E_VSI_VMDQ2:
11406 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11407 if (!vsi->num_tx_desc)
11408 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11409 I40E_REQ_DESCRIPTOR_MULTIPLE);
11410 if (!vsi->num_rx_desc)
11411 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11412 I40E_REQ_DESCRIPTOR_MULTIPLE);
11413 vsi->num_q_vectors = pf->num_vmdq_msix;
11414 break;
11415
11416 case I40E_VSI_SRIOV:
11417 vsi->alloc_queue_pairs = pf->num_vf_qps;
11418 if (!vsi->num_tx_desc)
11419 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11420 I40E_REQ_DESCRIPTOR_MULTIPLE);
11421 if (!vsi->num_rx_desc)
11422 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11423 I40E_REQ_DESCRIPTOR_MULTIPLE);
11424 break;
11425
11426 default:
11427 WARN_ON(1);
11428 return -ENODATA;
11429 }
11430
11431 if (is_kdump_kernel()) {
11432 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;
11433 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;
11434 }
11435
11436 return 0;
11437 }
11438
11439 /**
11440 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11441 * @vsi: VSI pointer
11442 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11443 *
11444 * On error: returns error code (negative)
11445 * On success: returns 0
11446 **/
i40e_vsi_alloc_arrays(struct i40e_vsi * vsi,bool alloc_qvectors)11447 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11448 {
11449 struct i40e_ring **next_rings;
11450 int size;
11451 int ret = 0;
11452
11453 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11454 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11455 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11456 vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11457 if (!vsi->tx_rings)
11458 return -ENOMEM;
11459 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11460 if (i40e_enabled_xdp_vsi(vsi)) {
11461 vsi->xdp_rings = next_rings;
11462 next_rings += vsi->alloc_queue_pairs;
11463 }
11464 vsi->rx_rings = next_rings;
11465
11466 if (alloc_qvectors) {
11467 /* allocate memory for q_vector pointers */
11468 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11469 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11470 if (!vsi->q_vectors) {
11471 ret = -ENOMEM;
11472 goto err_vectors;
11473 }
11474 }
11475 return ret;
11476
11477 err_vectors:
11478 kfree(vsi->tx_rings);
11479 return ret;
11480 }
11481
11482 /**
11483 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11484 * @pf: board private structure
11485 * @type: type of VSI
11486 *
11487 * On error: returns error code (negative)
11488 * On success: returns vsi index in PF (positive)
11489 **/
i40e_vsi_mem_alloc(struct i40e_pf * pf,enum i40e_vsi_type type)11490 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11491 {
11492 int ret = -ENODEV;
11493 struct i40e_vsi *vsi;
11494 int vsi_idx;
11495 int i;
11496
11497 /* Need to protect the allocation of the VSIs at the PF level */
11498 mutex_lock(&pf->switch_mutex);
11499
11500 /* VSI list may be fragmented if VSI creation/destruction has
11501 * been happening. We can afford to do a quick scan to look
11502 * for any free VSIs in the list.
11503 *
11504 * find next empty vsi slot, looping back around if necessary
11505 */
11506 i = pf->next_vsi;
11507 while (i < pf->num_alloc_vsi && pf->vsi[i])
11508 i++;
11509 if (i >= pf->num_alloc_vsi) {
11510 i = 0;
11511 while (i < pf->next_vsi && pf->vsi[i])
11512 i++;
11513 }
11514
11515 if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11516 vsi_idx = i; /* Found one! */
11517 } else {
11518 ret = -ENODEV;
11519 goto unlock_pf; /* out of VSI slots! */
11520 }
11521 pf->next_vsi = ++i;
11522
11523 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
11524 if (!vsi) {
11525 ret = -ENOMEM;
11526 goto unlock_pf;
11527 }
11528 vsi->type = type;
11529 vsi->back = pf;
11530 set_bit(__I40E_VSI_DOWN, vsi->state);
11531 vsi->flags = 0;
11532 vsi->idx = vsi_idx;
11533 vsi->int_rate_limit = 0;
11534 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11535 pf->rss_table_size : 64;
11536 vsi->netdev_registered = false;
11537 vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11538 hash_init(vsi->mac_filter_hash);
11539 vsi->irqs_ready = false;
11540
11541 if (type == I40E_VSI_MAIN) {
11542 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
11543 if (!vsi->af_xdp_zc_qps)
11544 goto err_rings;
11545 }
11546
11547 ret = i40e_set_num_rings_in_vsi(vsi);
11548 if (ret)
11549 goto err_rings;
11550
11551 ret = i40e_vsi_alloc_arrays(vsi, true);
11552 if (ret)
11553 goto err_rings;
11554
11555 /* Setup default MSIX irq handler for VSI */
11556 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
11557
11558 /* Initialize VSI lock */
11559 spin_lock_init(&vsi->mac_filter_hash_lock);
11560 pf->vsi[vsi_idx] = vsi;
11561 ret = vsi_idx;
11562 goto unlock_pf;
11563
11564 err_rings:
11565 bitmap_free(vsi->af_xdp_zc_qps);
11566 pf->next_vsi = i - 1;
11567 kfree(vsi);
11568 unlock_pf:
11569 mutex_unlock(&pf->switch_mutex);
11570 return ret;
11571 }
11572
11573 /**
11574 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11575 * @vsi: VSI pointer
11576 * @free_qvectors: a bool to specify if q_vectors need to be freed.
11577 *
11578 * On error: returns error code (negative)
11579 * On success: returns 0
11580 **/
i40e_vsi_free_arrays(struct i40e_vsi * vsi,bool free_qvectors)11581 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11582 {
11583 /* free the ring and vector containers */
11584 if (free_qvectors) {
11585 kfree(vsi->q_vectors);
11586 vsi->q_vectors = NULL;
11587 }
11588 kfree(vsi->tx_rings);
11589 vsi->tx_rings = NULL;
11590 vsi->rx_rings = NULL;
11591 vsi->xdp_rings = NULL;
11592 }
11593
11594 /**
11595 * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11596 * and lookup table
11597 * @vsi: Pointer to VSI structure
11598 */
i40e_clear_rss_config_user(struct i40e_vsi * vsi)11599 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11600 {
11601 if (!vsi)
11602 return;
11603
11604 kfree(vsi->rss_hkey_user);
11605 vsi->rss_hkey_user = NULL;
11606
11607 kfree(vsi->rss_lut_user);
11608 vsi->rss_lut_user = NULL;
11609 }
11610
11611 /**
11612 * i40e_vsi_clear - Deallocate the VSI provided
11613 * @vsi: the VSI being un-configured
11614 **/
i40e_vsi_clear(struct i40e_vsi * vsi)11615 static int i40e_vsi_clear(struct i40e_vsi *vsi)
11616 {
11617 struct i40e_pf *pf;
11618
11619 if (!vsi)
11620 return 0;
11621
11622 if (!vsi->back)
11623 goto free_vsi;
11624 pf = vsi->back;
11625
11626 mutex_lock(&pf->switch_mutex);
11627 if (!pf->vsi[vsi->idx]) {
11628 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11629 vsi->idx, vsi->idx, vsi->type);
11630 goto unlock_vsi;
11631 }
11632
11633 if (pf->vsi[vsi->idx] != vsi) {
11634 dev_err(&pf->pdev->dev,
11635 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11636 pf->vsi[vsi->idx]->idx,
11637 pf->vsi[vsi->idx]->type,
11638 vsi->idx, vsi->type);
11639 goto unlock_vsi;
11640 }
11641
11642 /* updates the PF for this cleared vsi */
11643 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
11644 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
11645
11646 bitmap_free(vsi->af_xdp_zc_qps);
11647 i40e_vsi_free_arrays(vsi, true);
11648 i40e_clear_rss_config_user(vsi);
11649
11650 pf->vsi[vsi->idx] = NULL;
11651 if (vsi->idx < pf->next_vsi)
11652 pf->next_vsi = vsi->idx;
11653
11654 unlock_vsi:
11655 mutex_unlock(&pf->switch_mutex);
11656 free_vsi:
11657 kfree(vsi);
11658
11659 return 0;
11660 }
11661
11662 /**
11663 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11664 * @vsi: the VSI being cleaned
11665 **/
i40e_vsi_clear_rings(struct i40e_vsi * vsi)11666 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11667 {
11668 int i;
11669
11670 if (vsi->tx_rings && vsi->tx_rings[0]) {
11671 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11672 kfree_rcu(vsi->tx_rings[i], rcu);
11673 WRITE_ONCE(vsi->tx_rings[i], NULL);
11674 WRITE_ONCE(vsi->rx_rings[i], NULL);
11675 if (vsi->xdp_rings)
11676 WRITE_ONCE(vsi->xdp_rings[i], NULL);
11677 }
11678 }
11679 }
11680
11681 /**
11682 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11683 * @vsi: the VSI being configured
11684 **/
i40e_alloc_rings(struct i40e_vsi * vsi)11685 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11686 {
11687 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11688 struct i40e_pf *pf = vsi->back;
11689 struct i40e_ring *ring;
11690
11691 /* Set basic values in the rings to be used later during open() */
11692 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11693 /* allocate space for both Tx and Rx in one shot */
11694 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11695 if (!ring)
11696 goto err_out;
11697
11698 ring->queue_index = i;
11699 ring->reg_idx = vsi->base_queue + i;
11700 ring->ring_active = false;
11701 ring->vsi = vsi;
11702 ring->netdev = vsi->netdev;
11703 ring->dev = &pf->pdev->dev;
11704 ring->count = vsi->num_tx_desc;
11705 ring->size = 0;
11706 ring->dcb_tc = 0;
11707 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11708 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11709 ring->itr_setting = pf->tx_itr_default;
11710 WRITE_ONCE(vsi->tx_rings[i], ring++);
11711
11712 if (!i40e_enabled_xdp_vsi(vsi))
11713 goto setup_rx;
11714
11715 ring->queue_index = vsi->alloc_queue_pairs + i;
11716 ring->reg_idx = vsi->base_queue + ring->queue_index;
11717 ring->ring_active = false;
11718 ring->vsi = vsi;
11719 ring->netdev = NULL;
11720 ring->dev = &pf->pdev->dev;
11721 ring->count = vsi->num_tx_desc;
11722 ring->size = 0;
11723 ring->dcb_tc = 0;
11724 if (test_bit(I40E_HW_CAP_WB_ON_ITR, vsi->back->hw.caps))
11725 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11726 set_ring_xdp(ring);
11727 ring->itr_setting = pf->tx_itr_default;
11728 WRITE_ONCE(vsi->xdp_rings[i], ring++);
11729
11730 setup_rx:
11731 ring->queue_index = i;
11732 ring->reg_idx = vsi->base_queue + i;
11733 ring->ring_active = false;
11734 ring->vsi = vsi;
11735 ring->netdev = vsi->netdev;
11736 ring->dev = &pf->pdev->dev;
11737 ring->count = vsi->num_rx_desc;
11738 ring->size = 0;
11739 ring->dcb_tc = 0;
11740 ring->itr_setting = pf->rx_itr_default;
11741 WRITE_ONCE(vsi->rx_rings[i], ring);
11742 }
11743
11744 return 0;
11745
11746 err_out:
11747 i40e_vsi_clear_rings(vsi);
11748 return -ENOMEM;
11749 }
11750
11751 /**
11752 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11753 * @pf: board private structure
11754 * @vectors: the number of MSI-X vectors to request
11755 *
11756 * Returns the number of vectors reserved, or error
11757 **/
i40e_reserve_msix_vectors(struct i40e_pf * pf,int vectors)11758 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11759 {
11760 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11761 I40E_MIN_MSIX, vectors);
11762 if (vectors < 0) {
11763 dev_info(&pf->pdev->dev,
11764 "MSI-X vector reservation failed: %d\n", vectors);
11765 vectors = 0;
11766 }
11767
11768 return vectors;
11769 }
11770
11771 /**
11772 * i40e_init_msix - Setup the MSIX capability
11773 * @pf: board private structure
11774 *
11775 * Work with the OS to set up the MSIX vectors needed.
11776 *
11777 * Returns the number of vectors reserved or negative on failure
11778 **/
i40e_init_msix(struct i40e_pf * pf)11779 static int i40e_init_msix(struct i40e_pf *pf)
11780 {
11781 struct i40e_hw *hw = &pf->hw;
11782 int cpus, extra_vectors;
11783 int vectors_left;
11784 int v_budget, i;
11785 int v_actual;
11786 int iwarp_requested = 0;
11787
11788 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
11789 return -ENODEV;
11790
11791 /* The number of vectors we'll request will be comprised of:
11792 * - Add 1 for "other" cause for Admin Queue events, etc.
11793 * - The number of LAN queue pairs
11794 * - Queues being used for RSS.
11795 * We don't need as many as max_rss_size vectors.
11796 * use rss_size instead in the calculation since that
11797 * is governed by number of cpus in the system.
11798 * - assumes symmetric Tx/Rx pairing
11799 * - The number of VMDq pairs
11800 * - The CPU count within the NUMA node if iWARP is enabled
11801 * Once we count this up, try the request.
11802 *
11803 * If we can't get what we want, we'll simplify to nearly nothing
11804 * and try again. If that still fails, we punt.
11805 */
11806 vectors_left = hw->func_caps.num_msix_vectors;
11807 v_budget = 0;
11808
11809 /* reserve one vector for miscellaneous handler */
11810 if (vectors_left) {
11811 v_budget++;
11812 vectors_left--;
11813 }
11814
11815 /* reserve some vectors for the main PF traffic queues. Initially we
11816 * only reserve at most 50% of the available vectors, in the case that
11817 * the number of online CPUs is large. This ensures that we can enable
11818 * extra features as well. Once we've enabled the other features, we
11819 * will use any remaining vectors to reach as close as we can to the
11820 * number of online CPUs.
11821 */
11822 cpus = num_online_cpus();
11823 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11824 vectors_left -= pf->num_lan_msix;
11825
11826 /* reserve one vector for sideband flow director */
11827 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
11828 if (vectors_left) {
11829 pf->num_fdsb_msix = 1;
11830 v_budget++;
11831 vectors_left--;
11832 } else {
11833 pf->num_fdsb_msix = 0;
11834 }
11835 }
11836
11837 /* can we reserve enough for iWARP? */
11838 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11839 iwarp_requested = pf->num_iwarp_msix;
11840
11841 if (!vectors_left)
11842 pf->num_iwarp_msix = 0;
11843 else if (vectors_left < pf->num_iwarp_msix)
11844 pf->num_iwarp_msix = 1;
11845 v_budget += pf->num_iwarp_msix;
11846 vectors_left -= pf->num_iwarp_msix;
11847 }
11848
11849 /* any vectors left over go for VMDq support */
11850 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags)) {
11851 if (!vectors_left) {
11852 pf->num_vmdq_msix = 0;
11853 pf->num_vmdq_qps = 0;
11854 } else {
11855 int vmdq_vecs_wanted =
11856 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11857 int vmdq_vecs =
11858 min_t(int, vectors_left, vmdq_vecs_wanted);
11859
11860 /* if we're short on vectors for what's desired, we limit
11861 * the queues per vmdq. If this is still more than are
11862 * available, the user will need to change the number of
11863 * queues/vectors used by the PF later with the ethtool
11864 * channels command
11865 */
11866 if (vectors_left < vmdq_vecs_wanted) {
11867 pf->num_vmdq_qps = 1;
11868 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11869 vmdq_vecs = min_t(int,
11870 vectors_left,
11871 vmdq_vecs_wanted);
11872 }
11873 pf->num_vmdq_msix = pf->num_vmdq_qps;
11874
11875 v_budget += vmdq_vecs;
11876 vectors_left -= vmdq_vecs;
11877 }
11878 }
11879
11880 /* On systems with a large number of SMP cores, we previously limited
11881 * the number of vectors for num_lan_msix to be at most 50% of the
11882 * available vectors, to allow for other features. Now, we add back
11883 * the remaining vectors. However, we ensure that the total
11884 * num_lan_msix will not exceed num_online_cpus(). To do this, we
11885 * calculate the number of vectors we can add without going over the
11886 * cap of CPUs. For systems with a small number of CPUs this will be
11887 * zero.
11888 */
11889 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11890 pf->num_lan_msix += extra_vectors;
11891 vectors_left -= extra_vectors;
11892
11893 WARN(vectors_left < 0,
11894 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11895
11896 v_budget += pf->num_lan_msix;
11897 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11898 GFP_KERNEL);
11899 if (!pf->msix_entries)
11900 return -ENOMEM;
11901
11902 for (i = 0; i < v_budget; i++)
11903 pf->msix_entries[i].entry = i;
11904 v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11905
11906 if (v_actual < I40E_MIN_MSIX) {
11907 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
11908 kfree(pf->msix_entries);
11909 pf->msix_entries = NULL;
11910 pci_disable_msix(pf->pdev);
11911 return -ENODEV;
11912
11913 } else if (v_actual == I40E_MIN_MSIX) {
11914 /* Adjust for minimal MSIX use */
11915 pf->num_vmdq_vsis = 0;
11916 pf->num_vmdq_qps = 0;
11917 pf->num_lan_qps = 1;
11918 pf->num_lan_msix = 1;
11919
11920 } else if (v_actual != v_budget) {
11921 /* If we have limited resources, we will start with no vectors
11922 * for the special features and then allocate vectors to some
11923 * of these features based on the policy and at the end disable
11924 * the features that did not get any vectors.
11925 */
11926 int vec;
11927
11928 dev_info(&pf->pdev->dev,
11929 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11930 v_actual, v_budget);
11931 /* reserve the misc vector */
11932 vec = v_actual - 1;
11933
11934 /* Scale vector usage down */
11935 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */
11936 pf->num_vmdq_vsis = 1;
11937 pf->num_vmdq_qps = 1;
11938
11939 /* partition out the remaining vectors */
11940 switch (vec) {
11941 case 2:
11942 pf->num_lan_msix = 1;
11943 break;
11944 case 3:
11945 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11946 pf->num_lan_msix = 1;
11947 pf->num_iwarp_msix = 1;
11948 } else {
11949 pf->num_lan_msix = 2;
11950 }
11951 break;
11952 default:
11953 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
11954 pf->num_iwarp_msix = min_t(int, (vec / 3),
11955 iwarp_requested);
11956 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11957 I40E_DEFAULT_NUM_VMDQ_VSI);
11958 } else {
11959 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11960 I40E_DEFAULT_NUM_VMDQ_VSI);
11961 }
11962 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
11963 pf->num_fdsb_msix = 1;
11964 vec--;
11965 }
11966 pf->num_lan_msix = min_t(int,
11967 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11968 pf->num_lan_msix);
11969 pf->num_lan_qps = pf->num_lan_msix;
11970 break;
11971 }
11972 }
11973
11974 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) && pf->num_fdsb_msix == 0) {
11975 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11976 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
11977 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
11978 }
11979 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) && pf->num_vmdq_msix == 0) {
11980 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11981 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
11982 }
11983
11984 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags) &&
11985 pf->num_iwarp_msix == 0) {
11986 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11987 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
11988 }
11989 i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11990 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11991 pf->num_lan_msix,
11992 pf->num_vmdq_msix * pf->num_vmdq_vsis,
11993 pf->num_fdsb_msix,
11994 pf->num_iwarp_msix);
11995
11996 return v_actual;
11997 }
11998
11999 /**
12000 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
12001 * @vsi: the VSI being configured
12002 * @v_idx: index of the vector in the vsi struct
12003 *
12004 * We allocate one q_vector. If allocation fails we return -ENOMEM.
12005 **/
i40e_vsi_alloc_q_vector(struct i40e_vsi * vsi,int v_idx)12006 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
12007 {
12008 struct i40e_q_vector *q_vector;
12009
12010 /* allocate q_vector */
12011 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
12012 if (!q_vector)
12013 return -ENOMEM;
12014
12015 q_vector->vsi = vsi;
12016 q_vector->v_idx = v_idx;
12017 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
12018
12019 if (vsi->netdev)
12020 netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll);
12021
12022 /* tie q_vector and vsi together */
12023 vsi->q_vectors[v_idx] = q_vector;
12024
12025 return 0;
12026 }
12027
12028 /**
12029 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
12030 * @vsi: the VSI being configured
12031 *
12032 * We allocate one q_vector per queue interrupt. If allocation fails we
12033 * return -ENOMEM.
12034 **/
i40e_vsi_alloc_q_vectors(struct i40e_vsi * vsi)12035 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
12036 {
12037 struct i40e_pf *pf = vsi->back;
12038 int err, v_idx, num_q_vectors;
12039
12040 /* if not MSIX, give the one vector only to the LAN VSI */
12041 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
12042 num_q_vectors = vsi->num_q_vectors;
12043 else if (vsi->type == I40E_VSI_MAIN)
12044 num_q_vectors = 1;
12045 else
12046 return -EINVAL;
12047
12048 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
12049 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
12050 if (err)
12051 goto err_out;
12052 }
12053
12054 return 0;
12055
12056 err_out:
12057 while (v_idx--)
12058 i40e_free_q_vector(vsi, v_idx);
12059
12060 return err;
12061 }
12062
12063 /**
12064 * i40e_init_interrupt_scheme - Determine proper interrupt scheme
12065 * @pf: board private structure to initialize
12066 **/
i40e_init_interrupt_scheme(struct i40e_pf * pf)12067 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
12068 {
12069 int vectors = 0;
12070 ssize_t size;
12071
12072 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12073 vectors = i40e_init_msix(pf);
12074 if (vectors < 0) {
12075 clear_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12076 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
12077 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
12078 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
12079 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
12080 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
12081 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12082 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
12083 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
12084 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12085
12086 /* rework the queue expectations without MSIX */
12087 i40e_determine_queue_usage(pf);
12088 }
12089 }
12090
12091 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
12092 test_bit(I40E_FLAG_MSI_ENA, pf->flags)) {
12093 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
12094 vectors = pci_enable_msi(pf->pdev);
12095 if (vectors < 0) {
12096 dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
12097 vectors);
12098 clear_bit(I40E_FLAG_MSI_ENA, pf->flags);
12099 }
12100 vectors = 1; /* one MSI or Legacy vector */
12101 }
12102
12103 if (!test_bit(I40E_FLAG_MSI_ENA, pf->flags) &&
12104 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
12105 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
12106
12107 /* set up vector assignment tracking */
12108 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
12109 pf->irq_pile = kzalloc(size, GFP_KERNEL);
12110 if (!pf->irq_pile)
12111 return -ENOMEM;
12112
12113 pf->irq_pile->num_entries = vectors;
12114
12115 /* track first vector for misc interrupts, ignore return */
12116 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
12117
12118 return 0;
12119 }
12120
12121 /**
12122 * i40e_restore_interrupt_scheme - Restore the interrupt scheme
12123 * @pf: private board data structure
12124 *
12125 * Restore the interrupt scheme that was cleared when we suspended the
12126 * device. This should be called during resume to re-allocate the q_vectors
12127 * and reacquire IRQs.
12128 */
i40e_restore_interrupt_scheme(struct i40e_pf * pf)12129 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
12130 {
12131 struct i40e_vsi *vsi;
12132 int err, i;
12133
12134 /* We cleared the MSI and MSI-X flags when disabling the old interrupt
12135 * scheme. We need to re-enabled them here in order to attempt to
12136 * re-acquire the MSI or MSI-X vectors
12137 */
12138 set_bit(I40E_FLAG_MSI_ENA, pf->flags);
12139 set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12140
12141 err = i40e_init_interrupt_scheme(pf);
12142 if (err)
12143 return err;
12144
12145 /* Now that we've re-acquired IRQs, we need to remap the vectors and
12146 * rings together again.
12147 */
12148 i40e_pf_for_each_vsi(pf, i, vsi) {
12149 err = i40e_vsi_alloc_q_vectors(vsi);
12150 if (err)
12151 goto err_unwind;
12152
12153 i40e_vsi_map_rings_to_vectors(vsi);
12154 }
12155
12156 err = i40e_setup_misc_vector(pf);
12157 if (err)
12158 goto err_unwind;
12159
12160 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags))
12161 i40e_client_update_msix_info(pf);
12162
12163 return 0;
12164
12165 err_unwind:
12166 while (i--) {
12167 if (pf->vsi[i])
12168 i40e_vsi_free_q_vectors(pf->vsi[i]);
12169 }
12170
12171 return err;
12172 }
12173
12174 /**
12175 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
12176 * non queue events in recovery mode
12177 * @pf: board private structure
12178 *
12179 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
12180 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
12181 * This is handled differently than in recovery mode since no Tx/Rx resources
12182 * are being allocated.
12183 **/
i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf * pf)12184 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
12185 {
12186 int err;
12187
12188 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
12189 err = i40e_setup_misc_vector(pf);
12190
12191 if (err) {
12192 dev_info(&pf->pdev->dev,
12193 "MSI-X misc vector request failed, error %d\n",
12194 err);
12195 return err;
12196 }
12197 } else {
12198 u32 flags = test_bit(I40E_FLAG_MSI_ENA, pf->flags) ? 0 : IRQF_SHARED;
12199
12200 err = request_irq(pf->pdev->irq, i40e_intr, flags,
12201 pf->int_name, pf);
12202
12203 if (err) {
12204 dev_info(&pf->pdev->dev,
12205 "MSI/legacy misc vector request failed, error %d\n",
12206 err);
12207 return err;
12208 }
12209 i40e_enable_misc_int_causes(pf);
12210 i40e_irq_dynamic_enable_icr0(pf);
12211 }
12212
12213 return 0;
12214 }
12215
12216 /**
12217 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
12218 * @pf: board private structure
12219 *
12220 * This sets up the handler for MSIX 0, which is used to manage the
12221 * non-queue interrupts, e.g. AdminQ and errors. This is not used
12222 * when in MSI or Legacy interrupt mode.
12223 **/
i40e_setup_misc_vector(struct i40e_pf * pf)12224 static int i40e_setup_misc_vector(struct i40e_pf *pf)
12225 {
12226 struct i40e_hw *hw = &pf->hw;
12227 int err = 0;
12228
12229 /* Only request the IRQ once, the first time through. */
12230 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
12231 err = request_irq(pf->msix_entries[0].vector,
12232 i40e_intr, 0, pf->int_name, pf);
12233 if (err) {
12234 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
12235 dev_info(&pf->pdev->dev,
12236 "request_irq for %s failed: %d\n",
12237 pf->int_name, err);
12238 return -EFAULT;
12239 }
12240 }
12241
12242 i40e_enable_misc_int_causes(pf);
12243
12244 /* associate no queues to the misc vector */
12245 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
12246 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
12247
12248 i40e_flush(hw);
12249
12250 i40e_irq_dynamic_enable_icr0(pf);
12251
12252 return err;
12253 }
12254
12255 /**
12256 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
12257 * @vsi: Pointer to vsi structure
12258 * @seed: Buffter to store the hash keys
12259 * @lut: Buffer to store the lookup table entries
12260 * @lut_size: Size of buffer to store the lookup table entries
12261 *
12262 * Return 0 on success, negative on failure
12263 */
i40e_get_rss_aq(struct i40e_vsi * vsi,const u8 * seed,u8 * lut,u16 lut_size)12264 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
12265 u8 *lut, u16 lut_size)
12266 {
12267 struct i40e_pf *pf = vsi->back;
12268 struct i40e_hw *hw = &pf->hw;
12269 int ret = 0;
12270
12271 if (seed) {
12272 ret = i40e_aq_get_rss_key(hw, vsi->id,
12273 (struct i40e_aqc_get_set_rss_key_data *)seed);
12274 if (ret) {
12275 dev_info(&pf->pdev->dev,
12276 "Cannot get RSS key, err %pe aq_err %s\n",
12277 ERR_PTR(ret),
12278 i40e_aq_str(&pf->hw,
12279 pf->hw.aq.asq_last_status));
12280 return ret;
12281 }
12282 }
12283
12284 if (lut) {
12285 bool pf_lut = vsi->type == I40E_VSI_MAIN;
12286
12287 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
12288 if (ret) {
12289 dev_info(&pf->pdev->dev,
12290 "Cannot get RSS lut, err %pe aq_err %s\n",
12291 ERR_PTR(ret),
12292 i40e_aq_str(&pf->hw,
12293 pf->hw.aq.asq_last_status));
12294 return ret;
12295 }
12296 }
12297
12298 return ret;
12299 }
12300
12301 /**
12302 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
12303 * @vsi: Pointer to vsi structure
12304 * @seed: RSS hash seed
12305 * @lut: Lookup table
12306 * @lut_size: Lookup table size
12307 *
12308 * Returns 0 on success, negative on failure
12309 **/
i40e_config_rss_reg(struct i40e_vsi * vsi,const u8 * seed,const u8 * lut,u16 lut_size)12310 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
12311 const u8 *lut, u16 lut_size)
12312 {
12313 struct i40e_pf *pf = vsi->back;
12314 struct i40e_hw *hw = &pf->hw;
12315 u16 vf_id = vsi->vf_id;
12316 u8 i;
12317
12318 /* Fill out hash function seed */
12319 if (seed) {
12320 u32 *seed_dw = (u32 *)seed;
12321
12322 if (vsi->type == I40E_VSI_MAIN) {
12323 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12324 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
12325 } else if (vsi->type == I40E_VSI_SRIOV) {
12326 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
12327 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
12328 } else {
12329 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
12330 }
12331 }
12332
12333 if (lut) {
12334 u32 *lut_dw = (u32 *)lut;
12335
12336 if (vsi->type == I40E_VSI_MAIN) {
12337 if (lut_size != I40E_HLUT_ARRAY_SIZE)
12338 return -EINVAL;
12339 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12340 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
12341 } else if (vsi->type == I40E_VSI_SRIOV) {
12342 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
12343 return -EINVAL;
12344 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12345 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
12346 } else {
12347 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12348 }
12349 }
12350 i40e_flush(hw);
12351
12352 return 0;
12353 }
12354
12355 /**
12356 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
12357 * @vsi: Pointer to VSI structure
12358 * @seed: Buffer to store the keys
12359 * @lut: Buffer to store the lookup table entries
12360 * @lut_size: Size of buffer to store the lookup table entries
12361 *
12362 * Returns 0 on success, negative on failure
12363 */
i40e_get_rss_reg(struct i40e_vsi * vsi,u8 * seed,u8 * lut,u16 lut_size)12364 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
12365 u8 *lut, u16 lut_size)
12366 {
12367 struct i40e_pf *pf = vsi->back;
12368 struct i40e_hw *hw = &pf->hw;
12369 u16 i;
12370
12371 if (seed) {
12372 u32 *seed_dw = (u32 *)seed;
12373
12374 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12375 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
12376 }
12377 if (lut) {
12378 u32 *lut_dw = (u32 *)lut;
12379
12380 if (lut_size != I40E_HLUT_ARRAY_SIZE)
12381 return -EINVAL;
12382 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12383 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
12384 }
12385
12386 return 0;
12387 }
12388
12389 /**
12390 * i40e_config_rss - Configure RSS keys and lut
12391 * @vsi: Pointer to VSI structure
12392 * @seed: RSS hash seed
12393 * @lut: Lookup table
12394 * @lut_size: Lookup table size
12395 *
12396 * Returns 0 on success, negative on failure
12397 */
i40e_config_rss(struct i40e_vsi * vsi,u8 * seed,u8 * lut,u16 lut_size)12398 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12399 {
12400 struct i40e_pf *pf = vsi->back;
12401
12402 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12403 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12404 else
12405 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12406 }
12407
12408 /**
12409 * i40e_get_rss - Get RSS keys and lut
12410 * @vsi: Pointer to VSI structure
12411 * @seed: Buffer to store the keys
12412 * @lut: Buffer to store the lookup table entries
12413 * @lut_size: Size of buffer to store the lookup table entries
12414 *
12415 * Returns 0 on success, negative on failure
12416 */
i40e_get_rss(struct i40e_vsi * vsi,u8 * seed,u8 * lut,u16 lut_size)12417 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12418 {
12419 struct i40e_pf *pf = vsi->back;
12420
12421 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps))
12422 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12423 else
12424 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12425 }
12426
12427 /**
12428 * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12429 * @pf: Pointer to board private structure
12430 * @lut: Lookup table
12431 * @rss_table_size: Lookup table size
12432 * @rss_size: Range of queue number for hashing
12433 */
i40e_fill_rss_lut(struct i40e_pf * pf,u8 * lut,u16 rss_table_size,u16 rss_size)12434 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12435 u16 rss_table_size, u16 rss_size)
12436 {
12437 u16 i;
12438
12439 for (i = 0; i < rss_table_size; i++)
12440 lut[i] = i % rss_size;
12441 }
12442
12443 /**
12444 * i40e_pf_config_rss - Prepare for RSS if used
12445 * @pf: board private structure
12446 **/
i40e_pf_config_rss(struct i40e_pf * pf)12447 static int i40e_pf_config_rss(struct i40e_pf *pf)
12448 {
12449 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
12450 u8 seed[I40E_HKEY_ARRAY_SIZE];
12451 u8 *lut;
12452 struct i40e_hw *hw = &pf->hw;
12453 u32 reg_val;
12454 u64 hena;
12455 int ret;
12456
12457 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12458 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12459 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12460 hena |= i40e_pf_get_default_rss_hena(pf);
12461
12462 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
12463 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
12464
12465 /* Determine the RSS table size based on the hardware capabilities */
12466 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12467 reg_val = (pf->rss_table_size == 512) ?
12468 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12469 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12470 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12471
12472 /* Determine the RSS size of the VSI */
12473 if (!vsi->rss_size) {
12474 u16 qcount;
12475 /* If the firmware does something weird during VSI init, we
12476 * could end up with zero TCs. Check for that to avoid
12477 * divide-by-zero. It probably won't pass traffic, but it also
12478 * won't panic.
12479 */
12480 qcount = vsi->num_queue_pairs /
12481 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12482 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12483 }
12484 if (!vsi->rss_size)
12485 return -EINVAL;
12486
12487 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
12488 if (!lut)
12489 return -ENOMEM;
12490
12491 /* Use user configured lut if there is one, otherwise use default */
12492 if (vsi->rss_lut_user)
12493 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12494 else
12495 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
12496
12497 /* Use user configured hash key if there is one, otherwise
12498 * use default.
12499 */
12500 if (vsi->rss_hkey_user)
12501 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12502 else
12503 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
12504 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
12505 kfree(lut);
12506
12507 return ret;
12508 }
12509
12510 /**
12511 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12512 * @pf: board private structure
12513 * @queue_count: the requested queue count for rss.
12514 *
12515 * returns 0 if rss is not enabled, if enabled returns the final rss queue
12516 * count which may be different from the requested queue count.
12517 * Note: expects to be called while under rtnl_lock()
12518 **/
i40e_reconfig_rss_queues(struct i40e_pf * pf,int queue_count)12519 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12520 {
12521 struct i40e_vsi *vsi = i40e_pf_get_main_vsi(pf);
12522 int new_rss_size;
12523
12524 if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags))
12525 return 0;
12526
12527 queue_count = min_t(int, queue_count, num_online_cpus());
12528 new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12529
12530 if (queue_count != vsi->num_queue_pairs) {
12531 u16 qcount;
12532
12533 vsi->req_queue_pairs = queue_count;
12534 i40e_prep_for_reset(pf);
12535 if (test_bit(__I40E_IN_REMOVE, pf->state))
12536 return pf->alloc_rss_size;
12537
12538 pf->alloc_rss_size = new_rss_size;
12539
12540 i40e_reset_and_rebuild(pf, true, true);
12541
12542 /* Discard the user configured hash keys and lut, if less
12543 * queues are enabled.
12544 */
12545 if (queue_count < vsi->rss_size) {
12546 i40e_clear_rss_config_user(vsi);
12547 dev_dbg(&pf->pdev->dev,
12548 "discard user configured hash keys and lut\n");
12549 }
12550
12551 /* Reset vsi->rss_size, as number of enabled queues changed */
12552 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12553 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12554
12555 i40e_pf_config_rss(pf);
12556 }
12557 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n",
12558 vsi->req_queue_pairs, pf->rss_size_max);
12559 return pf->alloc_rss_size;
12560 }
12561
12562 /**
12563 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12564 * @pf: board private structure
12565 **/
i40e_get_partition_bw_setting(struct i40e_pf * pf)12566 int i40e_get_partition_bw_setting(struct i40e_pf *pf)
12567 {
12568 bool min_valid, max_valid;
12569 u32 max_bw, min_bw;
12570 int status;
12571
12572 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
12573 &min_valid, &max_valid);
12574
12575 if (!status) {
12576 if (min_valid)
12577 pf->min_bw = min_bw;
12578 if (max_valid)
12579 pf->max_bw = max_bw;
12580 }
12581
12582 return status;
12583 }
12584
12585 /**
12586 * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12587 * @pf: board private structure
12588 **/
i40e_set_partition_bw_setting(struct i40e_pf * pf)12589 int i40e_set_partition_bw_setting(struct i40e_pf *pf)
12590 {
12591 struct i40e_aqc_configure_partition_bw_data bw_data;
12592 int status;
12593
12594 memset(&bw_data, 0, sizeof(bw_data));
12595
12596 /* Set the valid bit for this PF */
12597 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12598 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12599 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12600
12601 /* Set the new bandwidths */
12602 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
12603
12604 return status;
12605 }
12606
12607 /**
12608 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12609 * @pf: board private structure
12610 **/
i40e_commit_partition_bw_setting(struct i40e_pf * pf)12611 int i40e_commit_partition_bw_setting(struct i40e_pf *pf)
12612 {
12613 /* Commit temporary BW setting to permanent NVM image */
12614 enum i40e_admin_queue_err last_aq_status;
12615 u16 nvm_word;
12616 int ret;
12617
12618 if (pf->hw.partition_id != 1) {
12619 dev_info(&pf->pdev->dev,
12620 "Commit BW only works on partition 1! This is partition %d",
12621 pf->hw.partition_id);
12622 ret = -EOPNOTSUPP;
12623 goto bw_commit_out;
12624 }
12625
12626 /* Acquire NVM for read access */
12627 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
12628 last_aq_status = pf->hw.aq.asq_last_status;
12629 if (ret) {
12630 dev_info(&pf->pdev->dev,
12631 "Cannot acquire NVM for read access, err %pe aq_err %s\n",
12632 ERR_PTR(ret),
12633 i40e_aq_str(&pf->hw, last_aq_status));
12634 goto bw_commit_out;
12635 }
12636
12637 /* Read word 0x10 of NVM - SW compatibility word 1 */
12638 ret = i40e_aq_read_nvm(&pf->hw,
12639 I40E_SR_NVM_CONTROL_WORD,
12640 0x10, sizeof(nvm_word), &nvm_word,
12641 false, NULL);
12642 /* Save off last admin queue command status before releasing
12643 * the NVM
12644 */
12645 last_aq_status = pf->hw.aq.asq_last_status;
12646 i40e_release_nvm(&pf->hw);
12647 if (ret) {
12648 dev_info(&pf->pdev->dev, "NVM read error, err %pe aq_err %s\n",
12649 ERR_PTR(ret),
12650 i40e_aq_str(&pf->hw, last_aq_status));
12651 goto bw_commit_out;
12652 }
12653
12654 /* Wait a bit for NVM release to complete */
12655 msleep(50);
12656
12657 /* Acquire NVM for write access */
12658 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
12659 last_aq_status = pf->hw.aq.asq_last_status;
12660 if (ret) {
12661 dev_info(&pf->pdev->dev,
12662 "Cannot acquire NVM for write access, err %pe aq_err %s\n",
12663 ERR_PTR(ret),
12664 i40e_aq_str(&pf->hw, last_aq_status));
12665 goto bw_commit_out;
12666 }
12667 /* Write it back out unchanged to initiate update NVM,
12668 * which will force a write of the shadow (alt) RAM to
12669 * the NVM - thus storing the bandwidth values permanently.
12670 */
12671 ret = i40e_aq_update_nvm(&pf->hw,
12672 I40E_SR_NVM_CONTROL_WORD,
12673 0x10, sizeof(nvm_word),
12674 &nvm_word, true, 0, NULL);
12675 /* Save off last admin queue command status before releasing
12676 * the NVM
12677 */
12678 last_aq_status = pf->hw.aq.asq_last_status;
12679 i40e_release_nvm(&pf->hw);
12680 if (ret)
12681 dev_info(&pf->pdev->dev,
12682 "BW settings NOT SAVED, err %pe aq_err %s\n",
12683 ERR_PTR(ret),
12684 i40e_aq_str(&pf->hw, last_aq_status));
12685 bw_commit_out:
12686
12687 return ret;
12688 }
12689
12690 /**
12691 * i40e_is_total_port_shutdown_enabled - read NVM and return value
12692 * if total port shutdown feature is enabled for this PF
12693 * @pf: board private structure
12694 **/
i40e_is_total_port_shutdown_enabled(struct i40e_pf * pf)12695 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12696 {
12697 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4)
12698 #define I40E_FEATURES_ENABLE_PTR 0x2A
12699 #define I40E_CURRENT_SETTING_PTR 0x2B
12700 #define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D
12701 #define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1
12702 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0)
12703 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4
12704 u16 sr_emp_sr_settings_ptr = 0;
12705 u16 features_enable = 0;
12706 u16 link_behavior = 0;
12707 int read_status = 0;
12708 bool ret = false;
12709
12710 read_status = i40e_read_nvm_word(&pf->hw,
12711 I40E_SR_EMP_SR_SETTINGS_PTR,
12712 &sr_emp_sr_settings_ptr);
12713 if (read_status)
12714 goto err_nvm;
12715 read_status = i40e_read_nvm_word(&pf->hw,
12716 sr_emp_sr_settings_ptr +
12717 I40E_FEATURES_ENABLE_PTR,
12718 &features_enable);
12719 if (read_status)
12720 goto err_nvm;
12721 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12722 read_status = i40e_read_nvm_module_data(&pf->hw,
12723 I40E_SR_EMP_SR_SETTINGS_PTR,
12724 I40E_CURRENT_SETTING_PTR,
12725 I40E_LINK_BEHAVIOR_WORD_OFFSET,
12726 I40E_LINK_BEHAVIOR_WORD_LENGTH,
12727 &link_behavior);
12728 if (read_status)
12729 goto err_nvm;
12730 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12731 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12732 }
12733 return ret;
12734
12735 err_nvm:
12736 dev_warn(&pf->pdev->dev,
12737 "total-port-shutdown feature is off due to read nvm error: %pe\n",
12738 ERR_PTR(read_status));
12739 return ret;
12740 }
12741
12742 /**
12743 * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12744 * @pf: board private structure to initialize
12745 *
12746 * i40e_sw_init initializes the Adapter private data structure.
12747 * Fields are initialized based on PCI device information and
12748 * OS network device settings (MTU size).
12749 **/
i40e_sw_init(struct i40e_pf * pf)12750 static int i40e_sw_init(struct i40e_pf *pf)
12751 {
12752 int err = 0;
12753 int size;
12754 u16 pow;
12755
12756 /* Set default capability flags */
12757 bitmap_zero(pf->flags, I40E_PF_FLAGS_NBITS);
12758 set_bit(I40E_FLAG_MSI_ENA, pf->flags);
12759 set_bit(I40E_FLAG_MSIX_ENA, pf->flags);
12760
12761 /* Set default ITR */
12762 pf->rx_itr_default = I40E_ITR_RX_DEF;
12763 pf->tx_itr_default = I40E_ITR_TX_DEF;
12764
12765 /* Depending on PF configurations, it is possible that the RSS
12766 * maximum might end up larger than the available queues
12767 */
12768 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12769 pf->alloc_rss_size = 1;
12770 pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12771 pf->rss_size_max = min_t(int, pf->rss_size_max,
12772 pf->hw.func_caps.num_tx_qp);
12773
12774 /* find the next higher power-of-2 of num cpus */
12775 pow = roundup_pow_of_two(num_online_cpus());
12776 pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12777
12778 if (pf->hw.func_caps.rss) {
12779 set_bit(I40E_FLAG_RSS_ENA, pf->flags);
12780 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12781 num_online_cpus());
12782 }
12783
12784 /* MFP mode enabled */
12785 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12786 set_bit(I40E_FLAG_MFP_ENA, pf->flags);
12787 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12788 if (i40e_get_partition_bw_setting(pf)) {
12789 dev_warn(&pf->pdev->dev,
12790 "Could not get partition bw settings\n");
12791 } else {
12792 dev_info(&pf->pdev->dev,
12793 "Partition BW Min = %8.8x, Max = %8.8x\n",
12794 pf->min_bw, pf->max_bw);
12795
12796 /* nudge the Tx scheduler */
12797 i40e_set_partition_bw_setting(pf);
12798 }
12799 }
12800
12801 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12802 (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12803 set_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
12804 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
12805 pf->hw.num_partitions > 1)
12806 dev_info(&pf->pdev->dev,
12807 "Flow Director Sideband mode Disabled in MFP mode\n");
12808 else
12809 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12810 pf->fdir_pf_filter_count =
12811 pf->hw.func_caps.fd_filters_guaranteed;
12812 pf->hw.fdir_shared_filter_count =
12813 pf->hw.func_caps.fd_filters_best_effort;
12814 }
12815
12816 /* Enable HW ATR eviction if possible */
12817 if (test_bit(I40E_HW_CAP_ATR_EVICT, pf->hw.caps))
12818 set_bit(I40E_FLAG_HW_ATR_EVICT_ENA, pf->flags);
12819
12820 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12821 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12822 set_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
12823 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12824 }
12825
12826 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12827 set_bit(I40E_FLAG_IWARP_ENA, pf->flags);
12828 /* IWARP needs one extra vector for CQP just like MISC.*/
12829 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12830 }
12831 /* Stopping FW LLDP engine is supported on XL710 and X722
12832 * starting from FW versions determined in i40e_init_adminq.
12833 * Stopping the FW LLDP engine is not supported on XL710
12834 * if NPAR is functioning so unset this hw flag in this case.
12835 */
12836 if (pf->hw.mac.type == I40E_MAC_XL710 &&
12837 pf->hw.func_caps.npar_enable)
12838 clear_bit(I40E_HW_CAP_FW_LLDP_STOPPABLE, pf->hw.caps);
12839
12840 #ifdef CONFIG_PCI_IOV
12841 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12842 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12843 set_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
12844 pf->num_req_vfs = min_t(int,
12845 pf->hw.func_caps.num_vfs,
12846 I40E_MAX_VF_COUNT);
12847 }
12848 #endif /* CONFIG_PCI_IOV */
12849 pf->lan_veb = I40E_NO_VEB;
12850 pf->lan_vsi = I40E_NO_VSI;
12851
12852 /* By default FW has this off for performance reasons */
12853 clear_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
12854
12855 /* set up queue assignment tracking */
12856 size = sizeof(struct i40e_lump_tracking)
12857 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12858 pf->qp_pile = kzalloc(size, GFP_KERNEL);
12859 if (!pf->qp_pile) {
12860 err = -ENOMEM;
12861 goto sw_init_done;
12862 }
12863 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12864
12865 pf->tx_timeout_recovery_level = 1;
12866
12867 if (pf->hw.mac.type != I40E_MAC_X722 &&
12868 i40e_is_total_port_shutdown_enabled(pf)) {
12869 /* Link down on close must be on when total port shutdown
12870 * is enabled for a given port
12871 */
12872 set_bit(I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);
12873 set_bit(I40E_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);
12874 dev_info(&pf->pdev->dev,
12875 "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12876 }
12877 mutex_init(&pf->switch_mutex);
12878
12879 sw_init_done:
12880 return err;
12881 }
12882
12883 /**
12884 * i40e_set_ntuple - set the ntuple feature flag and take action
12885 * @pf: board private structure to initialize
12886 * @features: the feature set that the stack is suggesting
12887 *
12888 * returns a bool to indicate if reset needs to happen
12889 **/
i40e_set_ntuple(struct i40e_pf * pf,netdev_features_t features)12890 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12891 {
12892 bool need_reset = false;
12893
12894 /* Check if Flow Director n-tuple support was enabled or disabled. If
12895 * the state changed, we need to reset.
12896 */
12897 if (features & NETIF_F_NTUPLE) {
12898 /* Enable filters and mark for reset */
12899 if (!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags))
12900 need_reset = true;
12901 /* enable FD_SB only if there is MSI-X vector and no cloud
12902 * filters exist
12903 */
12904 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12905 set_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12906 clear_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12907 }
12908 } else {
12909 /* turn off filters, mark for reset and clear SW filter list */
12910 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
12911 need_reset = true;
12912 i40e_fdir_filter_exit(pf);
12913 }
12914 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
12915 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12916 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
12917
12918 /* reset fd counters */
12919 pf->fd_add_err = 0;
12920 pf->fd_atr_cnt = 0;
12921 /* if ATR was auto disabled it can be re-enabled. */
12922 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12923 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
12924 (I40E_DEBUG_FD & pf->hw.debug_mask))
12925 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12926 }
12927 return need_reset;
12928 }
12929
12930 /**
12931 * i40e_clear_rss_lut - clear the rx hash lookup table
12932 * @vsi: the VSI being configured
12933 **/
i40e_clear_rss_lut(struct i40e_vsi * vsi)12934 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12935 {
12936 struct i40e_pf *pf = vsi->back;
12937 struct i40e_hw *hw = &pf->hw;
12938 u16 vf_id = vsi->vf_id;
12939 u8 i;
12940
12941 if (vsi->type == I40E_VSI_MAIN) {
12942 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12943 wr32(hw, I40E_PFQF_HLUT(i), 0);
12944 } else if (vsi->type == I40E_VSI_SRIOV) {
12945 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12946 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12947 } else {
12948 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12949 }
12950 }
12951
12952 /**
12953 * i40e_set_loopback - turn on/off loopback mode on underlying PF
12954 * @vsi: ptr to VSI
12955 * @ena: flag to indicate the on/off setting
12956 */
i40e_set_loopback(struct i40e_vsi * vsi,bool ena)12957 static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena)
12958 {
12959 bool if_running = netif_running(vsi->netdev) &&
12960 !test_and_set_bit(__I40E_VSI_DOWN, vsi->state);
12961 int ret;
12962
12963 if (if_running)
12964 i40e_down(vsi);
12965
12966 ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);
12967 if (ret)
12968 netdev_err(vsi->netdev, "Failed to toggle loopback state\n");
12969 if (if_running)
12970 i40e_up(vsi);
12971
12972 return ret;
12973 }
12974
12975 /**
12976 * i40e_set_features - set the netdev feature flags
12977 * @netdev: ptr to the netdev being adjusted
12978 * @features: the feature set that the stack is suggesting
12979 * Note: expects to be called while under rtnl_lock()
12980 **/
i40e_set_features(struct net_device * netdev,netdev_features_t features)12981 static int i40e_set_features(struct net_device *netdev,
12982 netdev_features_t features)
12983 {
12984 struct i40e_netdev_priv *np = netdev_priv(netdev);
12985 struct i40e_vsi *vsi = np->vsi;
12986 struct i40e_pf *pf = vsi->back;
12987 bool need_reset;
12988
12989 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12990 i40e_pf_config_rss(pf);
12991 else if (!(features & NETIF_F_RXHASH) &&
12992 netdev->features & NETIF_F_RXHASH)
12993 i40e_clear_rss_lut(vsi);
12994
12995 if (features & NETIF_F_HW_VLAN_CTAG_RX)
12996 i40e_vlan_stripping_enable(vsi);
12997 else
12998 i40e_vlan_stripping_disable(vsi);
12999
13000 if (!(features & NETIF_F_HW_TC) &&
13001 (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
13002 dev_err(&pf->pdev->dev,
13003 "Offloaded tc filters active, can't turn hw_tc_offload off");
13004 return -EINVAL;
13005 }
13006
13007 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
13008 i40e_del_all_macvlans(vsi);
13009
13010 need_reset = i40e_set_ntuple(pf, features);
13011
13012 if (need_reset)
13013 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
13014
13015 if ((features ^ netdev->features) & NETIF_F_LOOPBACK)
13016 return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK));
13017
13018 return 0;
13019 }
13020
i40e_udp_tunnel_set_port(struct net_device * netdev,unsigned int table,unsigned int idx,struct udp_tunnel_info * ti)13021 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
13022 unsigned int table, unsigned int idx,
13023 struct udp_tunnel_info *ti)
13024 {
13025 struct i40e_netdev_priv *np = netdev_priv(netdev);
13026 struct i40e_hw *hw = &np->vsi->back->hw;
13027 u8 type, filter_index;
13028 int ret;
13029
13030 type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
13031 I40E_AQC_TUNNEL_TYPE_NGE;
13032
13033 ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
13034 NULL);
13035 if (ret) {
13036 netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n",
13037 ERR_PTR(ret),
13038 i40e_aq_str(hw, hw->aq.asq_last_status));
13039 return -EIO;
13040 }
13041
13042 udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
13043 return 0;
13044 }
13045
i40e_udp_tunnel_unset_port(struct net_device * netdev,unsigned int table,unsigned int idx,struct udp_tunnel_info * ti)13046 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
13047 unsigned int table, unsigned int idx,
13048 struct udp_tunnel_info *ti)
13049 {
13050 struct i40e_netdev_priv *np = netdev_priv(netdev);
13051 struct i40e_hw *hw = &np->vsi->back->hw;
13052 int ret;
13053
13054 ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
13055 if (ret) {
13056 netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n",
13057 ERR_PTR(ret),
13058 i40e_aq_str(hw, hw->aq.asq_last_status));
13059 return -EIO;
13060 }
13061
13062 return 0;
13063 }
13064
i40e_get_phys_port_id(struct net_device * netdev,struct netdev_phys_item_id * ppid)13065 static int i40e_get_phys_port_id(struct net_device *netdev,
13066 struct netdev_phys_item_id *ppid)
13067 {
13068 struct i40e_netdev_priv *np = netdev_priv(netdev);
13069 struct i40e_pf *pf = np->vsi->back;
13070 struct i40e_hw *hw = &pf->hw;
13071
13072 if (!test_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps))
13073 return -EOPNOTSUPP;
13074
13075 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
13076 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
13077
13078 return 0;
13079 }
13080
13081 /**
13082 * i40e_ndo_fdb_add - add an entry to the hardware database
13083 * @ndm: the input from the stack
13084 * @tb: pointer to array of nladdr (unused)
13085 * @dev: the net device pointer
13086 * @addr: the MAC address entry being added
13087 * @vid: VLAN ID
13088 * @flags: instructions from stack about fdb operation
13089 * @extack: netlink extended ack, unused currently
13090 */
i40e_ndo_fdb_add(struct ndmsg * ndm,struct nlattr * tb[],struct net_device * dev,const unsigned char * addr,u16 vid,u16 flags,struct netlink_ext_ack * extack)13091 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
13092 struct net_device *dev,
13093 const unsigned char *addr, u16 vid,
13094 u16 flags,
13095 struct netlink_ext_ack *extack)
13096 {
13097 struct i40e_netdev_priv *np = netdev_priv(dev);
13098 struct i40e_pf *pf = np->vsi->back;
13099 int err = 0;
13100
13101 if (!test_bit(I40E_FLAG_SRIOV_ENA, pf->flags))
13102 return -EOPNOTSUPP;
13103
13104 if (vid) {
13105 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
13106 return -EINVAL;
13107 }
13108
13109 /* Hardware does not support aging addresses so if a
13110 * ndm_state is given only allow permanent addresses
13111 */
13112 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
13113 netdev_info(dev, "FDB only supports static addresses\n");
13114 return -EINVAL;
13115 }
13116
13117 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
13118 err = dev_uc_add_excl(dev, addr);
13119 else if (is_multicast_ether_addr(addr))
13120 err = dev_mc_add_excl(dev, addr);
13121 else
13122 err = -EINVAL;
13123
13124 /* Only return duplicate errors if NLM_F_EXCL is set */
13125 if (err == -EEXIST && !(flags & NLM_F_EXCL))
13126 err = 0;
13127
13128 return err;
13129 }
13130
13131 /**
13132 * i40e_ndo_bridge_setlink - Set the hardware bridge mode
13133 * @dev: the netdev being configured
13134 * @nlh: RTNL message
13135 * @flags: bridge flags
13136 * @extack: netlink extended ack
13137 *
13138 * Inserts a new hardware bridge if not already created and
13139 * enables the bridging mode requested (VEB or VEPA). If the
13140 * hardware bridge has already been inserted and the request
13141 * is to change the mode then that requires a PF reset to
13142 * allow rebuild of the components with required hardware
13143 * bridge mode enabled.
13144 *
13145 * Note: expects to be called while under rtnl_lock()
13146 **/
i40e_ndo_bridge_setlink(struct net_device * dev,struct nlmsghdr * nlh,u16 flags,struct netlink_ext_ack * extack)13147 static int i40e_ndo_bridge_setlink(struct net_device *dev,
13148 struct nlmsghdr *nlh,
13149 u16 flags,
13150 struct netlink_ext_ack *extack)
13151 {
13152 struct i40e_netdev_priv *np = netdev_priv(dev);
13153 struct i40e_vsi *vsi = np->vsi;
13154 struct i40e_pf *pf = vsi->back;
13155 struct nlattr *attr, *br_spec;
13156 struct i40e_veb *veb;
13157 int rem;
13158
13159 /* Only for PF VSI for now */
13160 if (vsi->type != I40E_VSI_MAIN)
13161 return -EOPNOTSUPP;
13162
13163 /* Find the HW bridge for PF VSI */
13164 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
13165
13166 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13167 if (!br_spec)
13168 return -EINVAL;
13169
13170 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {
13171 __u16 mode = nla_get_u16(attr);
13172
13173 if ((mode != BRIDGE_MODE_VEPA) &&
13174 (mode != BRIDGE_MODE_VEB))
13175 return -EINVAL;
13176
13177 /* Insert a new HW bridge */
13178 if (!veb) {
13179 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,
13180 vsi->tc_config.enabled_tc);
13181 if (veb) {
13182 veb->bridge_mode = mode;
13183 i40e_config_bridge_mode(veb);
13184 } else {
13185 /* No Bridge HW offload available */
13186 return -ENOENT;
13187 }
13188 break;
13189 } else if (mode != veb->bridge_mode) {
13190 /* Existing HW bridge but different mode needs reset */
13191 veb->bridge_mode = mode;
13192 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
13193 if (mode == BRIDGE_MODE_VEB)
13194 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
13195 else
13196 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
13197 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
13198 break;
13199 }
13200 }
13201
13202 return 0;
13203 }
13204
13205 /**
13206 * i40e_ndo_bridge_getlink - Get the hardware bridge mode
13207 * @skb: skb buff
13208 * @pid: process id
13209 * @seq: RTNL message seq #
13210 * @dev: the netdev being configured
13211 * @filter_mask: unused
13212 * @nlflags: netlink flags passed in
13213 *
13214 * Return the mode in which the hardware bridge is operating in
13215 * i.e VEB or VEPA.
13216 **/
i40e_ndo_bridge_getlink(struct sk_buff * skb,u32 pid,u32 seq,struct net_device * dev,u32 __always_unused filter_mask,int nlflags)13217 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13218 struct net_device *dev,
13219 u32 __always_unused filter_mask,
13220 int nlflags)
13221 {
13222 struct i40e_netdev_priv *np = netdev_priv(dev);
13223 struct i40e_vsi *vsi = np->vsi;
13224 struct i40e_pf *pf = vsi->back;
13225 struct i40e_veb *veb;
13226
13227 /* Only for PF VSI for now */
13228 if (vsi->type != I40E_VSI_MAIN)
13229 return -EOPNOTSUPP;
13230
13231 /* Find the HW bridge for the PF VSI */
13232 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
13233 if (!veb)
13234 return 0;
13235
13236 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
13237 0, 0, nlflags, filter_mask, NULL);
13238 }
13239
13240 /**
13241 * i40e_features_check - Validate encapsulated packet conforms to limits
13242 * @skb: skb buff
13243 * @dev: This physical port's netdev
13244 * @features: Offload features that the stack believes apply
13245 **/
i40e_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)13246 static netdev_features_t i40e_features_check(struct sk_buff *skb,
13247 struct net_device *dev,
13248 netdev_features_t features)
13249 {
13250 size_t len;
13251
13252 /* No point in doing any of this if neither checksum nor GSO are
13253 * being requested for this frame. We can rule out both by just
13254 * checking for CHECKSUM_PARTIAL
13255 */
13256 if (skb->ip_summed != CHECKSUM_PARTIAL)
13257 return features;
13258
13259 /* We cannot support GSO if the MSS is going to be less than
13260 * 64 bytes. If it is then we need to drop support for GSO.
13261 */
13262 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
13263 features &= ~NETIF_F_GSO_MASK;
13264
13265 /* MACLEN can support at most 63 words */
13266 len = skb_network_offset(skb);
13267 if (len & ~(63 * 2))
13268 goto out_err;
13269
13270 /* IPLEN and EIPLEN can support at most 127 dwords */
13271 len = skb_network_header_len(skb);
13272 if (len & ~(127 * 4))
13273 goto out_err;
13274
13275 if (skb->encapsulation) {
13276 /* L4TUNLEN can support 127 words */
13277 len = skb_inner_network_header(skb) - skb_transport_header(skb);
13278 if (len & ~(127 * 2))
13279 goto out_err;
13280
13281 /* IPLEN can support at most 127 dwords */
13282 len = skb_inner_transport_header(skb) -
13283 skb_inner_network_header(skb);
13284 if (len & ~(127 * 4))
13285 goto out_err;
13286 }
13287
13288 /* No need to validate L4LEN as TCP is the only protocol with a
13289 * flexible value and we support all possible values supported
13290 * by TCP, which is at most 15 dwords
13291 */
13292
13293 return features;
13294 out_err:
13295 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13296 }
13297
13298 /**
13299 * i40e_xdp_setup - add/remove an XDP program
13300 * @vsi: VSI to changed
13301 * @prog: XDP program
13302 * @extack: netlink extended ack
13303 **/
i40e_xdp_setup(struct i40e_vsi * vsi,struct bpf_prog * prog,struct netlink_ext_ack * extack)13304 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
13305 struct netlink_ext_ack *extack)
13306 {
13307 int frame_size = i40e_max_vsi_frame_size(vsi, prog);
13308 struct i40e_pf *pf = vsi->back;
13309 struct bpf_prog *old_prog;
13310 bool need_reset;
13311 int i;
13312
13313 /* VSI shall be deleted in a moment, block loading new programs */
13314 if (prog && test_bit(__I40E_IN_REMOVE, pf->state))
13315 return -EINVAL;
13316
13317 /* Don't allow frames that span over multiple buffers */
13318 if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) {
13319 NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags");
13320 return -EINVAL;
13321 }
13322
13323 /* When turning XDP on->off/off->on we reset and rebuild the rings. */
13324 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
13325 if (need_reset)
13326 i40e_prep_for_reset(pf);
13327
13328 old_prog = xchg(&vsi->xdp_prog, prog);
13329
13330 if (need_reset) {
13331 if (!prog) {
13332 xdp_features_clear_redirect_target(vsi->netdev);
13333 /* Wait until ndo_xsk_wakeup completes. */
13334 synchronize_rcu();
13335 }
13336 i40e_reset_and_rebuild(pf, true, true);
13337 }
13338
13339 if (!i40e_enabled_xdp_vsi(vsi) && prog) {
13340 if (i40e_realloc_rx_bi_zc(vsi, true))
13341 return -ENOMEM;
13342 } else if (i40e_enabled_xdp_vsi(vsi) && !prog) {
13343 if (i40e_realloc_rx_bi_zc(vsi, false))
13344 return -ENOMEM;
13345 }
13346
13347 for (i = 0; i < vsi->num_queue_pairs; i++)
13348 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
13349
13350 if (old_prog)
13351 bpf_prog_put(old_prog);
13352
13353 /* Kick start the NAPI context if there is an AF_XDP socket open
13354 * on that queue id. This so that receiving will start.
13355 */
13356 if (need_reset && prog) {
13357 for (i = 0; i < vsi->num_queue_pairs; i++)
13358 if (vsi->xdp_rings[i]->xsk_pool)
13359 (void)i40e_xsk_wakeup(vsi->netdev, i,
13360 XDP_WAKEUP_RX);
13361 xdp_features_set_redirect_target(vsi->netdev, true);
13362 }
13363
13364 return 0;
13365 }
13366
13367 /**
13368 * i40e_enter_busy_conf - Enters busy config state
13369 * @vsi: vsi
13370 *
13371 * Returns 0 on success, <0 for failure.
13372 **/
i40e_enter_busy_conf(struct i40e_vsi * vsi)13373 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
13374 {
13375 struct i40e_pf *pf = vsi->back;
13376 int timeout = 50;
13377
13378 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
13379 timeout--;
13380 if (!timeout)
13381 return -EBUSY;
13382 usleep_range(1000, 2000);
13383 }
13384
13385 return 0;
13386 }
13387
13388 /**
13389 * i40e_exit_busy_conf - Exits busy config state
13390 * @vsi: vsi
13391 **/
i40e_exit_busy_conf(struct i40e_vsi * vsi)13392 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13393 {
13394 struct i40e_pf *pf = vsi->back;
13395
13396 clear_bit(__I40E_CONFIG_BUSY, pf->state);
13397 }
13398
13399 /**
13400 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13401 * @vsi: vsi
13402 * @queue_pair: queue pair
13403 **/
i40e_queue_pair_reset_stats(struct i40e_vsi * vsi,int queue_pair)13404 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13405 {
13406 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13407 sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13408 memset(&vsi->tx_rings[queue_pair]->stats, 0,
13409 sizeof(vsi->tx_rings[queue_pair]->stats));
13410 if (i40e_enabled_xdp_vsi(vsi)) {
13411 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13412 sizeof(vsi->xdp_rings[queue_pair]->stats));
13413 }
13414 }
13415
13416 /**
13417 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13418 * @vsi: vsi
13419 * @queue_pair: queue pair
13420 **/
i40e_queue_pair_clean_rings(struct i40e_vsi * vsi,int queue_pair)13421 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13422 {
13423 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
13424 if (i40e_enabled_xdp_vsi(vsi)) {
13425 /* Make sure that in-progress ndo_xdp_xmit calls are
13426 * completed.
13427 */
13428 synchronize_rcu();
13429 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
13430 }
13431 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13432 }
13433
13434 /**
13435 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13436 * @vsi: vsi
13437 * @queue_pair: queue pair
13438 * @enable: true for enable, false for disable
13439 **/
i40e_queue_pair_toggle_napi(struct i40e_vsi * vsi,int queue_pair,bool enable)13440 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13441 bool enable)
13442 {
13443 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13444 struct i40e_q_vector *q_vector = rxr->q_vector;
13445
13446 if (!vsi->netdev)
13447 return;
13448
13449 /* All rings in a qp belong to the same qvector. */
13450 if (q_vector->rx.ring || q_vector->tx.ring) {
13451 if (enable)
13452 napi_enable(&q_vector->napi);
13453 else
13454 napi_disable(&q_vector->napi);
13455 }
13456 }
13457
13458 /**
13459 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13460 * @vsi: vsi
13461 * @queue_pair: queue pair
13462 * @enable: true for enable, false for disable
13463 *
13464 * Returns 0 on success, <0 on failure.
13465 **/
i40e_queue_pair_toggle_rings(struct i40e_vsi * vsi,int queue_pair,bool enable)13466 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13467 bool enable)
13468 {
13469 struct i40e_pf *pf = vsi->back;
13470 int pf_q, ret = 0;
13471
13472 pf_q = vsi->base_queue + queue_pair;
13473 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
13474 false /*is xdp*/, enable);
13475 if (ret) {
13476 dev_info(&pf->pdev->dev,
13477 "VSI seid %d Tx ring %d %sable timeout\n",
13478 vsi->seid, pf_q, (enable ? "en" : "dis"));
13479 return ret;
13480 }
13481
13482 i40e_control_rx_q(pf, pf_q, enable);
13483 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13484 if (ret) {
13485 dev_info(&pf->pdev->dev,
13486 "VSI seid %d Rx ring %d %sable timeout\n",
13487 vsi->seid, pf_q, (enable ? "en" : "dis"));
13488 return ret;
13489 }
13490
13491 /* Due to HW errata, on Rx disable only, the register can
13492 * indicate done before it really is. Needs 50ms to be sure
13493 */
13494 if (!enable)
13495 mdelay(50);
13496
13497 if (!i40e_enabled_xdp_vsi(vsi))
13498 return ret;
13499
13500 ret = i40e_control_wait_tx_q(vsi->seid, pf,
13501 pf_q + vsi->alloc_queue_pairs,
13502 true /*is xdp*/, enable);
13503 if (ret) {
13504 dev_info(&pf->pdev->dev,
13505 "VSI seid %d XDP Tx ring %d %sable timeout\n",
13506 vsi->seid, pf_q, (enable ? "en" : "dis"));
13507 }
13508
13509 return ret;
13510 }
13511
13512 /**
13513 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13514 * @vsi: vsi
13515 * @queue_pair: queue_pair
13516 **/
i40e_queue_pair_enable_irq(struct i40e_vsi * vsi,int queue_pair)13517 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13518 {
13519 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13520 struct i40e_pf *pf = vsi->back;
13521 struct i40e_hw *hw = &pf->hw;
13522
13523 /* All rings in a qp belong to the same qvector. */
13524 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
13525 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
13526 else
13527 i40e_irq_dynamic_enable_icr0(pf);
13528
13529 i40e_flush(hw);
13530 }
13531
13532 /**
13533 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13534 * @vsi: vsi
13535 * @queue_pair: queue_pair
13536 **/
i40e_queue_pair_disable_irq(struct i40e_vsi * vsi,int queue_pair)13537 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13538 {
13539 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13540 struct i40e_pf *pf = vsi->back;
13541 struct i40e_hw *hw = &pf->hw;
13542
13543 /* For simplicity, instead of removing the qp interrupt causes
13544 * from the interrupt linked list, we simply disable the interrupt, and
13545 * leave the list intact.
13546 *
13547 * All rings in a qp belong to the same qvector.
13548 */
13549 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
13550 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13551
13552 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13553 i40e_flush(hw);
13554 synchronize_irq(pf->msix_entries[intpf].vector);
13555 } else {
13556 /* Legacy and MSI mode - this stops all interrupt handling */
13557 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13558 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13559 i40e_flush(hw);
13560 synchronize_irq(pf->pdev->irq);
13561 }
13562 }
13563
13564 /**
13565 * i40e_queue_pair_disable - Disables a queue pair
13566 * @vsi: vsi
13567 * @queue_pair: queue pair
13568 *
13569 * Returns 0 on success, <0 on failure.
13570 **/
i40e_queue_pair_disable(struct i40e_vsi * vsi,int queue_pair)13571 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13572 {
13573 int err;
13574
13575 err = i40e_enter_busy_conf(vsi);
13576 if (err)
13577 return err;
13578
13579 i40e_queue_pair_disable_irq(vsi, queue_pair);
13580 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
13581 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
13582 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13583 i40e_queue_pair_clean_rings(vsi, queue_pair);
13584 i40e_queue_pair_reset_stats(vsi, queue_pair);
13585
13586 return err;
13587 }
13588
13589 /**
13590 * i40e_queue_pair_enable - Enables a queue pair
13591 * @vsi: vsi
13592 * @queue_pair: queue pair
13593 *
13594 * Returns 0 on success, <0 on failure.
13595 **/
i40e_queue_pair_enable(struct i40e_vsi * vsi,int queue_pair)13596 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13597 {
13598 int err;
13599
13600 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
13601 if (err)
13602 return err;
13603
13604 if (i40e_enabled_xdp_vsi(vsi)) {
13605 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
13606 if (err)
13607 return err;
13608 }
13609
13610 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
13611 if (err)
13612 return err;
13613
13614 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
13615 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
13616 i40e_queue_pair_enable_irq(vsi, queue_pair);
13617
13618 i40e_exit_busy_conf(vsi);
13619
13620 return err;
13621 }
13622
13623 /**
13624 * i40e_xdp - implements ndo_bpf for i40e
13625 * @dev: netdevice
13626 * @xdp: XDP command
13627 **/
i40e_xdp(struct net_device * dev,struct netdev_bpf * xdp)13628 static int i40e_xdp(struct net_device *dev,
13629 struct netdev_bpf *xdp)
13630 {
13631 struct i40e_netdev_priv *np = netdev_priv(dev);
13632 struct i40e_vsi *vsi = np->vsi;
13633
13634 if (vsi->type != I40E_VSI_MAIN)
13635 return -EINVAL;
13636
13637 switch (xdp->command) {
13638 case XDP_SETUP_PROG:
13639 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);
13640 case XDP_SETUP_XSK_POOL:
13641 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13642 xdp->xsk.queue_id);
13643 default:
13644 return -EINVAL;
13645 }
13646 }
13647
13648 static const struct net_device_ops i40e_netdev_ops = {
13649 .ndo_open = i40e_open,
13650 .ndo_stop = i40e_close,
13651 .ndo_start_xmit = i40e_lan_xmit_frame,
13652 .ndo_get_stats64 = i40e_get_netdev_stats_struct,
13653 .ndo_set_rx_mode = i40e_set_rx_mode,
13654 .ndo_validate_addr = eth_validate_addr,
13655 .ndo_set_mac_address = i40e_set_mac,
13656 .ndo_change_mtu = i40e_change_mtu,
13657 .ndo_eth_ioctl = i40e_ioctl,
13658 .ndo_tx_timeout = i40e_tx_timeout,
13659 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid,
13660 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid,
13661 #ifdef CONFIG_NET_POLL_CONTROLLER
13662 .ndo_poll_controller = i40e_netpoll,
13663 #endif
13664 .ndo_setup_tc = __i40e_setup_tc,
13665 .ndo_select_queue = i40e_lan_select_queue,
13666 .ndo_set_features = i40e_set_features,
13667 .ndo_set_vf_mac = i40e_ndo_set_vf_mac,
13668 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan,
13669 .ndo_get_vf_stats = i40e_get_vf_stats,
13670 .ndo_set_vf_rate = i40e_ndo_set_vf_bw,
13671 .ndo_get_vf_config = i40e_ndo_get_vf_config,
13672 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state,
13673 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk,
13674 .ndo_set_vf_trust = i40e_ndo_set_vf_trust,
13675 .ndo_get_phys_port_id = i40e_get_phys_port_id,
13676 .ndo_fdb_add = i40e_ndo_fdb_add,
13677 .ndo_features_check = i40e_features_check,
13678 .ndo_bridge_getlink = i40e_ndo_bridge_getlink,
13679 .ndo_bridge_setlink = i40e_ndo_bridge_setlink,
13680 .ndo_bpf = i40e_xdp,
13681 .ndo_xdp_xmit = i40e_xdp_xmit,
13682 .ndo_xsk_wakeup = i40e_xsk_wakeup,
13683 .ndo_dfwd_add_station = i40e_fwd_add,
13684 .ndo_dfwd_del_station = i40e_fwd_del,
13685 };
13686
13687 /**
13688 * i40e_config_netdev - Setup the netdev flags
13689 * @vsi: the VSI being configured
13690 *
13691 * Returns 0 on success, negative value on failure
13692 **/
i40e_config_netdev(struct i40e_vsi * vsi)13693 static int i40e_config_netdev(struct i40e_vsi *vsi)
13694 {
13695 struct i40e_pf *pf = vsi->back;
13696 struct i40e_hw *hw = &pf->hw;
13697 struct i40e_netdev_priv *np;
13698 struct net_device *netdev;
13699 u8 broadcast[ETH_ALEN];
13700 u8 mac_addr[ETH_ALEN];
13701 int etherdev_size;
13702 netdev_features_t hw_enc_features;
13703 netdev_features_t hw_features;
13704
13705 etherdev_size = sizeof(struct i40e_netdev_priv);
13706 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13707 if (!netdev)
13708 return -ENOMEM;
13709
13710 vsi->netdev = netdev;
13711 np = netdev_priv(netdev);
13712 np->vsi = vsi;
13713
13714 hw_enc_features = NETIF_F_SG |
13715 NETIF_F_HW_CSUM |
13716 NETIF_F_HIGHDMA |
13717 NETIF_F_SOFT_FEATURES |
13718 NETIF_F_TSO |
13719 NETIF_F_TSO_ECN |
13720 NETIF_F_TSO6 |
13721 NETIF_F_GSO_GRE |
13722 NETIF_F_GSO_GRE_CSUM |
13723 NETIF_F_GSO_PARTIAL |
13724 NETIF_F_GSO_IPXIP4 |
13725 NETIF_F_GSO_IPXIP6 |
13726 NETIF_F_GSO_UDP_TUNNEL |
13727 NETIF_F_GSO_UDP_TUNNEL_CSUM |
13728 NETIF_F_GSO_UDP_L4 |
13729 NETIF_F_SCTP_CRC |
13730 NETIF_F_RXHASH |
13731 NETIF_F_RXCSUM |
13732 0;
13733
13734 if (!test_bit(I40E_HW_CAP_OUTER_UDP_CSUM, pf->hw.caps))
13735 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13736
13737 netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13738
13739 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13740
13741 netdev->hw_enc_features |= hw_enc_features;
13742
13743 /* record features VLANs can make use of */
13744 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13745
13746 #define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
13747 NETIF_F_GSO_GRE_CSUM | \
13748 NETIF_F_GSO_IPXIP4 | \
13749 NETIF_F_GSO_IPXIP6 | \
13750 NETIF_F_GSO_UDP_TUNNEL | \
13751 NETIF_F_GSO_UDP_TUNNEL_CSUM)
13752
13753 netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES;
13754 netdev->features |= NETIF_F_GSO_PARTIAL |
13755 I40E_GSO_PARTIAL_FEATURES;
13756
13757 netdev->mpls_features |= NETIF_F_SG;
13758 netdev->mpls_features |= NETIF_F_HW_CSUM;
13759 netdev->mpls_features |= NETIF_F_TSO;
13760 netdev->mpls_features |= NETIF_F_TSO6;
13761 netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES;
13762
13763 /* enable macvlan offloads */
13764 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13765
13766 hw_features = hw_enc_features |
13767 NETIF_F_HW_VLAN_CTAG_TX |
13768 NETIF_F_HW_VLAN_CTAG_RX;
13769
13770 if (!test_bit(I40E_FLAG_MFP_ENA, pf->flags))
13771 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13772
13773 netdev->hw_features |= hw_features | NETIF_F_LOOPBACK;
13774
13775 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13776 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13777
13778 netdev->features &= ~NETIF_F_HW_TC;
13779
13780 if (vsi->type == I40E_VSI_MAIN) {
13781 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13782 ether_addr_copy(mac_addr, hw->mac.perm_addr);
13783 /* The following steps are necessary for two reasons. First,
13784 * some older NVM configurations load a default MAC-VLAN
13785 * filter that will accept any tagged packet, and we want to
13786 * replace this with a normal filter. Additionally, it is
13787 * possible our MAC address was provided by the platform using
13788 * Open Firmware or similar.
13789 *
13790 * Thus, we need to remove the default filter and install one
13791 * specific to the MAC address.
13792 */
13793 i40e_rm_default_mac_filter(vsi, mac_addr);
13794 spin_lock_bh(&vsi->mac_filter_hash_lock);
13795 i40e_add_mac_filter(vsi, mac_addr);
13796 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13797
13798 netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
13799 NETDEV_XDP_ACT_REDIRECT |
13800 NETDEV_XDP_ACT_XSK_ZEROCOPY |
13801 NETDEV_XDP_ACT_RX_SG;
13802 netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD;
13803 } else {
13804 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13805 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13806 * the end, which is 4 bytes long, so force truncation of the
13807 * original name by IFNAMSIZ - 4
13808 */
13809 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
13810
13811 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", IFNAMSIZ - 4,
13812 main_vsi->netdev->name);
13813 eth_random_addr(mac_addr);
13814
13815 spin_lock_bh(&vsi->mac_filter_hash_lock);
13816 i40e_add_mac_filter(vsi, mac_addr);
13817 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13818 }
13819
13820 /* Add the broadcast filter so that we initially will receive
13821 * broadcast packets. Note that when a new VLAN is first added the
13822 * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13823 * specific filters as part of transitioning into "vlan" operation.
13824 * When more VLANs are added, the driver will copy each existing MAC
13825 * filter and add it for the new VLAN.
13826 *
13827 * Broadcast filters are handled specially by
13828 * i40e_sync_filters_subtask, as the driver must to set the broadcast
13829 * promiscuous bit instead of adding this directly as a MAC/VLAN
13830 * filter. The subtask will update the correct broadcast promiscuous
13831 * bits as VLANs become active or inactive.
13832 */
13833 eth_broadcast_addr(broadcast);
13834 spin_lock_bh(&vsi->mac_filter_hash_lock);
13835 i40e_add_mac_filter(vsi, broadcast);
13836 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13837
13838 eth_hw_addr_set(netdev, mac_addr);
13839 ether_addr_copy(netdev->perm_addr, mac_addr);
13840
13841 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13842 netdev->neigh_priv_len = sizeof(u32) * 4;
13843
13844 netdev->priv_flags |= IFF_UNICAST_FLT;
13845 netdev->priv_flags |= IFF_SUPP_NOFCS;
13846 /* Setup netdev TC information */
13847 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13848
13849 netdev->netdev_ops = &i40e_netdev_ops;
13850 netdev->watchdog_timeo = 5 * HZ;
13851 i40e_set_ethtool_ops(netdev);
13852
13853 /* MTU range: 68 - 9706 */
13854 netdev->min_mtu = ETH_MIN_MTU;
13855 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13856
13857 return 0;
13858 }
13859
13860 /**
13861 * i40e_vsi_delete - Delete a VSI from the switch
13862 * @vsi: the VSI being removed
13863 *
13864 * Returns 0 on success, negative value on failure
13865 **/
i40e_vsi_delete(struct i40e_vsi * vsi)13866 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13867 {
13868 /* remove default VSI is not allowed */
13869 if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13870 return;
13871
13872 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13873 }
13874
13875 /**
13876 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13877 * @vsi: the VSI being queried
13878 *
13879 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13880 **/
i40e_is_vsi_uplink_mode_veb(struct i40e_vsi * vsi)13881 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13882 {
13883 struct i40e_veb *veb;
13884 struct i40e_pf *pf = vsi->back;
13885
13886 /* Uplink is not a bridge so default to VEB */
13887 if (vsi->veb_idx >= I40E_MAX_VEB)
13888 return 1;
13889
13890 veb = pf->veb[vsi->veb_idx];
13891 if (!veb) {
13892 dev_info(&pf->pdev->dev,
13893 "There is no veb associated with the bridge\n");
13894 return -ENOENT;
13895 }
13896
13897 /* Uplink is a bridge in VEPA mode */
13898 if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13899 return 0;
13900 } else {
13901 /* Uplink is a bridge in VEB mode */
13902 return 1;
13903 }
13904
13905 /* VEPA is now default bridge, so return 0 */
13906 return 0;
13907 }
13908
13909 /**
13910 * i40e_add_vsi - Add a VSI to the switch
13911 * @vsi: the VSI being configured
13912 *
13913 * This initializes a VSI context depending on the VSI type to be added and
13914 * passes it down to the add_vsi aq command.
13915 **/
i40e_add_vsi(struct i40e_vsi * vsi)13916 static int i40e_add_vsi(struct i40e_vsi *vsi)
13917 {
13918 int ret = -ENODEV;
13919 struct i40e_pf *pf = vsi->back;
13920 struct i40e_hw *hw = &pf->hw;
13921 struct i40e_vsi_context ctxt;
13922 struct i40e_mac_filter *f;
13923 struct hlist_node *h;
13924 int bkt;
13925
13926 u8 enabled_tc = 0x1; /* TC0 enabled */
13927 int f_count = 0;
13928
13929 memset(&ctxt, 0, sizeof(ctxt));
13930 switch (vsi->type) {
13931 case I40E_VSI_MAIN:
13932 /* The PF's main VSI is already setup as part of the
13933 * device initialization, so we'll not bother with
13934 * the add_vsi call, but we will retrieve the current
13935 * VSI context.
13936 */
13937 ctxt.seid = pf->main_vsi_seid;
13938 ctxt.pf_num = pf->hw.pf_id;
13939 ctxt.vf_num = 0;
13940 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13941 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13942 if (ret) {
13943 dev_info(&pf->pdev->dev,
13944 "couldn't get PF vsi config, err %pe aq_err %s\n",
13945 ERR_PTR(ret),
13946 i40e_aq_str(&pf->hw,
13947 pf->hw.aq.asq_last_status));
13948 return -ENOENT;
13949 }
13950 vsi->info = ctxt.info;
13951 vsi->info.valid_sections = 0;
13952
13953 vsi->seid = ctxt.seid;
13954 vsi->id = ctxt.vsi_number;
13955
13956 enabled_tc = i40e_pf_get_tc_map(pf);
13957
13958 /* Source pruning is enabled by default, so the flag is
13959 * negative logic - if it's set, we need to fiddle with
13960 * the VSI to disable source pruning.
13961 */
13962 if (test_bit(I40E_FLAG_SOURCE_PRUNING_DIS, pf->flags)) {
13963 memset(&ctxt, 0, sizeof(ctxt));
13964 ctxt.seid = pf->main_vsi_seid;
13965 ctxt.pf_num = pf->hw.pf_id;
13966 ctxt.vf_num = 0;
13967 ctxt.info.valid_sections |=
13968 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13969 ctxt.info.switch_id =
13970 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13971 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13972 if (ret) {
13973 dev_info(&pf->pdev->dev,
13974 "update vsi failed, err %d aq_err %s\n",
13975 ret,
13976 i40e_aq_str(&pf->hw,
13977 pf->hw.aq.asq_last_status));
13978 ret = -ENOENT;
13979 goto err;
13980 }
13981 }
13982
13983 /* MFP mode setup queue map and update VSI */
13984 if (test_bit(I40E_FLAG_MFP_ENA, pf->flags) &&
13985 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13986 memset(&ctxt, 0, sizeof(ctxt));
13987 ctxt.seid = pf->main_vsi_seid;
13988 ctxt.pf_num = pf->hw.pf_id;
13989 ctxt.vf_num = 0;
13990 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13991 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13992 if (ret) {
13993 dev_info(&pf->pdev->dev,
13994 "update vsi failed, err %pe aq_err %s\n",
13995 ERR_PTR(ret),
13996 i40e_aq_str(&pf->hw,
13997 pf->hw.aq.asq_last_status));
13998 ret = -ENOENT;
13999 goto err;
14000 }
14001 /* update the local VSI info queue map */
14002 i40e_vsi_update_queue_map(vsi, &ctxt);
14003 vsi->info.valid_sections = 0;
14004 } else {
14005 /* Default/Main VSI is only enabled for TC0
14006 * reconfigure it to enable all TCs that are
14007 * available on the port in SFP mode.
14008 * For MFP case the iSCSI PF would use this
14009 * flow to enable LAN+iSCSI TC.
14010 */
14011 ret = i40e_vsi_config_tc(vsi, enabled_tc);
14012 if (ret) {
14013 /* Single TC condition is not fatal,
14014 * message and continue
14015 */
14016 dev_info(&pf->pdev->dev,
14017 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n",
14018 enabled_tc,
14019 ERR_PTR(ret),
14020 i40e_aq_str(&pf->hw,
14021 pf->hw.aq.asq_last_status));
14022 }
14023 }
14024 break;
14025
14026 case I40E_VSI_FDIR:
14027 ctxt.pf_num = hw->pf_id;
14028 ctxt.vf_num = 0;
14029 ctxt.uplink_seid = vsi->uplink_seid;
14030 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14031 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
14032 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags) &&
14033 (i40e_is_vsi_uplink_mode_veb(vsi))) {
14034 ctxt.info.valid_sections |=
14035 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14036 ctxt.info.switch_id =
14037 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14038 }
14039 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14040 break;
14041
14042 case I40E_VSI_VMDQ2:
14043 ctxt.pf_num = hw->pf_id;
14044 ctxt.vf_num = 0;
14045 ctxt.uplink_seid = vsi->uplink_seid;
14046 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14047 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
14048
14049 /* This VSI is connected to VEB so the switch_id
14050 * should be set to zero by default.
14051 */
14052 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14053 ctxt.info.valid_sections |=
14054 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14055 ctxt.info.switch_id =
14056 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14057 }
14058
14059 /* Setup the VSI tx/rx queue map for TC0 only for now */
14060 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14061 break;
14062
14063 case I40E_VSI_SRIOV:
14064 ctxt.pf_num = hw->pf_id;
14065 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
14066 ctxt.uplink_seid = vsi->uplink_seid;
14067 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14068 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
14069
14070 /* This VSI is connected to VEB so the switch_id
14071 * should be set to zero by default.
14072 */
14073 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14074 ctxt.info.valid_sections |=
14075 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14076 ctxt.info.switch_id =
14077 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14078 }
14079
14080 if (test_bit(I40E_FLAG_IWARP_ENA, vsi->back->flags)) {
14081 ctxt.info.valid_sections |=
14082 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
14083 ctxt.info.queueing_opt_flags |=
14084 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
14085 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
14086 }
14087
14088 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
14089 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
14090 if (pf->vf[vsi->vf_id].spoofchk) {
14091 ctxt.info.valid_sections |=
14092 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
14093 ctxt.info.sec_flags |=
14094 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
14095 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
14096 }
14097 /* Setup the VSI tx/rx queue map for TC0 only for now */
14098 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14099 break;
14100
14101 case I40E_VSI_IWARP:
14102 /* send down message to iWARP */
14103 break;
14104
14105 default:
14106 return -ENODEV;
14107 }
14108
14109 if (vsi->type != I40E_VSI_MAIN) {
14110 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
14111 if (ret) {
14112 dev_info(&vsi->back->pdev->dev,
14113 "add vsi failed, err %pe aq_err %s\n",
14114 ERR_PTR(ret),
14115 i40e_aq_str(&pf->hw,
14116 pf->hw.aq.asq_last_status));
14117 ret = -ENOENT;
14118 goto err;
14119 }
14120 vsi->info = ctxt.info;
14121 vsi->info.valid_sections = 0;
14122 vsi->seid = ctxt.seid;
14123 vsi->id = ctxt.vsi_number;
14124 }
14125
14126 spin_lock_bh(&vsi->mac_filter_hash_lock);
14127 vsi->active_filters = 0;
14128 /* If macvlan filters already exist, force them to get loaded */
14129 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
14130 f->state = I40E_FILTER_NEW;
14131 f_count++;
14132 }
14133 spin_unlock_bh(&vsi->mac_filter_hash_lock);
14134 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
14135
14136 if (f_count) {
14137 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
14138 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
14139 }
14140
14141 /* Update VSI BW information */
14142 ret = i40e_vsi_get_bw_info(vsi);
14143 if (ret) {
14144 dev_info(&pf->pdev->dev,
14145 "couldn't get vsi bw info, err %pe aq_err %s\n",
14146 ERR_PTR(ret),
14147 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14148 /* VSI is already added so not tearing that up */
14149 ret = 0;
14150 }
14151
14152 err:
14153 return ret;
14154 }
14155
14156 /**
14157 * i40e_vsi_release - Delete a VSI and free its resources
14158 * @vsi: the VSI being removed
14159 *
14160 * Returns 0 on success or < 0 on error
14161 **/
i40e_vsi_release(struct i40e_vsi * vsi)14162 int i40e_vsi_release(struct i40e_vsi *vsi)
14163 {
14164 struct i40e_mac_filter *f;
14165 struct hlist_node *h;
14166 struct i40e_veb *veb;
14167 struct i40e_pf *pf;
14168 u16 uplink_seid;
14169 int i, n, bkt;
14170
14171 pf = vsi->back;
14172
14173 /* release of a VEB-owner or last VSI is not allowed */
14174 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
14175 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
14176 vsi->seid, vsi->uplink_seid);
14177 return -ENODEV;
14178 }
14179 if (vsi->type == I40E_VSI_MAIN && !test_bit(__I40E_DOWN, pf->state)) {
14180 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
14181 return -ENODEV;
14182 }
14183 set_bit(__I40E_VSI_RELEASING, vsi->state);
14184 uplink_seid = vsi->uplink_seid;
14185
14186 if (vsi->type != I40E_VSI_SRIOV) {
14187 if (vsi->netdev_registered) {
14188 vsi->netdev_registered = false;
14189 if (vsi->netdev) {
14190 /* results in a call to i40e_close() */
14191 unregister_netdev(vsi->netdev);
14192 }
14193 } else {
14194 i40e_vsi_close(vsi);
14195 }
14196 i40e_vsi_disable_irq(vsi);
14197 }
14198
14199 if (vsi->type == I40E_VSI_MAIN)
14200 i40e_devlink_destroy_port(pf);
14201
14202 spin_lock_bh(&vsi->mac_filter_hash_lock);
14203
14204 /* clear the sync flag on all filters */
14205 if (vsi->netdev) {
14206 __dev_uc_unsync(vsi->netdev, NULL);
14207 __dev_mc_unsync(vsi->netdev, NULL);
14208 }
14209
14210 /* make sure any remaining filters are marked for deletion */
14211 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
14212 __i40e_del_filter(vsi, f);
14213
14214 spin_unlock_bh(&vsi->mac_filter_hash_lock);
14215
14216 i40e_sync_vsi_filters(vsi);
14217
14218 i40e_vsi_delete(vsi);
14219 i40e_vsi_free_q_vectors(vsi);
14220 if (vsi->netdev) {
14221 free_netdev(vsi->netdev);
14222 vsi->netdev = NULL;
14223 }
14224 i40e_vsi_clear_rings(vsi);
14225 i40e_vsi_clear(vsi);
14226
14227 /* If this was the last thing on the VEB, except for the
14228 * controlling VSI, remove the VEB, which puts the controlling
14229 * VSI onto the uplink port.
14230 *
14231 * Well, okay, there's one more exception here: don't remove
14232 * the floating VEBs yet. We'll wait for an explicit remove request
14233 * from up the network stack.
14234 */
14235 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);
14236 if (veb && veb->uplink_seid) {
14237 n = 0;
14238
14239 /* Count non-controlling VSIs present on the VEB */
14240 i40e_pf_for_each_vsi(pf, i, vsi)
14241 if (vsi->uplink_seid == uplink_seid &&
14242 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14243 n++;
14244
14245 /* If there is no VSI except the control one then release
14246 * the VEB and put the control VSI onto VEB uplink.
14247 */
14248 if (!n)
14249 i40e_veb_release(veb);
14250 }
14251
14252 return 0;
14253 }
14254
14255 /**
14256 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
14257 * @vsi: ptr to the VSI
14258 *
14259 * This should only be called after i40e_vsi_mem_alloc() which allocates the
14260 * corresponding SW VSI structure and initializes num_queue_pairs for the
14261 * newly allocated VSI.
14262 *
14263 * Returns 0 on success or negative on failure
14264 **/
i40e_vsi_setup_vectors(struct i40e_vsi * vsi)14265 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
14266 {
14267 int ret = -ENOENT;
14268 struct i40e_pf *pf = vsi->back;
14269
14270 if (vsi->q_vectors[0]) {
14271 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
14272 vsi->seid);
14273 return -EEXIST;
14274 }
14275
14276 if (vsi->base_vector) {
14277 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
14278 vsi->seid, vsi->base_vector);
14279 return -EEXIST;
14280 }
14281
14282 ret = i40e_vsi_alloc_q_vectors(vsi);
14283 if (ret) {
14284 dev_info(&pf->pdev->dev,
14285 "failed to allocate %d q_vector for VSI %d, ret=%d\n",
14286 vsi->num_q_vectors, vsi->seid, ret);
14287 vsi->num_q_vectors = 0;
14288 goto vector_setup_out;
14289 }
14290
14291 /* In Legacy mode, we do not have to get any other vector since we
14292 * piggyback on the misc/ICR0 for queue interrupts.
14293 */
14294 if (!test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
14295 return ret;
14296 if (vsi->num_q_vectors)
14297 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
14298 vsi->num_q_vectors, vsi->idx);
14299 if (vsi->base_vector < 0) {
14300 dev_info(&pf->pdev->dev,
14301 "failed to get tracking for %d vectors for VSI %d, err=%d\n",
14302 vsi->num_q_vectors, vsi->seid, vsi->base_vector);
14303 i40e_vsi_free_q_vectors(vsi);
14304 ret = -ENOENT;
14305 goto vector_setup_out;
14306 }
14307
14308 vector_setup_out:
14309 return ret;
14310 }
14311
14312 /**
14313 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
14314 * @vsi: pointer to the vsi.
14315 *
14316 * This re-allocates a vsi's queue resources.
14317 *
14318 * Returns pointer to the successfully allocated and configured VSI sw struct
14319 * on success, otherwise returns NULL on failure.
14320 **/
i40e_vsi_reinit_setup(struct i40e_vsi * vsi)14321 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
14322 {
14323 struct i40e_vsi *main_vsi;
14324 u16 alloc_queue_pairs;
14325 struct i40e_pf *pf;
14326 int ret;
14327
14328 if (!vsi)
14329 return NULL;
14330
14331 pf = vsi->back;
14332
14333 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
14334 i40e_vsi_clear_rings(vsi);
14335
14336 i40e_vsi_free_arrays(vsi, false);
14337 i40e_set_num_rings_in_vsi(vsi);
14338 ret = i40e_vsi_alloc_arrays(vsi, false);
14339 if (ret)
14340 goto err_vsi;
14341
14342 alloc_queue_pairs = vsi->alloc_queue_pairs *
14343 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14344
14345 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14346 if (ret < 0) {
14347 dev_info(&pf->pdev->dev,
14348 "failed to get tracking for %d queues for VSI %d err %d\n",
14349 alloc_queue_pairs, vsi->seid, ret);
14350 goto err_vsi;
14351 }
14352 vsi->base_queue = ret;
14353
14354 /* Update the FW view of the VSI. Force a reset of TC and queue
14355 * layout configurations.
14356 */
14357 main_vsi = i40e_pf_get_main_vsi(pf);
14358 main_vsi->seid = pf->main_vsi_seid;
14359 i40e_vsi_reconfig_tc(main_vsi);
14360
14361 if (vsi->type == I40E_VSI_MAIN)
14362 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
14363
14364 /* assign it some queues */
14365 ret = i40e_alloc_rings(vsi);
14366 if (ret)
14367 goto err_rings;
14368
14369 /* map all of the rings to the q_vectors */
14370 i40e_vsi_map_rings_to_vectors(vsi);
14371 return vsi;
14372
14373 err_rings:
14374 i40e_vsi_free_q_vectors(vsi);
14375 if (vsi->netdev_registered) {
14376 vsi->netdev_registered = false;
14377 unregister_netdev(vsi->netdev);
14378 free_netdev(vsi->netdev);
14379 vsi->netdev = NULL;
14380 }
14381 if (vsi->type == I40E_VSI_MAIN)
14382 i40e_devlink_destroy_port(pf);
14383 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14384 err_vsi:
14385 i40e_vsi_clear(vsi);
14386 return NULL;
14387 }
14388
14389 /**
14390 * i40e_vsi_setup - Set up a VSI by a given type
14391 * @pf: board private structure
14392 * @type: VSI type
14393 * @uplink_seid: the switch element to link to
14394 * @param1: usage depends upon VSI type. For VF types, indicates VF id
14395 *
14396 * This allocates the sw VSI structure and its queue resources, then add a VSI
14397 * to the identified VEB.
14398 *
14399 * Returns pointer to the successfully allocated and configure VSI sw struct on
14400 * success, otherwise returns NULL on failure.
14401 **/
i40e_vsi_setup(struct i40e_pf * pf,u8 type,u16 uplink_seid,u32 param1)14402 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
14403 u16 uplink_seid, u32 param1)
14404 {
14405 struct i40e_vsi *vsi = NULL;
14406 struct i40e_veb *veb = NULL;
14407 u16 alloc_queue_pairs;
14408 int v_idx;
14409 int ret;
14410
14411 /* The requested uplink_seid must be either
14412 * - the PF's port seid
14413 * no VEB is needed because this is the PF
14414 * or this is a Flow Director special case VSI
14415 * - seid of an existing VEB
14416 * - seid of a VSI that owns an existing VEB
14417 * - seid of a VSI that doesn't own a VEB
14418 * a new VEB is created and the VSI becomes the owner
14419 * - seid of the PF VSI, which is what creates the first VEB
14420 * this is a special case of the previous
14421 *
14422 * Find which uplink_seid we were given and create a new VEB if needed
14423 */
14424 veb = i40e_pf_get_veb_by_seid(pf, uplink_seid);
14425 if (!veb && uplink_seid != pf->mac_seid) {
14426 vsi = i40e_pf_get_vsi_by_seid(pf, uplink_seid);
14427 if (!vsi) {
14428 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14429 uplink_seid);
14430 return NULL;
14431 }
14432
14433 if (vsi->uplink_seid == pf->mac_seid)
14434 veb = i40e_veb_setup(pf, pf->mac_seid, vsi->seid,
14435 vsi->tc_config.enabled_tc);
14436 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14437 veb = i40e_veb_setup(pf, vsi->uplink_seid, vsi->seid,
14438 vsi->tc_config.enabled_tc);
14439 if (veb) {
14440 if (vsi->type != I40E_VSI_MAIN) {
14441 dev_info(&vsi->back->pdev->dev,
14442 "New VSI creation error, uplink seid of LAN VSI expected.\n");
14443 return NULL;
14444 }
14445 /* We come up by default in VEPA mode if SRIOV is not
14446 * already enabled, in which case we can't force VEPA
14447 * mode.
14448 */
14449 if (!test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags)) {
14450 veb->bridge_mode = BRIDGE_MODE_VEPA;
14451 clear_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
14452 }
14453 i40e_config_bridge_mode(veb);
14454 }
14455 veb = i40e_pf_get_veb_by_seid(pf, vsi->uplink_seid);
14456 if (!veb) {
14457 dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14458 return NULL;
14459 }
14460
14461 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14462 uplink_seid = veb->seid;
14463 }
14464
14465 /* get vsi sw struct */
14466 v_idx = i40e_vsi_mem_alloc(pf, type);
14467 if (v_idx < 0)
14468 goto err_alloc;
14469 vsi = pf->vsi[v_idx];
14470 if (!vsi)
14471 goto err_alloc;
14472 vsi->type = type;
14473 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14474
14475 if (type == I40E_VSI_MAIN)
14476 pf->lan_vsi = v_idx;
14477 else if (type == I40E_VSI_SRIOV)
14478 vsi->vf_id = param1;
14479 /* assign it some queues */
14480 alloc_queue_pairs = vsi->alloc_queue_pairs *
14481 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14482
14483 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14484 if (ret < 0) {
14485 dev_info(&pf->pdev->dev,
14486 "failed to get tracking for %d queues for VSI %d err=%d\n",
14487 alloc_queue_pairs, vsi->seid, ret);
14488 goto err_vsi;
14489 }
14490 vsi->base_queue = ret;
14491
14492 /* get a VSI from the hardware */
14493 vsi->uplink_seid = uplink_seid;
14494 ret = i40e_add_vsi(vsi);
14495 if (ret)
14496 goto err_vsi;
14497
14498 switch (vsi->type) {
14499 /* setup the netdev if needed */
14500 case I40E_VSI_MAIN:
14501 case I40E_VSI_VMDQ2:
14502 ret = i40e_config_netdev(vsi);
14503 if (ret)
14504 goto err_netdev;
14505 ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
14506 if (ret)
14507 goto err_netdev;
14508 if (vsi->type == I40E_VSI_MAIN) {
14509 ret = i40e_devlink_create_port(pf);
14510 if (ret)
14511 goto err_netdev;
14512 SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);
14513 }
14514 ret = register_netdev(vsi->netdev);
14515 if (ret)
14516 goto err_dl_port;
14517 vsi->netdev_registered = true;
14518 netif_carrier_off(vsi->netdev);
14519 #ifdef CONFIG_I40E_DCB
14520 /* Setup DCB netlink interface */
14521 i40e_dcbnl_setup(vsi);
14522 #endif /* CONFIG_I40E_DCB */
14523 fallthrough;
14524 case I40E_VSI_FDIR:
14525 /* set up vectors and rings if needed */
14526 ret = i40e_vsi_setup_vectors(vsi);
14527 if (ret)
14528 goto err_msix;
14529
14530 ret = i40e_alloc_rings(vsi);
14531 if (ret)
14532 goto err_rings;
14533
14534 /* map all of the rings to the q_vectors */
14535 i40e_vsi_map_rings_to_vectors(vsi);
14536
14537 i40e_vsi_reset_stats(vsi);
14538 break;
14539 default:
14540 /* no netdev or rings for the other VSI types */
14541 break;
14542 }
14543
14544 if (test_bit(I40E_HW_CAP_RSS_AQ, pf->hw.caps) &&
14545 vsi->type == I40E_VSI_VMDQ2) {
14546 ret = i40e_vsi_config_rss(vsi);
14547 if (ret)
14548 goto err_config;
14549 }
14550 return vsi;
14551
14552 err_config:
14553 i40e_vsi_clear_rings(vsi);
14554 err_rings:
14555 i40e_vsi_free_q_vectors(vsi);
14556 err_msix:
14557 if (vsi->netdev_registered) {
14558 vsi->netdev_registered = false;
14559 unregister_netdev(vsi->netdev);
14560 free_netdev(vsi->netdev);
14561 vsi->netdev = NULL;
14562 }
14563 err_dl_port:
14564 if (vsi->type == I40E_VSI_MAIN)
14565 i40e_devlink_destroy_port(pf);
14566 err_netdev:
14567 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14568 err_vsi:
14569 i40e_vsi_clear(vsi);
14570 err_alloc:
14571 return NULL;
14572 }
14573
14574 /**
14575 * i40e_veb_get_bw_info - Query VEB BW information
14576 * @veb: the veb to query
14577 *
14578 * Query the Tx scheduler BW configuration data for given VEB
14579 **/
i40e_veb_get_bw_info(struct i40e_veb * veb)14580 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14581 {
14582 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14583 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14584 struct i40e_pf *pf = veb->pf;
14585 struct i40e_hw *hw = &pf->hw;
14586 u32 tc_bw_max;
14587 int ret = 0;
14588 int i;
14589
14590 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
14591 &bw_data, NULL);
14592 if (ret) {
14593 dev_info(&pf->pdev->dev,
14594 "query veb bw config failed, err %pe aq_err %s\n",
14595 ERR_PTR(ret),
14596 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14597 goto out;
14598 }
14599
14600 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
14601 &ets_data, NULL);
14602 if (ret) {
14603 dev_info(&pf->pdev->dev,
14604 "query veb bw ets config failed, err %pe aq_err %s\n",
14605 ERR_PTR(ret),
14606 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14607 goto out;
14608 }
14609
14610 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14611 veb->bw_max_quanta = ets_data.tc_bw_max;
14612 veb->is_abs_credits = bw_data.absolute_credits_enable;
14613 veb->enabled_tc = ets_data.tc_valid_bits;
14614 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14615 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14616 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14617 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14618 veb->bw_tc_limit_credits[i] =
14619 le16_to_cpu(bw_data.tc_bw_limits[i]);
14620 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14621 }
14622
14623 out:
14624 return ret;
14625 }
14626
14627 /**
14628 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14629 * @pf: board private structure
14630 *
14631 * On error: returns error code (negative)
14632 * On success: returns vsi index in PF (positive)
14633 **/
i40e_veb_mem_alloc(struct i40e_pf * pf)14634 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14635 {
14636 int ret = -ENOENT;
14637 struct i40e_veb *veb;
14638 int i;
14639
14640 /* Need to protect the allocation of switch elements at the PF level */
14641 mutex_lock(&pf->switch_mutex);
14642
14643 /* VEB list may be fragmented if VEB creation/destruction has
14644 * been happening. We can afford to do a quick scan to look
14645 * for any free slots in the list.
14646 *
14647 * find next empty veb slot, looping back around if necessary
14648 */
14649 i = 0;
14650 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14651 i++;
14652 if (i >= I40E_MAX_VEB) {
14653 ret = -ENOMEM;
14654 goto err_alloc_veb; /* out of VEB slots! */
14655 }
14656
14657 veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14658 if (!veb) {
14659 ret = -ENOMEM;
14660 goto err_alloc_veb;
14661 }
14662 veb->pf = pf;
14663 veb->idx = i;
14664 veb->enabled_tc = 1;
14665
14666 pf->veb[i] = veb;
14667 ret = i;
14668 err_alloc_veb:
14669 mutex_unlock(&pf->switch_mutex);
14670 return ret;
14671 }
14672
14673 /**
14674 * i40e_switch_branch_release - Delete a branch of the switch tree
14675 * @branch: where to start deleting
14676 *
14677 * This uses recursion to find the tips of the branch to be
14678 * removed, deleting until we get back to and can delete this VEB.
14679 **/
i40e_switch_branch_release(struct i40e_veb * branch)14680 static void i40e_switch_branch_release(struct i40e_veb *branch)
14681 {
14682 struct i40e_pf *pf = branch->pf;
14683 u16 branch_seid = branch->seid;
14684 u16 veb_idx = branch->idx;
14685 struct i40e_vsi *vsi;
14686 struct i40e_veb *veb;
14687 int i;
14688
14689 /* release any VEBs on this VEB - RECURSION */
14690 i40e_pf_for_each_veb(pf, i, veb)
14691 if (veb->uplink_seid == branch->seid)
14692 i40e_switch_branch_release(veb);
14693
14694 /* Release the VSIs on this VEB, but not the owner VSI.
14695 *
14696 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14697 * the VEB itself, so don't use (*branch) after this loop.
14698 */
14699 i40e_pf_for_each_vsi(pf, i, vsi)
14700 if (vsi->uplink_seid == branch_seid &&
14701 (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14702 i40e_vsi_release(vsi);
14703
14704 /* There's one corner case where the VEB might not have been
14705 * removed, so double check it here and remove it if needed.
14706 * This case happens if the veb was created from the debugfs
14707 * commands and no VSIs were added to it.
14708 */
14709 if (pf->veb[veb_idx])
14710 i40e_veb_release(pf->veb[veb_idx]);
14711 }
14712
14713 /**
14714 * i40e_veb_clear - remove veb struct
14715 * @veb: the veb to remove
14716 **/
i40e_veb_clear(struct i40e_veb * veb)14717 static void i40e_veb_clear(struct i40e_veb *veb)
14718 {
14719 if (!veb)
14720 return;
14721
14722 if (veb->pf) {
14723 struct i40e_pf *pf = veb->pf;
14724
14725 mutex_lock(&pf->switch_mutex);
14726 if (pf->veb[veb->idx] == veb)
14727 pf->veb[veb->idx] = NULL;
14728 mutex_unlock(&pf->switch_mutex);
14729 }
14730
14731 kfree(veb);
14732 }
14733
14734 /**
14735 * i40e_veb_release - Delete a VEB and free its resources
14736 * @veb: the VEB being removed
14737 **/
i40e_veb_release(struct i40e_veb * veb)14738 void i40e_veb_release(struct i40e_veb *veb)
14739 {
14740 struct i40e_vsi *vsi, *vsi_it;
14741 struct i40e_pf *pf;
14742 int i, n = 0;
14743
14744 pf = veb->pf;
14745
14746 /* find the remaining VSI and check for extras */
14747 i40e_pf_for_each_vsi(pf, i, vsi_it)
14748 if (vsi_it->uplink_seid == veb->seid) {
14749 if (vsi_it->flags & I40E_VSI_FLAG_VEB_OWNER)
14750 vsi = vsi_it;
14751 n++;
14752 }
14753
14754 /* Floating VEB has to be empty and regular one must have
14755 * single owner VSI.
14756 */
14757 if ((veb->uplink_seid && n != 1) || (!veb->uplink_seid && n != 0)) {
14758 dev_info(&pf->pdev->dev,
14759 "can't remove VEB %d with %d VSIs left\n",
14760 veb->seid, n);
14761 return;
14762 }
14763
14764 /* For regular VEB move the owner VSI to uplink port */
14765 if (veb->uplink_seid) {
14766 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14767 vsi->uplink_seid = veb->uplink_seid;
14768 vsi->veb_idx = I40E_NO_VEB;
14769 }
14770
14771 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14772 i40e_veb_clear(veb);
14773 }
14774
14775 /**
14776 * i40e_add_veb - create the VEB in the switch
14777 * @veb: the VEB to be instantiated
14778 * @vsi: the controlling VSI
14779 **/
i40e_add_veb(struct i40e_veb * veb,struct i40e_vsi * vsi)14780 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14781 {
14782 struct i40e_pf *pf = veb->pf;
14783 bool enable_stats = !!test_bit(I40E_FLAG_VEB_STATS_ENA, pf->flags);
14784 int ret;
14785
14786 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi ? vsi->seid : 0,
14787 veb->enabled_tc, vsi ? false : true,
14788 &veb->seid, enable_stats, NULL);
14789
14790 /* get a VEB from the hardware */
14791 if (ret) {
14792 dev_info(&pf->pdev->dev,
14793 "couldn't add VEB, err %pe aq_err %s\n",
14794 ERR_PTR(ret),
14795 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14796 return -EPERM;
14797 }
14798
14799 /* get statistics counter */
14800 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14801 &veb->stats_idx, NULL, NULL, NULL);
14802 if (ret) {
14803 dev_info(&pf->pdev->dev,
14804 "couldn't get VEB statistics idx, err %pe aq_err %s\n",
14805 ERR_PTR(ret),
14806 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14807 return -EPERM;
14808 }
14809 ret = i40e_veb_get_bw_info(veb);
14810 if (ret) {
14811 dev_info(&pf->pdev->dev,
14812 "couldn't get VEB bw info, err %pe aq_err %s\n",
14813 ERR_PTR(ret),
14814 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14815 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14816 return -ENOENT;
14817 }
14818
14819 if (vsi) {
14820 vsi->uplink_seid = veb->seid;
14821 vsi->veb_idx = veb->idx;
14822 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14823 }
14824
14825 return 0;
14826 }
14827
14828 /**
14829 * i40e_veb_setup - Set up a VEB
14830 * @pf: board private structure
14831 * @uplink_seid: the switch element to link to
14832 * @vsi_seid: the initial VSI seid
14833 * @enabled_tc: Enabled TC bit-map
14834 *
14835 * This allocates the sw VEB structure and links it into the switch
14836 * It is possible and legal for this to be a duplicate of an already
14837 * existing VEB. It is also possible for both uplink and vsi seids
14838 * to be zero, in order to create a floating VEB.
14839 *
14840 * Returns pointer to the successfully allocated VEB sw struct on
14841 * success, otherwise returns NULL on failure.
14842 **/
i40e_veb_setup(struct i40e_pf * pf,u16 uplink_seid,u16 vsi_seid,u8 enabled_tc)14843 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 uplink_seid,
14844 u16 vsi_seid, u8 enabled_tc)
14845 {
14846 struct i40e_vsi *vsi = NULL;
14847 struct i40e_veb *veb;
14848 int veb_idx;
14849 int ret;
14850
14851 /* if one seid is 0, the other must be 0 to create a floating relay */
14852 if ((uplink_seid == 0 || vsi_seid == 0) &&
14853 (uplink_seid + vsi_seid != 0)) {
14854 dev_info(&pf->pdev->dev,
14855 "one, not both seid's are 0: uplink=%d vsi=%d\n",
14856 uplink_seid, vsi_seid);
14857 return NULL;
14858 }
14859
14860 /* make sure there is such a vsi and uplink */
14861 if (vsi_seid) {
14862 vsi = i40e_pf_get_vsi_by_seid(pf, vsi_seid);
14863 if (!vsi) {
14864 dev_err(&pf->pdev->dev, "vsi seid %d not found\n",
14865 vsi_seid);
14866 return NULL;
14867 }
14868 }
14869
14870 /* get veb sw struct */
14871 veb_idx = i40e_veb_mem_alloc(pf);
14872 if (veb_idx < 0)
14873 goto err_alloc;
14874 veb = pf->veb[veb_idx];
14875 veb->uplink_seid = uplink_seid;
14876 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14877
14878 /* create the VEB in the switch */
14879 ret = i40e_add_veb(veb, vsi);
14880 if (ret)
14881 goto err_veb;
14882
14883 if (vsi && vsi->idx == pf->lan_vsi)
14884 pf->lan_veb = veb->idx;
14885
14886 return veb;
14887
14888 err_veb:
14889 i40e_veb_clear(veb);
14890 err_alloc:
14891 return NULL;
14892 }
14893
14894 /**
14895 * i40e_setup_pf_switch_element - set PF vars based on switch type
14896 * @pf: board private structure
14897 * @ele: element we are building info from
14898 * @num_reported: total number of elements
14899 * @printconfig: should we print the contents
14900 *
14901 * helper function to assist in extracting a few useful SEID values.
14902 **/
i40e_setup_pf_switch_element(struct i40e_pf * pf,struct i40e_aqc_switch_config_element_resp * ele,u16 num_reported,bool printconfig)14903 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14904 struct i40e_aqc_switch_config_element_resp *ele,
14905 u16 num_reported, bool printconfig)
14906 {
14907 u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14908 u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14909 u8 element_type = ele->element_type;
14910 u16 seid = le16_to_cpu(ele->seid);
14911 struct i40e_veb *veb;
14912
14913 if (printconfig)
14914 dev_info(&pf->pdev->dev,
14915 "type=%d seid=%d uplink=%d downlink=%d\n",
14916 element_type, seid, uplink_seid, downlink_seid);
14917
14918 switch (element_type) {
14919 case I40E_SWITCH_ELEMENT_TYPE_MAC:
14920 pf->mac_seid = seid;
14921 break;
14922 case I40E_SWITCH_ELEMENT_TYPE_VEB:
14923 /* Main VEB? */
14924 if (uplink_seid != pf->mac_seid)
14925 break;
14926 veb = i40e_pf_get_main_veb(pf);
14927 if (!veb) {
14928 int v;
14929
14930 /* find existing or else empty VEB */
14931 veb = i40e_pf_get_veb_by_seid(pf, seid);
14932 if (veb) {
14933 pf->lan_veb = veb->idx;
14934 } else {
14935 v = i40e_veb_mem_alloc(pf);
14936 if (v < 0)
14937 break;
14938 pf->lan_veb = v;
14939 }
14940 }
14941
14942 /* Try to get again main VEB as pf->lan_veb may have changed */
14943 veb = i40e_pf_get_main_veb(pf);
14944 if (!veb)
14945 break;
14946
14947 veb->seid = seid;
14948 veb->uplink_seid = pf->mac_seid;
14949 veb->pf = pf;
14950 break;
14951 case I40E_SWITCH_ELEMENT_TYPE_VSI:
14952 if (num_reported != 1)
14953 break;
14954 /* This is immediately after a reset so we can assume this is
14955 * the PF's VSI
14956 */
14957 pf->mac_seid = uplink_seid;
14958 pf->main_vsi_seid = seid;
14959 if (printconfig)
14960 dev_info(&pf->pdev->dev,
14961 "pf_seid=%d main_vsi_seid=%d\n",
14962 downlink_seid, pf->main_vsi_seid);
14963 break;
14964 case I40E_SWITCH_ELEMENT_TYPE_PF:
14965 case I40E_SWITCH_ELEMENT_TYPE_VF:
14966 case I40E_SWITCH_ELEMENT_TYPE_EMP:
14967 case I40E_SWITCH_ELEMENT_TYPE_BMC:
14968 case I40E_SWITCH_ELEMENT_TYPE_PE:
14969 case I40E_SWITCH_ELEMENT_TYPE_PA:
14970 /* ignore these for now */
14971 break;
14972 default:
14973 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14974 element_type, seid);
14975 break;
14976 }
14977 }
14978
14979 /**
14980 * i40e_fetch_switch_configuration - Get switch config from firmware
14981 * @pf: board private structure
14982 * @printconfig: should we print the contents
14983 *
14984 * Get the current switch configuration from the device and
14985 * extract a few useful SEID values.
14986 **/
i40e_fetch_switch_configuration(struct i40e_pf * pf,bool printconfig)14987 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14988 {
14989 struct i40e_aqc_get_switch_config_resp *sw_config;
14990 u16 next_seid = 0;
14991 int ret = 0;
14992 u8 *aq_buf;
14993 int i;
14994
14995 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14996 if (!aq_buf)
14997 return -ENOMEM;
14998
14999 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
15000 do {
15001 u16 num_reported, num_total;
15002
15003 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
15004 I40E_AQ_LARGE_BUF,
15005 &next_seid, NULL);
15006 if (ret) {
15007 dev_info(&pf->pdev->dev,
15008 "get switch config failed err %d aq_err %s\n",
15009 ret,
15010 i40e_aq_str(&pf->hw,
15011 pf->hw.aq.asq_last_status));
15012 kfree(aq_buf);
15013 return -ENOENT;
15014 }
15015
15016 num_reported = le16_to_cpu(sw_config->header.num_reported);
15017 num_total = le16_to_cpu(sw_config->header.num_total);
15018
15019 if (printconfig)
15020 dev_info(&pf->pdev->dev,
15021 "header: %d reported %d total\n",
15022 num_reported, num_total);
15023
15024 for (i = 0; i < num_reported; i++) {
15025 struct i40e_aqc_switch_config_element_resp *ele =
15026 &sw_config->element[i];
15027
15028 i40e_setup_pf_switch_element(pf, ele, num_reported,
15029 printconfig);
15030 }
15031 } while (next_seid != 0);
15032
15033 kfree(aq_buf);
15034 return ret;
15035 }
15036
15037 /**
15038 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
15039 * @pf: board private structure
15040 * @reinit: if the Main VSI needs to re-initialized.
15041 * @lock_acquired: indicates whether or not the lock has been acquired
15042 *
15043 * Returns 0 on success, negative value on failure
15044 **/
i40e_setup_pf_switch(struct i40e_pf * pf,bool reinit,bool lock_acquired)15045 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
15046 {
15047 struct i40e_vsi *main_vsi;
15048 u16 flags = 0;
15049 int ret;
15050
15051 /* find out what's out there already */
15052 ret = i40e_fetch_switch_configuration(pf, false);
15053 if (ret) {
15054 dev_info(&pf->pdev->dev,
15055 "couldn't fetch switch config, err %pe aq_err %s\n",
15056 ERR_PTR(ret),
15057 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15058 return ret;
15059 }
15060 i40e_pf_reset_stats(pf);
15061
15062 /* set the switch config bit for the whole device to
15063 * support limited promisc or true promisc
15064 * when user requests promisc. The default is limited
15065 * promisc.
15066 */
15067
15068 if ((pf->hw.pf_id == 0) &&
15069 !test_bit(I40E_FLAG_TRUE_PROMISC_ENA, pf->flags)) {
15070 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15071 pf->last_sw_conf_flags = flags;
15072 }
15073
15074 if (pf->hw.pf_id == 0) {
15075 u16 valid_flags;
15076
15077 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15078 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
15079 NULL);
15080 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
15081 dev_info(&pf->pdev->dev,
15082 "couldn't set switch config bits, err %pe aq_err %s\n",
15083 ERR_PTR(ret),
15084 i40e_aq_str(&pf->hw,
15085 pf->hw.aq.asq_last_status));
15086 /* not a fatal problem, just keep going */
15087 }
15088 pf->last_sw_conf_valid_flags = valid_flags;
15089 }
15090
15091 /* first time setup */
15092 main_vsi = i40e_pf_get_main_vsi(pf);
15093 if (!main_vsi || reinit) {
15094 struct i40e_veb *veb;
15095 u16 uplink_seid;
15096
15097 /* Set up the PF VSI associated with the PF's main VSI
15098 * that is already in the HW switch
15099 */
15100 veb = i40e_pf_get_main_veb(pf);
15101 if (veb)
15102 uplink_seid = veb->seid;
15103 else
15104 uplink_seid = pf->mac_seid;
15105 if (!main_vsi)
15106 main_vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN,
15107 uplink_seid, 0);
15108 else if (reinit)
15109 main_vsi = i40e_vsi_reinit_setup(main_vsi);
15110 if (!main_vsi) {
15111 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
15112 i40e_cloud_filter_exit(pf);
15113 i40e_fdir_teardown(pf);
15114 return -EAGAIN;
15115 }
15116 } else {
15117 /* force a reset of TC and queue layout configurations */
15118 main_vsi->seid = pf->main_vsi_seid;
15119 i40e_vsi_reconfig_tc(main_vsi);
15120 }
15121 i40e_vlan_stripping_disable(main_vsi);
15122
15123 i40e_fdir_sb_setup(pf);
15124
15125 /* Setup static PF queue filter control settings */
15126 ret = i40e_setup_pf_filter_control(pf);
15127 if (ret) {
15128 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
15129 ret);
15130 /* Failure here should not stop continuing other steps */
15131 }
15132
15133 /* enable RSS in the HW, even for only one queue, as the stack can use
15134 * the hash
15135 */
15136 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15137 i40e_pf_config_rss(pf);
15138
15139 /* fill in link information and enable LSE reporting */
15140 i40e_link_event(pf);
15141
15142 i40e_ptp_init(pf);
15143
15144 if (!lock_acquired)
15145 rtnl_lock();
15146
15147 /* repopulate tunnel port filters */
15148 udp_tunnel_nic_reset_ntf(main_vsi->netdev);
15149
15150 if (!lock_acquired)
15151 rtnl_unlock();
15152
15153 return ret;
15154 }
15155
15156 /**
15157 * i40e_determine_queue_usage - Work out queue distribution
15158 * @pf: board private structure
15159 **/
i40e_determine_queue_usage(struct i40e_pf * pf)15160 static void i40e_determine_queue_usage(struct i40e_pf *pf)
15161 {
15162 int queues_left;
15163 int q_max;
15164
15165 pf->num_lan_qps = 0;
15166
15167 /* Find the max queues to be put into basic use. We'll always be
15168 * using TC0, whether or not DCB is running, and TC0 will get the
15169 * big RSS set.
15170 */
15171 queues_left = pf->hw.func_caps.num_tx_qp;
15172
15173 if ((queues_left == 1) ||
15174 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
15175 /* one qp for PF, no queues for anything else */
15176 queues_left = 0;
15177 pf->alloc_rss_size = pf->num_lan_qps = 1;
15178
15179 /* make sure all the fancies are disabled */
15180 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
15181 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
15182 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15183 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
15184 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15185 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15186 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
15187 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
15188 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15189 } else if (!test_bit(I40E_FLAG_RSS_ENA, pf->flags) &&
15190 !test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) &&
15191 !test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags) &&
15192 !test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags)) {
15193 /* one qp for PF */
15194 pf->alloc_rss_size = pf->num_lan_qps = 1;
15195 queues_left -= pf->num_lan_qps;
15196
15197 clear_bit(I40E_FLAG_RSS_ENA, pf->flags);
15198 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
15199 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15200 clear_bit(I40E_FLAG_FD_ATR_ENA, pf->flags);
15201 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15202 clear_bit(I40E_FLAG_VMDQ_ENA, pf->flags);
15203 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15204 } else {
15205 /* Not enough queues for all TCs */
15206 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags) &&
15207 queues_left < I40E_MAX_TRAFFIC_CLASS) {
15208 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15209 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15210 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
15211 }
15212
15213 /* limit lan qps to the smaller of qps, cpus or msix */
15214 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
15215 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
15216 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
15217 pf->num_lan_qps = q_max;
15218
15219 queues_left -= pf->num_lan_qps;
15220 }
15221
15222 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15223 if (queues_left > 1) {
15224 queues_left -= 1; /* save 1 queue for FD */
15225 } else {
15226 clear_bit(I40E_FLAG_FD_SB_ENA, pf->flags);
15227 set_bit(I40E_FLAG_FD_SB_INACTIVE, pf->flags);
15228 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
15229 }
15230 }
15231
15232 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15233 pf->num_vf_qps && pf->num_req_vfs && queues_left) {
15234 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
15235 (queues_left / pf->num_vf_qps));
15236 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
15237 }
15238
15239 if (test_bit(I40E_FLAG_VMDQ_ENA, pf->flags) &&
15240 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
15241 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
15242 (queues_left / pf->num_vmdq_qps));
15243 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
15244 }
15245
15246 pf->queues_left = queues_left;
15247 dev_dbg(&pf->pdev->dev,
15248 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
15249 pf->hw.func_caps.num_tx_qp,
15250 !!test_bit(I40E_FLAG_FD_SB_ENA, pf->flags),
15251 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
15252 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
15253 queues_left);
15254 }
15255
15256 /**
15257 * i40e_setup_pf_filter_control - Setup PF static filter control
15258 * @pf: PF to be setup
15259 *
15260 * i40e_setup_pf_filter_control sets up a PF's initial filter control
15261 * settings. If PE/FCoE are enabled then it will also set the per PF
15262 * based filter sizes required for them. It also enables Flow director,
15263 * ethertype and macvlan type filter settings for the pf.
15264 *
15265 * Returns 0 on success, negative on failure
15266 **/
i40e_setup_pf_filter_control(struct i40e_pf * pf)15267 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
15268 {
15269 struct i40e_filter_control_settings *settings = &pf->filter_settings;
15270
15271 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
15272
15273 /* Flow Director is enabled */
15274 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags) ||
15275 test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15276 settings->enable_fdir = true;
15277
15278 /* Ethtype and MACVLAN filters enabled for PF */
15279 settings->enable_ethtype = true;
15280 settings->enable_macvlan = true;
15281
15282 if (i40e_set_filter_control(&pf->hw, settings))
15283 return -ENOENT;
15284
15285 return 0;
15286 }
15287
15288 #define INFO_STRING_LEN 255
15289 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
i40e_print_features(struct i40e_pf * pf)15290 static void i40e_print_features(struct i40e_pf *pf)
15291 {
15292 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
15293 struct i40e_hw *hw = &pf->hw;
15294 char *buf;
15295 int i;
15296
15297 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
15298 if (!buf)
15299 return;
15300
15301 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
15302 #ifdef CONFIG_PCI_IOV
15303 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
15304 #endif
15305 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
15306 pf->hw.func_caps.num_vsis, main_vsi->num_queue_pairs);
15307 if (test_bit(I40E_FLAG_RSS_ENA, pf->flags))
15308 i += scnprintf(&buf[i], REMAIN(i), " RSS");
15309 if (test_bit(I40E_FLAG_FD_ATR_ENA, pf->flags))
15310 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
15311 if (test_bit(I40E_FLAG_FD_SB_ENA, pf->flags)) {
15312 i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
15313 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
15314 }
15315 if (test_bit(I40E_FLAG_DCB_CAPABLE, pf->flags))
15316 i += scnprintf(&buf[i], REMAIN(i), " DCB");
15317 i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
15318 i += scnprintf(&buf[i], REMAIN(i), " Geneve");
15319 if (test_bit(I40E_FLAG_PTP_ENA, pf->flags))
15320 i += scnprintf(&buf[i], REMAIN(i), " PTP");
15321 if (test_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags))
15322 i += scnprintf(&buf[i], REMAIN(i), " VEB");
15323 else
15324 i += scnprintf(&buf[i], REMAIN(i), " VEPA");
15325
15326 dev_info(&pf->pdev->dev, "%s\n", buf);
15327 kfree(buf);
15328 WARN_ON(i > INFO_STRING_LEN);
15329 }
15330
15331 /**
15332 * i40e_get_platform_mac_addr - get platform-specific MAC address
15333 * @pdev: PCI device information struct
15334 * @pf: board private structure
15335 *
15336 * Look up the MAC address for the device. First we'll try
15337 * eth_platform_get_mac_address, which will check Open Firmware, or arch
15338 * specific fallback. Otherwise, we'll default to the stored value in
15339 * firmware.
15340 **/
i40e_get_platform_mac_addr(struct pci_dev * pdev,struct i40e_pf * pf)15341 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
15342 {
15343 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
15344 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
15345 }
15346
15347 /**
15348 * i40e_set_fec_in_flags - helper function for setting FEC options in flags
15349 * @fec_cfg: FEC option to set in flags
15350 * @flags: ptr to flags in which we set FEC option
15351 **/
i40e_set_fec_in_flags(u8 fec_cfg,unsigned long * flags)15352 void i40e_set_fec_in_flags(u8 fec_cfg, unsigned long *flags)
15353 {
15354 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) {
15355 set_bit(I40E_FLAG_RS_FEC, flags);
15356 set_bit(I40E_FLAG_BASE_R_FEC, flags);
15357 }
15358 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
15359 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
15360 set_bit(I40E_FLAG_RS_FEC, flags);
15361 clear_bit(I40E_FLAG_BASE_R_FEC, flags);
15362 }
15363 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
15364 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
15365 set_bit(I40E_FLAG_BASE_R_FEC, flags);
15366 clear_bit(I40E_FLAG_RS_FEC, flags);
15367 }
15368 if (fec_cfg == 0) {
15369 clear_bit(I40E_FLAG_RS_FEC, flags);
15370 clear_bit(I40E_FLAG_BASE_R_FEC, flags);
15371 }
15372 }
15373
15374 /**
15375 * i40e_check_recovery_mode - check if we are running transition firmware
15376 * @pf: board private structure
15377 *
15378 * Check registers indicating the firmware runs in recovery mode. Sets the
15379 * appropriate driver state.
15380 *
15381 * Returns true if the recovery mode was detected, false otherwise
15382 **/
i40e_check_recovery_mode(struct i40e_pf * pf)15383 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
15384 {
15385 u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
15386
15387 if (val & I40E_GL_FWSTS_FWS1B_MASK) {
15388 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
15389 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
15390 set_bit(__I40E_RECOVERY_MODE, pf->state);
15391
15392 return true;
15393 }
15394 if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15395 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
15396
15397 return false;
15398 }
15399
15400 /**
15401 * i40e_pf_loop_reset - perform reset in a loop.
15402 * @pf: board private structure
15403 *
15404 * This function is useful when a NIC is about to enter recovery mode.
15405 * When a NIC's internal data structures are corrupted the NIC's
15406 * firmware is going to enter recovery mode.
15407 * Right after a POR it takes about 7 minutes for firmware to enter
15408 * recovery mode. Until that time a NIC is in some kind of intermediate
15409 * state. After that time period the NIC almost surely enters
15410 * recovery mode. The only way for a driver to detect intermediate
15411 * state is to issue a series of pf-resets and check a return value.
15412 * If a PF reset returns success then the firmware could be in recovery
15413 * mode so the caller of this code needs to check for recovery mode
15414 * if this function returns success. There is a little chance that
15415 * firmware will hang in intermediate state forever.
15416 * Since waiting 7 minutes is quite a lot of time this function waits
15417 * 10 seconds and then gives up by returning an error.
15418 *
15419 * Return 0 on success, negative on failure.
15420 **/
i40e_pf_loop_reset(struct i40e_pf * pf)15421 static int i40e_pf_loop_reset(struct i40e_pf *pf)
15422 {
15423 /* wait max 10 seconds for PF reset to succeed */
15424 const unsigned long time_end = jiffies + 10 * HZ;
15425 struct i40e_hw *hw = &pf->hw;
15426 int ret;
15427
15428 ret = i40e_pf_reset(hw);
15429 while (ret != 0 && time_before(jiffies, time_end)) {
15430 usleep_range(10000, 20000);
15431 ret = i40e_pf_reset(hw);
15432 }
15433
15434 if (ret == 0)
15435 pf->pfr_count++;
15436 else
15437 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15438
15439 return ret;
15440 }
15441
15442 /**
15443 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15444 * @pf: board private structure
15445 *
15446 * Check FW registers to determine if FW issued unexpected EMP Reset.
15447 * Every time when unexpected EMP Reset occurs the FW increments
15448 * a counter of unexpected EMP Resets. When the counter reaches 10
15449 * the FW should enter the Recovery mode
15450 *
15451 * Returns true if FW issued unexpected EMP Reset
15452 **/
i40e_check_fw_empr(struct i40e_pf * pf)15453 static bool i40e_check_fw_empr(struct i40e_pf *pf)
15454 {
15455 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15456 I40E_GL_FWSTS_FWS1B_MASK;
15457 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15458 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15459 }
15460
15461 /**
15462 * i40e_handle_resets - handle EMP resets and PF resets
15463 * @pf: board private structure
15464 *
15465 * Handle both EMP resets and PF resets and conclude whether there are
15466 * any issues regarding these resets. If there are any issues then
15467 * generate log entry.
15468 *
15469 * Return 0 if NIC is healthy or negative value when there are issues
15470 * with resets
15471 **/
i40e_handle_resets(struct i40e_pf * pf)15472 static int i40e_handle_resets(struct i40e_pf *pf)
15473 {
15474 const int pfr = i40e_pf_loop_reset(pf);
15475 const bool is_empr = i40e_check_fw_empr(pf);
15476
15477 if (is_empr || pfr != 0)
15478 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15479
15480 return is_empr ? -EIO : pfr;
15481 }
15482
15483 /**
15484 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15485 * @pf: board private structure
15486 * @hw: ptr to the hardware info
15487 *
15488 * This function does a minimal setup of all subsystems needed for running
15489 * recovery mode.
15490 *
15491 * Returns 0 on success, negative on failure
15492 **/
i40e_init_recovery_mode(struct i40e_pf * pf,struct i40e_hw * hw)15493 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15494 {
15495 struct i40e_vsi *vsi;
15496 int err;
15497 int v_idx;
15498
15499 pci_set_drvdata(pf->pdev, pf);
15500 pci_save_state(pf->pdev);
15501
15502 /* set up periodic task facility */
15503 timer_setup(&pf->service_timer, i40e_service_timer, 0);
15504 pf->service_timer_period = HZ;
15505
15506 INIT_WORK(&pf->service_task, i40e_service_task);
15507 clear_bit(__I40E_SERVICE_SCHED, pf->state);
15508
15509 err = i40e_init_interrupt_scheme(pf);
15510 if (err)
15511 goto err_switch_setup;
15512
15513 /* The number of VSIs reported by the FW is the minimum guaranteed
15514 * to us; HW supports far more and we share the remaining pool with
15515 * the other PFs. We allocate space for more than the guarantee with
15516 * the understanding that we might not get them all later.
15517 */
15518 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15519 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15520 else
15521 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15522
15523 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15524 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15525 GFP_KERNEL);
15526 if (!pf->vsi) {
15527 err = -ENOMEM;
15528 goto err_switch_setup;
15529 }
15530
15531 /* We allocate one VSI which is needed as absolute minimum
15532 * in order to register the netdev
15533 */
15534 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
15535 if (v_idx < 0) {
15536 err = v_idx;
15537 goto err_switch_setup;
15538 }
15539 pf->lan_vsi = v_idx;
15540 vsi = pf->vsi[v_idx];
15541 if (!vsi) {
15542 err = -EFAULT;
15543 goto err_switch_setup;
15544 }
15545 vsi->alloc_queue_pairs = 1;
15546 err = i40e_config_netdev(vsi);
15547 if (err)
15548 goto err_switch_setup;
15549 err = register_netdev(vsi->netdev);
15550 if (err)
15551 goto err_switch_setup;
15552 vsi->netdev_registered = true;
15553 i40e_dbg_pf_init(pf);
15554
15555 err = i40e_setup_misc_vector_for_recovery_mode(pf);
15556 if (err)
15557 goto err_switch_setup;
15558
15559 /* tell the firmware that we're starting */
15560 i40e_send_version(pf);
15561
15562 /* since everything's happy, start the service_task timer */
15563 mod_timer(&pf->service_timer,
15564 round_jiffies(jiffies + pf->service_timer_period));
15565
15566 return 0;
15567
15568 err_switch_setup:
15569 i40e_reset_interrupt_capability(pf);
15570 timer_shutdown_sync(&pf->service_timer);
15571 i40e_shutdown_adminq(hw);
15572 iounmap(hw->hw_addr);
15573 pci_release_mem_regions(pf->pdev);
15574 pci_disable_device(pf->pdev);
15575 i40e_free_pf(pf);
15576
15577 return err;
15578 }
15579
15580 /**
15581 * i40e_set_subsystem_device_id - set subsystem device id
15582 * @hw: pointer to the hardware info
15583 *
15584 * Set PCI subsystem device id either from a pci_dev structure or
15585 * a specific FW register.
15586 **/
i40e_set_subsystem_device_id(struct i40e_hw * hw)15587 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)
15588 {
15589 struct i40e_pf *pf = i40e_hw_to_pf(hw);
15590
15591 hw->subsystem_device_id = pf->pdev->subsystem_device ?
15592 pf->pdev->subsystem_device :
15593 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);
15594 }
15595
15596 /**
15597 * i40e_probe - Device initialization routine
15598 * @pdev: PCI device information struct
15599 * @ent: entry in i40e_pci_tbl
15600 *
15601 * i40e_probe initializes a PF identified by a pci_dev structure.
15602 * The OS initialization, configuring of the PF private structure,
15603 * and a hardware reset occur.
15604 *
15605 * Returns 0 on success, negative on failure
15606 **/
i40e_probe(struct pci_dev * pdev,const struct pci_device_id * ent)15607 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15608 {
15609 struct i40e_aq_get_phy_abilities_resp abilities;
15610 #ifdef CONFIG_I40E_DCB
15611 enum i40e_get_fw_lldp_status_resp lldp_status;
15612 #endif /* CONFIG_I40E_DCB */
15613 struct i40e_vsi *vsi;
15614 struct i40e_pf *pf;
15615 struct i40e_hw *hw;
15616 u16 wol_nvm_bits;
15617 char nvm_ver[32];
15618 u16 link_status;
15619 #ifdef CONFIG_I40E_DCB
15620 int status;
15621 #endif /* CONFIG_I40E_DCB */
15622 int err;
15623 u32 val;
15624
15625 err = pci_enable_device_mem(pdev);
15626 if (err)
15627 return err;
15628
15629 /* set up for high or low dma */
15630 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
15631 if (err) {
15632 dev_err(&pdev->dev,
15633 "DMA configuration failed: 0x%x\n", err);
15634 goto err_dma;
15635 }
15636
15637 /* set up pci connections */
15638 err = pci_request_mem_regions(pdev, i40e_driver_name);
15639 if (err) {
15640 dev_info(&pdev->dev,
15641 "pci_request_selected_regions failed %d\n", err);
15642 goto err_pci_reg;
15643 }
15644
15645 pci_set_master(pdev);
15646
15647 /* Now that we have a PCI connection, we need to do the
15648 * low level device setup. This is primarily setting up
15649 * the Admin Queue structures and then querying for the
15650 * device's current profile information.
15651 */
15652 pf = i40e_alloc_pf(&pdev->dev);
15653 if (!pf) {
15654 err = -ENOMEM;
15655 goto err_pf_alloc;
15656 }
15657 pf->next_vsi = 0;
15658 pf->pdev = pdev;
15659 set_bit(__I40E_DOWN, pf->state);
15660
15661 hw = &pf->hw;
15662
15663 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15664 I40E_MAX_CSR_SPACE);
15665 /* We believe that the highest register to read is
15666 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15667 * is not less than that before mapping to prevent a
15668 * kernel panic.
15669 */
15670 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15671 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15672 pf->ioremap_len);
15673 err = -ENOMEM;
15674 goto err_ioremap;
15675 }
15676 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15677 if (!hw->hw_addr) {
15678 err = -EIO;
15679 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15680 (unsigned int)pci_resource_start(pdev, 0),
15681 pf->ioremap_len, err);
15682 goto err_ioremap;
15683 }
15684 hw->vendor_id = pdev->vendor;
15685 hw->device_id = pdev->device;
15686 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15687 hw->subsystem_vendor_id = pdev->subsystem_vendor;
15688 i40e_set_subsystem_device_id(hw);
15689 hw->bus.device = PCI_SLOT(pdev->devfn);
15690 hw->bus.func = PCI_FUNC(pdev->devfn);
15691 hw->bus.bus_id = pdev->bus->number;
15692
15693 /* Select something other than the 802.1ad ethertype for the
15694 * switch to use internally and drop on ingress.
15695 */
15696 hw->switch_tag = 0xffff;
15697 hw->first_tag = ETH_P_8021AD;
15698 hw->second_tag = ETH_P_8021Q;
15699
15700 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15701 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15702 INIT_LIST_HEAD(&pf->ddp_old_prof);
15703
15704 /* set up the locks for the AQ, do this only once in probe
15705 * and destroy them only once in remove
15706 */
15707 mutex_init(&hw->aq.asq_mutex);
15708 mutex_init(&hw->aq.arq_mutex);
15709
15710 pf->msg_enable = netif_msg_init(debug,
15711 NETIF_MSG_DRV |
15712 NETIF_MSG_PROBE |
15713 NETIF_MSG_LINK);
15714 if (debug < -1)
15715 pf->hw.debug_mask = debug;
15716
15717 /* do a special CORER for clearing PXE mode once at init */
15718 if (hw->revision_id == 0 &&
15719 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15720 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15721 i40e_flush(hw);
15722 msleep(200);
15723 pf->corer_count++;
15724
15725 i40e_clear_pxe_mode(hw);
15726 }
15727
15728 /* Reset here to make sure all is clean and to define PF 'n' */
15729 i40e_clear_hw(hw);
15730
15731 err = i40e_set_mac_type(hw);
15732 if (err) {
15733 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15734 err);
15735 goto err_pf_reset;
15736 }
15737
15738 err = i40e_handle_resets(pf);
15739 if (err)
15740 goto err_pf_reset;
15741
15742 i40e_check_recovery_mode(pf);
15743
15744 if (is_kdump_kernel()) {
15745 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;
15746 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;
15747 } else {
15748 hw->aq.num_arq_entries = I40E_AQ_LEN;
15749 hw->aq.num_asq_entries = I40E_AQ_LEN;
15750 }
15751 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15752 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15753
15754 snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15755 "%s-%s:misc",
15756 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15757
15758 err = i40e_init_shared_code(hw);
15759 if (err) {
15760 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15761 err);
15762 goto err_pf_reset;
15763 }
15764
15765 /* set up a default setting for link flow control */
15766 pf->hw.fc.requested_mode = I40E_FC_NONE;
15767
15768 err = i40e_init_adminq(hw);
15769 if (err) {
15770 if (err == -EIO)
15771 dev_info(&pdev->dev,
15772 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15773 hw->aq.api_maj_ver,
15774 hw->aq.api_min_ver,
15775 I40E_FW_API_VERSION_MAJOR,
15776 I40E_FW_MINOR_VERSION(hw));
15777 else
15778 dev_info(&pdev->dev,
15779 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15780
15781 goto err_pf_reset;
15782 }
15783 i40e_get_oem_version(hw);
15784 i40e_get_pba_string(hw);
15785
15786 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15787 i40e_nvm_version_str(hw, nvm_ver, sizeof(nvm_ver));
15788 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15789 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15790 hw->aq.api_maj_ver, hw->aq.api_min_ver, nvm_ver,
15791 hw->vendor_id, hw->device_id, hw->subsystem_vendor_id,
15792 hw->subsystem_device_id);
15793
15794 if (i40e_is_aq_api_ver_ge(hw, I40E_FW_API_VERSION_MAJOR,
15795 I40E_FW_MINOR_VERSION(hw) + 1))
15796 dev_dbg(&pdev->dev,
15797 "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
15798 hw->aq.api_maj_ver,
15799 hw->aq.api_min_ver,
15800 I40E_FW_API_VERSION_MAJOR,
15801 I40E_FW_MINOR_VERSION(hw));
15802 else if (i40e_is_aq_api_ver_lt(hw, 1, 4))
15803 dev_info(&pdev->dev,
15804 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15805 hw->aq.api_maj_ver,
15806 hw->aq.api_min_ver,
15807 I40E_FW_API_VERSION_MAJOR,
15808 I40E_FW_MINOR_VERSION(hw));
15809
15810 i40e_verify_eeprom(pf);
15811
15812 /* Rev 0 hardware was never productized */
15813 if (hw->revision_id < 1)
15814 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15815
15816 i40e_clear_pxe_mode(hw);
15817
15818 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15819 if (err)
15820 goto err_adminq_setup;
15821
15822 err = i40e_sw_init(pf);
15823 if (err) {
15824 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15825 goto err_sw_init;
15826 }
15827
15828 if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15829 return i40e_init_recovery_mode(pf, hw);
15830
15831 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15832 hw->func_caps.num_rx_qp, 0, 0);
15833 if (err) {
15834 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15835 goto err_init_lan_hmc;
15836 }
15837
15838 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15839 if (err) {
15840 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15841 err = -ENOENT;
15842 goto err_configure_lan_hmc;
15843 }
15844
15845 /* Disable LLDP for NICs that have firmware versions lower than v4.3.
15846 * Ignore error return codes because if it was already disabled via
15847 * hardware settings this will fail
15848 */
15849 if (test_bit(I40E_HW_CAP_STOP_FW_LLDP, pf->hw.caps)) {
15850 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15851 i40e_aq_stop_lldp(hw, true, false, NULL);
15852 }
15853
15854 /* allow a platform config to override the HW addr */
15855 i40e_get_platform_mac_addr(pdev, pf);
15856
15857 if (!is_valid_ether_addr(hw->mac.addr)) {
15858 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15859 err = -EIO;
15860 goto err_mac_addr;
15861 }
15862 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15863 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15864 i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15865 if (is_valid_ether_addr(hw->mac.port_addr))
15866 set_bit(I40E_HW_CAP_PORT_ID_VALID, pf->hw.caps);
15867
15868 i40e_ptp_alloc_pins(pf);
15869 pci_set_drvdata(pdev, pf);
15870 pci_save_state(pdev);
15871
15872 #ifdef CONFIG_I40E_DCB
15873 status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
15874 (!status &&
15875 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15876 (clear_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags)) :
15877 (set_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags));
15878 dev_info(&pdev->dev,
15879 test_bit(I40E_FLAG_FW_LLDP_DIS, pf->flags) ?
15880 "FW LLDP is disabled\n" :
15881 "FW LLDP is enabled\n");
15882
15883 /* Enable FW to write default DCB config on link-up */
15884 i40e_aq_set_dcb_parameters(hw, true, NULL);
15885
15886 err = i40e_init_pf_dcb(pf);
15887 if (err) {
15888 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15889 clear_bit(I40E_FLAG_DCB_CAPABLE, pf->flags);
15890 clear_bit(I40E_FLAG_DCB_ENA, pf->flags);
15891 /* Continue without DCB enabled */
15892 }
15893 #endif /* CONFIG_I40E_DCB */
15894
15895 /* set up periodic task facility */
15896 timer_setup(&pf->service_timer, i40e_service_timer, 0);
15897 pf->service_timer_period = HZ;
15898
15899 INIT_WORK(&pf->service_task, i40e_service_task);
15900 clear_bit(__I40E_SERVICE_SCHED, pf->state);
15901
15902 /* NVM bit on means WoL disabled for the port */
15903 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15904 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15905 pf->wol_en = false;
15906 else
15907 pf->wol_en = true;
15908 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15909
15910 /* set up the main switch operations */
15911 i40e_determine_queue_usage(pf);
15912 err = i40e_init_interrupt_scheme(pf);
15913 if (err)
15914 goto err_switch_setup;
15915
15916 /* Reduce Tx and Rx pairs for kdump
15917 * When MSI-X is enabled, it's not allowed to use more TC queue
15918 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus
15919 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.
15920 */
15921 if (is_kdump_kernel())
15922 pf->num_lan_msix = 1;
15923
15924 pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15925 pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15926 pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15927 pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15928 pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15929 pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15930 UDP_TUNNEL_TYPE_GENEVE;
15931
15932 /* The number of VSIs reported by the FW is the minimum guaranteed
15933 * to us; HW supports far more and we share the remaining pool with
15934 * the other PFs. We allocate space for more than the guarantee with
15935 * the understanding that we might not get them all later.
15936 */
15937 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15938 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15939 else
15940 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15941 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15942 dev_warn(&pf->pdev->dev,
15943 "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15944 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15945 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15946 }
15947
15948 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15949 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15950 GFP_KERNEL);
15951 if (!pf->vsi) {
15952 err = -ENOMEM;
15953 goto err_switch_setup;
15954 }
15955
15956 #ifdef CONFIG_PCI_IOV
15957 /* prep for VF support */
15958 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
15959 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
15960 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15961 if (pci_num_vf(pdev))
15962 set_bit(I40E_FLAG_VEB_MODE_ENA, pf->flags);
15963 }
15964 #endif
15965 err = i40e_setup_pf_switch(pf, false, false);
15966 if (err) {
15967 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15968 goto err_vsis;
15969 }
15970
15971 vsi = i40e_pf_get_main_vsi(pf);
15972 INIT_LIST_HEAD(&vsi->ch_list);
15973
15974 /* if FDIR VSI was set up, start it now */
15975 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
15976 if (vsi)
15977 i40e_vsi_open(vsi);
15978
15979 /* The driver only wants link up/down and module qualification
15980 * reports from firmware. Note the negative logic.
15981 */
15982 err = i40e_aq_set_phy_int_mask(&pf->hw,
15983 ~(I40E_AQ_EVENT_LINK_UPDOWN |
15984 I40E_AQ_EVENT_MEDIA_NA |
15985 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15986 if (err)
15987 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
15988 ERR_PTR(err),
15989 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15990
15991 /* Reconfigure hardware for allowing smaller MSS in the case
15992 * of TSO, so that we avoid the MDD being fired and causing
15993 * a reset in the case of small MSS+TSO.
15994 */
15995 val = rd32(hw, I40E_REG_MSS);
15996 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15997 val &= ~I40E_REG_MSS_MIN_MASK;
15998 val |= I40E_64BYTE_MSS;
15999 wr32(hw, I40E_REG_MSS, val);
16000 }
16001
16002 if (test_bit(I40E_HW_CAP_RESTART_AUTONEG, pf->hw.caps)) {
16003 msleep(75);
16004 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
16005 if (err)
16006 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
16007 ERR_PTR(err),
16008 i40e_aq_str(&pf->hw,
16009 pf->hw.aq.asq_last_status));
16010 }
16011 /* The main driver is (mostly) up and happy. We need to set this state
16012 * before setting up the misc vector or we get a race and the vector
16013 * ends up disabled forever.
16014 */
16015 clear_bit(__I40E_DOWN, pf->state);
16016
16017 /* In case of MSIX we are going to setup the misc vector right here
16018 * to handle admin queue events etc. In case of legacy and MSI
16019 * the misc functionality and queue processing is combined in
16020 * the same vector and that gets setup at open.
16021 */
16022 if (test_bit(I40E_FLAG_MSIX_ENA, pf->flags)) {
16023 err = i40e_setup_misc_vector(pf);
16024 if (err) {
16025 dev_info(&pdev->dev,
16026 "setup of misc vector failed: %d\n", err);
16027 i40e_cloud_filter_exit(pf);
16028 i40e_fdir_teardown(pf);
16029 goto err_vsis;
16030 }
16031 }
16032
16033 #ifdef CONFIG_PCI_IOV
16034 /* prep for VF support */
16035 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags) &&
16036 test_bit(I40E_FLAG_MSIX_ENA, pf->flags) &&
16037 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
16038 /* disable link interrupts for VFs */
16039 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
16040 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
16041 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
16042 i40e_flush(hw);
16043
16044 if (pci_num_vf(pdev)) {
16045 dev_info(&pdev->dev,
16046 "Active VFs found, allocating resources.\n");
16047 err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
16048 if (err)
16049 dev_info(&pdev->dev,
16050 "Error %d allocating resources for existing VFs\n",
16051 err);
16052 }
16053 }
16054 #endif /* CONFIG_PCI_IOV */
16055
16056 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16057 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
16058 pf->num_iwarp_msix,
16059 I40E_IWARP_IRQ_PILE_ID);
16060 if (pf->iwarp_base_vector < 0) {
16061 dev_info(&pdev->dev,
16062 "failed to get tracking for %d vectors for IWARP err=%d\n",
16063 pf->num_iwarp_msix, pf->iwarp_base_vector);
16064 clear_bit(I40E_FLAG_IWARP_ENA, pf->flags);
16065 }
16066 }
16067
16068 i40e_dbg_pf_init(pf);
16069
16070 /* tell the firmware that we're starting */
16071 i40e_send_version(pf);
16072
16073 /* since everything's happy, start the service_task timer */
16074 mod_timer(&pf->service_timer,
16075 round_jiffies(jiffies + pf->service_timer_period));
16076
16077 /* add this PF to client device list and launch a client service task */
16078 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16079 err = i40e_lan_add_device(pf);
16080 if (err)
16081 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
16082 err);
16083 }
16084
16085 #define PCI_SPEED_SIZE 8
16086 #define PCI_WIDTH_SIZE 8
16087 /* Devices on the IOSF bus do not have this information
16088 * and will report PCI Gen 1 x 1 by default so don't bother
16089 * checking them.
16090 */
16091 if (!test_bit(I40E_HW_CAP_NO_PCI_LINK_CHECK, pf->hw.caps)) {
16092 char speed[PCI_SPEED_SIZE] = "Unknown";
16093 char width[PCI_WIDTH_SIZE] = "Unknown";
16094
16095 /* Get the negotiated link width and speed from PCI config
16096 * space
16097 */
16098 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
16099 &link_status);
16100
16101 i40e_set_pci_config_data(hw, link_status);
16102
16103 switch (hw->bus.speed) {
16104 case i40e_bus_speed_8000:
16105 strscpy(speed, "8.0", PCI_SPEED_SIZE); break;
16106 case i40e_bus_speed_5000:
16107 strscpy(speed, "5.0", PCI_SPEED_SIZE); break;
16108 case i40e_bus_speed_2500:
16109 strscpy(speed, "2.5", PCI_SPEED_SIZE); break;
16110 default:
16111 break;
16112 }
16113 switch (hw->bus.width) {
16114 case i40e_bus_width_pcie_x8:
16115 strscpy(width, "8", PCI_WIDTH_SIZE); break;
16116 case i40e_bus_width_pcie_x4:
16117 strscpy(width, "4", PCI_WIDTH_SIZE); break;
16118 case i40e_bus_width_pcie_x2:
16119 strscpy(width, "2", PCI_WIDTH_SIZE); break;
16120 case i40e_bus_width_pcie_x1:
16121 strscpy(width, "1", PCI_WIDTH_SIZE); break;
16122 default:
16123 break;
16124 }
16125
16126 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
16127 speed, width);
16128
16129 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
16130 hw->bus.speed < i40e_bus_speed_8000) {
16131 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
16132 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
16133 }
16134 }
16135
16136 /* get the requested speeds from the fw */
16137 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
16138 if (err)
16139 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %pe last_status = %s\n",
16140 ERR_PTR(err),
16141 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16142 pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
16143
16144 /* set the FEC config due to the board capabilities */
16145 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, pf->flags);
16146
16147 /* get the supported phy types from the fw */
16148 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
16149 if (err)
16150 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %pe last_status = %s\n",
16151 ERR_PTR(err),
16152 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16153
16154 /* make sure the MFS hasn't been set lower than the default */
16155 #define MAX_FRAME_SIZE_DEFAULT 0x2600
16156 val = FIELD_GET(I40E_PRTGL_SAH_MFS_MASK,
16157 rd32(&pf->hw, I40E_PRTGL_SAH));
16158 if (val < MAX_FRAME_SIZE_DEFAULT)
16159 dev_warn(&pdev->dev, "MFS for port %x (%d) has been set below the default (%d)\n",
16160 pf->hw.port, val, MAX_FRAME_SIZE_DEFAULT);
16161
16162 /* Add a filter to drop all Flow control frames from any VSI from being
16163 * transmitted. By doing so we stop a malicious VF from sending out
16164 * PAUSE or PFC frames and potentially controlling traffic for other
16165 * PF/VF VSIs.
16166 * The FW can still send Flow control frames if enabled.
16167 */
16168 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
16169 pf->main_vsi_seid);
16170
16171 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
16172 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
16173 set_bit(I40E_HW_CAP_PHY_CONTROLS_LEDS, pf->hw.caps);
16174 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
16175 set_bit(I40E_HW_CAP_CRT_RETIMER, pf->hw.caps);
16176 /* print a string summarizing features */
16177 i40e_print_features(pf);
16178
16179 i40e_devlink_register(pf);
16180
16181 return 0;
16182
16183 /* Unwind what we've done if something failed in the setup */
16184 err_vsis:
16185 set_bit(__I40E_DOWN, pf->state);
16186 i40e_clear_interrupt_scheme(pf);
16187 kfree(pf->vsi);
16188 err_switch_setup:
16189 i40e_reset_interrupt_capability(pf);
16190 timer_shutdown_sync(&pf->service_timer);
16191 err_mac_addr:
16192 err_configure_lan_hmc:
16193 (void)i40e_shutdown_lan_hmc(hw);
16194 err_init_lan_hmc:
16195 kfree(pf->qp_pile);
16196 err_sw_init:
16197 err_adminq_setup:
16198 err_pf_reset:
16199 iounmap(hw->hw_addr);
16200 err_ioremap:
16201 i40e_free_pf(pf);
16202 err_pf_alloc:
16203 pci_release_mem_regions(pdev);
16204 err_pci_reg:
16205 err_dma:
16206 pci_disable_device(pdev);
16207 return err;
16208 }
16209
16210 /**
16211 * i40e_remove - Device removal routine
16212 * @pdev: PCI device information struct
16213 *
16214 * i40e_remove is called by the PCI subsystem to alert the driver
16215 * that is should release a PCI device. This could be caused by a
16216 * Hot-Plug event, or because the driver is going to be removed from
16217 * memory.
16218 **/
i40e_remove(struct pci_dev * pdev)16219 static void i40e_remove(struct pci_dev *pdev)
16220 {
16221 struct i40e_pf *pf = pci_get_drvdata(pdev);
16222 struct i40e_hw *hw = &pf->hw;
16223 struct i40e_vsi *vsi;
16224 struct i40e_veb *veb;
16225 int ret_code;
16226 int i;
16227
16228 i40e_devlink_unregister(pf);
16229
16230 i40e_dbg_pf_exit(pf);
16231
16232 i40e_ptp_stop(pf);
16233
16234 /* Disable RSS in hw */
16235 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
16236 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
16237
16238 /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE
16239 * flags, once they are set, i40e_rebuild should not be called as
16240 * i40e_prep_for_reset always returns early.
16241 */
16242 while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
16243 usleep_range(1000, 2000);
16244 set_bit(__I40E_IN_REMOVE, pf->state);
16245
16246 if (test_bit(I40E_FLAG_SRIOV_ENA, pf->flags)) {
16247 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
16248 i40e_free_vfs(pf);
16249 clear_bit(I40E_FLAG_SRIOV_ENA, pf->flags);
16250 }
16251 /* no more scheduling of any task */
16252 set_bit(__I40E_SUSPENDED, pf->state);
16253 set_bit(__I40E_DOWN, pf->state);
16254 if (pf->service_timer.function)
16255 timer_shutdown_sync(&pf->service_timer);
16256 if (pf->service_task.func)
16257 cancel_work_sync(&pf->service_task);
16258
16259 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
16260 struct i40e_vsi *vsi = pf->vsi[0];
16261
16262 /* We know that we have allocated only one vsi for this PF,
16263 * it was just for registering netdevice, so the interface
16264 * could be visible in the 'ifconfig' output
16265 */
16266 unregister_netdev(vsi->netdev);
16267 free_netdev(vsi->netdev);
16268
16269 goto unmap;
16270 }
16271
16272 /* Client close must be called explicitly here because the timer
16273 * has been stopped.
16274 */
16275 i40e_notify_client_of_netdev_close(pf, false);
16276
16277 i40e_fdir_teardown(pf);
16278
16279 /* If there is a switch structure or any orphans, remove them.
16280 * This will leave only the PF's VSI remaining.
16281 */
16282 i40e_pf_for_each_veb(pf, i, veb)
16283 if (veb->uplink_seid == pf->mac_seid ||
16284 veb->uplink_seid == 0)
16285 i40e_switch_branch_release(veb);
16286
16287 /* Now we can shutdown the PF's VSIs, just before we kill
16288 * adminq and hmc.
16289 */
16290 i40e_pf_for_each_vsi(pf, i, vsi) {
16291 i40e_vsi_close(vsi);
16292 i40e_vsi_release(vsi);
16293 pf->vsi[i] = NULL;
16294 }
16295
16296 i40e_cloud_filter_exit(pf);
16297
16298 /* remove attached clients */
16299 if (test_bit(I40E_FLAG_IWARP_ENA, pf->flags)) {
16300 ret_code = i40e_lan_del_device(pf);
16301 if (ret_code)
16302 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
16303 ret_code);
16304 }
16305
16306 /* shutdown and destroy the HMC */
16307 if (hw->hmc.hmc_obj) {
16308 ret_code = i40e_shutdown_lan_hmc(hw);
16309 if (ret_code)
16310 dev_warn(&pdev->dev,
16311 "Failed to destroy the HMC resources: %d\n",
16312 ret_code);
16313 }
16314
16315 unmap:
16316 /* Free MSI/legacy interrupt 0 when in recovery mode. */
16317 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16318 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16319 free_irq(pf->pdev->irq, pf);
16320
16321 /* shutdown the adminq */
16322 i40e_shutdown_adminq(hw);
16323
16324 /* destroy the locks only once, here */
16325 mutex_destroy(&hw->aq.arq_mutex);
16326 mutex_destroy(&hw->aq.asq_mutex);
16327
16328 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
16329 rtnl_lock();
16330 i40e_clear_interrupt_scheme(pf);
16331 i40e_pf_for_each_vsi(pf, i, vsi) {
16332 if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
16333 i40e_vsi_clear_rings(vsi);
16334
16335 i40e_vsi_clear(vsi);
16336 pf->vsi[i] = NULL;
16337 }
16338 rtnl_unlock();
16339
16340 i40e_pf_for_each_veb(pf, i, veb) {
16341 kfree(veb);
16342 pf->veb[i] = NULL;
16343 }
16344
16345 kfree(pf->qp_pile);
16346 kfree(pf->vsi);
16347
16348 iounmap(hw->hw_addr);
16349 i40e_free_pf(pf);
16350 pci_release_mem_regions(pdev);
16351
16352 pci_disable_device(pdev);
16353 }
16354
16355 /**
16356 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16357 * using the mac_address_write admin q function
16358 * @pf: pointer to i40e_pf struct
16359 **/
i40e_enable_mc_magic_wake(struct i40e_pf * pf)16360 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16361 {
16362 struct i40e_vsi *main_vsi = i40e_pf_get_main_vsi(pf);
16363 struct i40e_hw *hw = &pf->hw;
16364 u8 mac_addr[6];
16365 u16 flags = 0;
16366 int ret;
16367
16368 /* Get current MAC address in case it's an LAA */
16369 if (main_vsi && main_vsi->netdev) {
16370 ether_addr_copy(mac_addr, main_vsi->netdev->dev_addr);
16371 } else {
16372 dev_err(&pf->pdev->dev,
16373 "Failed to retrieve MAC address; using default\n");
16374 ether_addr_copy(mac_addr, hw->mac.addr);
16375 }
16376
16377 /* The FW expects the mac address write cmd to first be called with
16378 * one of these flags before calling it again with the multicast
16379 * enable flags.
16380 */
16381 flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16382
16383 if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16384 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16385
16386 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16387 if (ret) {
16388 dev_err(&pf->pdev->dev,
16389 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16390 return;
16391 }
16392
16393 flags = I40E_AQC_MC_MAG_EN
16394 | I40E_AQC_WOL_PRESERVE_ON_PFR
16395 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16396 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16397 if (ret)
16398 dev_err(&pf->pdev->dev,
16399 "Failed to enable Multicast Magic Packet wake up\n");
16400 }
16401
16402 /**
16403 * i40e_io_suspend - suspend all IO operations
16404 * @pf: pointer to i40e_pf struct
16405 *
16406 **/
i40e_io_suspend(struct i40e_pf * pf)16407 static int i40e_io_suspend(struct i40e_pf *pf)
16408 {
16409 struct i40e_hw *hw = &pf->hw;
16410
16411 set_bit(__I40E_DOWN, pf->state);
16412
16413 /* Ensure service task will not be running */
16414 del_timer_sync(&pf->service_timer);
16415 cancel_work_sync(&pf->service_task);
16416
16417 /* Client close must be called explicitly here because the timer
16418 * has been stopped.
16419 */
16420 i40e_notify_client_of_netdev_close(pf, false);
16421
16422 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16423 pf->wol_en)
16424 i40e_enable_mc_magic_wake(pf);
16425
16426 /* Since we're going to destroy queues during the
16427 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16428 * whole section
16429 */
16430 rtnl_lock();
16431
16432 i40e_prep_for_reset(pf);
16433
16434 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16435 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16436
16437 /* Clear the interrupt scheme and release our IRQs so that the system
16438 * can safely hibernate even when there are a large number of CPUs.
16439 * Otherwise hibernation might fail when mapping all the vectors back
16440 * to CPU0.
16441 */
16442 i40e_clear_interrupt_scheme(pf);
16443
16444 rtnl_unlock();
16445
16446 return 0;
16447 }
16448
16449 /**
16450 * i40e_io_resume - resume IO operations
16451 * @pf: pointer to i40e_pf struct
16452 *
16453 **/
i40e_io_resume(struct i40e_pf * pf)16454 static int i40e_io_resume(struct i40e_pf *pf)
16455 {
16456 struct device *dev = &pf->pdev->dev;
16457 int err;
16458
16459 /* We need to hold the RTNL lock prior to restoring interrupt schemes,
16460 * since we're going to be restoring queues
16461 */
16462 rtnl_lock();
16463
16464 /* We cleared the interrupt scheme when we suspended, so we need to
16465 * restore it now to resume device functionality.
16466 */
16467 err = i40e_restore_interrupt_scheme(pf);
16468 if (err) {
16469 dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16470 err);
16471 }
16472
16473 clear_bit(__I40E_DOWN, pf->state);
16474 i40e_reset_and_rebuild(pf, false, true);
16475
16476 rtnl_unlock();
16477
16478 /* Clear suspended state last after everything is recovered */
16479 clear_bit(__I40E_SUSPENDED, pf->state);
16480
16481 /* Restart the service task */
16482 mod_timer(&pf->service_timer,
16483 round_jiffies(jiffies + pf->service_timer_period));
16484
16485 return 0;
16486 }
16487
16488 /**
16489 * i40e_pci_error_detected - warning that something funky happened in PCI land
16490 * @pdev: PCI device information struct
16491 * @error: the type of PCI error
16492 *
16493 * Called to warn that something happened and the error handling steps
16494 * are in progress. Allows the driver to quiesce things, be ready for
16495 * remediation.
16496 **/
i40e_pci_error_detected(struct pci_dev * pdev,pci_channel_state_t error)16497 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
16498 pci_channel_state_t error)
16499 {
16500 struct i40e_pf *pf = pci_get_drvdata(pdev);
16501
16502 dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
16503
16504 if (!pf) {
16505 dev_info(&pdev->dev,
16506 "Cannot recover - error happened during device probe\n");
16507 return PCI_ERS_RESULT_DISCONNECT;
16508 }
16509
16510 /* shutdown all operations */
16511 if (!test_bit(__I40E_SUSPENDED, pf->state))
16512 i40e_io_suspend(pf);
16513
16514 /* Request a slot reset */
16515 return PCI_ERS_RESULT_NEED_RESET;
16516 }
16517
16518 /**
16519 * i40e_pci_error_slot_reset - a PCI slot reset just happened
16520 * @pdev: PCI device information struct
16521 *
16522 * Called to find if the driver can work with the device now that
16523 * the pci slot has been reset. If a basic connection seems good
16524 * (registers are readable and have sane content) then return a
16525 * happy little PCI_ERS_RESULT_xxx.
16526 **/
i40e_pci_error_slot_reset(struct pci_dev * pdev)16527 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
16528 {
16529 struct i40e_pf *pf = pci_get_drvdata(pdev);
16530 pci_ers_result_t result;
16531 u32 reg;
16532
16533 dev_dbg(&pdev->dev, "%s\n", __func__);
16534 /* enable I/O and memory of the device */
16535 if (pci_enable_device(pdev)) {
16536 dev_info(&pdev->dev,
16537 "Cannot re-enable PCI device after reset.\n");
16538 result = PCI_ERS_RESULT_DISCONNECT;
16539 } else {
16540 pci_set_master(pdev);
16541 pci_restore_state(pdev);
16542 pci_save_state(pdev);
16543 pci_wake_from_d3(pdev, false);
16544
16545 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
16546 if (reg == 0)
16547 result = PCI_ERS_RESULT_RECOVERED;
16548 else
16549 result = PCI_ERS_RESULT_DISCONNECT;
16550 }
16551
16552 return result;
16553 }
16554
16555 /**
16556 * i40e_pci_error_reset_prepare - prepare device driver for pci reset
16557 * @pdev: PCI device information struct
16558 */
i40e_pci_error_reset_prepare(struct pci_dev * pdev)16559 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
16560 {
16561 struct i40e_pf *pf = pci_get_drvdata(pdev);
16562
16563 i40e_prep_for_reset(pf);
16564 }
16565
16566 /**
16567 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16568 * @pdev: PCI device information struct
16569 */
i40e_pci_error_reset_done(struct pci_dev * pdev)16570 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
16571 {
16572 struct i40e_pf *pf = pci_get_drvdata(pdev);
16573
16574 if (test_bit(__I40E_IN_REMOVE, pf->state))
16575 return;
16576
16577 i40e_reset_and_rebuild(pf, false, false);
16578 #ifdef CONFIG_PCI_IOV
16579 i40e_restore_all_vfs_msi_state(pdev);
16580 #endif /* CONFIG_PCI_IOV */
16581 }
16582
16583 /**
16584 * i40e_pci_error_resume - restart operations after PCI error recovery
16585 * @pdev: PCI device information struct
16586 *
16587 * Called to allow the driver to bring things back up after PCI error
16588 * and/or reset recovery has finished.
16589 **/
i40e_pci_error_resume(struct pci_dev * pdev)16590 static void i40e_pci_error_resume(struct pci_dev *pdev)
16591 {
16592 struct i40e_pf *pf = pci_get_drvdata(pdev);
16593
16594 dev_dbg(&pdev->dev, "%s\n", __func__);
16595 if (test_bit(__I40E_SUSPENDED, pf->state))
16596 return;
16597
16598 i40e_io_resume(pf);
16599 }
16600
16601 /**
16602 * i40e_shutdown - PCI callback for shutting down
16603 * @pdev: PCI device information struct
16604 **/
i40e_shutdown(struct pci_dev * pdev)16605 static void i40e_shutdown(struct pci_dev *pdev)
16606 {
16607 struct i40e_pf *pf = pci_get_drvdata(pdev);
16608 struct i40e_hw *hw = &pf->hw;
16609
16610 set_bit(__I40E_SUSPENDED, pf->state);
16611 set_bit(__I40E_DOWN, pf->state);
16612
16613 del_timer_sync(&pf->service_timer);
16614 cancel_work_sync(&pf->service_task);
16615 i40e_cloud_filter_exit(pf);
16616 i40e_fdir_teardown(pf);
16617
16618 /* Client close must be called explicitly here because the timer
16619 * has been stopped.
16620 */
16621 i40e_notify_client_of_netdev_close(pf, false);
16622
16623 if (test_bit(I40E_HW_CAP_WOL_MC_MAGIC_PKT_WAKE, pf->hw.caps) &&
16624 pf->wol_en)
16625 i40e_enable_mc_magic_wake(pf);
16626
16627 i40e_prep_for_reset(pf);
16628
16629 wr32(hw, I40E_PFPM_APM,
16630 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16631 wr32(hw, I40E_PFPM_WUFC,
16632 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16633
16634 /* Free MSI/legacy interrupt 0 when in recovery mode. */
16635 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16636 !test_bit(I40E_FLAG_MSIX_ENA, pf->flags))
16637 free_irq(pf->pdev->irq, pf);
16638
16639 /* Since we're going to destroy queues during the
16640 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16641 * whole section
16642 */
16643 rtnl_lock();
16644 i40e_clear_interrupt_scheme(pf);
16645 rtnl_unlock();
16646
16647 if (system_state == SYSTEM_POWER_OFF) {
16648 pci_wake_from_d3(pdev, pf->wol_en);
16649 pci_set_power_state(pdev, PCI_D3hot);
16650 }
16651 }
16652
16653 /**
16654 * i40e_suspend - PM callback for moving to D3
16655 * @dev: generic device information structure
16656 **/
i40e_suspend(struct device * dev)16657 static int i40e_suspend(struct device *dev)
16658 {
16659 struct i40e_pf *pf = dev_get_drvdata(dev);
16660
16661 /* If we're already suspended, then there is nothing to do */
16662 if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
16663 return 0;
16664 return i40e_io_suspend(pf);
16665 }
16666
16667 /**
16668 * i40e_resume - PM callback for waking up from D3
16669 * @dev: generic device information structure
16670 **/
i40e_resume(struct device * dev)16671 static int i40e_resume(struct device *dev)
16672 {
16673 struct i40e_pf *pf = dev_get_drvdata(dev);
16674
16675 /* If we're not suspended, then there is nothing to do */
16676 if (!test_bit(__I40E_SUSPENDED, pf->state))
16677 return 0;
16678 return i40e_io_resume(pf);
16679 }
16680
16681 static const struct pci_error_handlers i40e_err_handler = {
16682 .error_detected = i40e_pci_error_detected,
16683 .slot_reset = i40e_pci_error_slot_reset,
16684 .reset_prepare = i40e_pci_error_reset_prepare,
16685 .reset_done = i40e_pci_error_reset_done,
16686 .resume = i40e_pci_error_resume,
16687 };
16688
16689 static DEFINE_SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16690
16691 static struct pci_driver i40e_driver = {
16692 .name = i40e_driver_name,
16693 .id_table = i40e_pci_tbl,
16694 .probe = i40e_probe,
16695 .remove = i40e_remove,
16696 .driver.pm = pm_sleep_ptr(&i40e_pm_ops),
16697 .shutdown = i40e_shutdown,
16698 .err_handler = &i40e_err_handler,
16699 .sriov_configure = i40e_pci_sriov_configure,
16700 };
16701
16702 /**
16703 * i40e_init_module - Driver registration routine
16704 *
16705 * i40e_init_module is the first routine called when the driver is
16706 * loaded. All it does is register with the PCI subsystem.
16707 **/
i40e_init_module(void)16708 static int __init i40e_init_module(void)
16709 {
16710 int err;
16711
16712 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16713 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16714
16715 /* There is no need to throttle the number of active tasks because
16716 * each device limits its own task using a state bit for scheduling
16717 * the service task, and the device tasks do not interfere with each
16718 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16719 * since we need to be able to guarantee forward progress even under
16720 * memory pressure.
16721 */
16722 i40e_wq = alloc_workqueue("%s", 0, 0, i40e_driver_name);
16723 if (!i40e_wq) {
16724 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16725 return -ENOMEM;
16726 }
16727
16728 i40e_dbg_init();
16729 err = pci_register_driver(&i40e_driver);
16730 if (err) {
16731 destroy_workqueue(i40e_wq);
16732 i40e_dbg_exit();
16733 return err;
16734 }
16735
16736 return 0;
16737 }
16738 module_init(i40e_init_module);
16739
16740 /**
16741 * i40e_exit_module - Driver exit cleanup routine
16742 *
16743 * i40e_exit_module is called just before the driver is removed
16744 * from memory.
16745 **/
i40e_exit_module(void)16746 static void __exit i40e_exit_module(void)
16747 {
16748 pci_unregister_driver(&i40e_driver);
16749 destroy_workqueue(i40e_wq);
16750 ida_destroy(&i40e_client_ida);
16751 i40e_dbg_exit();
16752 }
16753 module_exit(i40e_exit_module);
16754