1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */
3
4 #include <linux/etherdevice.h>
5 #include <linux/ipv6.h>
6 #include <linux/types.h>
7 #include <net/netdev_queues.h>
8
9 #include "fbnic.h"
10 #include "fbnic_netdev.h"
11 #include "fbnic_txrx.h"
12
__fbnic_open(struct fbnic_net * fbn)13 int __fbnic_open(struct fbnic_net *fbn)
14 {
15 struct fbnic_dev *fbd = fbn->fbd;
16 int err;
17
18 err = fbnic_alloc_napi_vectors(fbn);
19 if (err)
20 return err;
21
22 err = fbnic_alloc_resources(fbn);
23 if (err)
24 goto free_napi_vectors;
25
26 err = fbnic_set_netif_queues(fbn);
27 if (err)
28 goto free_resources;
29
30 /* Send ownership message and flush to verify FW has seen it */
31 err = fbnic_fw_xmit_ownership_msg(fbd, true);
32 if (err) {
33 dev_warn(fbd->dev,
34 "Error %d sending host ownership message to the firmware\n",
35 err);
36 goto err_reset_queues;
37 }
38
39 err = fbnic_time_start(fbn);
40 if (err)
41 goto release_ownership;
42
43 err = fbnic_fw_init_heartbeat(fbd, false);
44 if (err)
45 goto time_stop;
46
47 err = fbnic_mac_request_irq(fbd);
48 if (err)
49 goto time_stop;
50
51 /* Pull the BMC config and initialize the RPC */
52 fbnic_bmc_rpc_init(fbd);
53 fbnic_rss_reinit(fbd, fbn);
54
55 phylink_resume(fbn->phylink);
56
57 return 0;
58 time_stop:
59 fbnic_time_stop(fbn);
60 release_ownership:
61 fbnic_fw_xmit_ownership_msg(fbn->fbd, false);
62 err_reset_queues:
63 fbnic_reset_netif_queues(fbn);
64 free_resources:
65 fbnic_free_resources(fbn);
66 free_napi_vectors:
67 fbnic_free_napi_vectors(fbn);
68 return err;
69 }
70
fbnic_open(struct net_device * netdev)71 static int fbnic_open(struct net_device *netdev)
72 {
73 struct fbnic_net *fbn = netdev_priv(netdev);
74 int err;
75
76 fbnic_napi_name_irqs(fbn->fbd);
77
78 err = __fbnic_open(fbn);
79 if (!err)
80 fbnic_up(fbn);
81
82 return err;
83 }
84
fbnic_stop(struct net_device * netdev)85 static int fbnic_stop(struct net_device *netdev)
86 {
87 struct fbnic_net *fbn = netdev_priv(netdev);
88
89 fbnic_mac_free_irq(fbn->fbd);
90 phylink_suspend(fbn->phylink, fbnic_bmc_present(fbn->fbd));
91
92 fbnic_down(fbn);
93
94 fbnic_time_stop(fbn);
95 fbnic_fw_xmit_ownership_msg(fbn->fbd, false);
96
97 fbnic_reset_netif_queues(fbn);
98 fbnic_free_resources(fbn);
99 fbnic_free_napi_vectors(fbn);
100
101 return 0;
102 }
103
fbnic_uc_sync(struct net_device * netdev,const unsigned char * addr)104 static int fbnic_uc_sync(struct net_device *netdev, const unsigned char *addr)
105 {
106 struct fbnic_net *fbn = netdev_priv(netdev);
107 struct fbnic_mac_addr *avail_addr;
108
109 if (WARN_ON(!is_valid_ether_addr(addr)))
110 return -EADDRNOTAVAIL;
111
112 avail_addr = __fbnic_uc_sync(fbn->fbd, addr);
113 if (!avail_addr)
114 return -ENOSPC;
115
116 /* Add type flag indicating this address is in use by the host */
117 set_bit(FBNIC_MAC_ADDR_T_UNICAST, avail_addr->act_tcam);
118
119 return 0;
120 }
121
fbnic_uc_unsync(struct net_device * netdev,const unsigned char * addr)122 static int fbnic_uc_unsync(struct net_device *netdev, const unsigned char *addr)
123 {
124 struct fbnic_net *fbn = netdev_priv(netdev);
125 struct fbnic_dev *fbd = fbn->fbd;
126 int i, ret;
127
128 /* Scan from middle of list to bottom, filling bottom up.
129 * Skip the first entry which is reserved for dev_addr and
130 * leave the last entry to use for promiscuous filtering.
131 */
132 for (i = fbd->mac_addr_boundary, ret = -ENOENT;
133 i < FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX && ret; i++) {
134 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i];
135
136 if (!ether_addr_equal(mac_addr->value.addr8, addr))
137 continue;
138
139 ret = __fbnic_uc_unsync(mac_addr);
140 }
141
142 return ret;
143 }
144
fbnic_mc_sync(struct net_device * netdev,const unsigned char * addr)145 static int fbnic_mc_sync(struct net_device *netdev, const unsigned char *addr)
146 {
147 struct fbnic_net *fbn = netdev_priv(netdev);
148 struct fbnic_mac_addr *avail_addr;
149
150 if (WARN_ON(!is_multicast_ether_addr(addr)))
151 return -EADDRNOTAVAIL;
152
153 avail_addr = __fbnic_mc_sync(fbn->fbd, addr);
154 if (!avail_addr)
155 return -ENOSPC;
156
157 /* Add type flag indicating this address is in use by the host */
158 set_bit(FBNIC_MAC_ADDR_T_MULTICAST, avail_addr->act_tcam);
159
160 return 0;
161 }
162
fbnic_mc_unsync(struct net_device * netdev,const unsigned char * addr)163 static int fbnic_mc_unsync(struct net_device *netdev, const unsigned char *addr)
164 {
165 struct fbnic_net *fbn = netdev_priv(netdev);
166 struct fbnic_dev *fbd = fbn->fbd;
167 int i, ret;
168
169 /* Scan from middle of list to top, filling top down.
170 * Skip over the address reserved for the BMC MAC and
171 * exclude index 0 as that belongs to the broadcast address
172 */
173 for (i = fbd->mac_addr_boundary, ret = -ENOENT;
174 --i > FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX && ret;) {
175 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i];
176
177 if (!ether_addr_equal(mac_addr->value.addr8, addr))
178 continue;
179
180 ret = __fbnic_mc_unsync(mac_addr);
181 }
182
183 return ret;
184 }
185
__fbnic_set_rx_mode(struct fbnic_dev * fbd,struct netdev_hw_addr_list * uc,struct netdev_hw_addr_list * mc)186 void __fbnic_set_rx_mode(struct fbnic_dev *fbd,
187 struct netdev_hw_addr_list *uc,
188 struct netdev_hw_addr_list *mc)
189 {
190 bool uc_promisc = false, mc_promisc = false;
191 struct net_device *netdev = fbd->netdev;
192 struct fbnic_mac_addr *mac_addr;
193 int err;
194
195 /* Populate host address from dev_addr */
196 mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX];
197 if (!ether_addr_equal(mac_addr->value.addr8, netdev->dev_addr) ||
198 mac_addr->state != FBNIC_TCAM_S_VALID) {
199 ether_addr_copy(mac_addr->value.addr8, netdev->dev_addr);
200 mac_addr->state = FBNIC_TCAM_S_UPDATE;
201 set_bit(FBNIC_MAC_ADDR_T_UNICAST, mac_addr->act_tcam);
202 }
203
204 /* Populate broadcast address if broadcast is enabled */
205 mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX];
206 if (netdev->flags & IFF_BROADCAST) {
207 if (!is_broadcast_ether_addr(mac_addr->value.addr8) ||
208 mac_addr->state != FBNIC_TCAM_S_VALID) {
209 eth_broadcast_addr(mac_addr->value.addr8);
210 mac_addr->state = FBNIC_TCAM_S_ADD;
211 }
212 set_bit(FBNIC_MAC_ADDR_T_BROADCAST, mac_addr->act_tcam);
213 } else if (mac_addr->state == FBNIC_TCAM_S_VALID) {
214 __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BROADCAST);
215 }
216
217 /* Synchronize unicast and multicast address lists */
218 err = __hw_addr_sync_dev(uc, netdev, fbnic_uc_sync, fbnic_uc_unsync);
219 if (err == -ENOSPC)
220 uc_promisc = true;
221 err = __hw_addr_sync_dev(mc, netdev, fbnic_mc_sync, fbnic_mc_unsync);
222 if (err == -ENOSPC)
223 mc_promisc = true;
224
225 uc_promisc |= !!(netdev->flags & IFF_PROMISC);
226 mc_promisc |= !!(netdev->flags & IFF_ALLMULTI) || uc_promisc;
227
228 /* Update the promiscuous rules */
229 fbnic_promisc_sync(fbd, uc_promisc, mc_promisc);
230
231 /* Add rules for BMC all multicast if it is enabled */
232 fbnic_bmc_rpc_all_multi_config(fbd, mc_promisc);
233
234 /* Sift out any unshared BMC rules and place them in BMC only section */
235 fbnic_sift_macda(fbd);
236
237 /* Write updates to hardware */
238 fbnic_write_rules(fbd);
239 fbnic_write_macda(fbd);
240 fbnic_write_tce_tcam(fbd);
241 }
242
fbnic_set_rx_mode(struct net_device * netdev,struct netdev_hw_addr_list * uc,struct netdev_hw_addr_list * mc)243 static int fbnic_set_rx_mode(struct net_device *netdev,
244 struct netdev_hw_addr_list *uc,
245 struct netdev_hw_addr_list *mc)
246 {
247 struct fbnic_net *fbn = netdev_priv(netdev);
248 struct fbnic_dev *fbd = fbn->fbd;
249
250 /* No need to update the hardware if we are not running */
251 if (netif_running(netdev))
252 __fbnic_set_rx_mode(fbd, uc, mc);
253
254 return 0;
255 }
256
fbnic_set_mac(struct net_device * netdev,void * p)257 static int fbnic_set_mac(struct net_device *netdev, void *p)
258 {
259 struct fbnic_net *fbn = netdev_priv(netdev);
260 struct sockaddr *addr = p;
261
262 if (!is_valid_ether_addr(addr->sa_data))
263 return -EADDRNOTAVAIL;
264
265 eth_hw_addr_set(netdev, addr->sa_data);
266
267 if (netif_running(netdev)) {
268 netif_addr_lock_bh(netdev);
269 __fbnic_set_rx_mode(fbn->fbd, &netdev->uc, &netdev->mc);
270 netif_addr_unlock_bh(netdev);
271 }
272
273 return 0;
274 }
275
fbnic_change_mtu(struct net_device * dev,int new_mtu)276 static int fbnic_change_mtu(struct net_device *dev, int new_mtu)
277 {
278 struct fbnic_net *fbn = netdev_priv(dev);
279
280 if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) {
281 dev_err(&dev->dev,
282 "MTU %d is larger than HDS threshold %d in XDP mode\n",
283 new_mtu, fbn->hds_thresh);
284
285 return -EINVAL;
286 }
287
288 WRITE_ONCE(dev->mtu, new_mtu);
289
290 return 0;
291 }
292
fbnic_clear_rx_mode(struct fbnic_dev * fbd)293 void fbnic_clear_rx_mode(struct fbnic_dev *fbd)
294 {
295 struct net_device *netdev = fbd->netdev;
296 int idx;
297
298 for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) {
299 struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx];
300
301 if (mac_addr->state != FBNIC_TCAM_S_VALID)
302 continue;
303
304 bitmap_clear(mac_addr->act_tcam,
305 FBNIC_MAC_ADDR_T_HOST_START,
306 FBNIC_MAC_ADDR_T_HOST_LEN);
307
308 if (bitmap_empty(mac_addr->act_tcam,
309 FBNIC_RPC_TCAM_ACT_NUM_ENTRIES))
310 mac_addr->state = FBNIC_TCAM_S_DELETE;
311 }
312
313 /* Write updates to hardware */
314 fbnic_write_macda(fbd);
315
316 netif_addr_lock_bh(netdev);
317 __dev_uc_unsync(netdev, NULL);
318 __dev_mc_unsync(netdev, NULL);
319 netif_addr_unlock_bh(netdev);
320 }
321
fbnic_hwtstamp_get(struct net_device * netdev,struct kernel_hwtstamp_config * config)322 static int fbnic_hwtstamp_get(struct net_device *netdev,
323 struct kernel_hwtstamp_config *config)
324 {
325 struct fbnic_net *fbn = netdev_priv(netdev);
326
327 *config = fbn->hwtstamp_config;
328
329 return 0;
330 }
331
fbnic_hwtstamp_set(struct net_device * netdev,struct kernel_hwtstamp_config * config,struct netlink_ext_ack * extack)332 static int fbnic_hwtstamp_set(struct net_device *netdev,
333 struct kernel_hwtstamp_config *config,
334 struct netlink_ext_ack *extack)
335 {
336 struct fbnic_net *fbn = netdev_priv(netdev);
337 int old_rx_filter;
338
339 if (config->source != HWTSTAMP_SOURCE_NETDEV)
340 return -EOPNOTSUPP;
341
342 if (!kernel_hwtstamp_config_changed(config, &fbn->hwtstamp_config))
343 return 0;
344
345 /* Upscale the filters */
346 switch (config->rx_filter) {
347 case HWTSTAMP_FILTER_NONE:
348 case HWTSTAMP_FILTER_ALL:
349 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
350 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
351 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
352 case HWTSTAMP_FILTER_PTP_V2_EVENT:
353 break;
354 case HWTSTAMP_FILTER_NTP_ALL:
355 config->rx_filter = HWTSTAMP_FILTER_ALL;
356 break;
357 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
358 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
359 config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
360 break;
361 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
362 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
363 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
364 break;
365 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
366 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
367 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
368 break;
369 case HWTSTAMP_FILTER_PTP_V2_SYNC:
370 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
371 config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
372 break;
373 default:
374 return -ERANGE;
375 }
376
377 /* Configure */
378 old_rx_filter = fbn->hwtstamp_config.rx_filter;
379 memcpy(&fbn->hwtstamp_config, config, sizeof(*config));
380
381 if (old_rx_filter != config->rx_filter && netif_running(fbn->netdev)) {
382 fbnic_rss_reinit(fbn->fbd, fbn);
383 fbnic_write_rules(fbn->fbd);
384 }
385
386 /* Save / report back filter configuration
387 * Note that our filter configuration is inexact. Instead of
388 * filtering for a specific UDP port or L2 Ethertype we are
389 * filtering in all UDP or all non-IP packets for timestamping. So
390 * if anything other than FILTER_ALL is requested we report
391 * FILTER_SOME indicating that we will be timestamping a few
392 * additional packets.
393 */
394 if (config->rx_filter > HWTSTAMP_FILTER_ALL)
395 config->rx_filter = HWTSTAMP_FILTER_SOME;
396
397 return 0;
398 }
399
fbnic_get_stats64(struct net_device * dev,struct rtnl_link_stats64 * stats64)400 static void fbnic_get_stats64(struct net_device *dev,
401 struct rtnl_link_stats64 *stats64)
402 {
403 u64 rx_bytes, rx_packets, rx_dropped = 0, rx_errors = 0;
404 u64 rx_over = 0, rx_missed = 0, rx_length = 0;
405 u64 tx_bytes, tx_packets, tx_dropped = 0;
406 struct fbnic_net *fbn = netdev_priv(dev);
407 struct fbnic_dev *fbd = fbn->fbd;
408 struct fbnic_queue_stats *stats;
409
410 unsigned int start, i;
411
412 fbnic_get_hw_stats(fbd);
413
414 stats = &fbn->tx_stats;
415
416 tx_bytes = stats->bytes;
417 tx_packets = stats->packets;
418 tx_dropped = stats->dropped;
419
420 /* Record drops from Tx HW Datapath */
421 spin_lock(&fbd->hw_stats.lock);
422 tx_dropped += fbd->hw_stats.tmi.drop.frames.value +
423 fbd->hw_stats.tti.cm_drop.frames.value +
424 fbd->hw_stats.tti.frame_drop.frames.value +
425 fbd->hw_stats.tti.tbi_drop.frames.value;
426 spin_unlock(&fbd->hw_stats.lock);
427
428 stats64->tx_bytes = tx_bytes;
429 stats64->tx_packets = tx_packets;
430 stats64->tx_dropped = tx_dropped;
431
432 for (i = 0; i < fbn->num_tx_queues; i++) {
433 struct fbnic_ring *txr = fbn->tx[i];
434
435 if (!txr)
436 continue;
437
438 stats = &txr->stats;
439 do {
440 start = u64_stats_fetch_begin(&stats->syncp);
441 tx_bytes = stats->bytes;
442 tx_packets = stats->packets;
443 tx_dropped = stats->dropped;
444 } while (u64_stats_fetch_retry(&stats->syncp, start));
445
446 stats64->tx_bytes += tx_bytes;
447 stats64->tx_packets += tx_packets;
448 stats64->tx_dropped += tx_dropped;
449 }
450
451 stats = &fbn->rx_stats;
452
453 rx_bytes = stats->bytes;
454 rx_packets = stats->packets;
455 rx_dropped = stats->dropped;
456
457 spin_lock(&fbd->hw_stats.lock);
458 /* Record drops for the host FIFOs.
459 * 4: network to Host, 6: BMC to Host
460 * Exclude the BMC and MC FIFOs as those stats may contain drops
461 * due to unrelated items such as TCAM misses. They are still
462 * accessible through the ethtool stats.
463 */
464 i = FBNIC_RXB_FIFO_HOST;
465 rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value;
466 i = FBNIC_RXB_FIFO_BMC_TO_HOST;
467 rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value;
468
469 for (i = 0; i < fbd->max_num_queues; i++) {
470 /* Report packets dropped due to CQ/BDQ being full/empty */
471 rx_over += fbd->hw_stats.hw_q[i].rde_pkt_cq_drop.value;
472 rx_over += fbd->hw_stats.hw_q[i].rde_pkt_bdq_drop.value;
473
474 /* Report packets with errors */
475 rx_errors += fbd->hw_stats.hw_q[i].rde_pkt_err.value;
476 }
477 spin_unlock(&fbd->hw_stats.lock);
478
479 stats64->rx_bytes = rx_bytes;
480 stats64->rx_packets = rx_packets;
481 stats64->rx_dropped = rx_dropped;
482 stats64->rx_over_errors = rx_over;
483 stats64->rx_errors = rx_errors;
484 stats64->rx_missed_errors = rx_missed;
485
486 for (i = 0; i < fbn->num_rx_queues; i++) {
487 struct fbnic_ring *xdpr = fbn->tx[FBNIC_MAX_TXQS + i];
488 struct fbnic_ring *rxr = fbn->rx[i];
489
490 if (!rxr)
491 continue;
492
493 stats = &rxr->stats;
494 do {
495 start = u64_stats_fetch_begin(&stats->syncp);
496 rx_bytes = stats->bytes;
497 rx_packets = stats->packets;
498 rx_dropped = stats->dropped;
499 rx_length = stats->rx.length_errors;
500 } while (u64_stats_fetch_retry(&stats->syncp, start));
501
502 stats64->rx_bytes += rx_bytes;
503 stats64->rx_packets += rx_packets;
504 stats64->rx_dropped += rx_dropped;
505 stats64->rx_errors += rx_length;
506 stats64->rx_length_errors += rx_length;
507
508 if (!xdpr)
509 continue;
510
511 stats = &xdpr->stats;
512 do {
513 start = u64_stats_fetch_begin(&stats->syncp);
514 tx_bytes = stats->bytes;
515 tx_packets = stats->packets;
516 tx_dropped = stats->dropped;
517 } while (u64_stats_fetch_retry(&stats->syncp, start));
518
519 stats64->tx_bytes += tx_bytes;
520 stats64->tx_packets += tx_packets;
521 stats64->tx_dropped += tx_dropped;
522 }
523 }
524
fbnic_check_split_frames(struct bpf_prog * prog,unsigned int mtu,u32 hds_thresh)525 bool fbnic_check_split_frames(struct bpf_prog *prog, unsigned int mtu,
526 u32 hds_thresh)
527 {
528 if (!prog)
529 return false;
530
531 if (prog->aux->xdp_has_frags)
532 return false;
533
534 return mtu + ETH_HLEN > hds_thresh;
535 }
536
fbnic_bpf(struct net_device * netdev,struct netdev_bpf * bpf)537 static int fbnic_bpf(struct net_device *netdev, struct netdev_bpf *bpf)
538 {
539 struct bpf_prog *prog = bpf->prog, *prev_prog;
540 struct fbnic_net *fbn = netdev_priv(netdev);
541
542 if (bpf->command != XDP_SETUP_PROG)
543 return -EINVAL;
544
545 if (fbnic_check_split_frames(prog, netdev->mtu,
546 fbn->hds_thresh)) {
547 NL_SET_ERR_MSG_MOD(bpf->extack,
548 "MTU too high, or HDS threshold is too low for single buffer XDP");
549 return -EOPNOTSUPP;
550 }
551
552 prev_prog = xchg(&fbn->xdp_prog, prog);
553 if (prev_prog)
554 bpf_prog_put(prev_prog);
555
556 return 0;
557 }
558
559 static const struct net_device_ops fbnic_netdev_ops = {
560 .ndo_open = fbnic_open,
561 .ndo_stop = fbnic_stop,
562 .ndo_validate_addr = eth_validate_addr,
563 .ndo_start_xmit = fbnic_xmit_frame,
564 .ndo_features_check = fbnic_features_check,
565 .ndo_set_mac_address = fbnic_set_mac,
566 .ndo_change_mtu = fbnic_change_mtu,
567 .ndo_set_rx_mode_async = fbnic_set_rx_mode,
568 .ndo_get_stats64 = fbnic_get_stats64,
569 .ndo_bpf = fbnic_bpf,
570 .ndo_hwtstamp_get = fbnic_hwtstamp_get,
571 .ndo_hwtstamp_set = fbnic_hwtstamp_set,
572 };
573
fbnic_get_queue_stats_rx(struct net_device * dev,int idx,struct netdev_queue_stats_rx * rx)574 static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx,
575 struct netdev_queue_stats_rx *rx)
576 {
577 u64 bytes, packets, alloc_fail, alloc_fail_bdq;
578 struct fbnic_net *fbn = netdev_priv(dev);
579 struct fbnic_ring *rxr = fbn->rx[idx];
580 struct fbnic_dev *fbd = fbn->fbd;
581 struct fbnic_queue_stats *stats;
582 u64 csum_complete, csum_none;
583 struct fbnic_q_triad *qt;
584 unsigned int start;
585
586 if (!rxr)
587 return;
588
589 /* fbn->rx points to completion queues */
590 qt = container_of(rxr, struct fbnic_q_triad, cmpl);
591
592 stats = &rxr->stats;
593 do {
594 start = u64_stats_fetch_begin(&stats->syncp);
595 bytes = stats->bytes;
596 packets = stats->packets;
597 alloc_fail = stats->rx.alloc_failed;
598 csum_complete = stats->rx.csum_complete;
599 csum_none = stats->rx.csum_none;
600 } while (u64_stats_fetch_retry(&stats->syncp, start));
601
602 stats = &qt->sub0.stats;
603 do {
604 start = u64_stats_fetch_begin(&stats->syncp);
605 alloc_fail_bdq = stats->bdq.alloc_failed;
606 } while (u64_stats_fetch_retry(&stats->syncp, start));
607 alloc_fail += alloc_fail_bdq;
608
609 stats = &qt->sub1.stats;
610 do {
611 start = u64_stats_fetch_begin(&stats->syncp);
612 alloc_fail_bdq = stats->bdq.alloc_failed;
613 } while (u64_stats_fetch_retry(&stats->syncp, start));
614 alloc_fail += alloc_fail_bdq;
615
616 rx->bytes = bytes;
617 rx->packets = packets;
618 rx->alloc_fail = alloc_fail;
619 rx->csum_complete = csum_complete;
620 rx->csum_none = csum_none;
621
622 fbnic_get_hw_q_stats(fbd, fbd->hw_stats.hw_q);
623
624 spin_lock(&fbd->hw_stats.lock);
625 rx->hw_drop_overruns = fbd->hw_stats.hw_q[idx].rde_pkt_cq_drop.value +
626 fbd->hw_stats.hw_q[idx].rde_pkt_bdq_drop.value;
627 rx->hw_drops = fbd->hw_stats.hw_q[idx].rde_pkt_err.value +
628 rx->hw_drop_overruns;
629 spin_unlock(&fbd->hw_stats.lock);
630 }
631
fbnic_get_queue_stats_tx(struct net_device * dev,int idx,struct netdev_queue_stats_tx * tx)632 static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx,
633 struct netdev_queue_stats_tx *tx)
634 {
635 struct fbnic_net *fbn = netdev_priv(dev);
636 struct fbnic_ring *txr = fbn->tx[idx];
637 struct fbnic_queue_stats *stats;
638 u64 stop, wake, csum, lso;
639 struct fbnic_ring *xdpr;
640 unsigned int start;
641 u64 bytes, packets;
642
643 if (!txr)
644 return;
645
646 stats = &txr->stats;
647 do {
648 start = u64_stats_fetch_begin(&stats->syncp);
649 bytes = stats->bytes;
650 packets = stats->packets;
651 csum = stats->twq.csum_partial;
652 lso = stats->twq.lso;
653 stop = stats->twq.stop;
654 wake = stats->twq.wake;
655 } while (u64_stats_fetch_retry(&stats->syncp, start));
656
657 tx->bytes = bytes;
658 tx->packets = packets;
659 tx->needs_csum = csum + lso;
660 tx->hw_gso_wire_packets = lso;
661 tx->stop = stop;
662 tx->wake = wake;
663
664 xdpr = fbn->tx[FBNIC_MAX_TXQS + idx];
665 if (xdpr) {
666 stats = &xdpr->stats;
667 do {
668 start = u64_stats_fetch_begin(&stats->syncp);
669 bytes = stats->bytes;
670 packets = stats->packets;
671 } while (u64_stats_fetch_retry(&stats->syncp, start));
672
673 tx->bytes += bytes;
674 tx->packets += packets;
675 }
676 }
677
fbnic_get_base_stats(struct net_device * dev,struct netdev_queue_stats_rx * rx,struct netdev_queue_stats_tx * tx)678 static void fbnic_get_base_stats(struct net_device *dev,
679 struct netdev_queue_stats_rx *rx,
680 struct netdev_queue_stats_tx *tx)
681 {
682 struct fbnic_net *fbn = netdev_priv(dev);
683
684 tx->bytes = fbn->tx_stats.bytes;
685 tx->packets = fbn->tx_stats.packets;
686 tx->needs_csum = fbn->tx_stats.twq.csum_partial + fbn->tx_stats.twq.lso;
687 tx->hw_gso_wire_packets = fbn->tx_stats.twq.lso;
688 tx->stop = fbn->tx_stats.twq.stop;
689 tx->wake = fbn->tx_stats.twq.wake;
690
691 rx->bytes = fbn->rx_stats.bytes;
692 rx->packets = fbn->rx_stats.packets;
693 rx->alloc_fail = fbn->rx_stats.rx.alloc_failed +
694 fbn->bdq_stats.bdq.alloc_failed;
695 rx->csum_complete = fbn->rx_stats.rx.csum_complete;
696 rx->csum_none = fbn->rx_stats.rx.csum_none;
697 }
698
699 static const struct netdev_stat_ops fbnic_stat_ops = {
700 .get_queue_stats_rx = fbnic_get_queue_stats_rx,
701 .get_queue_stats_tx = fbnic_get_queue_stats_tx,
702 .get_base_stats = fbnic_get_base_stats,
703 };
704
fbnic_reset_queues(struct fbnic_net * fbn,unsigned int tx,unsigned int rx)705 void fbnic_reset_queues(struct fbnic_net *fbn,
706 unsigned int tx, unsigned int rx)
707 {
708 struct fbnic_dev *fbd = fbn->fbd;
709 unsigned int max_napis;
710
711 max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS;
712
713 tx = min(tx, max_napis);
714 fbn->num_tx_queues = tx;
715
716 rx = min(rx, max_napis);
717 fbn->num_rx_queues = rx;
718
719 fbn->num_napi = max(tx, rx);
720 }
721
722 /**
723 * fbnic_netdev_free - Free the netdev associate with fbnic
724 * @fbd: Driver specific structure to free netdev from
725 *
726 * Allocate and initialize the netdev and netdev private structure. Bind
727 * together the hardware, netdev, and pci data structures.
728 **/
fbnic_netdev_free(struct fbnic_dev * fbd)729 void fbnic_netdev_free(struct fbnic_dev *fbd)
730 {
731 fbnic_phylink_destroy(fbd->netdev);
732
733 free_netdev(fbd->netdev);
734 fbd->netdev = NULL;
735 }
736
737 /**
738 * fbnic_netdev_alloc - Allocate a netdev and associate with fbnic
739 * @fbd: Driver specific structure to associate netdev with
740 *
741 * Allocate and initialize the netdev and netdev private structure. Bind
742 * together the hardware, netdev, and pci data structures.
743 *
744 * Return: Pointer to net_device on success, NULL on failure
745 **/
fbnic_netdev_alloc(struct fbnic_dev * fbd)746 struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd)
747 {
748 struct net_device *netdev;
749 struct fbnic_net *fbn;
750 int default_queues;
751
752 netdev = alloc_etherdev_mq(sizeof(*fbn), FBNIC_MAX_RXQS);
753 if (!netdev)
754 return NULL;
755
756 SET_NETDEV_DEV(netdev, fbd->dev);
757 fbd->netdev = netdev;
758
759 netdev->netdev_ops = &fbnic_netdev_ops;
760 netdev->stat_ops = &fbnic_stat_ops;
761 netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops;
762 netdev->netmem_tx = NETMEM_TX_DMA;
763
764 fbnic_set_ethtool_ops(netdev);
765
766 fbn = netdev_priv(netdev);
767
768 fbn->netdev = netdev;
769 fbn->fbd = fbd;
770
771 fbn->txq_size = FBNIC_TXQ_SIZE_DEFAULT;
772 fbn->hpq_size = FBNIC_HPQ_SIZE_DEFAULT;
773 fbn->ppq_size = FBNIC_PPQ_SIZE_DEFAULT;
774 fbn->rcq_size = FBNIC_RCQ_SIZE_DEFAULT;
775
776 fbn->tx_usecs = FBNIC_TX_USECS_DEFAULT;
777 fbn->rx_usecs = FBNIC_RX_USECS_DEFAULT;
778 fbn->rx_max_frames = FBNIC_RX_FRAMES_DEFAULT;
779
780 /* Initialize the hds_thresh */
781 netdev->cfg->hds_thresh = FBNIC_HDS_THRESH_DEFAULT;
782 fbn->hds_thresh = FBNIC_HDS_THRESH_DEFAULT;
783
784 default_queues = netif_get_num_default_rss_queues();
785 if (default_queues > fbd->max_num_queues)
786 default_queues = fbd->max_num_queues;
787
788 fbnic_reset_queues(fbn, default_queues, default_queues);
789
790 fbnic_reset_indir_tbl(fbn);
791 fbnic_rss_key_fill(fbn->rss_key);
792 fbnic_rss_init_en_mask(fbn);
793
794 netdev->priv_flags |= IFF_UNICAST_FLT;
795
796 netdev->gso_partial_features =
797 NETIF_F_GSO_GRE |
798 NETIF_F_GSO_GRE_CSUM |
799 NETIF_F_GSO_IPXIP4 |
800 NETIF_F_GSO_UDP_TUNNEL |
801 NETIF_F_GSO_UDP_TUNNEL_CSUM;
802
803 netdev->features |=
804 netdev->gso_partial_features |
805 FBNIC_TUN_GSO_FEATURES |
806 NETIF_F_RXHASH |
807 NETIF_F_SG |
808 NETIF_F_HW_CSUM |
809 NETIF_F_RXCSUM |
810 NETIF_F_TSO |
811 NETIF_F_TSO_ECN |
812 NETIF_F_TSO6 |
813 NETIF_F_GSO_PARTIAL |
814 NETIF_F_GSO_UDP_L4;
815
816 netdev->hw_features |= netdev->features;
817 netdev->vlan_features |= netdev->features;
818 netdev->hw_enc_features |= netdev->features;
819 netdev->features |= NETIF_F_NTUPLE;
820
821 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG;
822
823 netdev->min_mtu = IPV6_MIN_MTU;
824 netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN;
825
826 /* TBD: This is workaround for BMC as phylink doesn't have support
827 * for leavling the link enabled if a BMC is present.
828 */
829 netdev->ethtool->wol_enabled = true;
830
831 netif_carrier_off(netdev);
832
833 netif_tx_stop_all_queues(netdev);
834
835 if (fbnic_phylink_create(netdev)) {
836 free_netdev(netdev);
837 fbd->netdev = NULL;
838 return NULL;
839 }
840
841 return netdev;
842 }
843
fbnic_dsn_to_mac_addr(u64 dsn,char * addr)844 static int fbnic_dsn_to_mac_addr(u64 dsn, char *addr)
845 {
846 addr[0] = (dsn >> 56) & 0xFF;
847 addr[1] = (dsn >> 48) & 0xFF;
848 addr[2] = (dsn >> 40) & 0xFF;
849 addr[3] = (dsn >> 16) & 0xFF;
850 addr[4] = (dsn >> 8) & 0xFF;
851 addr[5] = dsn & 0xFF;
852
853 return is_valid_ether_addr(addr) ? 0 : -EINVAL;
854 }
855
856 /**
857 * fbnic_netdev_register - Initialize general software structures
858 * @netdev: Netdev containing structure to initialize and register
859 *
860 * Initialize the MAC address for the netdev and register it.
861 *
862 * Return: 0 on success, negative on failure
863 **/
fbnic_netdev_register(struct net_device * netdev)864 int fbnic_netdev_register(struct net_device *netdev)
865 {
866 struct fbnic_net *fbn = netdev_priv(netdev);
867 struct fbnic_dev *fbd = fbn->fbd;
868 u64 dsn = fbd->dsn;
869 u8 addr[ETH_ALEN];
870 int err;
871
872 err = fbnic_dsn_to_mac_addr(dsn, addr);
873 if (!err) {
874 ether_addr_copy(netdev->perm_addr, addr);
875 eth_hw_addr_set(netdev, addr);
876 } else {
877 /* A randomly assigned MAC address will cause provisioning
878 * issues so instead just fail to spawn the netdev and
879 * avoid any confusion.
880 */
881 dev_err(fbd->dev, "MAC addr %pM invalid\n", addr);
882 return err;
883 }
884
885 return register_netdev(netdev);
886 }
887
fbnic_netdev_unregister(struct net_device * netdev)888 void fbnic_netdev_unregister(struct net_device *netdev)
889 {
890 unregister_netdev(netdev);
891 }
892