1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2023-2024 Google LLC
5 *
6 * Redistribution and use in source and binary forms, with or without modification,
7 * are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * 3. Neither the name of the copyright holder nor the names of its contributors
17 * may be used to endorse or promote products derived from this software without
18 * specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
22 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
24 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
25 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
27 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31 #include "gve.h"
32 #include "gve_adminq.h"
33 #include "gve_dqo.h"
34
35 #define GVE_DRIVER_VERSION "GVE-FBSD-1.3.2\n"
36 #define GVE_VERSION_MAJOR 1
37 #define GVE_VERSION_MINOR 3
38 #define GVE_VERSION_SUB 2
39
40 #define GVE_DEFAULT_RX_COPYBREAK 256
41
42 /* Devices supported by this driver. */
43 static struct gve_dev {
44 uint16_t vendor_id;
45 uint16_t device_id;
46 const char *name;
47 } gve_devs[] = {
48 { PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC, "gVNIC" }
49 };
50
51 struct sx gve_global_lock;
52
53 static int
gve_verify_driver_compatibility(struct gve_priv * priv)54 gve_verify_driver_compatibility(struct gve_priv *priv)
55 {
56 int err;
57 struct gve_driver_info *driver_info;
58 struct gve_dma_handle driver_info_mem;
59
60 err = gve_dma_alloc_coherent(priv, sizeof(struct gve_driver_info),
61 PAGE_SIZE, &driver_info_mem);
62
63 if (err != 0)
64 return (ENOMEM);
65
66 driver_info = driver_info_mem.cpu_addr;
67
68 *driver_info = (struct gve_driver_info) {
69 .os_type = 3, /* Freebsd */
70 .driver_major = GVE_VERSION_MAJOR,
71 .driver_minor = GVE_VERSION_MINOR,
72 .driver_sub = GVE_VERSION_SUB,
73 .os_version_major = htobe32(FBSD_VERSION_MAJOR),
74 .os_version_minor = htobe32(FBSD_VERSION_MINOR),
75 .os_version_sub = htobe32(FBSD_VERSION_PATCH),
76 .driver_capability_flags = {
77 htobe64(GVE_DRIVER_CAPABILITY_FLAGS1),
78 htobe64(GVE_DRIVER_CAPABILITY_FLAGS2),
79 htobe64(GVE_DRIVER_CAPABILITY_FLAGS3),
80 htobe64(GVE_DRIVER_CAPABILITY_FLAGS4),
81 },
82 };
83
84 snprintf(driver_info->os_version_str1, sizeof(driver_info->os_version_str1),
85 "FreeBSD %u", __FreeBSD_version);
86
87 bus_dmamap_sync(driver_info_mem.tag, driver_info_mem.map,
88 BUS_DMASYNC_PREREAD);
89
90 err = gve_adminq_verify_driver_compatibility(priv,
91 sizeof(struct gve_driver_info), driver_info_mem.bus_addr);
92
93 /* It's ok if the device doesn't support this */
94 if (err == EOPNOTSUPP)
95 err = 0;
96
97 gve_dma_free_coherent(&driver_info_mem);
98
99 return (err);
100 }
101
102 static int
gve_up(struct gve_priv * priv)103 gve_up(struct gve_priv *priv)
104 {
105 if_t ifp = priv->ifp;
106 int err;
107
108 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
109
110 if (device_is_attached(priv->dev) == 0) {
111 device_printf(priv->dev, "Cannot bring the iface up when detached\n");
112 return (ENXIO);
113 }
114
115 if (gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP))
116 return (0);
117
118 if_clearhwassist(ifp);
119 if (if_getcapenable(ifp) & IFCAP_TXCSUM)
120 if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
121 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
122 if_sethwassistbits(ifp, CSUM_IP6_TCP | CSUM_IP6_UDP, 0);
123 if (if_getcapenable(ifp) & IFCAP_TSO4)
124 if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
125 if (if_getcapenable(ifp) & IFCAP_TSO6)
126 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
127
128 if (gve_is_qpl(priv)) {
129 err = gve_register_qpls(priv);
130 if (err != 0)
131 goto reset;
132 }
133
134 err = gve_create_rx_rings(priv);
135 if (err != 0)
136 goto reset;
137
138 err = gve_create_tx_rings(priv);
139 if (err != 0)
140 goto reset;
141
142 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
143
144 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
145 if_link_state_change(ifp, LINK_STATE_UP);
146 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
147 }
148
149 gve_unmask_all_queue_irqs(priv);
150 gve_set_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP);
151 priv->interface_up_cnt++;
152 return (0);
153
154 reset:
155 gve_schedule_reset(priv);
156 return (err);
157 }
158
159 static void
gve_down(struct gve_priv * priv)160 gve_down(struct gve_priv *priv)
161 {
162 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
163
164 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP))
165 return;
166
167 if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
168 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
169 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
170 }
171
172 if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
173
174 if (gve_destroy_rx_rings(priv) != 0)
175 goto reset;
176
177 if (gve_destroy_tx_rings(priv) != 0)
178 goto reset;
179
180 if (gve_is_qpl(priv)) {
181 if (gve_unregister_qpls(priv) != 0)
182 goto reset;
183 }
184
185 if (gve_is_gqi(priv))
186 gve_mask_all_queue_irqs(priv);
187 gve_clear_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP);
188 priv->interface_down_cnt++;
189 return;
190
191 reset:
192 gve_schedule_reset(priv);
193 }
194
195 static int
gve_set_mtu(if_t ifp,uint32_t new_mtu)196 gve_set_mtu(if_t ifp, uint32_t new_mtu)
197 {
198 struct gve_priv *priv = if_getsoftc(ifp);
199 const uint32_t max_problem_range = 8227;
200 const uint32_t min_problem_range = 7822;
201 int err;
202
203 if ((new_mtu > priv->max_mtu) || (new_mtu < ETHERMIN)) {
204 device_printf(priv->dev, "Invalid new MTU setting. new mtu: %d max mtu: %d min mtu: %d\n",
205 new_mtu, priv->max_mtu, ETHERMIN);
206 return (EINVAL);
207 }
208
209 /*
210 * When hardware LRO is enabled in DQ mode, MTUs within the range
211 * [7822, 8227] trigger hardware issues which cause a drastic drop
212 * in throughput.
213 */
214 if (!gve_is_gqi(priv) && !gve_disable_hw_lro &&
215 new_mtu >= min_problem_range && new_mtu <= max_problem_range) {
216 device_printf(priv->dev,
217 "Cannot set to MTU to %d within the range [%d, %d] while hardware LRO is enabled\n",
218 new_mtu, min_problem_range, max_problem_range);
219 return (EINVAL);
220 }
221
222 err = gve_adminq_set_mtu(priv, new_mtu);
223 if (err == 0) {
224 if (bootverbose)
225 device_printf(priv->dev, "MTU set to %d\n", new_mtu);
226 if_setmtu(ifp, new_mtu);
227 } else {
228 device_printf(priv->dev, "Failed to set MTU to %d\n", new_mtu);
229 }
230
231 return (err);
232 }
233
234 static void
gve_init(void * arg)235 gve_init(void *arg)
236 {
237 struct gve_priv *priv = (struct gve_priv *)arg;
238
239 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP)) {
240 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
241 gve_up(priv);
242 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
243 }
244 }
245
246 static int
gve_ioctl(if_t ifp,u_long command,caddr_t data)247 gve_ioctl(if_t ifp, u_long command, caddr_t data)
248 {
249 struct gve_priv *priv;
250 struct ifreq *ifr;
251 int rc = 0;
252
253 priv = if_getsoftc(ifp);
254 ifr = (struct ifreq *)data;
255
256 switch (command) {
257 case SIOCSIFMTU:
258 if (if_getmtu(ifp) == ifr->ifr_mtu)
259 break;
260 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
261 gve_down(priv);
262 gve_set_mtu(ifp, ifr->ifr_mtu);
263 rc = gve_up(priv);
264 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
265 break;
266
267 case SIOCSIFFLAGS:
268 if ((if_getflags(ifp) & IFF_UP) != 0) {
269 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
270 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
271 rc = gve_up(priv);
272 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
273 }
274 } else {
275 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
276 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
277 gve_down(priv);
278 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
279 }
280 }
281 break;
282
283 case SIOCSIFCAP:
284 if (ifr->ifr_reqcap == if_getcapenable(ifp))
285 break;
286 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
287 gve_down(priv);
288 if_setcapenable(ifp, ifr->ifr_reqcap);
289 rc = gve_up(priv);
290 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
291 break;
292
293 case SIOCSIFMEDIA:
294 /* FALLTHROUGH */
295 case SIOCGIFMEDIA:
296 rc = ifmedia_ioctl(ifp, ifr, &priv->media, command);
297 break;
298
299 default:
300 rc = ether_ioctl(ifp, command, data);
301 break;
302 }
303
304 return (rc);
305 }
306
307 static int
gve_media_change(if_t ifp)308 gve_media_change(if_t ifp)
309 {
310 struct gve_priv *priv = if_getsoftc(ifp);
311
312 device_printf(priv->dev, "Media change not supported\n");
313 return (0);
314 }
315
316 static void
gve_media_status(if_t ifp,struct ifmediareq * ifmr)317 gve_media_status(if_t ifp, struct ifmediareq *ifmr)
318 {
319 struct gve_priv *priv = if_getsoftc(ifp);
320
321 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
322
323 ifmr->ifm_status = IFM_AVALID;
324 ifmr->ifm_active = IFM_ETHER;
325
326 if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
327 ifmr->ifm_status |= IFM_ACTIVE;
328 ifmr->ifm_active |= IFM_AUTO;
329 } else {
330 ifmr->ifm_active |= IFM_NONE;
331 }
332
333 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
334 }
335
336 static uint64_t
gve_get_counter(if_t ifp,ift_counter cnt)337 gve_get_counter(if_t ifp, ift_counter cnt)
338 {
339 struct gve_priv *priv;
340 uint64_t rpackets = 0;
341 uint64_t tpackets = 0;
342 uint64_t rbytes = 0;
343 uint64_t tbytes = 0;
344 uint64_t rx_dropped_pkt = 0;
345 uint64_t tx_dropped_pkt = 0;
346
347 priv = if_getsoftc(ifp);
348
349 gve_accum_stats(priv, &rpackets, &rbytes, &rx_dropped_pkt, &tpackets,
350 &tbytes, &tx_dropped_pkt);
351
352 switch (cnt) {
353 case IFCOUNTER_IPACKETS:
354 return (rpackets);
355
356 case IFCOUNTER_OPACKETS:
357 return (tpackets);
358
359 case IFCOUNTER_IBYTES:
360 return (rbytes);
361
362 case IFCOUNTER_OBYTES:
363 return (tbytes);
364
365 case IFCOUNTER_IQDROPS:
366 return (rx_dropped_pkt);
367
368 case IFCOUNTER_OQDROPS:
369 return (tx_dropped_pkt);
370
371 default:
372 return (if_get_counter_default(ifp, cnt));
373 }
374 }
375
376 static void
gve_setup_ifnet(device_t dev,struct gve_priv * priv)377 gve_setup_ifnet(device_t dev, struct gve_priv *priv)
378 {
379 int caps = 0;
380 if_t ifp;
381
382 ifp = priv->ifp = if_alloc(IFT_ETHER);
383 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
384 if_setsoftc(ifp, priv);
385 if_setdev(ifp, dev);
386 if_setinitfn(ifp, gve_init);
387 if_setioctlfn(ifp, gve_ioctl);
388 if_settransmitfn(ifp, gve_xmit_ifp);
389 if_setqflushfn(ifp, gve_qflush);
390
391 /*
392 * Set TSO limits, must match the arguments to bus_dma_tag_create
393 * when creating tx->dqo.buf_dmatag. Only applies to the RDA mode
394 * because in QPL we copy the entire packet into the bounce buffer
395 * and thus it does not matter how fragmented the mbuf is.
396 */
397 if (!gve_is_gqi(priv) && !gve_is_qpl(priv)) {
398 if_sethwtsomaxsegcount(ifp, GVE_TX_MAX_DATA_DESCS_DQO);
399 if_sethwtsomaxsegsize(ifp, GVE_TX_MAX_BUF_SIZE_DQO);
400 }
401 if_sethwtsomax(ifp, GVE_TSO_MAXSIZE_DQO);
402
403 #if __FreeBSD_version >= 1400086
404 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
405 #else
406 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | IFF_KNOWSEPOCH);
407 #endif
408
409 ifmedia_init(&priv->media, IFM_IMASK, gve_media_change, gve_media_status);
410 if_setgetcounterfn(ifp, gve_get_counter);
411
412 caps = IFCAP_RXCSUM |
413 IFCAP_TXCSUM |
414 IFCAP_TXCSUM_IPV6 |
415 IFCAP_TSO |
416 IFCAP_LRO;
417
418 if ((priv->supported_features & GVE_SUP_JUMBO_FRAMES_MASK) != 0)
419 caps |= IFCAP_JUMBO_MTU;
420
421 if_setcapabilities(ifp, caps);
422 if_setcapenable(ifp, caps);
423
424 if (bootverbose)
425 device_printf(priv->dev, "Setting initial MTU to %d\n", priv->max_mtu);
426 if_setmtu(ifp, priv->max_mtu);
427
428 ether_ifattach(ifp, priv->mac);
429
430 ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
431 ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO);
432 }
433
434 static int
gve_alloc_counter_array(struct gve_priv * priv)435 gve_alloc_counter_array(struct gve_priv *priv)
436 {
437 int err;
438
439 err = gve_dma_alloc_coherent(priv, sizeof(uint32_t) * priv->num_event_counters,
440 PAGE_SIZE, &priv->counter_array_mem);
441 if (err != 0)
442 return (err);
443
444 priv->counters = priv->counter_array_mem.cpu_addr;
445 return (0);
446 }
447
448 static void
gve_free_counter_array(struct gve_priv * priv)449 gve_free_counter_array(struct gve_priv *priv)
450 {
451 if (priv->counters != NULL)
452 gve_dma_free_coherent(&priv->counter_array_mem);
453 priv->counter_array_mem = (struct gve_dma_handle){};
454 }
455
456 static int
gve_alloc_irq_db_array(struct gve_priv * priv)457 gve_alloc_irq_db_array(struct gve_priv *priv)
458 {
459 int err;
460
461 err = gve_dma_alloc_coherent(priv,
462 sizeof(struct gve_irq_db) * (priv->num_queues), PAGE_SIZE,
463 &priv->irqs_db_mem);
464 if (err != 0)
465 return (err);
466
467 priv->irq_db_indices = priv->irqs_db_mem.cpu_addr;
468 return (0);
469 }
470
471 static void
gve_free_irq_db_array(struct gve_priv * priv)472 gve_free_irq_db_array(struct gve_priv *priv)
473 {
474 if (priv->irq_db_indices != NULL)
475 gve_dma_free_coherent(&priv->irqs_db_mem);
476 priv->irqs_db_mem = (struct gve_dma_handle){};
477 }
478
479 static void
gve_free_rings(struct gve_priv * priv)480 gve_free_rings(struct gve_priv *priv)
481 {
482 gve_free_irqs(priv);
483 gve_free_tx_rings(priv);
484 gve_free_rx_rings(priv);
485 if (gve_is_qpl(priv))
486 gve_free_qpls(priv);
487 }
488
489 static int
gve_alloc_rings(struct gve_priv * priv)490 gve_alloc_rings(struct gve_priv *priv)
491 {
492 int err;
493
494 if (gve_is_qpl(priv)) {
495 err = gve_alloc_qpls(priv);
496 if (err != 0)
497 goto abort;
498 }
499
500 err = gve_alloc_rx_rings(priv);
501 if (err != 0)
502 goto abort;
503
504 err = gve_alloc_tx_rings(priv);
505 if (err != 0)
506 goto abort;
507
508 err = gve_alloc_irqs(priv);
509 if (err != 0)
510 goto abort;
511
512 return (0);
513
514 abort:
515 gve_free_rings(priv);
516 return (err);
517 }
518
519 static void
gve_deconfigure_and_free_device_resources(struct gve_priv * priv)520 gve_deconfigure_and_free_device_resources(struct gve_priv *priv)
521 {
522 int err;
523
524 if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK)) {
525 err = gve_adminq_deconfigure_device_resources(priv);
526 if (err != 0) {
527 device_printf(priv->dev, "Failed to deconfigure device resources: err=%d\n",
528 err);
529 return;
530 }
531 if (bootverbose)
532 device_printf(priv->dev, "Deconfigured device resources\n");
533 gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
534 }
535
536 gve_free_irq_db_array(priv);
537 gve_free_counter_array(priv);
538
539 if (priv->ptype_lut_dqo) {
540 free(priv->ptype_lut_dqo, M_GVE);
541 priv->ptype_lut_dqo = NULL;
542 }
543 }
544
545 static int
gve_alloc_and_configure_device_resources(struct gve_priv * priv)546 gve_alloc_and_configure_device_resources(struct gve_priv *priv)
547 {
548 int err;
549
550 if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK))
551 return (0);
552
553 err = gve_alloc_counter_array(priv);
554 if (err != 0)
555 return (err);
556
557 err = gve_alloc_irq_db_array(priv);
558 if (err != 0)
559 goto abort;
560
561 err = gve_adminq_configure_device_resources(priv);
562 if (err != 0) {
563 device_printf(priv->dev, "Failed to configure device resources: err=%d\n",
564 err);
565 err = (ENXIO);
566 goto abort;
567 }
568
569 if (!gve_is_gqi(priv)) {
570 priv->ptype_lut_dqo = malloc(sizeof(*priv->ptype_lut_dqo), M_GVE,
571 M_WAITOK | M_ZERO);
572
573 err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo);
574 if (err != 0) {
575 device_printf(priv->dev, "Failed to configure ptype lut: err=%d\n",
576 err);
577 goto abort;
578 }
579 }
580
581 gve_set_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
582 if (bootverbose)
583 device_printf(priv->dev, "Configured device resources\n");
584 return (0);
585
586 abort:
587 gve_deconfigure_and_free_device_resources(priv);
588 return (err);
589 }
590
591 static void
gve_set_queue_cnts(struct gve_priv * priv)592 gve_set_queue_cnts(struct gve_priv *priv)
593 {
594 priv->tx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_TX_QUEUES);
595 priv->rx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_RX_QUEUES);
596 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
597 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
598
599 if (priv->default_num_queues > 0) {
600 priv->tx_cfg.num_queues = MIN(priv->default_num_queues,
601 priv->tx_cfg.num_queues);
602 priv->rx_cfg.num_queues = MIN(priv->default_num_queues,
603 priv->rx_cfg.num_queues);
604 }
605
606 priv->num_queues = priv->tx_cfg.num_queues + priv->rx_cfg.num_queues;
607 priv->mgmt_msix_idx = priv->num_queues;
608 }
609
610 static int
gve_alloc_adminq_and_describe_device(struct gve_priv * priv)611 gve_alloc_adminq_and_describe_device(struct gve_priv *priv)
612 {
613 int err;
614
615 if ((err = gve_adminq_alloc(priv)) != 0)
616 return (err);
617
618 if ((err = gve_verify_driver_compatibility(priv)) != 0) {
619 device_printf(priv->dev,
620 "Failed to verify driver compatibility: err=%d\n", err);
621 goto abort;
622 }
623
624 if ((err = gve_adminq_describe_device(priv)) != 0)
625 goto abort;
626
627 gve_set_queue_cnts(priv);
628
629 priv->num_registered_pages = 0;
630 return (0);
631
632 abort:
633 gve_release_adminq(priv);
634 return (err);
635 }
636
637 void
gve_schedule_reset(struct gve_priv * priv)638 gve_schedule_reset(struct gve_priv *priv)
639 {
640 if (gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET))
641 return;
642
643 device_printf(priv->dev, "Scheduling reset task!\n");
644 gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
645 taskqueue_enqueue(priv->service_tq, &priv->service_task);
646 }
647
648 static void
gve_destroy(struct gve_priv * priv)649 gve_destroy(struct gve_priv *priv)
650 {
651 gve_down(priv);
652 gve_deconfigure_and_free_device_resources(priv);
653 gve_release_adminq(priv);
654 }
655
656 static void
gve_restore(struct gve_priv * priv)657 gve_restore(struct gve_priv *priv)
658 {
659 int err;
660
661 err = gve_adminq_alloc(priv);
662 if (err != 0)
663 goto abort;
664
665 err = gve_adminq_configure_device_resources(priv);
666 if (err != 0) {
667 device_printf(priv->dev, "Failed to configure device resources: err=%d\n",
668 err);
669 err = (ENXIO);
670 goto abort;
671 }
672 if (!gve_is_gqi(priv)) {
673 err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo);
674 if (err != 0) {
675 device_printf(priv->dev, "Failed to configure ptype lut: err=%d\n",
676 err);
677 goto abort;
678 }
679 }
680
681 err = gve_up(priv);
682 if (err != 0)
683 goto abort;
684
685 return;
686
687 abort:
688 device_printf(priv->dev, "Restore failed!\n");
689 return;
690 }
691
692 static void
gve_clear_device_resources(struct gve_priv * priv)693 gve_clear_device_resources(struct gve_priv *priv)
694 {
695 int i;
696
697 for (i = 0; i < priv->num_event_counters; i++)
698 priv->counters[i] = 0;
699 bus_dmamap_sync(priv->counter_array_mem.tag, priv->counter_array_mem.map,
700 BUS_DMASYNC_PREWRITE);
701
702 for (i = 0; i < priv->num_queues; i++)
703 priv->irq_db_indices[i] = (struct gve_irq_db){};
704 bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,
705 BUS_DMASYNC_PREWRITE);
706
707 if (priv->ptype_lut_dqo)
708 *priv->ptype_lut_dqo = (struct gve_ptype_lut){0};
709 }
710
711 static void
gve_handle_reset(struct gve_priv * priv)712 gve_handle_reset(struct gve_priv *priv)
713 {
714 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_DO_RESET))
715 return;
716
717 gve_clear_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
718 gve_set_state_flag(priv, GVE_STATE_FLAG_IN_RESET);
719
720 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
721
722 if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
723 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
724 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
725
726 /*
727 * Releasing the adminq causes the NIC to destroy all resources
728 * registered with it, so by clearing the flags beneath we cause
729 * the subsequent gve_down call below to not attempt to tell the
730 * NIC to destroy these resources again.
731 *
732 * The call to gve_down is needed in the first place to refresh
733 * the state and the DMA-able memory within each driver ring.
734 */
735 gve_release_adminq(priv);
736 gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
737 gve_clear_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK);
738 gve_clear_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK);
739 gve_clear_state_flag(priv, GVE_STATE_FLAG_TX_RINGS_OK);
740
741 gve_down(priv);
742 gve_clear_device_resources(priv);
743
744 gve_restore(priv);
745
746 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
747
748 priv->reset_cnt++;
749 gve_clear_state_flag(priv, GVE_STATE_FLAG_IN_RESET);
750 }
751
752 static void
gve_handle_link_status(struct gve_priv * priv)753 gve_handle_link_status(struct gve_priv *priv)
754 {
755 uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS);
756 bool link_up = status & GVE_DEVICE_STATUS_LINK_STATUS;
757
758 if (link_up == gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP))
759 return;
760
761 if (link_up) {
762 if (bootverbose)
763 device_printf(priv->dev, "Device link is up.\n");
764 if_link_state_change(priv->ifp, LINK_STATE_UP);
765 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
766 } else {
767 device_printf(priv->dev, "Device link is down.\n");
768 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
769 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
770 }
771 }
772
773 static void
gve_service_task(void * arg,int pending)774 gve_service_task(void *arg, int pending)
775 {
776 struct gve_priv *priv = (struct gve_priv *)arg;
777 uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS);
778
779 if (((GVE_DEVICE_STATUS_RESET_MASK & status) != 0) &&
780 !gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET)) {
781 device_printf(priv->dev, "Device requested reset\n");
782 gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
783 }
784
785 gve_handle_reset(priv);
786 gve_handle_link_status(priv);
787 }
788
789 static int
gve_probe(device_t dev)790 gve_probe(device_t dev)
791 {
792 uint16_t deviceid, vendorid;
793 int i;
794
795 vendorid = pci_get_vendor(dev);
796 deviceid = pci_get_device(dev);
797
798 for (i = 0; i < nitems(gve_devs); i++) {
799 if (vendorid == gve_devs[i].vendor_id &&
800 deviceid == gve_devs[i].device_id) {
801 device_set_desc(dev, gve_devs[i].name);
802 return (BUS_PROBE_DEFAULT);
803 }
804 }
805 return (ENXIO);
806 }
807
808 static void
gve_free_sys_res_mem(struct gve_priv * priv)809 gve_free_sys_res_mem(struct gve_priv *priv)
810 {
811 if (priv->msix_table != NULL)
812 bus_release_resource(priv->dev, SYS_RES_MEMORY,
813 rman_get_rid(priv->msix_table), priv->msix_table);
814
815 if (priv->db_bar != NULL)
816 bus_release_resource(priv->dev, SYS_RES_MEMORY,
817 rman_get_rid(priv->db_bar), priv->db_bar);
818
819 if (priv->reg_bar != NULL)
820 bus_release_resource(priv->dev, SYS_RES_MEMORY,
821 rman_get_rid(priv->reg_bar), priv->reg_bar);
822 }
823
824 static int
gve_attach(device_t dev)825 gve_attach(device_t dev)
826 {
827 struct gve_priv *priv;
828 int rid;
829 int err;
830
831 snprintf(gve_version, sizeof(gve_version), "%d.%d.%d",
832 GVE_VERSION_MAJOR, GVE_VERSION_MINOR, GVE_VERSION_SUB);
833
834 priv = device_get_softc(dev);
835 priv->dev = dev;
836 GVE_IFACE_LOCK_INIT(priv->gve_iface_lock);
837
838 pci_enable_busmaster(dev);
839
840 rid = PCIR_BAR(GVE_REGISTER_BAR);
841 priv->reg_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
842 &rid, RF_ACTIVE);
843 if (priv->reg_bar == NULL) {
844 device_printf(dev, "Failed to allocate BAR0\n");
845 err = ENXIO;
846 goto abort;
847 }
848
849 rid = PCIR_BAR(GVE_DOORBELL_BAR);
850 priv->db_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
851 &rid, RF_ACTIVE);
852 if (priv->db_bar == NULL) {
853 device_printf(dev, "Failed to allocate BAR2\n");
854 err = ENXIO;
855 goto abort;
856 }
857
858 rid = pci_msix_table_bar(priv->dev);
859 priv->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
860 &rid, RF_ACTIVE);
861 if (priv->msix_table == NULL) {
862 device_printf(dev, "Failed to allocate msix table\n");
863 err = ENXIO;
864 goto abort;
865 }
866
867 err = gve_alloc_adminq_and_describe_device(priv);
868 if (err != 0)
869 goto abort;
870
871 err = gve_alloc_and_configure_device_resources(priv);
872 if (err != 0)
873 goto abort;
874
875 err = gve_alloc_rings(priv);
876 if (err != 0)
877 goto abort;
878
879 gve_setup_ifnet(dev, priv);
880
881 priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
882
883 bus_write_multi_1(priv->reg_bar, DRIVER_VERSION, GVE_DRIVER_VERSION,
884 sizeof(GVE_DRIVER_VERSION) - 1);
885
886 TASK_INIT(&priv->service_task, 0, gve_service_task, priv);
887 priv->service_tq = taskqueue_create("gve service", M_WAITOK | M_ZERO,
888 taskqueue_thread_enqueue, &priv->service_tq);
889 taskqueue_start_threads(&priv->service_tq, 1, PI_NET, "%s service tq",
890 device_get_nameunit(priv->dev));
891
892 gve_setup_sysctl(priv);
893
894 if (bootverbose)
895 device_printf(priv->dev, "Successfully attached %s", GVE_DRIVER_VERSION);
896 return (0);
897
898 abort:
899 gve_free_rings(priv);
900 gve_deconfigure_and_free_device_resources(priv);
901 gve_release_adminq(priv);
902 gve_free_sys_res_mem(priv);
903 GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock);
904 return (err);
905 }
906
907 static int
gve_detach(device_t dev)908 gve_detach(device_t dev)
909 {
910 struct gve_priv *priv = device_get_softc(dev);
911 if_t ifp = priv->ifp;
912 int error;
913
914 error = bus_generic_detach(dev);
915 if (error != 0)
916 return (error);
917
918 ether_ifdetach(ifp);
919
920 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
921 gve_destroy(priv);
922 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
923
924 gve_free_rings(priv);
925 gve_free_sys_res_mem(priv);
926 GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock);
927
928 while (taskqueue_cancel(priv->service_tq, &priv->service_task, NULL))
929 taskqueue_drain(priv->service_tq, &priv->service_task);
930 taskqueue_free(priv->service_tq);
931
932 if_free(ifp);
933 return (0);
934 }
935
936 static device_method_t gve_methods[] = {
937 DEVMETHOD(device_probe, gve_probe),
938 DEVMETHOD(device_attach, gve_attach),
939 DEVMETHOD(device_detach, gve_detach),
940 DEVMETHOD_END
941 };
942
943 static driver_t gve_driver = {
944 "gve",
945 gve_methods,
946 sizeof(struct gve_priv)
947 };
948
949 #if __FreeBSD_version < 1301503
950 static devclass_t gve_devclass;
951
952 DRIVER_MODULE(gve, pci, gve_driver, gve_devclass, 0, 0);
953 #else
954 DRIVER_MODULE(gve, pci, gve_driver, 0, 0);
955 #endif
956 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, gve, gve_devs,
957 nitems(gve_devs));
958