1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2023-2024 Google LLC
5 *
6 * Redistribution and use in source and binary forms, with or without modification,
7 * are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * 3. Neither the name of the copyright holder nor the names of its contributors
17 * may be used to endorse or promote products derived from this software without
18 * specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
22 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
24 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
25 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
27 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
29 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 */
31 #include "gve.h"
32 #include "gve_adminq.h"
33 #include "gve_dqo.h"
34
35 #define GVE_DRIVER_VERSION "GVE-FBSD-1.3.3\n"
36 #define GVE_VERSION_MAJOR 1
37 #define GVE_VERSION_MINOR 3
38 #define GVE_VERSION_SUB 3
39
40 #define GVE_DEFAULT_RX_COPYBREAK 256
41
42 /* Devices supported by this driver. */
43 static struct gve_dev {
44 uint16_t vendor_id;
45 uint16_t device_id;
46 const char *name;
47 } gve_devs[] = {
48 { PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC, "gVNIC" }
49 };
50
51 struct sx gve_global_lock;
52
53 static int
gve_verify_driver_compatibility(struct gve_priv * priv)54 gve_verify_driver_compatibility(struct gve_priv *priv)
55 {
56 int err;
57 struct gve_driver_info *driver_info;
58 struct gve_dma_handle driver_info_mem;
59
60 err = gve_dma_alloc_coherent(priv, sizeof(struct gve_driver_info),
61 PAGE_SIZE, &driver_info_mem);
62
63 if (err != 0)
64 return (ENOMEM);
65
66 driver_info = driver_info_mem.cpu_addr;
67
68 *driver_info = (struct gve_driver_info) {
69 .os_type = 3, /* Freebsd */
70 .driver_major = GVE_VERSION_MAJOR,
71 .driver_minor = GVE_VERSION_MINOR,
72 .driver_sub = GVE_VERSION_SUB,
73 .os_version_major = htobe32(FBSD_VERSION_MAJOR),
74 .os_version_minor = htobe32(FBSD_VERSION_MINOR),
75 .os_version_sub = htobe32(FBSD_VERSION_PATCH),
76 .driver_capability_flags = {
77 htobe64(GVE_DRIVER_CAPABILITY_FLAGS1),
78 htobe64(GVE_DRIVER_CAPABILITY_FLAGS2),
79 htobe64(GVE_DRIVER_CAPABILITY_FLAGS3),
80 htobe64(GVE_DRIVER_CAPABILITY_FLAGS4),
81 },
82 };
83
84 snprintf(driver_info->os_version_str1, sizeof(driver_info->os_version_str1),
85 "FreeBSD %u", __FreeBSD_version);
86
87 bus_dmamap_sync(driver_info_mem.tag, driver_info_mem.map,
88 BUS_DMASYNC_PREREAD);
89
90 err = gve_adminq_verify_driver_compatibility(priv,
91 sizeof(struct gve_driver_info), driver_info_mem.bus_addr);
92
93 /* It's ok if the device doesn't support this */
94 if (err == EOPNOTSUPP)
95 err = 0;
96
97 gve_dma_free_coherent(&driver_info_mem);
98
99 return (err);
100 }
101
102 static int
gve_up(struct gve_priv * priv)103 gve_up(struct gve_priv *priv)
104 {
105 if_t ifp = priv->ifp;
106 int err;
107
108 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
109
110 if (device_is_attached(priv->dev) == 0) {
111 device_printf(priv->dev, "Cannot bring the iface up when detached\n");
112 return (ENXIO);
113 }
114
115 if (gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP))
116 return (0);
117
118 if_clearhwassist(ifp);
119 if (if_getcapenable(ifp) & IFCAP_TXCSUM)
120 if_sethwassistbits(ifp, CSUM_TCP | CSUM_UDP, 0);
121 if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
122 if_sethwassistbits(ifp, CSUM_IP6_TCP | CSUM_IP6_UDP, 0);
123 if (if_getcapenable(ifp) & IFCAP_TSO4)
124 if_sethwassistbits(ifp, CSUM_IP_TSO, 0);
125 if (if_getcapenable(ifp) & IFCAP_TSO6)
126 if_sethwassistbits(ifp, CSUM_IP6_TSO, 0);
127
128 if (gve_is_qpl(priv)) {
129 err = gve_register_qpls(priv);
130 if (err != 0)
131 goto reset;
132 }
133
134 err = gve_create_rx_rings(priv);
135 if (err != 0)
136 goto reset;
137
138 err = gve_create_tx_rings(priv);
139 if (err != 0)
140 goto reset;
141
142 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
143
144 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
145 if_link_state_change(ifp, LINK_STATE_UP);
146 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
147 }
148
149 gve_unmask_all_queue_irqs(priv);
150 gve_set_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP);
151 priv->interface_up_cnt++;
152 return (0);
153
154 reset:
155 gve_schedule_reset(priv);
156 return (err);
157 }
158
159 static void
gve_down(struct gve_priv * priv)160 gve_down(struct gve_priv *priv)
161 {
162 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
163
164 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP))
165 return;
166
167 if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
168 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
169 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
170 }
171
172 if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
173
174 if (gve_destroy_rx_rings(priv) != 0)
175 goto reset;
176
177 if (gve_destroy_tx_rings(priv) != 0)
178 goto reset;
179
180 if (gve_is_qpl(priv)) {
181 if (gve_unregister_qpls(priv) != 0)
182 goto reset;
183 }
184
185 if (gve_is_gqi(priv))
186 gve_mask_all_queue_irqs(priv);
187 gve_clear_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP);
188 priv->interface_down_cnt++;
189 return;
190
191 reset:
192 gve_schedule_reset(priv);
193 }
194
195 int
gve_adjust_rx_queues(struct gve_priv * priv,uint16_t new_queue_cnt)196 gve_adjust_rx_queues(struct gve_priv *priv, uint16_t new_queue_cnt)
197 {
198 int err;
199
200 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
201
202 gve_down(priv);
203
204 if (new_queue_cnt < priv->rx_cfg.num_queues) {
205 /*
206 * Freeing a ring still preserves its ntfy_id,
207 * which is needed if we create the ring again.
208 */
209 gve_free_rx_rings(priv, new_queue_cnt, priv->rx_cfg.num_queues);
210 } else {
211 err = gve_alloc_rx_rings(priv, priv->rx_cfg.num_queues, new_queue_cnt);
212 if (err != 0) {
213 device_printf(priv->dev, "Failed to allocate new queues");
214 /* Failed to allocate rings, start back up with old ones */
215 gve_up(priv);
216 return (err);
217
218 }
219 }
220 priv->rx_cfg.num_queues = new_queue_cnt;
221
222 err = gve_up(priv);
223 if (err != 0)
224 gve_schedule_reset(priv);
225
226 return (err);
227 }
228
229 int
gve_adjust_tx_queues(struct gve_priv * priv,uint16_t new_queue_cnt)230 gve_adjust_tx_queues(struct gve_priv *priv, uint16_t new_queue_cnt)
231 {
232 int err;
233
234 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
235
236 gve_down(priv);
237
238 if (new_queue_cnt < priv->tx_cfg.num_queues) {
239 /*
240 * Freeing a ring still preserves its ntfy_id,
241 * which is needed if we create the ring again.
242 */
243 gve_free_tx_rings(priv, new_queue_cnt, priv->tx_cfg.num_queues);
244 } else {
245 err = gve_alloc_tx_rings(priv, priv->tx_cfg.num_queues, new_queue_cnt);
246 if (err != 0) {
247 device_printf(priv->dev, "Failed to allocate new queues");
248 /* Failed to allocate rings, start back up with old ones */
249 gve_up(priv);
250 return (err);
251
252 }
253 }
254 priv->tx_cfg.num_queues = new_queue_cnt;
255
256 err = gve_up(priv);
257 if (err != 0)
258 gve_schedule_reset(priv);
259
260 return (err);
261 }
262
263 int
gve_adjust_ring_sizes(struct gve_priv * priv,uint16_t new_desc_cnt,bool is_rx)264 gve_adjust_ring_sizes(struct gve_priv *priv, uint16_t new_desc_cnt, bool is_rx)
265 {
266 int err;
267 uint16_t prev_desc_cnt;
268
269 GVE_IFACE_LOCK_ASSERT(priv->gve_iface_lock);
270
271 gve_down(priv);
272
273 if (is_rx) {
274 gve_free_rx_rings(priv, 0, priv->rx_cfg.num_queues);
275 prev_desc_cnt = priv->rx_desc_cnt;
276 priv->rx_desc_cnt = new_desc_cnt;
277 err = gve_alloc_rx_rings(priv, 0, priv->rx_cfg.num_queues);
278 if (err != 0) {
279 device_printf(priv->dev,
280 "Failed to allocate rings. Trying to start back up with previous ring size.");
281 priv->rx_desc_cnt = prev_desc_cnt;
282 err = gve_alloc_rx_rings(priv, 0, priv->rx_cfg.num_queues);
283 }
284 } else {
285 gve_free_tx_rings(priv, 0, priv->tx_cfg.num_queues);
286 prev_desc_cnt = priv->tx_desc_cnt;
287 priv->tx_desc_cnt = new_desc_cnt;
288 err = gve_alloc_tx_rings(priv, 0, priv->tx_cfg.num_queues);
289 if (err != 0) {
290 device_printf(priv->dev,
291 "Failed to allocate rings. Trying to start back up with previous ring size.");
292 priv->tx_desc_cnt = prev_desc_cnt;
293 err = gve_alloc_tx_rings(priv, 0, priv->tx_cfg.num_queues);
294 }
295 }
296
297 if (err != 0) {
298 device_printf(priv->dev, "Failed to allocate rings! Cannot start device back up!");
299 return (err);
300 }
301
302 err = gve_up(priv);
303 if (err != 0) {
304 gve_schedule_reset(priv);
305 return (err);
306 }
307
308 return (0);
309 }
310
311 static int
gve_set_mtu(if_t ifp,uint32_t new_mtu)312 gve_set_mtu(if_t ifp, uint32_t new_mtu)
313 {
314 struct gve_priv *priv = if_getsoftc(ifp);
315 const uint32_t max_problem_range = 8227;
316 const uint32_t min_problem_range = 7822;
317 int err;
318
319 if ((new_mtu > priv->max_mtu) || (new_mtu < ETHERMIN)) {
320 device_printf(priv->dev, "Invalid new MTU setting. new mtu: %d max mtu: %d min mtu: %d\n",
321 new_mtu, priv->max_mtu, ETHERMIN);
322 return (EINVAL);
323 }
324
325 /*
326 * When hardware LRO is enabled in DQ mode, MTUs within the range
327 * [7822, 8227] trigger hardware issues which cause a drastic drop
328 * in throughput.
329 */
330 if (!gve_is_gqi(priv) && !gve_disable_hw_lro &&
331 new_mtu >= min_problem_range && new_mtu <= max_problem_range) {
332 device_printf(priv->dev,
333 "Cannot set to MTU to %d within the range [%d, %d] while hardware LRO is enabled\n",
334 new_mtu, min_problem_range, max_problem_range);
335 return (EINVAL);
336 }
337
338 err = gve_adminq_set_mtu(priv, new_mtu);
339 if (err == 0) {
340 if (bootverbose)
341 device_printf(priv->dev, "MTU set to %d\n", new_mtu);
342 if_setmtu(ifp, new_mtu);
343 } else {
344 device_printf(priv->dev, "Failed to set MTU to %d\n", new_mtu);
345 }
346
347 return (err);
348 }
349
350 static void
gve_init(void * arg)351 gve_init(void *arg)
352 {
353 struct gve_priv *priv = (struct gve_priv *)arg;
354
355 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_QUEUES_UP)) {
356 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
357 gve_up(priv);
358 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
359 }
360 }
361
362 static int
gve_ioctl(if_t ifp,u_long command,caddr_t data)363 gve_ioctl(if_t ifp, u_long command, caddr_t data)
364 {
365 struct gve_priv *priv;
366 struct ifreq *ifr;
367 int rc = 0;
368
369 priv = if_getsoftc(ifp);
370 ifr = (struct ifreq *)data;
371
372 switch (command) {
373 case SIOCSIFMTU:
374 if (if_getmtu(ifp) == ifr->ifr_mtu)
375 break;
376 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
377 gve_down(priv);
378 gve_set_mtu(ifp, ifr->ifr_mtu);
379 rc = gve_up(priv);
380 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
381 break;
382
383 case SIOCSIFFLAGS:
384 if ((if_getflags(ifp) & IFF_UP) != 0) {
385 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
386 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
387 rc = gve_up(priv);
388 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
389 }
390 } else {
391 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
392 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
393 gve_down(priv);
394 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
395 }
396 }
397 break;
398
399 case SIOCSIFCAP:
400 if (ifr->ifr_reqcap == if_getcapenable(ifp))
401 break;
402 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
403 gve_down(priv);
404 if_setcapenable(ifp, ifr->ifr_reqcap);
405 rc = gve_up(priv);
406 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
407 break;
408
409 case SIOCSIFMEDIA:
410 /* FALLTHROUGH */
411 case SIOCGIFMEDIA:
412 rc = ifmedia_ioctl(ifp, ifr, &priv->media, command);
413 break;
414
415 default:
416 rc = ether_ioctl(ifp, command, data);
417 break;
418 }
419
420 return (rc);
421 }
422
423 static int
gve_media_change(if_t ifp)424 gve_media_change(if_t ifp)
425 {
426 struct gve_priv *priv = if_getsoftc(ifp);
427
428 device_printf(priv->dev, "Media change not supported\n");
429 return (0);
430 }
431
432 static void
gve_media_status(if_t ifp,struct ifmediareq * ifmr)433 gve_media_status(if_t ifp, struct ifmediareq *ifmr)
434 {
435 struct gve_priv *priv = if_getsoftc(ifp);
436
437 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
438
439 ifmr->ifm_status = IFM_AVALID;
440 ifmr->ifm_active = IFM_ETHER;
441
442 if (gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP)) {
443 ifmr->ifm_status |= IFM_ACTIVE;
444 ifmr->ifm_active |= IFM_AUTO;
445 } else {
446 ifmr->ifm_active |= IFM_NONE;
447 }
448
449 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
450 }
451
452 static uint64_t
gve_get_counter(if_t ifp,ift_counter cnt)453 gve_get_counter(if_t ifp, ift_counter cnt)
454 {
455 struct gve_priv *priv;
456 uint64_t rpackets = 0;
457 uint64_t tpackets = 0;
458 uint64_t rbytes = 0;
459 uint64_t tbytes = 0;
460 uint64_t rx_dropped_pkt = 0;
461 uint64_t tx_dropped_pkt = 0;
462
463 priv = if_getsoftc(ifp);
464
465 gve_accum_stats(priv, &rpackets, &rbytes, &rx_dropped_pkt, &tpackets,
466 &tbytes, &tx_dropped_pkt);
467
468 switch (cnt) {
469 case IFCOUNTER_IPACKETS:
470 return (rpackets);
471
472 case IFCOUNTER_OPACKETS:
473 return (tpackets);
474
475 case IFCOUNTER_IBYTES:
476 return (rbytes);
477
478 case IFCOUNTER_OBYTES:
479 return (tbytes);
480
481 case IFCOUNTER_IQDROPS:
482 return (rx_dropped_pkt);
483
484 case IFCOUNTER_OQDROPS:
485 return (tx_dropped_pkt);
486
487 default:
488 return (if_get_counter_default(ifp, cnt));
489 }
490 }
491
492 static void
gve_setup_ifnet(device_t dev,struct gve_priv * priv)493 gve_setup_ifnet(device_t dev, struct gve_priv *priv)
494 {
495 int caps = 0;
496 if_t ifp;
497
498 ifp = priv->ifp = if_alloc(IFT_ETHER);
499 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
500 if_setsoftc(ifp, priv);
501 if_setdev(ifp, dev);
502 if_setinitfn(ifp, gve_init);
503 if_setioctlfn(ifp, gve_ioctl);
504 if_settransmitfn(ifp, gve_xmit_ifp);
505 if_setqflushfn(ifp, gve_qflush);
506
507 /*
508 * Set TSO limits, must match the arguments to bus_dma_tag_create
509 * when creating tx->dqo.buf_dmatag. Only applies to the RDA mode
510 * because in QPL we copy the entire packet into the bounce buffer
511 * and thus it does not matter how fragmented the mbuf is.
512 */
513 if (!gve_is_gqi(priv) && !gve_is_qpl(priv)) {
514 if_sethwtsomaxsegcount(ifp, GVE_TX_MAX_DATA_DESCS_DQO);
515 if_sethwtsomaxsegsize(ifp, GVE_TX_MAX_BUF_SIZE_DQO);
516 }
517 if_sethwtsomax(ifp, GVE_TSO_MAXSIZE_DQO);
518
519 #if __FreeBSD_version >= 1400086
520 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
521 #else
522 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST | IFF_KNOWSEPOCH);
523 #endif
524
525 ifmedia_init(&priv->media, IFM_IMASK, gve_media_change, gve_media_status);
526 if_setgetcounterfn(ifp, gve_get_counter);
527
528 caps = IFCAP_RXCSUM |
529 IFCAP_TXCSUM |
530 IFCAP_TXCSUM_IPV6 |
531 IFCAP_TSO |
532 IFCAP_LRO;
533
534 if ((priv->supported_features & GVE_SUP_JUMBO_FRAMES_MASK) != 0)
535 caps |= IFCAP_JUMBO_MTU;
536
537 if_setcapabilities(ifp, caps);
538 if_setcapenable(ifp, caps);
539
540 if (bootverbose)
541 device_printf(priv->dev, "Setting initial MTU to %d\n", priv->max_mtu);
542 if_setmtu(ifp, priv->max_mtu);
543
544 ether_ifattach(ifp, priv->mac);
545
546 ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
547 ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO);
548 }
549
550 static int
gve_alloc_counter_array(struct gve_priv * priv)551 gve_alloc_counter_array(struct gve_priv *priv)
552 {
553 int err;
554
555 err = gve_dma_alloc_coherent(priv, sizeof(uint32_t) * priv->num_event_counters,
556 PAGE_SIZE, &priv->counter_array_mem);
557 if (err != 0)
558 return (err);
559
560 priv->counters = priv->counter_array_mem.cpu_addr;
561 return (0);
562 }
563
564 static void
gve_free_counter_array(struct gve_priv * priv)565 gve_free_counter_array(struct gve_priv *priv)
566 {
567 if (priv->counters != NULL)
568 gve_dma_free_coherent(&priv->counter_array_mem);
569 priv->counter_array_mem = (struct gve_dma_handle){};
570 }
571
572 static int
gve_alloc_irq_db_array(struct gve_priv * priv)573 gve_alloc_irq_db_array(struct gve_priv *priv)
574 {
575 int err;
576
577 err = gve_dma_alloc_coherent(priv,
578 sizeof(struct gve_irq_db) * (priv->num_queues), PAGE_SIZE,
579 &priv->irqs_db_mem);
580 if (err != 0)
581 return (err);
582
583 priv->irq_db_indices = priv->irqs_db_mem.cpu_addr;
584 return (0);
585 }
586
587 static void
gve_free_irq_db_array(struct gve_priv * priv)588 gve_free_irq_db_array(struct gve_priv *priv)
589 {
590 if (priv->irq_db_indices != NULL)
591 gve_dma_free_coherent(&priv->irqs_db_mem);
592 priv->irqs_db_mem = (struct gve_dma_handle){};
593 }
594
595 static void
gve_free_rings(struct gve_priv * priv)596 gve_free_rings(struct gve_priv *priv)
597 {
598 gve_free_irqs(priv);
599
600 gve_free_tx_rings(priv, 0, priv->tx_cfg.num_queues);
601 free(priv->tx, M_GVE);
602 priv->tx = NULL;
603
604 gve_free_rx_rings(priv, 0, priv->rx_cfg.num_queues);
605 free(priv->rx, M_GVE);
606 priv->rx = NULL;
607 }
608
609 static int
gve_alloc_rings(struct gve_priv * priv)610 gve_alloc_rings(struct gve_priv *priv)
611 {
612 int err;
613
614 priv->rx = malloc(sizeof(struct gve_rx_ring) * priv->rx_cfg.max_queues,
615 M_GVE, M_WAITOK | M_ZERO);
616 err = gve_alloc_rx_rings(priv, 0, priv->rx_cfg.num_queues);
617 if (err != 0)
618 goto abort;
619
620 priv->tx = malloc(sizeof(struct gve_tx_ring) * priv->tx_cfg.max_queues,
621 M_GVE, M_WAITOK | M_ZERO);
622 err = gve_alloc_tx_rings(priv, 0, priv->tx_cfg.num_queues);
623 if (err != 0)
624 goto abort;
625
626 err = gve_alloc_irqs(priv);
627 if (err != 0)
628 goto abort;
629
630 return (0);
631
632 abort:
633 gve_free_rings(priv);
634 return (err);
635 }
636
637 static void
gve_deconfigure_and_free_device_resources(struct gve_priv * priv)638 gve_deconfigure_and_free_device_resources(struct gve_priv *priv)
639 {
640 int err;
641
642 if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK)) {
643 err = gve_adminq_deconfigure_device_resources(priv);
644 if (err != 0) {
645 device_printf(priv->dev, "Failed to deconfigure device resources: err=%d\n",
646 err);
647 return;
648 }
649 if (bootverbose)
650 device_printf(priv->dev, "Deconfigured device resources\n");
651 gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
652 }
653
654 gve_free_irq_db_array(priv);
655 gve_free_counter_array(priv);
656
657 if (priv->ptype_lut_dqo) {
658 free(priv->ptype_lut_dqo, M_GVE);
659 priv->ptype_lut_dqo = NULL;
660 }
661 }
662
663 static int
gve_alloc_and_configure_device_resources(struct gve_priv * priv)664 gve_alloc_and_configure_device_resources(struct gve_priv *priv)
665 {
666 int err;
667
668 if (gve_get_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK))
669 return (0);
670
671 err = gve_alloc_counter_array(priv);
672 if (err != 0)
673 return (err);
674
675 err = gve_alloc_irq_db_array(priv);
676 if (err != 0)
677 goto abort;
678
679 err = gve_adminq_configure_device_resources(priv);
680 if (err != 0) {
681 device_printf(priv->dev, "Failed to configure device resources: err=%d\n",
682 err);
683 err = (ENXIO);
684 goto abort;
685 }
686
687 if (!gve_is_gqi(priv)) {
688 priv->ptype_lut_dqo = malloc(sizeof(*priv->ptype_lut_dqo), M_GVE,
689 M_WAITOK | M_ZERO);
690
691 err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo);
692 if (err != 0) {
693 device_printf(priv->dev, "Failed to configure ptype lut: err=%d\n",
694 err);
695 goto abort;
696 }
697 }
698
699 gve_set_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
700 if (bootverbose)
701 device_printf(priv->dev, "Configured device resources\n");
702 return (0);
703
704 abort:
705 gve_deconfigure_and_free_device_resources(priv);
706 return (err);
707 }
708
709 static void
gve_set_queue_cnts(struct gve_priv * priv)710 gve_set_queue_cnts(struct gve_priv *priv)
711 {
712 priv->tx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_TX_QUEUES);
713 priv->rx_cfg.max_queues = gve_reg_bar_read_4(priv, MAX_RX_QUEUES);
714 priv->tx_cfg.num_queues = priv->tx_cfg.max_queues;
715 priv->rx_cfg.num_queues = priv->rx_cfg.max_queues;
716
717 if (priv->default_num_queues > 0) {
718 priv->tx_cfg.num_queues = MIN(priv->default_num_queues,
719 priv->tx_cfg.num_queues);
720 priv->rx_cfg.num_queues = MIN(priv->default_num_queues,
721 priv->rx_cfg.num_queues);
722 }
723
724 priv->num_queues = priv->tx_cfg.max_queues + priv->rx_cfg.max_queues;
725 priv->mgmt_msix_idx = priv->num_queues;
726 }
727
728 static int
gve_alloc_adminq_and_describe_device(struct gve_priv * priv)729 gve_alloc_adminq_and_describe_device(struct gve_priv *priv)
730 {
731 int err;
732
733 if ((err = gve_adminq_alloc(priv)) != 0)
734 return (err);
735
736 if ((err = gve_verify_driver_compatibility(priv)) != 0) {
737 device_printf(priv->dev,
738 "Failed to verify driver compatibility: err=%d\n", err);
739 goto abort;
740 }
741
742 if ((err = gve_adminq_describe_device(priv)) != 0)
743 goto abort;
744
745 gve_set_queue_cnts(priv);
746
747 priv->num_registered_pages = 0;
748 return (0);
749
750 abort:
751 gve_release_adminq(priv);
752 return (err);
753 }
754
755 void
gve_schedule_reset(struct gve_priv * priv)756 gve_schedule_reset(struct gve_priv *priv)
757 {
758 if (gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET))
759 return;
760
761 device_printf(priv->dev, "Scheduling reset task!\n");
762 gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
763 taskqueue_enqueue(priv->service_tq, &priv->service_task);
764 }
765
766 static void
gve_destroy(struct gve_priv * priv)767 gve_destroy(struct gve_priv *priv)
768 {
769 gve_down(priv);
770 gve_deconfigure_and_free_device_resources(priv);
771 gve_release_adminq(priv);
772 }
773
774 static void
gve_restore(struct gve_priv * priv)775 gve_restore(struct gve_priv *priv)
776 {
777 int err;
778
779 err = gve_adminq_alloc(priv);
780 if (err != 0)
781 goto abort;
782
783 err = gve_adminq_configure_device_resources(priv);
784 if (err != 0) {
785 device_printf(priv->dev, "Failed to configure device resources: err=%d\n",
786 err);
787 err = (ENXIO);
788 goto abort;
789 }
790 if (!gve_is_gqi(priv)) {
791 err = gve_adminq_get_ptype_map_dqo(priv, priv->ptype_lut_dqo);
792 if (err != 0) {
793 device_printf(priv->dev, "Failed to configure ptype lut: err=%d\n",
794 err);
795 goto abort;
796 }
797 }
798
799 err = gve_up(priv);
800 if (err != 0)
801 goto abort;
802
803 return;
804
805 abort:
806 device_printf(priv->dev, "Restore failed!\n");
807 return;
808 }
809
810 static void
gve_clear_device_resources(struct gve_priv * priv)811 gve_clear_device_resources(struct gve_priv *priv)
812 {
813 int i;
814
815 for (i = 0; i < priv->num_event_counters; i++)
816 priv->counters[i] = 0;
817 bus_dmamap_sync(priv->counter_array_mem.tag, priv->counter_array_mem.map,
818 BUS_DMASYNC_PREWRITE);
819
820 for (i = 0; i < priv->num_queues; i++)
821 priv->irq_db_indices[i] = (struct gve_irq_db){};
822 bus_dmamap_sync(priv->irqs_db_mem.tag, priv->irqs_db_mem.map,
823 BUS_DMASYNC_PREWRITE);
824
825 if (priv->ptype_lut_dqo)
826 *priv->ptype_lut_dqo = (struct gve_ptype_lut){0};
827 }
828
829 static void
gve_handle_reset(struct gve_priv * priv)830 gve_handle_reset(struct gve_priv *priv)
831 {
832 if (!gve_get_state_flag(priv, GVE_STATE_FLAG_DO_RESET))
833 return;
834
835 gve_clear_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
836 gve_set_state_flag(priv, GVE_STATE_FLAG_IN_RESET);
837
838 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
839
840 if_setdrvflagbits(priv->ifp, IFF_DRV_OACTIVE, IFF_DRV_RUNNING);
841 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
842 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
843
844 /*
845 * Releasing the adminq causes the NIC to destroy all resources
846 * registered with it, so by clearing the flags beneath we cause
847 * the subsequent gve_down call below to not attempt to tell the
848 * NIC to destroy these resources again.
849 *
850 * The call to gve_down is needed in the first place to refresh
851 * the state and the DMA-able memory within each driver ring.
852 */
853 gve_release_adminq(priv);
854 gve_clear_state_flag(priv, GVE_STATE_FLAG_RESOURCES_OK);
855 gve_clear_state_flag(priv, GVE_STATE_FLAG_QPLREG_OK);
856 gve_clear_state_flag(priv, GVE_STATE_FLAG_RX_RINGS_OK);
857 gve_clear_state_flag(priv, GVE_STATE_FLAG_TX_RINGS_OK);
858
859 gve_down(priv);
860 gve_clear_device_resources(priv);
861
862 gve_restore(priv);
863
864 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
865
866 priv->reset_cnt++;
867 gve_clear_state_flag(priv, GVE_STATE_FLAG_IN_RESET);
868 }
869
870 static void
gve_handle_link_status(struct gve_priv * priv)871 gve_handle_link_status(struct gve_priv *priv)
872 {
873 uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS);
874 bool link_up = status & GVE_DEVICE_STATUS_LINK_STATUS;
875
876 if (link_up == gve_get_state_flag(priv, GVE_STATE_FLAG_LINK_UP))
877 return;
878
879 if (link_up) {
880 if (bootverbose)
881 device_printf(priv->dev, "Device link is up.\n");
882 if_link_state_change(priv->ifp, LINK_STATE_UP);
883 gve_set_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
884 } else {
885 device_printf(priv->dev, "Device link is down.\n");
886 if_link_state_change(priv->ifp, LINK_STATE_DOWN);
887 gve_clear_state_flag(priv, GVE_STATE_FLAG_LINK_UP);
888 }
889 }
890
891 static void
gve_service_task(void * arg,int pending)892 gve_service_task(void *arg, int pending)
893 {
894 struct gve_priv *priv = (struct gve_priv *)arg;
895 uint32_t status = gve_reg_bar_read_4(priv, DEVICE_STATUS);
896
897 if (((GVE_DEVICE_STATUS_RESET_MASK & status) != 0) &&
898 !gve_get_state_flag(priv, GVE_STATE_FLAG_IN_RESET)) {
899 device_printf(priv->dev, "Device requested reset\n");
900 gve_set_state_flag(priv, GVE_STATE_FLAG_DO_RESET);
901 }
902
903 gve_handle_reset(priv);
904 gve_handle_link_status(priv);
905 }
906
907 static int
gve_probe(device_t dev)908 gve_probe(device_t dev)
909 {
910 uint16_t deviceid, vendorid;
911 int i;
912
913 vendorid = pci_get_vendor(dev);
914 deviceid = pci_get_device(dev);
915
916 for (i = 0; i < nitems(gve_devs); i++) {
917 if (vendorid == gve_devs[i].vendor_id &&
918 deviceid == gve_devs[i].device_id) {
919 device_set_desc(dev, gve_devs[i].name);
920 return (BUS_PROBE_DEFAULT);
921 }
922 }
923 return (ENXIO);
924 }
925
926 static void
gve_free_sys_res_mem(struct gve_priv * priv)927 gve_free_sys_res_mem(struct gve_priv *priv)
928 {
929 if (priv->msix_table != NULL)
930 bus_release_resource(priv->dev, SYS_RES_MEMORY,
931 rman_get_rid(priv->msix_table), priv->msix_table);
932
933 if (priv->db_bar != NULL)
934 bus_release_resource(priv->dev, SYS_RES_MEMORY,
935 rman_get_rid(priv->db_bar), priv->db_bar);
936
937 if (priv->reg_bar != NULL)
938 bus_release_resource(priv->dev, SYS_RES_MEMORY,
939 rman_get_rid(priv->reg_bar), priv->reg_bar);
940 }
941
942 static int
gve_attach(device_t dev)943 gve_attach(device_t dev)
944 {
945 struct gve_priv *priv;
946 int rid;
947 int err;
948
949 snprintf(gve_version, sizeof(gve_version), "%d.%d.%d",
950 GVE_VERSION_MAJOR, GVE_VERSION_MINOR, GVE_VERSION_SUB);
951
952 priv = device_get_softc(dev);
953 priv->dev = dev;
954 GVE_IFACE_LOCK_INIT(priv->gve_iface_lock);
955
956 pci_enable_busmaster(dev);
957
958 rid = PCIR_BAR(GVE_REGISTER_BAR);
959 priv->reg_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
960 &rid, RF_ACTIVE);
961 if (priv->reg_bar == NULL) {
962 device_printf(dev, "Failed to allocate BAR0\n");
963 err = ENXIO;
964 goto abort;
965 }
966
967 rid = PCIR_BAR(GVE_DOORBELL_BAR);
968 priv->db_bar = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
969 &rid, RF_ACTIVE);
970 if (priv->db_bar == NULL) {
971 device_printf(dev, "Failed to allocate BAR2\n");
972 err = ENXIO;
973 goto abort;
974 }
975
976 rid = pci_msix_table_bar(priv->dev);
977 priv->msix_table = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
978 &rid, RF_ACTIVE);
979 if (priv->msix_table == NULL) {
980 device_printf(dev, "Failed to allocate msix table\n");
981 err = ENXIO;
982 goto abort;
983 }
984
985 err = gve_alloc_adminq_and_describe_device(priv);
986 if (err != 0)
987 goto abort;
988
989 err = gve_alloc_and_configure_device_resources(priv);
990 if (err != 0)
991 goto abort;
992
993 err = gve_alloc_rings(priv);
994 if (err != 0)
995 goto abort;
996
997 gve_setup_ifnet(dev, priv);
998
999 priv->rx_copybreak = GVE_DEFAULT_RX_COPYBREAK;
1000
1001 bus_write_multi_1(priv->reg_bar, DRIVER_VERSION, GVE_DRIVER_VERSION,
1002 sizeof(GVE_DRIVER_VERSION) - 1);
1003
1004 TASK_INIT(&priv->service_task, 0, gve_service_task, priv);
1005 priv->service_tq = taskqueue_create("gve service", M_WAITOK | M_ZERO,
1006 taskqueue_thread_enqueue, &priv->service_tq);
1007 taskqueue_start_threads(&priv->service_tq, 1, PI_NET, "%s service tq",
1008 device_get_nameunit(priv->dev));
1009
1010 gve_setup_sysctl(priv);
1011
1012 if (bootverbose)
1013 device_printf(priv->dev, "Successfully attached %s", GVE_DRIVER_VERSION);
1014 return (0);
1015
1016 abort:
1017 gve_free_rings(priv);
1018 gve_deconfigure_and_free_device_resources(priv);
1019 gve_release_adminq(priv);
1020 gve_free_sys_res_mem(priv);
1021 GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock);
1022 return (err);
1023 }
1024
1025 static int
gve_detach(device_t dev)1026 gve_detach(device_t dev)
1027 {
1028 struct gve_priv *priv = device_get_softc(dev);
1029 if_t ifp = priv->ifp;
1030 int error;
1031
1032 error = bus_generic_detach(dev);
1033 if (error != 0)
1034 return (error);
1035
1036 ether_ifdetach(ifp);
1037
1038 GVE_IFACE_LOCK_LOCK(priv->gve_iface_lock);
1039 gve_destroy(priv);
1040 GVE_IFACE_LOCK_UNLOCK(priv->gve_iface_lock);
1041
1042 gve_free_rings(priv);
1043 gve_free_sys_res_mem(priv);
1044 GVE_IFACE_LOCK_DESTROY(priv->gve_iface_lock);
1045
1046 while (taskqueue_cancel(priv->service_tq, &priv->service_task, NULL))
1047 taskqueue_drain(priv->service_tq, &priv->service_task);
1048 taskqueue_free(priv->service_tq);
1049
1050 if_free(ifp);
1051 return (0);
1052 }
1053
1054 static device_method_t gve_methods[] = {
1055 DEVMETHOD(device_probe, gve_probe),
1056 DEVMETHOD(device_attach, gve_attach),
1057 DEVMETHOD(device_detach, gve_detach),
1058 DEVMETHOD_END
1059 };
1060
1061 static driver_t gve_driver = {
1062 "gve",
1063 gve_methods,
1064 sizeof(struct gve_priv)
1065 };
1066
1067 #if __FreeBSD_version < 1301503
1068 static devclass_t gve_devclass;
1069
1070 DRIVER_MODULE(gve, pci, gve_driver, gve_devclass, 0, 0);
1071 #else
1072 DRIVER_MODULE(gve, pci, gve_driver, 0, 0);
1073 #endif
1074 MODULE_PNP_INFO("U16:vendor;U16:device;D:#", pci, gve, gve_devs,
1075 nitems(gve_devs));
1076