1 /*
2 * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 *
6 * Apple T2 Buffer Copy Engine (BCE) PCI driver.
7 * Provides the transport layer for T2 coprocessor communication:
8 * mailbox handshake, DMA queue setup, and firmware keepalive.
9 */
10
11 #include <sys/param.h>
12 #include <sys/bus.h>
13 #include <sys/kernel.h>
14 #include <sys/lock.h>
15 #include <sys/malloc.h>
16 #include <sys/module.h>
17 #include <sys/mutex.h>
18 #include <sys/sema.h>
19 #include <sys/sysctl.h>
20 #include <sys/systm.h>
21 #include <sys/callout.h>
22 #include <sys/rman.h>
23 #include <sys/time.h>
24
25 #include <machine/bus.h>
26 #include <machine/resource.h>
27 #include <machine/atomic.h>
28
29 #include <dev/pci/pcivar.h>
30 #include <dev/pci/pcireg.h>
31
32 #include "apple_bce.h"
33 #include "apple_bce_mailbox.h"
34 #include "apple_bce_queue.h"
35 #include "apple_bce_vhci.h"
36
37 static int apple_bce_probe(device_t dev);
38 static int apple_bce_attach(device_t dev);
39 static int apple_bce_detach(device_t dev);
40 static int apple_bce_suspend(device_t dev);
41 static int apple_bce_resume(device_t dev);
42 static void apple_bce_timestamp_cb(void *arg);
43 static void apple_bce_timestamp_init(struct apple_bce_softc *sc);
44 static void apple_bce_timestamp_start(struct apple_bce_softc *sc,
45 int is_initial);
46 static void apple_bce_timestamp_stop(struct apple_bce_softc *sc);
47
48 /*
49 * Timestamp protocol -- T2 firmware expects periodic boottime updates.
50 *
51 * Register layout at BAR4 + 0xC000:
52 * regb + 0 (0xC000): high 32 bits / control
53 * regb + 8 (0xC008): low 32 bits / data
54 *
55 * Control opcodes (written to regb+8 with 0xFFFFFFFF to regb+0):
56 * -4 (0xFFFFFFFC): initial start
57 * -3 (0xFFFFFFFD): restart
58 * -2 (0xFFFFFFFE): stop
59 */
60 static void
apple_bce_timestamp_init(struct apple_bce_softc * sc)61 apple_bce_timestamp_init(struct apple_bce_softc *sc)
62 {
63 /* Read control register and barrier to sync with firmware */
64 bus_read_4(sc->sc_bar4, BCE_REG_TIMESTAMP);
65 mb();
66 }
67
68 static void
apple_bce_timestamp_start(struct apple_bce_softc * sc,int is_initial)69 apple_bce_timestamp_start(struct apple_bce_softc *sc, int is_initial)
70 {
71 /* Send start opcode: -4 for initial, -3 for restart */
72 bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8,
73 is_initial ? 0xFFFFFFFC : 0xFFFFFFFD);
74 bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP, 0xFFFFFFFF);
75
76 mtx_lock_spin(&sc->sc_timestamp_lock);
77 sc->sc_timestamp_stopped = 0;
78 mtx_unlock_spin(&sc->sc_timestamp_lock);
79
80 callout_reset(&sc->sc_timestamp_co, hz * BCE_TIMESTAMP_MS / 1000,
81 apple_bce_timestamp_cb, sc);
82 }
83
84 static void
apple_bce_timestamp_stop(struct apple_bce_softc * sc)85 apple_bce_timestamp_stop(struct apple_bce_softc *sc)
86 {
87 mtx_lock_spin(&sc->sc_timestamp_lock);
88 sc->sc_timestamp_stopped = 1;
89 mtx_unlock_spin(&sc->sc_timestamp_lock);
90
91 callout_drain(&sc->sc_timestamp_co);
92
93 /* Send stop opcode */
94 bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8, 0xFFFFFFFE);
95 bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP, 0xFFFFFFFF);
96 }
97
98 static void
apple_bce_timestamp_cb(void * arg)99 apple_bce_timestamp_cb(void *arg)
100 {
101 struct apple_bce_softc *sc = arg;
102 struct bintime bt;
103 uint64_t ns;
104
105 /* Read to sync, then barrier */
106 bus_read_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8);
107 mb();
108
109 /* Get boot time in nanoseconds */
110 binuptime(&bt);
111 ns = (uint64_t)bt.sec * 1000000000ULL +
112 (((uint64_t)(bt.frac >> 32) * 1000000000ULL) >> 32);
113
114 /* Write: low 32 bits to regb+8, high 32 bits to regb+0 */
115 bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP + 8, (uint32_t)ns);
116 bus_write_4(sc->sc_bar4, BCE_REG_TIMESTAMP, (uint32_t)(ns >> 32));
117
118 mtx_lock_spin(&sc->sc_timestamp_lock);
119 if (!sc->sc_timestamp_stopped)
120 callout_schedule(&sc->sc_timestamp_co,
121 hz * BCE_TIMESTAMP_MS / 1000);
122 mtx_unlock_spin(&sc->sc_timestamp_lock);
123 }
124
125 /*
126 * IRQ handlers.
127 */
128 static void
apple_bce_mbox_intr(void * arg)129 apple_bce_mbox_intr(void *arg)
130 {
131 struct apple_bce_softc *sc = arg;
132
133 bce_mailbox_handle_interrupt(&sc->sc_mbox);
134 }
135
136 static void
apple_bce_dma_intr(void * arg)137 apple_bce_dma_intr(void *arg)
138 {
139 struct apple_bce_softc *sc = arg;
140 int i;
141
142 /*
143 * Only process registered CQs. sc_cq_list[] contains only CQ
144 * pointers -- separate from sc_queues[] which mixes CQ and SQ.
145 */
146 mtx_lock(&sc->sc_queues_lock);
147 for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
148 if (sc->sc_cq_list[i] != NULL)
149 bce_handle_cq_completions(sc, sc->sc_cq_list[i]);
150 }
151 mtx_unlock(&sc->sc_queues_lock);
152 }
153
154 /*
155 * Enable bus master on PCI function 0 (NVMe).
156 * T2 requires function 0 to be bus master for DMA on function 1.
157 */
158 static int
apple_bce_enable_pci0_busmaster(device_t dev)159 apple_bce_enable_pci0_busmaster(device_t dev)
160 {
161 device_t pci0;
162 device_t bus;
163
164 bus = device_get_parent(dev);
165 if (bus == NULL)
166 return (ENXIO);
167
168 /*
169 * Find function 0 on the same bus/slot.
170 * Our device is function 1; function 0 is NVMe.
171 */
172 pci0 = pci_find_dbsf(pci_get_domain(dev), pci_get_bus(dev),
173 pci_get_slot(dev), 0);
174 if (pci0 == NULL) {
175 device_printf(dev, "cannot find PCI function 0\n");
176 return (ENXIO);
177 }
178
179 pci_enable_busmaster(pci0);
180 device_printf(dev, "enabled bus master on function 0 (%s)\n",
181 device_get_nameunit(pci0));
182 return (0);
183 }
184
185 /*
186 * Firmware handshake via mailbox.
187 */
188 static int
apple_bce_fw_handshake(struct apple_bce_softc * sc)189 apple_bce_fw_handshake(struct apple_bce_softc *sc)
190 {
191 uint64_t reply;
192 int error;
193
194 error = bce_mailbox_send(&sc->sc_mbox,
195 BCE_MB_MSG(BCE_MB_SET_FW_PROTOCOL_VER, BCE_FW_PROTOCOL_VER),
196 &reply, BCE_MBOX_TIMEOUT_MS);
197 if (error != 0) {
198 device_printf(sc->sc_dev, "firmware handshake timeout\n");
199 return (error);
200 }
201
202 if (BCE_MB_TYPE(reply) != BCE_MB_SET_FW_PROTOCOL_VER ||
203 BCE_MB_VALUE(reply) != BCE_FW_PROTOCOL_VER) {
204 device_printf(sc->sc_dev,
205 "firmware version mismatch: got type=%u val=0x%llx\n",
206 BCE_MB_TYPE(reply),
207 (unsigned long long)BCE_MB_VALUE(reply));
208 return (ENODEV);
209 }
210
211 device_printf(sc->sc_dev, "firmware handshake OK (protocol 0x%x)\n",
212 BCE_FW_PROTOCOL_VER);
213 return (0);
214 }
215
216 /*
217 * DMA callback for command queue registration.
218 */
219 static void
bce_reg_dma_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)220 bce_reg_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
221 {
222 struct { bus_addr_t addr; int error; } *cb = arg;
223
224 cb->error = error;
225 if (error == 0)
226 cb->addr = segs[0].ds_addr;
227 }
228
229 /*
230 * Register a command queue (CQ or SQ) with firmware via mailbox.
231 * The memcfg struct is DMA-mapped and its physical address sent
232 * in the mailbox message -- firmware reads the config from DMA.
233 */
234 static int
apple_bce_register_cmd_queue(struct apple_bce_softc * sc,struct bce_queue_memcfg * cfg,int is_sq)235 apple_bce_register_cmd_queue(struct apple_bce_softc *sc,
236 struct bce_queue_memcfg *cfg, int is_sq)
237 {
238 bus_dma_tag_t tag;
239 bus_dmamap_t map;
240 bus_addr_t paddr;
241 struct bce_queue_memcfg *dma_cfg;
242 struct {
243 bus_addr_t addr;
244 int error;
245 } cb;
246 uint64_t reply;
247 int error, cmd_type;
248
249 /* Allocate DMA-coherent buffer for memcfg (8-byte aligned) */
250 error = bus_dma_tag_create(sc->sc_dma_tag,
251 8, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
252 NULL, NULL, sizeof(*cfg), 1, sizeof(*cfg),
253 BUS_DMA_WAITOK, NULL, NULL, &tag);
254 if (error != 0)
255 return (error);
256
257 error = bus_dmamem_alloc(tag, (void **)&dma_cfg,
258 BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &map);
259 if (error != 0) {
260 bus_dma_tag_destroy(tag);
261 return (error);
262 }
263
264 /* Copy config and load DMA address */
265 memcpy(dma_cfg, cfg, sizeof(*cfg));
266
267 cb.error = 0;
268 error = bus_dmamap_load(tag, map, dma_cfg, sizeof(*cfg),
269 bce_reg_dma_cb, &cb, BUS_DMA_WAITOK);
270 if (error != 0 || cb.error != 0) {
271 bus_dmamem_free(tag, dma_cfg, map);
272 bus_dma_tag_destroy(tag);
273 return (error != 0 ? error : cb.error);
274 }
275 paddr = cb.addr;
276
277 /* Sync DMA buffer before device access */
278 bus_dmamap_sync(tag, map, BUS_DMASYNC_PREWRITE);
279
280 /* Send DMA address of memcfg to firmware */
281 cmd_type = is_sq ? BCE_MB_REGISTER_CMD_SQ : BCE_MB_REGISTER_CMD_CQ;
282 error = bce_mailbox_send(&sc->sc_mbox,
283 BCE_MB_MSG(cmd_type, paddr), &reply, BCE_MBOX_TIMEOUT_MS);
284
285 bus_dmamap_sync(tag, map, BUS_DMASYNC_POSTWRITE);
286 bus_dmamap_unload(tag, map);
287 bus_dmamem_free(tag, dma_cfg, map);
288 bus_dma_tag_destroy(tag);
289
290 if (error != 0)
291 return (error);
292
293 if (BCE_MB_TYPE(reply) != BCE_MB_REGISTER_QUEUE_REPLY) {
294 device_printf(sc->sc_dev,
295 "unexpected queue registration reply: type=%u\n",
296 BCE_MB_TYPE(reply));
297 return (EINVAL);
298 }
299
300 return (0);
301 }
302
303 /*
304 * Setup command queues (CQ qid=0, SQ qid=1).
305 */
306 static int
apple_bce_setup_cmd_queues(struct apple_bce_softc * sc)307 apple_bce_setup_cmd_queues(struct apple_bce_softc *sc)
308 {
309 struct bce_queue_memcfg cfg;
310 struct bce_queue_sq *sq;
311 int error;
312
313 /* Allocate command CQ (qid 0, 32 entries) */
314 sc->sc_cmd_cq = bce_alloc_cq(sc, 0, 32);
315 if (sc->sc_cmd_cq == NULL) {
316 device_printf(sc->sc_dev, "failed to allocate command CQ\n");
317 return (ENOMEM);
318 }
319
320 /* Register CQ with firmware via DMA-mapped memcfg */
321 bce_get_cq_memcfg(sc->sc_cmd_cq, &cfg);
322 error = apple_bce_register_cmd_queue(sc, &cfg, 0);
323 if (error != 0) {
324 device_printf(sc->sc_dev, "failed to register command CQ\n");
325 goto fail_cq;
326 }
327
328 /* Store CQ in queue registries */
329 mtx_lock(&sc->sc_queues_lock);
330 sc->sc_queues[0] = sc->sc_cmd_cq;
331 sc->sc_cq_list[0] = sc->sc_cmd_cq;
332 mtx_unlock(&sc->sc_queues_lock);
333
334 /* Allocate command SQ (qid 1, 64-byte elements, 32 entries) */
335 sq = bce_alloc_sq(sc, 1, BCE_CMD_SIZE, 32, NULL, NULL);
336 if (sq == NULL) {
337 device_printf(sc->sc_dev, "failed to allocate command SQ\n");
338 goto fail_cq;
339 }
340
341 /* Wrap SQ in command queue (sets completion callback internally) */
342 sc->sc_cmd_cmdq = bce_alloc_cmdq(sc, sq);
343 if (sc->sc_cmd_cmdq == NULL) {
344 bce_free_sq(sc, sq);
345 goto fail_cq;
346 }
347
348 /* Register SQ with firmware via DMA-mapped memcfg */
349 bce_get_sq_memcfg(sq, sc->sc_cmd_cq, &cfg);
350 error = apple_bce_register_cmd_queue(sc, &cfg, 1);
351 if (error != 0) {
352 device_printf(sc->sc_dev, "failed to register command SQ\n");
353 goto fail_sq;
354 }
355
356 /* Store SQ in queue registry */
357 mtx_lock(&sc->sc_queues_lock);
358 sc->sc_queues[1] = sq;
359 sc->sc_int_sq_list[0] = sq;
360 mtx_unlock(&sc->sc_queues_lock);
361
362 device_printf(sc->sc_dev, "command queues created (CQ=0, SQ=1)\n");
363 return (0);
364
365 fail_sq:
366 bce_free_cmdq(sc->sc_cmd_cmdq);
367 sc->sc_cmd_cmdq = NULL;
368 bce_free_sq(sc, sq);
369 fail_cq:
370 /* Clear CQ from registries before freeing */
371 mtx_lock(&sc->sc_queues_lock);
372 sc->sc_queues[0] = NULL;
373 sc->sc_cq_list[0] = NULL;
374 mtx_unlock(&sc->sc_queues_lock);
375 bce_free_cq(sc, sc->sc_cmd_cq);
376 sc->sc_cmd_cq = NULL;
377 return (error != 0 ? error : ENOMEM);
378 }
379
380 /*
381 * PCI probe.
382 */
383 static int
apple_bce_probe(device_t dev)384 apple_bce_probe(device_t dev)
385 {
386 if (pci_get_vendor(dev) != BCE_PCI_VENDOR_APPLE ||
387 pci_get_device(dev) != BCE_PCI_DEVICE_T2)
388 return (ENXIO);
389
390 /* Only attach to function 1 (BCE), not function 0 (NVMe) */
391 if (pci_get_function(dev) != 1)
392 return (ENXIO);
393
394 device_set_desc(dev, "Apple T2 Buffer Copy Engine");
395 return (BUS_PROBE_DEFAULT);
396 }
397
398 /*
399 * PCI attach.
400 */
401 static int
apple_bce_attach(device_t dev)402 apple_bce_attach(device_t dev)
403 {
404 struct apple_bce_softc *sc = device_get_softc(dev);
405 int error;
406
407 sc->sc_dev = dev;
408 pci_enable_busmaster(dev);
409 mtx_init(&sc->sc_queues_lock, "bce_queues", NULL, MTX_DEF);
410 mtx_init(&sc->sc_timestamp_lock, "bce_ts", NULL, MTX_SPIN);
411 callout_init(&sc->sc_timestamp_co, 1);
412
413 /* Map BAR2 (DMA) and BAR4 (mailbox) */
414 sc->sc_bar2_rid = PCIR_BAR(2);
415 sc->sc_bar2 = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
416 &sc->sc_bar2_rid, RF_ACTIVE);
417 if (sc->sc_bar2 == NULL) {
418 device_printf(dev, "cannot map BAR2 (DMA)\n");
419 error = ENXIO;
420 goto fail;
421 }
422
423 sc->sc_bar4_rid = PCIR_BAR(4);
424 sc->sc_bar4 = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
425 &sc->sc_bar4_rid, RF_ACTIVE);
426 if (sc->sc_bar4 == NULL) {
427 device_printf(dev, "cannot map BAR4 (mailbox)\n");
428 error = ENXIO;
429 goto fail;
430 }
431
432 /* Allocate MSI vectors (need at least 5: mbox=0, dma=4) */
433 sc->sc_msi_count = 8; /* Must be power of 2 for FreeBSD */
434 error = pci_alloc_msi(dev, &sc->sc_msi_count);
435 if (error != 0 || sc->sc_msi_count < 8) {
436 device_printf(dev, "cannot allocate MSI vectors: "
437 "error=%d got=%d\n", error, sc->sc_msi_count);
438 if (error == 0 && sc->sc_msi_count > 0)
439 pci_release_msi(dev);
440 sc->sc_msi_count = 0;
441 error = ENXIO;
442 goto fail;
443 }
444
445 /* Initialize mailbox before installing handlers */
446 bce_mailbox_init(&sc->sc_mbox, sc->sc_bar4);
447
448 /* Setup IRQ: vector 0 = mailbox, vector 4 = DMA */
449 sc->sc_irq_rid_mbox = 1; /* MSI vectors start at rid 1 */
450 sc->sc_irq_mbox = bus_alloc_resource_any(dev, SYS_RES_IRQ,
451 &sc->sc_irq_rid_mbox, RF_ACTIVE);
452 if (sc->sc_irq_mbox == NULL) {
453 device_printf(dev, "cannot allocate mailbox IRQ\n");
454 error = ENXIO;
455 goto fail;
456 }
457
458 error = bus_setup_intr(dev, sc->sc_irq_mbox,
459 INTR_TYPE_MISC | INTR_MPSAFE,
460 NULL, apple_bce_mbox_intr, sc, &sc->sc_irq_mbox_cookie);
461 if (error != 0) {
462 device_printf(dev, "cannot setup mailbox IRQ\n");
463 goto fail;
464 }
465
466 sc->sc_irq_rid_dma = 5; /* Vector 4 = rid 5 */
467 sc->sc_irq_dma = bus_alloc_resource_any(dev, SYS_RES_IRQ,
468 &sc->sc_irq_rid_dma, RF_ACTIVE);
469 if (sc->sc_irq_dma == NULL) {
470 device_printf(dev, "cannot allocate DMA IRQ\n");
471 error = ENXIO;
472 goto fail;
473 }
474
475 error = bus_setup_intr(dev, sc->sc_irq_dma,
476 INTR_TYPE_MISC | INTR_MPSAFE,
477 NULL, apple_bce_dma_intr, sc, &sc->sc_irq_dma_cookie);
478 if (error != 0) {
479 device_printf(dev, "cannot setup DMA IRQ\n");
480 goto fail;
481 }
482
483 /* Create parent DMA tag with 37-bit addressing limit */
484 error = bus_dma_tag_create(bus_get_dma_tag(dev),
485 1, 0, /* alignment, boundary */
486 (1ULL << 37) - 1, /* lowaddr: 37-bit limit */
487 BUS_SPACE_MAXADDR, /* highaddr */
488 NULL, NULL, /* filter */
489 BUS_SPACE_MAXSIZE, /* maxsize */
490 BUS_SPACE_UNRESTRICTED, /* nsegments */
491 BUS_SPACE_MAXSIZE, /* maxsegsize */
492 0, /* flags */
493 NULL, NULL, /* lockfunc */
494 &sc->sc_dma_tag);
495 if (error != 0) {
496 device_printf(dev, "cannot create DMA tag\n");
497 goto fail;
498 }
499
500 /*
501 * Enable bus master on function 0 (NVMe) -- T2 quirk.
502 * Must be done before firmware handshake; T2 rejects DMA
503 * on function 1 unless function 0 is bus master.
504 */
505 apple_bce_enable_pci0_busmaster(dev);
506
507 /* Initialize and start timestamp keepalive */
508 apple_bce_timestamp_init(sc);
509 apple_bce_timestamp_start(sc, 1);
510
511 /* Firmware handshake */
512 error = apple_bce_fw_handshake(sc);
513 if (error != 0)
514 goto fail;
515
516 /* Setup command queues */
517 error = apple_bce_setup_cmd_queues(sc);
518 if (error != 0)
519 goto fail;
520
521 device_printf(dev, "Apple T2 BCE initialized\n");
522
523 /* Create VHCI child for virtual USB */
524 error = bce_vhci_attach(sc);
525 if (error != 0)
526 goto fail;
527
528 return (0);
529
530 fail:
531 apple_bce_detach(dev);
532 return (error);
533 }
534
535 /*
536 * PCI detach.
537 */
538 static int
apple_bce_detach(device_t dev)539 apple_bce_detach(device_t dev)
540 {
541 struct apple_bce_softc *sc = device_get_softc(dev);
542
543 /* 0. Detach VHCI child first (before destroying parent resources) */
544 bce_vhci_detach(sc);
545
546 /* 1. Stop timestamp */
547 if (sc->sc_bar4 != NULL && mtx_initialized(&sc->sc_timestamp_lock))
548 apple_bce_timestamp_stop(sc);
549 else
550 callout_drain(&sc->sc_timestamp_co);
551
552 /* 2. Tear down IRQs first -- no more interrupts after this */
553 if (sc->sc_irq_dma_cookie != NULL) {
554 bus_teardown_intr(dev, sc->sc_irq_dma, sc->sc_irq_dma_cookie);
555 sc->sc_irq_dma_cookie = NULL;
556 }
557 if (sc->sc_irq_mbox_cookie != NULL) {
558 bus_teardown_intr(dev, sc->sc_irq_mbox,
559 sc->sc_irq_mbox_cookie);
560 sc->sc_irq_mbox_cookie = NULL;
561 }
562
563 /* 3. Free command queues (safe now -- no IRQs can fire) */
564 if (sc->sc_cmd_cmdq != NULL) {
565 struct bce_queue_sq *cmd_sq;
566
567 cmd_sq = sc->sc_cmd_cmdq->sq;
568 bce_free_cmdq(sc->sc_cmd_cmdq);
569 sc->sc_cmd_cmdq = NULL;
570 bce_free_sq(sc, cmd_sq);
571 }
572 if (sc->sc_cmd_cq != NULL) {
573 bce_free_cq(sc, sc->sc_cmd_cq);
574 sc->sc_cmd_cq = NULL;
575 }
576
577 /* 4. Clear queue registries */
578 if (mtx_initialized(&sc->sc_queues_lock)) {
579 mtx_lock(&sc->sc_queues_lock);
580 memset(sc->sc_queues, 0, sizeof(sc->sc_queues));
581 memset(sc->sc_cq_list, 0, sizeof(sc->sc_cq_list));
582 memset(sc->sc_int_sq_list, 0, sizeof(sc->sc_int_sq_list));
583 mtx_unlock(&sc->sc_queues_lock);
584 }
585
586 /* 5. Destroy mailbox */
587 bce_mailbox_destroy(&sc->sc_mbox);
588
589 /* 6. Release DMA tag */
590 if (sc->sc_dma_tag != NULL) {
591 bus_dma_tag_destroy(sc->sc_dma_tag);
592 sc->sc_dma_tag = NULL;
593 }
594
595 /* 7. Release IRQ resources */
596 if (sc->sc_irq_dma != NULL) {
597 bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irq_rid_dma,
598 sc->sc_irq_dma);
599 sc->sc_irq_dma = NULL;
600 }
601 if (sc->sc_irq_mbox != NULL) {
602 bus_release_resource(dev, SYS_RES_IRQ, sc->sc_irq_rid_mbox,
603 sc->sc_irq_mbox);
604 sc->sc_irq_mbox = NULL;
605 }
606 if (sc->sc_msi_count > 0) {
607 pci_release_msi(dev);
608 sc->sc_msi_count = 0;
609 }
610
611 /* 8. Release BARs */
612 if (sc->sc_bar4 != NULL) {
613 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_bar4_rid,
614 sc->sc_bar4);
615 sc->sc_bar4 = NULL;
616 }
617 if (sc->sc_bar2 != NULL) {
618 bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_bar2_rid,
619 sc->sc_bar2);
620 sc->sc_bar2 = NULL;
621 }
622
623 if (mtx_initialized(&sc->sc_timestamp_lock))
624 mtx_destroy(&sc->sc_timestamp_lock);
625 if (mtx_initialized(&sc->sc_queues_lock))
626 mtx_destroy(&sc->sc_queues_lock);
627
628 return (0);
629 }
630
631 static int
apple_bce_suspend(device_t dev)632 apple_bce_suspend(device_t dev)
633 {
634 struct apple_bce_softc *sc = device_get_softc(dev);
635 int error, restore_error;
636
637 apple_bce_timestamp_stop(sc);
638
639 error = bce_vhci_detach(sc);
640 if (error != 0) {
641 device_printf(dev, "failed to detach VHCI for suspend: %d\n",
642 error);
643 apple_bce_timestamp_start(sc, 0);
644 return (error);
645 }
646
647 error = bce_mailbox_send(&sc->sc_mbox,
648 BCE_MB_MSG(BCE_MB_SLEEP_NO_STATE, 0), NULL,
649 BCE_MBOX_TIMEOUT_MS);
650 if (error != 0) {
651 device_printf(dev,
652 "failed to send SLEEP_NO_STATE mailbox command: %d\n",
653 error);
654 restore_error = bce_vhci_attach(sc);
655 if (restore_error != 0) {
656 device_printf(dev,
657 "failed to reattach VHCI after suspend error: %d\n",
658 restore_error);
659 }
660 apple_bce_timestamp_start(sc, 0);
661 return (error);
662 }
663
664 return (0);
665 }
666
667 static int
apple_bce_resume(device_t dev)668 apple_bce_resume(device_t dev)
669 {
670 struct apple_bce_softc *sc = device_get_softc(dev);
671 uint64_t reply;
672 int error;
673
674 error = bce_mailbox_send(&sc->sc_mbox,
675 BCE_MB_MSG(BCE_MB_RESTORE_NO_STATE, 0), &reply,
676 BCE_MBOX_TIMEOUT_MS);
677 if (error != 0) {
678 device_printf(dev,
679 "failed to send RESTORE_NO_STATE mailbox command: %d\n",
680 error);
681 return (error);
682 }
683
684 if (BCE_MB_TYPE(reply) != BCE_MB_RESTORE_NO_STATE) {
685 device_printf(dev,
686 "unexpected RESTORE_NO_STATE reply: type=%u val=0x%llx\n",
687 BCE_MB_TYPE(reply),
688 (unsigned long long)BCE_MB_VALUE(reply));
689 return (EINVAL);
690 }
691
692 error = bce_vhci_attach(sc);
693 if (error != 0) {
694 device_printf(dev, "failed to reattach VHCI after resume: %d\n",
695 error);
696 apple_bce_timestamp_start(sc, 0);
697 return (error);
698 }
699
700 apple_bce_timestamp_start(sc, 0);
701 return (0);
702 }
703
704 static device_method_t apple_bce_methods[] = {
705 DEVMETHOD(device_probe, apple_bce_probe),
706 DEVMETHOD(device_attach, apple_bce_attach),
707 DEVMETHOD(device_detach, apple_bce_detach),
708 DEVMETHOD(device_suspend, apple_bce_suspend),
709 DEVMETHOD(device_resume, apple_bce_resume),
710 DEVMETHOD_END
711 };
712
713 static driver_t apple_bce_driver = {
714 "apple_bce",
715 apple_bce_methods,
716 sizeof(struct apple_bce_softc)
717 };
718
719 DRIVER_MODULE(apple_bce, pci, apple_bce_driver, NULL, NULL);
720 MODULE_DEPEND(apple_bce, pci, 1, 1, 1);
721 static const struct {
722 uint16_t vendor;
723 uint16_t device;
724 } apple_bce_pnp[] = {
725 { BCE_PCI_VENDOR_APPLE, BCE_PCI_DEVICE_T2 },
726 };
727
728 MODULE_PNP_INFO("U16:vendor;U16:device", pci, apple_bce, apple_bce_pnp,
729 nitems(apple_bce_pnp));
730