1 /*-
2 * Copyright (c) 2026 Abdelkader Boudih <freebsd@seuros.com>
3 *
4 * SPDX-License-Identifier: BSD-2-Clause
5 *
6 * Apple T2 BCE Virtual USB Host Controller Interface (VHCI).
7 */
8
9 #ifdef USB_GLOBAL_INCLUDE_FILE
10 #include USB_GLOBAL_INCLUDE_FILE
11 #else
12 #include <sys/stdint.h>
13 #include <sys/stddef.h>
14 #include <sys/param.h>
15 #include <sys/queue.h>
16 #include <sys/types.h>
17 #include <sys/systm.h>
18 #include <sys/kernel.h>
19 #include <sys/bus.h>
20 #include <sys/module.h>
21 #include <sys/lock.h>
22 #include <sys/mutex.h>
23 #include <sys/condvar.h>
24 #include <sys/sysctl.h>
25 #include <sys/sx.h>
26 #include <sys/unistd.h>
27 #include <sys/callout.h>
28 #include <sys/malloc.h>
29 #include <sys/priv.h>
30 #endif
31
32 #include <sys/sema.h>
33 #include <sys/taskqueue.h>
34 #include <sys/endian.h>
35 #include <machine/bus.h>
36 #include <machine/atomic.h>
37
38 #include <dev/usb/usb.h>
39 #include <dev/usb/usbdi.h>
40 #include <dev/usb/usb_core.h>
41 #include <dev/usb/usb_busdma.h>
42 #include <dev/usb/usb_process.h>
43 #include <dev/usb/usb_transfer.h>
44 #include <dev/usb/usb_device.h>
45 #include <dev/usb/usb_hub.h>
46 #include <dev/usb/usb_util.h>
47 #include <dev/usb/usb_controller.h>
48 #include <dev/usb/usb_bus.h>
49
50 #include "apple_bce.h"
51 #include "apple_bce_queue.h"
52 #include "apple_bce_vhci.h"
53
54 /*
55 * VHCI softc, defined here because it depends on USB headers.
56 */
57 struct bce_vhci_softc {
58 struct usb_bus sc_bus; /* Must be first */
59 struct usb_device *sc_devices[BCE_VHCI_MAX_DEVICES];
60 struct apple_bce_softc *sc_bce;
61 device_t sc_dev;
62
63 /* Controller state */
64 uint32_t sc_port_mask;
65 uint8_t sc_port_count;
66 int sc_started;
67
68 /* Port state */
69 uint32_t sc_port_status[BCE_VHCI_MAX_PORTS];
70 uint32_t sc_port_change[BCE_VHCI_MAX_PORTS];
71 uint8_t sc_port_power[BCE_VHCI_MAX_PORTS];
72
73 /* Hub scratch buffer (for descriptor/status responses) */
74 uint8_t sc_hub_idata[32];
75
76 /* Message queues (host -> device) */
77 struct bce_vhci_msg_queue msg_commands;
78 struct bce_vhci_msg_queue msg_system;
79 struct bce_vhci_msg_queue msg_isochronous;
80 struct bce_vhci_msg_queue msg_interrupt;
81 struct bce_vhci_msg_queue msg_asynchronous;
82
83 /* Event queues (device -> host), share a single CQ */
84 struct bce_queue_cq *ev_cq;
85 struct bce_vhci_evt_queue ev_commands;
86 struct bce_vhci_evt_queue ev_system;
87 struct bce_vhci_evt_queue ev_isochronous;
88 struct bce_vhci_evt_queue ev_interrupt;
89 struct bce_vhci_evt_queue ev_asynchronous;
90
91 /* Command execution (synchronous, wraps msg_commands) */
92 struct bce_vhci_cmd_queue cmd;
93
94 /* Queue ID bitmap (256 bits = BCE_MAX_QUEUE_COUNT) */
95 uint32_t sc_qid_bitmap[8];
96
97 /* Per-device state (indexed by firmware device ID) */
98 struct bce_vhci_device sc_devs[BCE_VHCI_MAX_DEVICES];
99 uint8_t sc_port_to_dev[BCE_VHCI_MAX_PORTS];
100
101 /* Deferred firmware event processing (from ev_commands) */
102 struct task sc_fwevt_task;
103 volatile int sc_detaching; /* Teardown guard */
104
105 /*
106 * Firmware event mailbox: ISR copies events here, task processes.
107 * Protected by sc_fwevt_lock. Ring of BCE_VHCI_EVT_PENDING entries.
108 */
109 struct mtx sc_fwevt_lock;
110 #define BCE_VHCI_FWEVT_RING (BCE_VHCI_EVT_PENDING + 1)
111 struct {
112 struct bce_vhci_message msg;
113 int needs_reply;
114 } sc_fwevt_ring[BCE_VHCI_FWEVT_RING];
115 uint32_t sc_fwevt_prod;
116 uint32_t sc_fwevt_cons;
117
118 /* Spinlock for msg_asynchronous writes (ISR + taskqueue context) */
119 struct mtx sc_async_lock;
120
121 /* Deferred endpoint reset (cannot sleep in pipe_start) */
122 struct task sc_reset_task;
123
124 /* Deferred endpoint create (cannot sleep in pipe_start) */
125 struct task sc_create_task;
126
127 /* Deferred port status change (ISR cannot call cmd_execute) */
128 struct task sc_port_chg_task;
129 volatile uint32_t sc_port_chg_mask;
130 };
131
132 /* Command timeout (ticks) */
133 #define BCE_VHCI_CMD_TIMEOUT_SHORT (hz * 2)
134 #define BCE_VHCI_CMD_TIMEOUT_LONG (hz * 30)
135
136 static usb_handle_req_t bce_vhci_roothub_exec;
137 static void bce_vhci_endpoint_init(struct usb_device *udev,
138 struct usb_endpoint_descriptor *edesc, struct usb_endpoint *ep);
139 static void bce_vhci_xfer_setup(struct usb_setup_params *parm);
140 static void bce_vhci_xfer_unsetup(struct usb_xfer *xfer);
141 static void bce_vhci_get_dma_delay(struct usb_device *udev, uint32_t *pus);
142
143 static void bce_vhci_pipe_open(struct usb_xfer *xfer);
144 static void bce_vhci_pipe_close(struct usb_xfer *xfer);
145 static void bce_vhci_pipe_enter(struct usb_xfer *xfer);
146 static void bce_vhci_pipe_start(struct usb_xfer *xfer);
147
148 static int bce_vhci_probe(device_t dev);
149 static int bce_vhci_attach_dev(device_t dev);
150 static int bce_vhci_detach_dev(device_t dev);
151
152 static int bce_vhci_alloc_qid(struct bce_vhci_softc *vhci);
153 static void bce_vhci_free_qid(struct bce_vhci_softc *vhci, int qid);
154 static int bce_vhci_create_queues(struct bce_vhci_softc *vhci);
155 static void bce_vhci_destroy_queues(struct bce_vhci_softc *vhci);
156 static int bce_vhci_start_controller(struct bce_vhci_softc *vhci);
157 static void bce_vhci_msg_queue_completion(struct bce_queue_sq *sq);
158 static void bce_vhci_ev_cmd_completion(struct bce_queue_sq *sq);
159 static void bce_vhci_ev_system_completion(struct bce_queue_sq *sq);
160 static void bce_vhci_ev_generic_completion(struct bce_queue_sq *sq);
161 static void bce_vhci_cmd_deliver_completion(struct bce_vhci_softc *vhci,
162 struct bce_vhci_message *msg);
163 static void bce_vhci_handle_port_status_change(struct bce_vhci_softc *vhci,
164 struct bce_vhci_message *msg);
165 static void bce_vhci_evt_queue_submit_pending(struct bce_vhci_softc *vhci,
166 struct bce_vhci_evt_queue *eq, uint32_t count);
167
168 static int bce_vhci_device_create(struct bce_vhci_softc *vhci, uint8_t port);
169 static void bce_vhci_device_destroy(struct bce_vhci_softc *vhci, uint8_t port);
170 static int bce_vhci_endpoint_create(struct bce_vhci_softc *vhci,
171 struct bce_vhci_device *dev, uint8_t ep_addr,
172 struct usb_endpoint_descriptor *edesc);
173 static void bce_vhci_endpoint_destroy(struct bce_vhci_softc *vhci,
174 struct bce_vhci_device *dev, uint8_t ep_addr);
175 static void bce_vhci_handle_transfer_request(struct bce_vhci_softc *vhci,
176 struct bce_vhci_message *msg);
177 static void bce_vhci_complete_ctrl_locked(struct bce_vhci_softc *vhci,
178 struct bce_vhci_transfer_queue *tq, struct bce_vhci_message *msg);
179 static void bce_vhci_handle_ctrl_status(struct bce_vhci_softc *vhci,
180 struct bce_vhci_message *msg);
181 static uint16_t bce_vhci_handle_endpoint_req_state(struct bce_vhci_softc *vhci,
182 struct bce_vhci_message *msg);
183 static uint16_t bce_vhci_handle_endpoint_set_state(struct bce_vhci_softc *vhci,
184 struct bce_vhci_message *msg);
185 static void bce_vhci_fwevt_task(void *arg, int pending);
186 static void bce_vhci_send_fw_event_reply(struct bce_vhci_softc *vhci,
187 struct bce_vhci_message *req, uint16_t status);
188 static void bce_vhci_tq_completion(struct bce_queue_sq *sq);
189 static void bce_vhci_reset_task(void *arg, int pending);
190 static void bce_vhci_create_task(void *arg, int pending);
191 static void bce_vhci_port_chg_task(void *arg, int pending);
192 static int bce_vhci_cmd_execute(struct bce_vhci_softc *vhci,
193 struct bce_vhci_message *req, struct bce_vhci_message *reply,
194 int timeout_ticks);
195
196 /*
197 * Convert USB endpoint address to tq[] index.
198 * ep0 (0x00) maps to index 0. For other endpoints, IN and OUT get
199 * separate slots: OUT 0x01 -> 1, IN 0x81 -> 2, OUT 0x02 -> 3, etc.
200 * Maximum index is 30 (ep 0x8F), fits in BCE_VHCI_MAX_ENDPOINTS (32).
201 */
202 static inline uint8_t
bce_vhci_ep_index(uint8_t ep_addr)203 bce_vhci_ep_index(uint8_t ep_addr)
204 {
205 uint8_t num;
206
207 num = ep_addr & 0x0F;
208 if (num == 0)
209 return (0);
210 return (num * 2 - ((ep_addr & 0x80) ? 0 : 1));
211 }
212
213 static const struct usb_bus_methods bce_vhci_bus_methods = {
214 .roothub_exec = bce_vhci_roothub_exec,
215 .endpoint_init = bce_vhci_endpoint_init,
216 .xfer_setup = bce_vhci_xfer_setup,
217 .xfer_unsetup = bce_vhci_xfer_unsetup,
218 .get_dma_delay = bce_vhci_get_dma_delay,
219 };
220
221 /*
222 * Generic pipe methods (all transfer types for now).
223 */
224 static const struct usb_pipe_methods bce_vhci_pipe_methods = {
225 .open = bce_vhci_pipe_open,
226 .close = bce_vhci_pipe_close,
227 .enter = bce_vhci_pipe_enter,
228 .start = bce_vhci_pipe_start,
229 };
230
231 /*
232 * Device methods.
233 */
234 static device_method_t bce_vhci_methods[] = {
235 DEVMETHOD(device_probe, bce_vhci_probe),
236 DEVMETHOD(device_attach, bce_vhci_attach_dev),
237 DEVMETHOD(device_detach, bce_vhci_detach_dev),
238 DEVMETHOD(device_suspend, bus_generic_suspend),
239 DEVMETHOD(device_resume, bus_generic_resume),
240 DEVMETHOD(device_shutdown, bus_generic_shutdown),
241
242 /* Bus interface for usbus child */
243 DEVMETHOD(bus_print_child, bus_generic_print_child),
244 DEVMETHOD_END
245 };
246
247 static driver_t bce_vhci_driver = {
248 .name = "bce_vhci",
249 .methods = bce_vhci_methods,
250 .size = sizeof(struct bce_vhci_softc),
251 };
252
253 DRIVER_MODULE(bce_vhci, apple_bce, bce_vhci_driver, 0, 0);
254 MODULE_DEPEND(bce_vhci, usb, 1, 1, 1);
255
256 /*
257 * Hub descriptor (USB 2.0 hub with per-port power switching)
258 */
259
260 /* Hub descriptor built dynamically in roothub_exec (port count varies) */
261
262 /* Device descriptor for the root hub */
263 static const struct usb_device_descriptor bce_vhci_devd = {
264 .bLength = sizeof(struct usb_device_descriptor),
265 .bDescriptorType = UDESC_DEVICE,
266 .bcdUSB = { 0x00, 0x02 }, /* USB 2.0 */
267 .bDeviceClass = UDCLASS_HUB,
268 .bDeviceSubClass = UDSUBCLASS_HUB,
269 .bDeviceProtocol = UDPROTO_HSHUBSTT,
270 .bMaxPacketSize = 64,
271 .idVendor = { 0x6b, 0x10 }, /* Apple 0x106b */
272 .idProduct = { 0x01, 0x18 }, /* T2 BCE 0x1801 */
273 .bcdDevice = { 0x00, 0x01 }, /* 1.00 */
274 .iManufacturer = 1,
275 .iProduct = 2,
276 .bNumConfigurations = 1,
277 };
278
279 static const struct usb_device_qualifier bce_vhci_odevd = {
280 .bLength = sizeof(struct usb_device_qualifier),
281 .bDescriptorType = UDESC_DEVICE_QUALIFIER,
282 .bcdUSB = { 0x00, 0x02 },
283 .bDeviceClass = UDCLASS_HUB,
284 .bDeviceSubClass = UDSUBCLASS_HUB,
285 .bDeviceProtocol = UDPROTO_HSHUBSTT,
286 .bMaxPacketSize0 = 0,
287 .bNumConfigurations = 0,
288 };
289
290 /* Configuration descriptor + interface + endpoint */
291 /* 9 + 9 + 7 = 25 bytes */
292 static const uint8_t bce_vhci_confd[] = {
293 /* Configuration descriptor */
294 0x09, 0x02, /* bLength, bDescriptorType */
295 0x19, 0x00, /* wTotalLength = 25 */
296 0x01, 0x01, 0x00, 0xC0, 0x00, /* nIntf, cfgVal, iCfg */
297 /* Interface descriptor */
298 0x09, 0x04, /* bLength, bDescriptorType */
299 0x00, 0x00, 0x01, 0x09, 0x00, 0x01, 0x00,
300 /* Endpoint descriptor (interrupt IN ep1) */
301 0x07, 0x05, /* bLength, bDescriptorType */
302 0x81, 0x03, 0x08, 0x00, 0xFF, /* addr, attr, maxPkt, interval */
303 };
304
305 struct bce_vhci_dma_cb_arg {
306 bus_addr_t addr;
307 int error;
308 };
309
310 static void
bce_vhci_dma_cb(void * arg,bus_dma_segment_t * segs,int nseg,int error)311 bce_vhci_dma_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
312 {
313 struct bce_vhci_dma_cb_arg *cb = arg;
314
315 cb->error = error;
316 if (error == 0)
317 cb->addr = segs[0].ds_addr;
318 }
319
320 /*
321 * Allocate a CQ + SQ pair and DMA message buffer for a host->device queue.
322 * Register it with firmware under the given name.
323 */
324 static int
bce_vhci_msg_queue_create(struct bce_vhci_softc * vhci,struct bce_vhci_msg_queue * mq,const char * name,int cq_qid,int sq_qid,bce_sq_completion_fn compl_fn,void * compl_arg)325 bce_vhci_msg_queue_create(struct bce_vhci_softc *vhci,
326 struct bce_vhci_msg_queue *mq, const char *name, int cq_qid, int sq_qid,
327 bce_sq_completion_fn compl_fn, void *compl_arg)
328 {
329 struct apple_bce_softc *sc = vhci->sc_bce;
330 struct bce_vhci_dma_cb_arg cb;
331 struct bce_queue_memcfg cfg;
332 uint32_t el_count = BCE_VHCI_MSG_QUEUE_EL;
333 uint32_t status;
334 int error, i;
335
336 memset(mq, 0, sizeof(*mq));
337 mq->el_count = el_count;
338
339 /* Allocate CQ */
340 mq->cq = bce_alloc_cq(sc, cq_qid, el_count);
341 if (mq->cq == NULL)
342 return (ENOMEM);
343
344 /* Register CQ with firmware via command path */
345 bce_get_cq_memcfg(mq->cq, &cfg);
346 /* CQ interrupt vector = 4 (DMA MSI) */
347 cfg.vector_or_cq = 4;
348 status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, NULL, 0);
349 if (status != 0) {
350 device_printf(vhci->sc_dev,
351 "failed to register CQ %d for %s: %u\n",
352 cq_qid, name, status);
353 error = EIO;
354 goto fail_cq;
355 }
356
357 /* Register CQ in parent's dispatch tables */
358 mtx_lock(&sc->sc_queues_lock);
359 sc->sc_queues[cq_qid] = mq->cq;
360 for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
361 if (sc->sc_cq_list[i] == NULL) {
362 sc->sc_cq_list[i] = mq->cq;
363 break;
364 }
365 }
366 if (i == BCE_MAX_CQ_COUNT) {
367 sc->sc_queues[cq_qid] = NULL;
368 mtx_unlock(&sc->sc_queues_lock);
369 device_printf(vhci->sc_dev,
370 "CQ list full for %s\n", name);
371 error = ENOSPC;
372 goto fail_cq_reg;
373 }
374 mtx_unlock(&sc->sc_queues_lock);
375
376 /* Allocate SQ (element size = bce_qe_submission = 32 bytes) */
377 mq->sq = bce_alloc_sq(sc, sq_qid,
378 sizeof(struct bce_qe_submission), el_count,
379 compl_fn, compl_arg);
380 if (mq->sq == NULL) {
381 error = ENOMEM;
382 goto fail_cq_reg;
383 }
384
385 /* Register SQ with firmware under the given name */
386 bce_get_sq_memcfg(mq->sq, mq->cq, &cfg);
387 status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 1);
388 if (status != 0) {
389 device_printf(vhci->sc_dev,
390 "failed to register SQ %d (%s): %u\n",
391 sq_qid, name, status);
392 error = EIO;
393 goto fail_sq;
394 }
395
396 /* Register SQ in parent's dispatch tables */
397 mtx_lock(&sc->sc_queues_lock);
398 sc->sc_queues[sq_qid] = mq->sq;
399 sc->sc_int_sq_list[sq_qid] = mq->sq;
400 mtx_unlock(&sc->sc_queues_lock);
401
402 /* Allocate DMA-coherent message buffer */
403 error = bus_dma_tag_create(sc->sc_dma_tag,
404 4, 0,
405 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
406 NULL, NULL,
407 el_count * sizeof(struct bce_vhci_message), 1,
408 el_count * sizeof(struct bce_vhci_message),
409 BUS_DMA_WAITOK,
410 NULL, NULL,
411 &mq->dma_tag);
412 if (error != 0)
413 goto fail_sq_reg;
414
415 error = bus_dmamem_alloc(mq->dma_tag, (void **)&mq->data,
416 BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
417 &mq->dma_map);
418 if (error != 0)
419 goto fail_dma_tag;
420
421 error = bus_dmamap_load(mq->dma_tag, mq->dma_map, mq->data,
422 el_count * sizeof(struct bce_vhci_message),
423 bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK);
424 if (error != 0 || cb.error != 0) {
425 error = error != 0 ? error : cb.error;
426 goto fail_dma_mem;
427 }
428 mq->dma_addr = cb.addr;
429
430 return (0);
431
432 fail_dma_mem:
433 bus_dmamem_free(mq->dma_tag, mq->data, mq->dma_map);
434 fail_dma_tag:
435 bus_dma_tag_destroy(mq->dma_tag);
436 fail_sq_reg:
437 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, sq_qid);
438 mtx_lock(&sc->sc_queues_lock);
439 sc->sc_queues[sq_qid] = NULL;
440 sc->sc_int_sq_list[sq_qid] = NULL;
441 mtx_unlock(&sc->sc_queues_lock);
442 fail_sq:
443 bce_free_sq(sc, mq->sq);
444 mq->sq = NULL;
445 fail_cq_reg:
446 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, cq_qid);
447 mtx_lock(&sc->sc_queues_lock);
448 sc->sc_queues[cq_qid] = NULL;
449 for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
450 if (sc->sc_cq_list[i] == mq->cq) {
451 sc->sc_cq_list[i] = NULL;
452 break;
453 }
454 }
455 mtx_unlock(&sc->sc_queues_lock);
456 fail_cq:
457 bce_free_cq(sc, mq->cq);
458 mq->cq = NULL;
459 return (error);
460 }
461
462 static void
bce_vhci_msg_queue_destroy(struct bce_vhci_softc * vhci,struct bce_vhci_msg_queue * mq)463 bce_vhci_msg_queue_destroy(struct bce_vhci_softc *vhci,
464 struct bce_vhci_msg_queue *mq)
465 {
466 struct apple_bce_softc *sc = vhci->sc_bce;
467 int i;
468
469 if (mq->cq == NULL)
470 return;
471
472 /*
473 * Unregister and free SQ before releasing the DMA buffer it
474 * references
475 */
476 if (mq->sq != NULL) {
477 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, mq->sq->qid);
478 mtx_lock(&sc->sc_queues_lock);
479 sc->sc_queues[mq->sq->qid] = NULL;
480 sc->sc_int_sq_list[mq->sq->qid] = NULL;
481 mtx_unlock(&sc->sc_queues_lock);
482 bce_free_sq(sc, mq->sq);
483 mq->sq = NULL;
484 }
485
486 /* Free DMA message buffer */
487 if (mq->data != NULL) {
488 bus_dmamap_unload(mq->dma_tag, mq->dma_map);
489 bus_dmamem_free(mq->dma_tag, mq->data, mq->dma_map);
490 bus_dma_tag_destroy(mq->dma_tag);
491 mq->data = NULL;
492 }
493
494 /* Unregister and free CQ */
495 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, mq->cq->qid);
496 mtx_lock(&sc->sc_queues_lock);
497 sc->sc_queues[mq->cq->qid] = NULL;
498 for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
499 if (sc->sc_cq_list[i] == mq->cq) {
500 sc->sc_cq_list[i] = NULL;
501 break;
502 }
503 }
504 mtx_unlock(&sc->sc_queues_lock);
505 bce_free_cq(sc, mq->cq);
506 mq->cq = NULL;
507 }
508
509 /*
510 * Write a message to a host->device queue.
511 * Caller must have reserved a submission slot.
512 */
513 static void
bce_vhci_msg_queue_write(struct bce_vhci_softc * vhci,struct bce_vhci_msg_queue * mq,struct bce_vhci_message * msg)514 bce_vhci_msg_queue_write(struct bce_vhci_softc *vhci,
515 struct bce_vhci_msg_queue *mq, struct bce_vhci_message *msg)
516 {
517 struct bce_qe_submission *s;
518 uint32_t sidx;
519
520 sidx = mq->sq->tail;
521 s = bce_next_submission(mq->sq);
522
523 /* Copy message into DMA buffer slot and sync for device access */
524 mq->data[sidx] = *msg;
525 bus_dmamap_sync(mq->dma_tag, mq->dma_map, BUS_DMASYNC_PREWRITE);
526
527 /* Fill SQ entry pointing to the DMA buffer slot */
528 s->length = sizeof(struct bce_vhci_message);
529 s->addr = mq->dma_addr +
530 sidx * sizeof(struct bce_vhci_message);
531 s->segl_addr = 0;
532 s->segl_length = 0;
533
534 bce_submit_to_device(vhci->sc_bce, mq->sq);
535 }
536
537 /*
538 * Message queue completion: consume completions and free slots.
539 */
540 static void
bce_vhci_msg_queue_completion(struct bce_queue_sq * sq)541 bce_vhci_msg_queue_completion(struct bce_queue_sq *sq)
542 {
543 struct bce_vhci_msg_queue *mq = sq->userdata;
544
545 while (sq->completion_cidx != sq->completion_tail) {
546 sq->completion_cidx =
547 (sq->completion_cidx + 1) % sq->el_count;
548 bce_notify_submission_complete(sq);
549 }
550 bus_dmamap_sync(mq->dma_tag, mq->dma_map, BUS_DMASYNC_POSTWRITE);
551 }
552
553 /*
554 * Allocate an SQ (paired with the shared ev_cq) and DMA buffer for a
555 * device->host event queue. Register with firmware and pre-submit
556 * receive buffers.
557 */
558 static int
bce_vhci_evt_queue_create(struct bce_vhci_softc * vhci,struct bce_vhci_evt_queue * eq,const char * name,int sq_qid,bce_sq_completion_fn compl_fn)559 bce_vhci_evt_queue_create(struct bce_vhci_softc *vhci,
560 struct bce_vhci_evt_queue *eq, const char *name, int sq_qid,
561 bce_sq_completion_fn compl_fn)
562 {
563 struct apple_bce_softc *sc = vhci->sc_bce;
564 struct bce_vhci_dma_cb_arg cb;
565 struct bce_queue_memcfg cfg;
566 uint32_t el_count = BCE_VHCI_EVT_QUEUE_EL;
567 uint32_t status;
568 int error;
569
570 memset(eq, 0, sizeof(*eq));
571 eq->el_count = el_count;
572 eq->userdata = vhci;
573
574 /* Allocate SQ (shared CQ = vhci->ev_cq) */
575 eq->sq = bce_alloc_sq(sc, sq_qid,
576 sizeof(struct bce_qe_submission), el_count,
577 compl_fn, eq);
578 if (eq->sq == NULL)
579 return (ENOMEM);
580
581 /* Register SQ with firmware (direction = from device = 0) */
582 bce_get_sq_memcfg(eq->sq, vhci->ev_cq, &cfg);
583 status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 0);
584 if (status != 0) {
585 device_printf(vhci->sc_dev,
586 "failed to register event SQ %d (%s): %u\n",
587 sq_qid, name, status);
588 error = EIO;
589 goto fail_sq;
590 }
591
592 /* Register SQ in dispatch tables */
593 mtx_lock(&sc->sc_queues_lock);
594 sc->sc_queues[sq_qid] = eq->sq;
595 sc->sc_int_sq_list[sq_qid] = eq->sq;
596 mtx_unlock(&sc->sc_queues_lock);
597
598 /* Allocate DMA-coherent receive buffer */
599 error = bus_dma_tag_create(sc->sc_dma_tag,
600 4, 0,
601 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
602 NULL, NULL,
603 el_count * sizeof(struct bce_vhci_message), 1,
604 el_count * sizeof(struct bce_vhci_message),
605 BUS_DMA_WAITOK,
606 NULL, NULL,
607 &eq->dma_tag);
608 if (error != 0)
609 goto fail_sq_reg;
610
611 error = bus_dmamem_alloc(eq->dma_tag, (void **)&eq->data,
612 BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
613 &eq->dma_map);
614 if (error != 0)
615 goto fail_dma_tag;
616
617 error = bus_dmamap_load(eq->dma_tag, eq->dma_map, eq->data,
618 el_count * sizeof(struct bce_vhci_message),
619 bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK);
620 if (error != 0 || cb.error != 0) {
621 error = error != 0 ? error : cb.error;
622 goto fail_dma_mem;
623 }
624 eq->dma_addr = cb.addr;
625
626 /* Pre-submit receive buffers */
627 bce_vhci_evt_queue_submit_pending(vhci, eq, BCE_VHCI_EVT_PENDING);
628
629 return (0);
630
631 fail_dma_mem:
632 bus_dmamem_free(eq->dma_tag, eq->data, eq->dma_map);
633 fail_dma_tag:
634 bus_dma_tag_destroy(eq->dma_tag);
635 fail_sq_reg:
636 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, sq_qid);
637 mtx_lock(&sc->sc_queues_lock);
638 sc->sc_queues[sq_qid] = NULL;
639 sc->sc_int_sq_list[sq_qid] = NULL;
640 mtx_unlock(&sc->sc_queues_lock);
641 fail_sq:
642 bce_free_sq(sc, eq->sq);
643 eq->sq = NULL;
644 return (error);
645 }
646
647 static void
bce_vhci_evt_queue_destroy(struct bce_vhci_softc * vhci,struct bce_vhci_evt_queue * eq)648 bce_vhci_evt_queue_destroy(struct bce_vhci_softc *vhci,
649 struct bce_vhci_evt_queue *eq)
650 {
651 struct apple_bce_softc *sc = vhci->sc_bce;
652
653 if (eq->sq == NULL)
654 return;
655
656 /* Unregister SQ from dispatch tables FIRST to stop IRQ callbacks */
657 bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, eq->sq->qid);
658 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, eq->sq->qid);
659 mtx_lock(&sc->sc_queues_lock);
660 sc->sc_queues[eq->sq->qid] = NULL;
661 sc->sc_int_sq_list[eq->sq->qid] = NULL;
662 mtx_unlock(&sc->sc_queues_lock);
663
664 /* Now safe to free DMA buffer; no IRQ can reference it */
665 if (eq->data != NULL) {
666 bus_dmamap_unload(eq->dma_tag, eq->dma_map);
667 bus_dmamem_free(eq->dma_tag, eq->data, eq->dma_map);
668 bus_dma_tag_destroy(eq->dma_tag);
669 eq->data = NULL;
670 }
671 bce_free_sq(sc, eq->sq);
672 eq->sq = NULL;
673 }
674
675 /*
676 * Submit empty receive buffers to an event queue so firmware can
677 * write messages into them.
678 */
679 static void
bce_vhci_evt_queue_submit_pending(struct bce_vhci_softc * vhci,struct bce_vhci_evt_queue * eq,uint32_t count)680 bce_vhci_evt_queue_submit_pending(struct bce_vhci_softc *vhci,
681 struct bce_vhci_evt_queue *eq, uint32_t count)
682 {
683 struct bce_qe_submission *s;
684 uint32_t idx;
685
686 bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_PREREAD);
687
688 while (count-- > 0) {
689 if (bce_reserve_submission(eq->sq) != 0) {
690 device_printf(vhci->sc_dev,
691 "cannot reserve event submission\n");
692 break;
693 }
694 idx = eq->sq->tail;
695 s = bce_next_submission(eq->sq);
696 s->length = sizeof(struct bce_vhci_message);
697 s->addr = eq->dma_addr +
698 idx * sizeof(struct bce_vhci_message);
699 s->segl_addr = 0;
700 s->segl_length = 0;
701 }
702 bce_submit_to_device(vhci->sc_bce, eq->sq);
703 }
704
705 /*
706 * Enqueue a firmware event into sc_fwevt_ring for deferred processing.
707 * Called from ISR context; returns 0 on success, -1 if ring is full.
708 */
709 static int
bce_vhci_fwevt_enqueue(struct bce_vhci_softc * vhci,struct bce_vhci_message * msg,int needs_reply)710 bce_vhci_fwevt_enqueue(struct bce_vhci_softc *vhci,
711 struct bce_vhci_message *msg, int needs_reply)
712 {
713 uint32_t next_prod;
714
715 mtx_lock_spin(&vhci->sc_fwevt_lock);
716 next_prod = (vhci->sc_fwevt_prod + 1) % BCE_VHCI_FWEVT_RING;
717 if (next_prod == vhci->sc_fwevt_cons) {
718 mtx_unlock_spin(&vhci->sc_fwevt_lock);
719 device_printf(vhci->sc_dev,
720 "fwevt ring full, dropping 0x%04x\n", msg->cmd);
721 return (-1);
722 }
723 vhci->sc_fwevt_ring[vhci->sc_fwevt_prod].msg = *msg;
724 vhci->sc_fwevt_ring[vhci->sc_fwevt_prod].needs_reply = needs_reply;
725 vhci->sc_fwevt_prod = next_prod;
726 mtx_unlock_spin(&vhci->sc_fwevt_lock);
727
728 if (vhci->sc_detaching == 0)
729 taskqueue_enqueue(taskqueue_thread, &vhci->sc_fwevt_task);
730 return (0);
731 }
732
733 /*
734 * Generic event queue completion: read messages and resubmit buffers.
735 * Used for system, isochronous, interrupt, and asynchronous event queues.
736 */
737 static void
bce_vhci_ev_generic_completion(struct bce_queue_sq * sq)738 bce_vhci_ev_generic_completion(struct bce_queue_sq *sq)
739 {
740 struct bce_vhci_evt_queue *eq = sq->userdata;
741 struct bce_vhci_softc *vhci = eq->userdata;
742 struct bce_vhci_message *msg;
743 uint32_t cnt = 0;
744
745 bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_POSTREAD);
746
747 while (sq->completion_cidx != sq->completion_tail) {
748 struct bce_sq_completion_data *cd;
749
750 cd = &sq->completion_data[sq->completion_cidx];
751 if (cd->status == BCE_COMP_ABORTED) {
752 sq->completion_cidx =
753 (sq->completion_cidx + 1) % sq->el_count;
754 bce_notify_submission_complete(sq);
755 cnt++;
756 continue;
757 }
758
759 msg = &eq->data[sq->head];
760 /*
761 * Route events to appropriate handlers.
762 * Strip 0x4000 flag; firmware uses it as a
763 * variant marker.
764 */
765 if (msg->cmd & BCE_VHCI_CMD_REPLY_FLAG)
766 bce_vhci_cmd_deliver_completion(vhci, msg);
767 else {
768 uint16_t base_cmd = msg->cmd &
769 ~BCE_VHCI_CMD_CANCEL_FLAG;
770
771 if (base_cmd == BCE_VHCI_CMD_PORT_STATUS_CHANGE)
772 bce_vhci_handle_port_status_change(vhci,
773 msg);
774 else if (base_cmd == BCE_VHCI_CMD_TRANSFER_REQUEST)
775 bce_vhci_handle_transfer_request(vhci, msg);
776 else if (base_cmd ==
777 BCE_VHCI_CMD_CTRL_TRANSFER_STATUS)
778 bce_vhci_handle_ctrl_status(vhci, msg);
779 else if (base_cmd ==
780 BCE_VHCI_CMD_ENDPOINT_REQ_STATE ||
781 base_cmd ==
782 BCE_VHCI_CMD_ENDPOINT_SET_STATE)
783 bce_vhci_fwevt_enqueue(vhci, msg, 0);
784 }
785
786 sq->completion_cidx =
787 (sq->completion_cidx + 1) % sq->el_count;
788 bce_notify_submission_complete(sq);
789 cnt++;
790 }
791
792 if (cnt > 0)
793 bce_vhci_evt_queue_submit_pending(vhci, eq, cnt);
794 }
795
796 /*
797 * Event queue completion for the firmware command channel (ev_commands).
798 *
799 * This ISR callback is the sole consumer of the ev_commands SQ ring.
800 * Command replies are delivered inline (semaphore post, ISR-safe).
801 * Firmware events are copied into sc_fwevt_ring and deferred to
802 * sc_fwevt_task which handles them in taskqueue_thread context
803 * (needed because ENDP_PAUSED handling calls bce_cmd_flush_queue).
804 */
805 static void
bce_vhci_ev_cmd_completion(struct bce_queue_sq * sq)806 bce_vhci_ev_cmd_completion(struct bce_queue_sq *sq)
807 {
808 struct bce_vhci_evt_queue *eq = sq->userdata;
809 struct bce_vhci_softc *vhci = eq->userdata;
810 struct bce_vhci_message *msg;
811 uint32_t cnt = 0;
812
813 bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_POSTREAD);
814
815 while (sq->completion_cidx != sq->completion_tail) {
816 struct bce_sq_completion_data *cd;
817
818 cd = &sq->completion_data[sq->completion_cidx];
819 if (cd->status == BCE_COMP_ABORTED) {
820 sq->completion_cidx =
821 (sq->completion_cidx + 1) % sq->el_count;
822 bce_notify_submission_complete(sq);
823 cnt++;
824 continue;
825 }
826
827 msg = &eq->data[sq->head];
828
829 if (msg->cmd & BCE_VHCI_CMD_REPLY_FLAG) {
830 /* Command reply: deliver inline (semaphore post) */
831 bce_vhci_cmd_deliver_completion(vhci, msg);
832 } else {
833 /* Firmware event: defer to taskqueue */
834 bce_vhci_fwevt_enqueue(vhci, msg, 1);
835 }
836
837 sq->completion_cidx =
838 (sq->completion_cidx + 1) % sq->el_count;
839 bce_notify_submission_complete(sq);
840 cnt++;
841 }
842
843 if (cnt > 0) {
844 bus_dmamap_sync(eq->dma_tag, eq->dma_map, BUS_DMASYNC_PREREAD);
845 bce_vhci_evt_queue_submit_pending(vhci, eq, cnt);
846 }
847
848 }
849
850 /*
851 * System event queue completion: handles command replies and
852 * port status change notifications.
853 */
854 static void
bce_vhci_ev_system_completion(struct bce_queue_sq * sq)855 bce_vhci_ev_system_completion(struct bce_queue_sq *sq)
856 {
857
858 /* Route through generic handler which checks for both */
859 bce_vhci_ev_generic_completion(sq);
860 }
861
862 /*
863 * Taskqueue handler for firmware events on ev_commands.
864 *
865 * Processes ENDPOINT_REQ_STATE / ENDPOINT_SET_STATE events from
866 * process context (not ISR). Implements cancel-pair detection:
867 * if two consecutive events are cmd + cmd|0x4000 with same param1,
868 * both are consumed with a single ABORT reply.
869 *
870 * Normal events are handled and replied with SUCCESS on msg_system.
871 */
872 static void
bce_vhci_fwevt_task(void * arg,int pending __unused)873 bce_vhci_fwevt_task(void *arg, int pending __unused)
874 {
875 struct bce_vhci_softc *vhci = arg;
876 struct bce_vhci_message msg;
877
878 if (vhci->sc_detaching)
879 return;
880
881 /*
882 * Process firmware events from the mailbox ring.
883 * The ISR is the sole consumer of the ev_commands SQ ring and
884 * copies events here; we process them in taskqueue context.
885 */
886 for (;;) {
887 uint16_t result;
888 int needs_reply;
889
890 mtx_lock_spin(&vhci->sc_fwevt_lock);
891 if (vhci->sc_fwevt_cons == vhci->sc_fwevt_prod) {
892 mtx_unlock_spin(&vhci->sc_fwevt_lock);
893 break;
894 }
895 msg = vhci->sc_fwevt_ring[vhci->sc_fwevt_cons].msg;
896 needs_reply =
897 vhci->sc_fwevt_ring[vhci->sc_fwevt_cons].needs_reply;
898 vhci->sc_fwevt_cons = (vhci->sc_fwevt_cons + 1) %
899 BCE_VHCI_FWEVT_RING;
900 mtx_unlock_spin(&vhci->sc_fwevt_lock);
901
902 if (msg.cmd & BCE_VHCI_CMD_CANCEL_FLAG) {
903 /* Firmware cancel; reply ABORT */
904 result = BCE_VHCI_ABORT;
905 } else if (msg.cmd == BCE_VHCI_CMD_ENDPOINT_REQ_STATE)
906 result = bce_vhci_handle_endpoint_req_state(vhci, &msg);
907 else if (msg.cmd == BCE_VHCI_CMD_ENDPOINT_SET_STATE)
908 result = bce_vhci_handle_endpoint_set_state(vhci, &msg);
909 else {
910 device_printf(vhci->sc_dev,
911 "unhandled fw event: 0x%04x\n", msg.cmd);
912 result = BCE_VHCI_BAD_ARGUMENT;
913 }
914 if (needs_reply)
915 bce_vhci_send_fw_event_reply(vhci, &msg, result);
916 }
917 }
918
919 /*
920 * Deliver a firmware reply to the synchronous command waiter.
921 */
922 static void
bce_vhci_cmd_deliver_completion(struct bce_vhci_softc * vhci,struct bce_vhci_message * msg)923 bce_vhci_cmd_deliver_completion(struct bce_vhci_softc *vhci,
924 struct bce_vhci_message *msg)
925 {
926 struct bce_vhci_cmd_queue *cq = &vhci->cmd;
927 int do_post = 0;
928
929 mtx_lock_spin(&cq->lock);
930 if (cq->pending != 0) {
931 uint16_t base_cmd;
932
933 /*
934 * Accept only replies matching the expected command
935 * (with REPLY_FLAG and optionally CANCEL_FLAG).
936 * Drop stale replies from timed-out commands.
937 */
938 base_cmd = msg->cmd & ~(BCE_VHCI_CMD_REPLY_FLAG |
939 BCE_VHCI_CMD_CANCEL_FLAG);
940 if (base_cmd == cq->expected_cmd) {
941 cq->response = *msg;
942 cq->pending = 0;
943 do_post = 1;
944 }
945 }
946 mtx_unlock_spin(&cq->lock);
947
948 /*
949 * sema_post uses MTX_DEF internally; must not be called under
950 * MTX_SPIN
951 */
952 if (do_post)
953 sema_post(&cq->completion);
954 }
955
956 /*
957 * Handle port status change event from firmware (ISR context).
958 * Cannot call cmd_execute here (sleeps), so defer to taskqueue.
959 */
960 static void
bce_vhci_handle_port_status_change(struct bce_vhci_softc * vhci,struct bce_vhci_message * msg)961 bce_vhci_handle_port_status_change(struct bce_vhci_softc *vhci,
962 struct bce_vhci_message *msg)
963 {
964 uint32_t port;
965
966 if (vhci->sc_detaching)
967 return;
968
969 port = msg->param1;
970 if (port >= vhci->sc_port_count)
971 return;
972
973 atomic_set_int(&vhci->sc_port_chg_mask, 1U << port);
974 taskqueue_enqueue(taskqueue_thread, &vhci->sc_port_chg_task);
975 }
976
977 /*
978 * Deferred port status change handler (taskqueue context, can sleep).
979 * Queries firmware for current port status and updates the cache.
980 */
981 static void
bce_vhci_port_chg_task(void * arg,int pending __unused)982 bce_vhci_port_chg_task(void *arg, int pending __unused)
983 {
984 struct bce_vhci_softc *vhci = arg;
985 struct bce_vhci_message cmd, reply;
986 uint32_t mask, port, port_status;
987 int error;
988
989 if (vhci->sc_detaching)
990 return;
991
992 mask = atomic_readandclear_int(&vhci->sc_port_chg_mask);
993
994 for (port = 0; mask != 0; port++, mask >>= 1) {
995 if ((mask & 1) == 0)
996 continue;
997
998 device_printf(vhci->sc_dev,
999 "port %u status change\n", port);
1000
1001 memset(&cmd, 0, sizeof(cmd));
1002 cmd.cmd = BCE_VHCI_CMD_PORT_STATUS;
1003 cmd.param1 = port;
1004
1005 error = bce_vhci_cmd_execute(vhci, &cmd, &reply,
1006 BCE_VHCI_CMD_TIMEOUT_SHORT);
1007
1008 USB_BUS_LOCK(&vhci->sc_bus);
1009 if (error == 0) {
1010 port_status = (uint32_t)reply.param2;
1011
1012 vhci->sc_port_status[port] = 0;
1013 if (vhci->sc_port_power[port])
1014 vhci->sc_port_status[port] |=
1015 UPS_PORT_POWER;
1016 if (port_status & BCE_VHCI_PORT_ENABLED)
1017 vhci->sc_port_status[port] |=
1018 UPS_PORT_ENABLED | UPS_HIGH_SPEED;
1019 if (port_status & BCE_VHCI_PORT_CONNECTED)
1020 vhci->sc_port_status[port] |=
1021 UPS_CURRENT_CONNECT_STATUS;
1022 if (port_status & BCE_VHCI_PORT_SUSPENDED)
1023 vhci->sc_port_status[port] |=
1024 UPS_SUSPEND;
1025 if (port_status & BCE_VHCI_PORT_OVERCURRENT)
1026 vhci->sc_port_status[port] |=
1027 UPS_OVERCURRENT_INDICATOR;
1028 }
1029 vhci->sc_port_change[port] |= UPS_C_CONNECT_STATUS;
1030 USB_BUS_UNLOCK(&vhci->sc_bus);
1031 }
1032
1033 /* Wake the USB hub poll */
1034 usb_needs_explore(&vhci->sc_bus, 0);
1035 }
1036
1037 /*
1038 * Deferred endpoint reset task.
1039 *
1040 * Called on taskqueue_thread (can sleep) after CTRL_TRANSFER_STATUS(STALL).
1041 * Flushes residual SQ entries and issues ENDPOINT_RESET (0x0044) to clear
1042 * firmware's stall state.
1043 *
1044 * After reset, clears tq->stalled so the USB stack's next retry succeeds.
1045 */
1046 static void
bce_vhci_reset_one_tq(struct bce_vhci_softc * vhci,struct bce_vhci_transfer_queue * tq)1047 bce_vhci_reset_one_tq(struct bce_vhci_softc *vhci,
1048 struct bce_vhci_transfer_queue *tq)
1049 {
1050 struct apple_bce_softc *sc = vhci->sc_bce;
1051 struct bce_vhci_message cmd, reply;
1052
1053 device_printf(vhci->sc_dev,
1054 "reset_task: flushing + ENDPOINT_RESET dev=%d ep=0x%02x\n",
1055 tq->dev_addr, tq->endp_addr);
1056
1057 /* Flush residual SQ submissions */
1058 if (tq->sq_in != NULL)
1059 bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, tq->sq_in->qid);
1060 if (tq->sq_out != NULL)
1061 bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, tq->sq_out->qid);
1062
1063 /* Issue ENDPOINT_RESET to clear firmware stall state */
1064 memset(&cmd, 0, sizeof(cmd));
1065 cmd.cmd = BCE_VHCI_CMD_ENDPOINT_RESET;
1066 cmd.param1 = tq->dev_addr | ((tq->endp_addr & 0x8F) << 8);
1067 bce_vhci_cmd_execute(vhci, &cmd, &reply, BCE_VHCI_CMD_TIMEOUT_SHORT);
1068
1069 device_printf(vhci->sc_dev,
1070 "reset_task: ENDPOINT_RESET done, clearing stall\n");
1071
1072 USB_BUS_LOCK(&vhci->sc_bus);
1073 tq->stalled = 0;
1074 USB_BUS_UNLOCK(&vhci->sc_bus);
1075 }
1076
1077 static void
bce_vhci_reset_task(void * arg,int pending __unused)1078 bce_vhci_reset_task(void *arg, int pending __unused)
1079 {
1080 struct bce_vhci_softc *vhci = arg;
1081 int i, j;
1082
1083 for (i = 0; i < BCE_VHCI_MAX_DEVICES; i++) {
1084 struct bce_vhci_device *dev = &vhci->sc_devs[i];
1085
1086 if (dev->allocated == 0)
1087 continue;
1088 for (j = 0; j < BCE_VHCI_MAX_ENDPOINTS; j++) {
1089 struct bce_vhci_transfer_queue *tq = &dev->tq[j];
1090
1091 if (tq->active == 0 || tq->stalled == 0)
1092 continue;
1093 bce_vhci_reset_one_tq(vhci, tq);
1094 }
1095 }
1096 }
1097
1098 /*
1099 * bce_vhci_create_task: deferred endpoint creation from taskqueue_thread.
1100 *
1101 * bce_vhci_pipe_start cannot call bce_vhci_endpoint_create directly because
1102 * it may be invoked from a USB callback (e.g. usbhid_intr_in_callback) that
1103 * holds a non-sleepable lock. Instead, pipe_start sets create_pending on the
1104 * tq and schedules this task. We scan all devices/endpoints, create any with
1105 * create_pending set, then return USB_ERR_STALLED from pipe_start so the USB
1106 * stack retries, at which point tq->active is set and we skip creation.
1107 */
1108 static void
bce_vhci_create_task(void * arg,int pending __unused)1109 bce_vhci_create_task(void *arg, int pending __unused)
1110 {
1111 struct bce_vhci_softc *vhci = arg;
1112 int i, j, ep_err;
1113
1114 for (i = 0; i < BCE_VHCI_MAX_DEVICES; i++) {
1115 struct bce_vhci_device *dev = &vhci->sc_devs[i];
1116
1117 if (dev->allocated == 0)
1118 continue;
1119
1120 for (j = 0; j < BCE_VHCI_MAX_ENDPOINTS; j++) {
1121 struct bce_vhci_transfer_queue *tq = &dev->tq[j];
1122 struct usb_endpoint_descriptor *edesc;
1123 struct usb_xfer *xfer;
1124 uint8_t ep_addr;
1125
1126 USB_BUS_LOCK(&vhci->sc_bus);
1127 if (tq->create_pending == 0 || tq->active) {
1128 tq->create_pending = 0;
1129 USB_BUS_UNLOCK(&vhci->sc_bus);
1130 continue;
1131 }
1132 tq->create_pending = 0;
1133 ep_addr = tq->endp_addr;
1134 edesc = tq->create_edesc;
1135 xfer = tq->create_xfer;
1136 /*
1137 * Do NOT clear create_xfer yet --
1138 * pipe_close may race
1139 */
1140 USB_BUS_UNLOCK(&vhci->sc_bus);
1141
1142 ep_err = bce_vhci_endpoint_create(vhci, dev,
1143 ep_addr, edesc);
1144
1145 /*
1146 * Re-check create_xfer under lock. Atomically
1147 * clear it and either install active_xfer or
1148 * complete with error; no gap for pipe_close.
1149 */
1150 USB_BUS_LOCK(&vhci->sc_bus);
1151 if (tq->create_xfer != xfer) {
1152 /*
1153 * Original xfer was closed; leave any
1154 * newer create_xfer for pipe_start retry.
1155 */
1156 USB_BUS_UNLOCK(&vhci->sc_bus);
1157 continue;
1158 }
1159
1160 if (ep_err != 0) {
1161 tq->create_xfer = NULL;
1162 if (xfer != NULL)
1163 usbd_transfer_done(xfer,
1164 USB_ERR_STALLED);
1165 USB_BUS_UNLOCK(&vhci->sc_bus);
1166 device_printf(vhci->sc_dev,
1167 "create_task: ep create "
1168 "failed: dev=%d ep=0x%02x "
1169 "err=%d\n",
1170 dev->fw_dev_id, ep_addr,
1171 ep_err);
1172 continue;
1173 }
1174
1175 if (xfer != NULL && (ep_addr & UE_DIR_IN)) {
1176 struct bce_vhci_message treq;
1177 struct bce_qe_submission *si;
1178 uint32_t len;
1179
1180 len = xfer->frlengths[0];
1181 if (len > BCE_VHCI_XFER_BUFSZ)
1182 len = BCE_VHCI_XFER_BUFSZ;
1183 /*
1184 * Handoff: clear create_xfer and
1185 * set active_xfer atomically.
1186 */
1187 tq->create_xfer = NULL;
1188 tq->active_xfer = xfer;
1189 tq->dma_inflight = 1;
1190 USB_BUS_UNLOCK(&vhci->sc_bus);
1191
1192 bus_dmamap_sync(tq->dma_tag,
1193 tq->dma_map,
1194 BUS_DMASYNC_PREREAD);
1195
1196 /* Reserve msg first, then SQ */
1197 memset(&treq, 0, sizeof(treq));
1198 treq.cmd =
1199 BCE_VHCI_CMD_TRANSFER_REQUEST;
1200 treq.param1 =
1201 ((uint32_t)ep_addr << 8) |
1202 dev->fw_dev_id;
1203 treq.param2 = len;
1204
1205 mtx_lock_spin(&vhci->sc_async_lock);
1206 if (bce_reserve_submission(
1207 vhci->msg_asynchronous.sq) != 0) {
1208 mtx_unlock_spin(
1209 &vhci->sc_async_lock);
1210 USB_BUS_LOCK(&vhci->sc_bus);
1211 if (tq->active_xfer == xfer) {
1212 tq->active_xfer = NULL;
1213 tq->dma_inflight = 0;
1214 usbd_transfer_done(xfer,
1215 USB_ERR_IOERROR);
1216 }
1217 USB_BUS_UNLOCK(&vhci->sc_bus);
1218 continue;
1219 }
1220 mtx_unlock_spin(&vhci->sc_async_lock);
1221 /* active_xfer already set above */
1222
1223 mtx_lock_spin(&tq->lock);
1224 if (bce_reserve_submission(
1225 tq->sq_in) == 0) {
1226 si = bce_next_submission(
1227 tq->sq_in);
1228 si->addr = tq->dma_addr;
1229 si->length = len;
1230 si->segl_addr = 0;
1231 si->segl_length = 0;
1232 bce_submit_to_device(
1233 vhci->sc_bce,
1234 tq->sq_in);
1235 mtx_unlock_spin(&tq->lock);
1236
1237 mtx_lock_spin(
1238 &vhci->sc_async_lock);
1239 bce_vhci_msg_queue_write(vhci,
1240 &vhci->msg_asynchronous,
1241 &treq);
1242 mtx_unlock_spin(
1243 &vhci->sc_async_lock);
1244 } else {
1245 mtx_unlock_spin(&tq->lock);
1246 /* Return reserved msg slot */
1247 mtx_lock_spin(
1248 &vhci->sc_async_lock);
1249 atomic_add_int(&vhci->
1250 msg_asynchronous.sq->
1251 available_commands, 1);
1252 mtx_unlock_spin(
1253 &vhci->sc_async_lock);
1254 USB_BUS_LOCK(&vhci->sc_bus);
1255 if (tq->active_xfer == xfer) {
1256 tq->active_xfer = NULL;
1257 tq->dma_inflight = 0;
1258 usbd_transfer_done(xfer,
1259 USB_ERR_IOERROR);
1260 }
1261 USB_BUS_UNLOCK(&vhci->sc_bus);
1262 }
1263 } else if (xfer != NULL &&
1264 (ep_addr & UE_DIR_IN) == 0) {
1265 /*
1266 * OUT endpoint: set active and submit.
1267 */
1268 struct bce_vhci_message treq;
1269 struct bce_qe_submission *so;
1270 uint32_t len;
1271
1272 len = xfer->frlengths[0];
1273 if (len > BCE_VHCI_XFER_BUFSZ)
1274 len = BCE_VHCI_XFER_BUFSZ;
1275
1276 tq->create_xfer = NULL;
1277 tq->active_xfer = xfer;
1278 tq->dma_inflight = 1;
1279
1280 if (len > 0) {
1281 usbd_copy_out(
1282 &xfer->frbuffers[0], 0,
1283 tq->dma_buf, len);
1284 }
1285 USB_BUS_UNLOCK(&vhci->sc_bus);
1286
1287 if (len > 0) {
1288 bus_dmamap_sync(tq->dma_tag,
1289 tq->dma_map,
1290 BUS_DMASYNC_PREWRITE);
1291 }
1292
1293 memset(&treq, 0, sizeof(treq));
1294 treq.cmd =
1295 BCE_VHCI_CMD_TRANSFER_REQUEST;
1296 treq.param1 =
1297 ((uint32_t)ep_addr << 8) |
1298 dev->fw_dev_id;
1299 treq.param2 = len;
1300
1301 mtx_lock_spin(&vhci->sc_async_lock);
1302 if (bce_reserve_submission(
1303 vhci->msg_asynchronous.sq) != 0) {
1304 mtx_unlock_spin(
1305 &vhci->sc_async_lock);
1306 USB_BUS_LOCK(&vhci->sc_bus);
1307 if (tq->active_xfer == xfer) {
1308 tq->active_xfer = NULL;
1309 tq->dma_inflight = 0;
1310 usbd_transfer_done(xfer,
1311 USB_ERR_IOERROR);
1312 }
1313 USB_BUS_UNLOCK(&vhci->sc_bus);
1314 continue;
1315 }
1316 mtx_unlock_spin(&vhci->sc_async_lock);
1317
1318 mtx_lock_spin(&tq->lock);
1319 if (bce_reserve_submission(
1320 tq->sq_out) == 0) {
1321 so = bce_next_submission(
1322 tq->sq_out);
1323 so->addr = tq->dma_addr;
1324 so->length = len;
1325 so->segl_addr = 0;
1326 so->segl_length = 0;
1327 bce_submit_to_device(
1328 vhci->sc_bce,
1329 tq->sq_out);
1330 mtx_unlock_spin(&tq->lock);
1331
1332 mtx_lock_spin(
1333 &vhci->sc_async_lock);
1334 bce_vhci_msg_queue_write(vhci,
1335 &vhci->msg_asynchronous,
1336 &treq);
1337 mtx_unlock_spin(
1338 &vhci->sc_async_lock);
1339 } else {
1340 mtx_unlock_spin(&tq->lock);
1341 mtx_lock_spin(
1342 &vhci->sc_async_lock);
1343 atomic_add_int(&vhci->
1344 msg_asynchronous.sq->
1345 available_commands, 1);
1346 mtx_unlock_spin(
1347 &vhci->sc_async_lock);
1348 USB_BUS_LOCK(&vhci->sc_bus);
1349 if (tq->active_xfer == xfer) {
1350 tq->active_xfer = NULL;
1351 tq->dma_inflight = 0;
1352 usbd_transfer_done(xfer,
1353 USB_ERR_IOERROR);
1354 }
1355 USB_BUS_UNLOCK(&vhci->sc_bus);
1356 }
1357 } else if (xfer != NULL) {
1358 tq->create_xfer = NULL;
1359 usbd_transfer_done(xfer,
1360 USB_ERR_STALLED);
1361 USB_BUS_UNLOCK(&vhci->sc_bus);
1362 } else {
1363 tq->create_xfer = NULL;
1364 USB_BUS_UNLOCK(&vhci->sc_bus);
1365 }
1366 }
1367 }
1368 }
1369
1370 /*
1371 * Execute a synchronous command: send on msg_commands, wait for reply
1372 * on ev_commands or ev_system.
1373 */
1374 static int
bce_vhci_cmd_execute(struct bce_vhci_softc * vhci,struct bce_vhci_message * req,struct bce_vhci_message * reply,int timeout_ticks)1375 bce_vhci_cmd_execute(struct bce_vhci_softc *vhci,
1376 struct bce_vhci_message *req, struct bce_vhci_message *reply,
1377 int timeout_ticks)
1378 {
1379 struct bce_vhci_cmd_queue *cq = &vhci->cmd;
1380 struct bce_vhci_message cancel;
1381 int error;
1382
1383 sx_xlock(&cq->exec_lock);
1384 mtx_lock_spin(&cq->lock);
1385
1386 /* Reserve a submission slot */
1387 if (bce_reserve_submission(cq->msg->sq) != 0) {
1388 mtx_unlock_spin(&cq->lock);
1389 sx_xunlock(&cq->exec_lock);
1390 return (EAGAIN);
1391 }
1392
1393 /* Setup completion state */
1394 cq->pending = 1;
1395 cq->expected_cmd = req->cmd;
1396 memset(&cq->response, 0, sizeof(cq->response));
1397
1398 mtx_unlock_spin(&cq->lock);
1399
1400 /* Send the command */
1401 bce_vhci_msg_queue_write(vhci, cq->msg, req);
1402
1403 /* Wait for reply */
1404 error = sema_timedwait(&cq->completion, timeout_ticks);
1405
1406 mtx_lock_spin(&cq->lock);
1407
1408 if (error != 0) {
1409 /*
1410 * Timeout: send cancellation and wait briefly.
1411 */
1412 device_printf(vhci->sc_dev,
1413 "cmd 0x%04x timeout, sending cancel\n", req->cmd);
1414
1415 if (bce_reserve_submission(cq->msg->sq) == 0) {
1416 cancel = *req;
1417 cancel.cmd |= BCE_VHCI_CMD_CANCEL_FLAG;
1418 cq->pending = 1;
1419 mtx_unlock_spin(&cq->lock);
1420
1421 bce_vhci_msg_queue_write(vhci, cq->msg, &cancel);
1422
1423 error = sema_timedwait(&cq->completion, hz);
1424
1425 mtx_lock_spin(&cq->lock);
1426 if (error != 0) {
1427 device_printf(vhci->sc_dev,
1428 "cmd cancel timeout, possible desync\n");
1429 cq->pending = 0;
1430 mtx_unlock_spin(&cq->lock);
1431 sx_xunlock(&cq->exec_lock);
1432 return (ETIMEDOUT);
1433 }
1434
1435 /*
1436 * Check if we got the cancel ack or the
1437 * original reply
1438 */
1439 if ((cq->response.cmd & ~BCE_VHCI_CMD_REPLY_FLAG) ==
1440 (req->cmd | BCE_VHCI_CMD_CANCEL_FLAG)) {
1441 cq->pending = 0;
1442 mtx_unlock_spin(&cq->lock);
1443 sx_xunlock(&cq->exec_lock);
1444 return (ETIMEDOUT);
1445 }
1446 /* Got original reply; fall through */
1447 } else {
1448 cq->pending = 0;
1449 mtx_unlock_spin(&cq->lock);
1450 sx_xunlock(&cq->exec_lock);
1451 return (ETIMEDOUT);
1452 }
1453 }
1454
1455 /* Copy reply before releasing the lock */
1456 {
1457 struct bce_vhci_message resp;
1458
1459 resp = cq->response;
1460 cq->pending = 0;
1461 mtx_unlock_spin(&cq->lock);
1462 sx_xunlock(&cq->exec_lock);
1463
1464 if (reply != NULL)
1465 *reply = resp;
1466
1467 /* Validate reply from local copy */
1468 if ((resp.cmd & ~BCE_VHCI_CMD_REPLY_FLAG) != req->cmd) {
1469 device_printf(vhci->sc_dev,
1470 "cmd mismatch: sent 0x%04x, got 0x%04x\n",
1471 req->cmd, resp.cmd);
1472 return (EIO);
1473 }
1474
1475 if (resp.status != BCE_VHCI_SUCCESS)
1476 return (resp.status);
1477 }
1478
1479 return (0);
1480 }
1481
1482 /*
1483 * Submit a pending IN xfer after the previous one completed.
1484 * Called under USB_BUS_LOCK. nxfer has been detached from
1485 * tq->pending_xfer by the caller.
1486 */
1487 static void
bce_vhci_submit_pending_in(struct bce_vhci_softc * vhci,struct bce_vhci_transfer_queue * tq,struct usb_xfer * nxfer)1488 bce_vhci_submit_pending_in(struct bce_vhci_softc *vhci,
1489 struct bce_vhci_transfer_queue *tq, struct usb_xfer *nxfer)
1490 {
1491 struct bce_vhci_message treq;
1492 struct bce_qe_submission *si;
1493 uint32_t nlen;
1494
1495 nlen = nxfer->frlengths[0];
1496 if (nlen > BCE_VHCI_XFER_BUFSZ)
1497 nlen = BCE_VHCI_XFER_BUFSZ;
1498
1499 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
1500 BUS_DMASYNC_PREREAD);
1501
1502 memset(&treq, 0, sizeof(treq));
1503 treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST;
1504 treq.param1 =
1505 ((uint32_t)tq->endp_addr << 8) | tq->dev_addr;
1506 treq.param2 = nlen;
1507
1508 /* Reserve msg first, then SQ */
1509 mtx_lock_spin(&vhci->sc_async_lock);
1510 if (bce_reserve_submission(
1511 vhci->msg_asynchronous.sq) != 0) {
1512 mtx_unlock_spin(&vhci->sc_async_lock);
1513 usbd_transfer_done(nxfer, USB_ERR_IOERROR);
1514 return;
1515 }
1516 mtx_unlock_spin(&vhci->sc_async_lock);
1517
1518 /*
1519 * Install active_xfer BEFORE ringing the doorbell.
1520 * A fast completion could otherwise see NULL and
1521 * discard the result.
1522 */
1523 tq->active_xfer = nxfer;
1524 tq->dma_inflight = 1;
1525
1526 mtx_lock_spin(&tq->lock);
1527 if (bce_reserve_submission(tq->sq_in) == 0) {
1528 si = bce_next_submission(tq->sq_in);
1529 si->addr = tq->dma_addr;
1530 si->length = nlen;
1531 si->segl_addr = 0;
1532 si->segl_length = 0;
1533 bce_submit_to_device(vhci->sc_bce, tq->sq_in);
1534 mtx_unlock_spin(&tq->lock);
1535
1536 mtx_lock_spin(&vhci->sc_async_lock);
1537 bce_vhci_msg_queue_write(vhci,
1538 &vhci->msg_asynchronous, &treq);
1539 mtx_unlock_spin(&vhci->sc_async_lock);
1540 } else {
1541 mtx_unlock_spin(&tq->lock);
1542 tq->active_xfer = NULL;
1543 tq->dma_inflight = 0;
1544 /* Return reserved msg slot */
1545 mtx_lock_spin(&vhci->sc_async_lock);
1546 atomic_add_int(
1547 &vhci->msg_asynchronous.sq->
1548 available_commands, 1);
1549 mtx_unlock_spin(&vhci->sc_async_lock);
1550 usbd_transfer_done(nxfer, USB_ERR_IOERROR);
1551 }
1552 }
1553
1554 /*
1555 * Submit a pending OUT xfer after the previous one completed.
1556 * Called under USB_BUS_LOCK. nxfer has been detached from
1557 * tq->pending_xfer by the caller.
1558 */
1559 static void
bce_vhci_submit_pending_out(struct bce_vhci_softc * vhci,struct bce_vhci_transfer_queue * tq,struct usb_xfer * nxfer)1560 bce_vhci_submit_pending_out(struct bce_vhci_softc *vhci,
1561 struct bce_vhci_transfer_queue *tq, struct usb_xfer *nxfer)
1562 {
1563 struct bce_vhci_message treq;
1564 struct bce_qe_submission *so;
1565 uint32_t nlen;
1566
1567 nlen = nxfer->frlengths[0];
1568 if (nlen > BCE_VHCI_XFER_BUFSZ)
1569 nlen = BCE_VHCI_XFER_BUFSZ;
1570
1571 usbd_copy_out(&nxfer->frbuffers[0], 0,
1572 tq->dma_buf, nlen);
1573 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
1574 BUS_DMASYNC_PREWRITE);
1575
1576 memset(&treq, 0, sizeof(treq));
1577 treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST;
1578 treq.param1 =
1579 ((uint32_t)tq->endp_addr << 8) | tq->dev_addr;
1580 treq.param2 = nlen;
1581
1582 /* Reserve msg first, then SQ */
1583 mtx_lock_spin(&vhci->sc_async_lock);
1584 if (bce_reserve_submission(
1585 vhci->msg_asynchronous.sq) != 0) {
1586 mtx_unlock_spin(&vhci->sc_async_lock);
1587 usbd_transfer_done(nxfer, USB_ERR_IOERROR);
1588 return;
1589 }
1590 mtx_unlock_spin(&vhci->sc_async_lock);
1591
1592 /*
1593 * Install active_xfer BEFORE ringing the doorbell.
1594 */
1595 tq->active_xfer = nxfer;
1596 tq->dma_inflight = 1;
1597
1598 mtx_lock_spin(&tq->lock);
1599 if (bce_reserve_submission(tq->sq_out) == 0) {
1600 so = bce_next_submission(tq->sq_out);
1601 so->addr = tq->dma_addr;
1602 so->length = nlen;
1603 so->segl_addr = 0;
1604 so->segl_length = 0;
1605 bce_submit_to_device(vhci->sc_bce, tq->sq_out);
1606 mtx_unlock_spin(&tq->lock);
1607
1608 mtx_lock_spin(&vhci->sc_async_lock);
1609 bce_vhci_msg_queue_write(vhci,
1610 &vhci->msg_asynchronous, &treq);
1611 mtx_unlock_spin(&vhci->sc_async_lock);
1612 } else {
1613 mtx_unlock_spin(&tq->lock);
1614 tq->active_xfer = NULL;
1615 tq->dma_inflight = 0;
1616 /* Return reserved msg slot */
1617 mtx_lock_spin(&vhci->sc_async_lock);
1618 atomic_add_int(
1619 &vhci->msg_asynchronous.sq->
1620 available_commands, 1);
1621 mtx_unlock_spin(&vhci->sc_async_lock);
1622 usbd_transfer_done(nxfer, USB_ERR_IOERROR);
1623 }
1624 }
1625
1626 /*
1627 * Transfer queue DMA completion callback. Fires when the firmware
1628 * has consumed (OUT) or filled (IN) a DMA buffer we submitted.
1629 *
1630 * For IN transfers, record the actual byte count from the completion
1631 * so that handle_ctrl_status knows how much data was received.
1632 */
1633 static void
bce_vhci_tq_completion(struct bce_queue_sq * sq)1634 bce_vhci_tq_completion(struct bce_queue_sq *sq)
1635 {
1636 struct bce_vhci_transfer_queue *tq = sq->userdata;
1637 struct bce_vhci_softc *vhci = tq->vhci;
1638
1639 while (sq->completion_cidx != sq->completion_tail) {
1640 struct bce_sq_completion_data *cd;
1641
1642 cd = &sq->completion_data[sq->completion_cidx];
1643
1644 /*
1645 * For IN SQ completions (device -> host), handle data.
1646 * BCE uses ithreaded MSI, so we can acquire USB_BUS_LOCK
1647 * (MTX_DEF) here. tq->lock (MTX_SPIN) nesting inside
1648 * USB_BUS_LOCK is valid.
1649 */
1650 if (sq == tq->sq_in && cd->status == BCE_COMP_SUCCESS) {
1651 if (tq->endp_addr == 0x00) {
1652 /*
1653 * Control transfer: just record data length.
1654 * Actual completion happens in
1655 * handle_ctrl_status. Clamp to DMA buffer.
1656 * USB_BUS_LOCK protects ctrl_actual and
1657 * ctrl_data_done against concurrent access
1658 * from handle_ctrl_status.
1659 *
1660 * If CTRL_TRANSFER_STATUS arrived first
1661 * (ctrl_status_pending), process it now
1662 * that data is ready.
1663 */
1664 uint32_t alen = (uint32_t)cd->data_size;
1665 if (alen > BCE_VHCI_XFER_BUFSZ)
1666 alen = BCE_VHCI_XFER_BUFSZ;
1667 USB_BUS_LOCK(&vhci->sc_bus);
1668 if (tq->active_xfer == NULL ||
1669 (tq->ctrl_state != BCE_VHCI_CTRL_STATUS &&
1670 tq->ctrl_state != BCE_VHCI_CTRL_DATA)) {
1671 tq->dma_inflight = 0;
1672 USB_BUS_UNLOCK(&vhci->sc_bus);
1673 goto next_compl;
1674 }
1675 if (alen > tq->ctrl_data_len)
1676 alen = tq->ctrl_data_len;
1677 tq->ctrl_actual = alen;
1678 tq->ctrl_data_done = 1;
1679 if (tq->ctrl_status_pending != 0) {
1680 tq->ctrl_status_pending = 0;
1681 bce_vhci_complete_ctrl_locked(
1682 vhci, tq,
1683 &tq->ctrl_status_msg);
1684 }
1685 USB_BUS_UNLOCK(&vhci->sc_bus);
1686 } else {
1687 /*
1688 * Interrupt/bulk IN transfer: data is ready.
1689 * Copy into xfer buffer and complete.
1690 */
1691 struct usb_xfer *xfer;
1692 uint32_t len = (uint32_t)cd->data_size;
1693
1694 USB_BUS_LOCK(&vhci->sc_bus);
1695 xfer = tq->active_xfer;
1696 tq->dma_inflight = 0;
1697 if (xfer != NULL) {
1698 bus_dmamap_sync(tq->dma_tag,
1699 tq->dma_map,
1700 BUS_DMASYNC_POSTREAD);
1701
1702 if (len > BCE_VHCI_XFER_BUFSZ)
1703 len = BCE_VHCI_XFER_BUFSZ;
1704 if (len > xfer->frlengths[0])
1705 len = xfer->frlengths[0];
1706
1707 usbd_copy_in(&xfer->frbuffers[0], 0,
1708 tq->dma_buf, len);
1709 xfer->frlengths[0] = len;
1710 xfer->aframes = xfer->nframes;
1711 tq->active_xfer = NULL;
1712
1713 /* Start next queued xfer if any */
1714 if (tq->pending_xfer != NULL) {
1715 struct usb_xfer *nxfer;
1716
1717 nxfer = tq->pending_xfer;
1718 tq->pending_xfer = NULL;
1719 bce_vhci_submit_pending_in(
1720 vhci, tq, nxfer);
1721 }
1722
1723 usbd_transfer_done(xfer,
1724 USB_ERR_NORMAL_COMPLETION);
1725 } else if (tq->pending_xfer != NULL) {
1726 /*
1727 * Stale completion from cancelled xfer.
1728 * DMA drained; start pending xfer now.
1729 */
1730 struct usb_xfer *nxfer;
1731
1732 nxfer = tq->pending_xfer;
1733 tq->pending_xfer = NULL;
1734 bce_vhci_submit_pending_in(
1735 vhci, tq, nxfer);
1736 }
1737 USB_BUS_UNLOCK(&vhci->sc_bus);
1738 }
1739 }
1740
1741 /*
1742 * For ep0 OUT SQ completion in CTRL_SETUP state:
1743 * setup packet DMA is done, start the data phase.
1744 * USB_BUS_LOCK protects ctrl_state against concurrent
1745 * access from pipe_start, pipe_close, and handle_ctrl_status.
1746 */
1747 if (sq == tq->sq_out && tq->endp_addr == 0x00 &&
1748 cd->status == BCE_COMP_SUCCESS) {
1749 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
1750 BUS_DMASYNC_POSTWRITE);
1751 USB_BUS_LOCK(&vhci->sc_bus);
1752 if (tq->active_xfer == NULL) {
1753 tq->dma_inflight = 0;
1754 USB_BUS_UNLOCK(&vhci->sc_bus);
1755 goto next_compl;
1756 }
1757 if (tq->ctrl_state == BCE_VHCI_CTRL_SETUP) {
1758 if (tq->ctrl_data_len > 0) {
1759 tq->ctrl_state = BCE_VHCI_CTRL_DATA;
1760 } else {
1761 tq->ctrl_state = BCE_VHCI_CTRL_STATUS;
1762 }
1763 /*
1764 * CTRL_TRANSFER_STATUS may have arrived
1765 * before setup DMA completed. Process
1766 * the deferred status now.
1767 */
1768 if (tq->ctrl_status_pending != 0) {
1769 tq->ctrl_status_pending = 0;
1770 bce_vhci_complete_ctrl_locked(
1771 vhci, tq,
1772 &tq->ctrl_status_msg);
1773 }
1774 } else if (tq->ctrl_state ==
1775 BCE_VHCI_CTRL_STATUS &&
1776 tq->ctrl_dir == UE_DIR_OUT) {
1777 /*
1778 * OUT data DMA done. Allow
1779 * CTRL_TRANSFER_STATUS to proceed.
1780 */
1781 tq->ctrl_data_done = 1;
1782 if (tq->ctrl_status_pending != 0) {
1783 tq->ctrl_status_pending = 0;
1784 bce_vhci_complete_ctrl_locked(
1785 vhci, tq,
1786 &tq->ctrl_status_msg);
1787 }
1788 }
1789 USB_BUS_UNLOCK(&vhci->sc_bus);
1790 /*
1791 * Data phase (both IN and OUT) is driven by firmware
1792 * TRANSFER_REQUEST events handled in
1793 * handle_transfer_request().
1794 */
1795 }
1796
1797 /*
1798 * For OUT SQ completions on non-control endpoints,
1799 * the firmware consumed our data; complete the xfer.
1800 */
1801 if (sq == tq->sq_out && tq->endp_addr != 0x00 &&
1802 cd->status == BCE_COMP_SUCCESS) {
1803 struct usb_xfer *xfer;
1804
1805 /*
1806 * POSTWRITE before CPU touches buffer again.
1807 */
1808 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
1809 BUS_DMASYNC_POSTWRITE);
1810
1811 USB_BUS_LOCK(&vhci->sc_bus);
1812 xfer = tq->active_xfer;
1813 tq->dma_inflight = 0;
1814 if (xfer != NULL) {
1815 tq->active_xfer = NULL;
1816
1817 /* Start next queued OUT xfer if any */
1818 if (tq->pending_xfer != NULL) {
1819 struct usb_xfer *nxfer;
1820
1821 nxfer = tq->pending_xfer;
1822 tq->pending_xfer = NULL;
1823 bce_vhci_submit_pending_out(
1824 vhci, tq, nxfer);
1825 }
1826
1827 xfer->aframes = xfer->nframes;
1828 usbd_transfer_done(xfer,
1829 USB_ERR_NORMAL_COMPLETION);
1830 } else if (tq->pending_xfer != NULL) {
1831 struct usb_xfer *nxfer;
1832
1833 nxfer = tq->pending_xfer;
1834 tq->pending_xfer = NULL;
1835 bce_vhci_submit_pending_out(
1836 vhci, tq, nxfer);
1837 }
1838 USB_BUS_UNLOCK(&vhci->sc_bus);
1839 }
1840
1841 /*
1842 * Handle SQ error completions. Clear dma_inflight
1843 * and complete active xfer with error so the endpoint
1844 * is not permanently stuck.
1845 */
1846 if (cd->status != BCE_COMP_SUCCESS) {
1847 struct usb_xfer *xfer, *pxfer;
1848
1849 USB_BUS_LOCK(&vhci->sc_bus);
1850 xfer = tq->active_xfer;
1851 pxfer = tq->pending_xfer;
1852 tq->dma_inflight = 0;
1853 if (xfer != NULL) {
1854 tq->active_xfer = NULL;
1855 tq->pending_xfer = NULL;
1856 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
1857 if (pxfer != NULL)
1858 usbd_transfer_done(pxfer,
1859 USB_ERR_IOERROR);
1860 usbd_transfer_done(xfer,
1861 USB_ERR_IOERROR);
1862 } else if (pxfer != NULL) {
1863 tq->pending_xfer = NULL;
1864 usbd_transfer_done(pxfer,
1865 USB_ERR_IOERROR);
1866 }
1867 USB_BUS_UNLOCK(&vhci->sc_bus);
1868 }
1869
1870 next_compl:
1871 sq->completion_cidx =
1872 (sq->completion_cidx + 1) % sq->el_count;
1873 bce_notify_submission_complete(sq);
1874 }
1875 }
1876
1877 /*
1878 * Create per-endpoint DMA transfer queues and register with firmware.
1879 */
1880 static int
bce_vhci_endpoint_create(struct bce_vhci_softc * vhci,struct bce_vhci_device * dev,uint8_t ep_addr,struct usb_endpoint_descriptor * edesc)1881 bce_vhci_endpoint_create(struct bce_vhci_softc *vhci,
1882 struct bce_vhci_device *dev, uint8_t ep_addr,
1883 struct usb_endpoint_descriptor *edesc)
1884 {
1885 struct apple_bce_softc *sc = vhci->sc_bce;
1886 struct bce_vhci_transfer_queue *tq;
1887 struct bce_queue_memcfg cfg;
1888 struct bce_vhci_dma_cb_arg cb;
1889 struct bce_vhci_message cmd, reply;
1890 char name[0x20];
1891 uint32_t status;
1892 int error, cq_qid, out_qid, in_qid, i;
1893 uint8_t ep_idx;
1894
1895 ep_idx = bce_vhci_ep_index(ep_addr);
1896 if (ep_idx >= BCE_VHCI_MAX_ENDPOINTS)
1897 return (EINVAL);
1898
1899 tq = &dev->tq[ep_idx];
1900 if (tq->active)
1901 return (EEXIST);
1902
1903 /*
1904 * Initialize runtime fields. Do NOT zero the whole struct:
1905 * create_xfer/create_pending are live state managed by
1906 * create_task under USB_BUS_LOCK.
1907 */
1908 tq->vhci = vhci;
1909 tq->dev_addr = dev->fw_dev_id;
1910 tq->endp_addr = ep_addr;
1911 tq->cq = NULL;
1912 tq->sq_in = NULL;
1913 tq->sq_out = NULL;
1914 tq->active_xfer = NULL;
1915 tq->pending_xfer = NULL;
1916 tq->paused_by = 0;
1917 tq->active = 0;
1918 tq->stalled = 0;
1919 tq->dma_inflight = 0;
1920
1921 /* Free leftover DMA buffer from previous incarnation */
1922 if (tq->dma_tag != NULL) {
1923 bus_dmamap_unload(tq->dma_tag, tq->dma_map);
1924 bus_dmamem_free(tq->dma_tag, tq->dma_buf, tq->dma_map);
1925 bus_dma_tag_destroy(tq->dma_tag);
1926 tq->dma_tag = NULL;
1927 }
1928 tq->dma_map = NULL;
1929 tq->dma_addr = 0;
1930 tq->dma_buf = NULL;
1931 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
1932 tq->ctrl_dir = 0;
1933 tq->ctrl_data_len = 0;
1934 tq->ctrl_actual = 0;
1935 tq->ctrl_data_done = 0;
1936 tq->ctrl_status_pending = 0;
1937 tq->evt_pending = 0;
1938 /* tq->lock initialized in device_create, valid for device lifetime */
1939
1940 /* Allocate DMA buffer for data transfers */
1941 error = bus_dma_tag_create(sc->sc_dma_tag,
1942 4, 0,
1943 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
1944 NULL, NULL,
1945 BCE_VHCI_XFER_BUFSZ, 1, BCE_VHCI_XFER_BUFSZ,
1946 BUS_DMA_WAITOK,
1947 NULL, NULL,
1948 &tq->dma_tag);
1949 if (error != 0)
1950 return (error);
1951
1952 error = bus_dmamem_alloc(tq->dma_tag, &tq->dma_buf,
1953 BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT,
1954 &tq->dma_map);
1955 if (error != 0)
1956 goto fail_tag;
1957
1958 error = bus_dmamap_load(tq->dma_tag, tq->dma_map, tq->dma_buf,
1959 BCE_VHCI_XFER_BUFSZ, bce_vhci_dma_cb, &cb, BUS_DMA_WAITOK);
1960 if (error != 0 || cb.error != 0) {
1961 error = error != 0 ? error : cb.error;
1962 goto fail_mem;
1963 }
1964 tq->dma_addr = cb.addr;
1965
1966 /* Allocate CQ for this endpoint */
1967 cq_qid = bce_vhci_alloc_qid(vhci);
1968 if (cq_qid < 0) {
1969 error = ENOSPC;
1970 goto fail_dma;
1971 }
1972 tq->cq = bce_alloc_cq(sc, cq_qid, BCE_VHCI_TQ_EL);
1973 if (tq->cq == NULL) {
1974 error = ENOMEM;
1975 goto fail_cq_alloc;
1976 }
1977
1978 bce_get_cq_memcfg(tq->cq, &cfg);
1979 cfg.vector_or_cq = 4;
1980 status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, NULL, 0);
1981 if (status != 0) {
1982 error = EIO;
1983 goto fail_cq;
1984 }
1985
1986 mtx_lock(&sc->sc_queues_lock);
1987 sc->sc_queues[cq_qid] = tq->cq;
1988 {
1989 int inserted = 0;
1990
1991 for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
1992 if (sc->sc_cq_list[i] == NULL) {
1993 sc->sc_cq_list[i] = tq->cq;
1994 inserted = 1;
1995 break;
1996 }
1997 }
1998 if (inserted == 0) {
1999 sc->sc_queues[cq_qid] = NULL;
2000 mtx_unlock(&sc->sc_queues_lock);
2001 device_printf(vhci->sc_dev,
2002 "CQ list full, cannot add endpoint CQ\n");
2003 error = ENOSPC;
2004 goto fail_cq_reg;
2005 }
2006 }
2007 mtx_unlock(&sc->sc_queues_lock);
2008
2009 /* Allocate OUT SQ (host -> device) */
2010 out_qid = bce_vhci_alloc_qid(vhci);
2011 if (out_qid < 0) {
2012 error = ENOSPC;
2013 goto fail_cq_reg;
2014 }
2015 tq->sq_out = bce_alloc_sq(sc, out_qid,
2016 sizeof(struct bce_qe_submission), BCE_VHCI_TQ_EL,
2017 bce_vhci_tq_completion, tq);
2018 if (tq->sq_out == NULL) {
2019 error = ENOMEM;
2020 goto fail_sq_out_alloc;
2021 }
2022
2023 snprintf(name, sizeof(name), "VHC1-%d-%02x",
2024 dev->fw_dev_id, ep_addr & 0x0F);
2025 bce_get_sq_memcfg(tq->sq_out, tq->cq, &cfg);
2026 status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 1);
2027 if (status != 0) {
2028 device_printf(vhci->sc_dev,
2029 "failed to register OUT SQ '%s': %u\n", name, status);
2030 error = EIO;
2031 goto fail_sq_out;
2032 }
2033
2034 mtx_lock(&sc->sc_queues_lock);
2035 sc->sc_queues[out_qid] = tq->sq_out;
2036 sc->sc_int_sq_list[out_qid] = tq->sq_out;
2037 mtx_unlock(&sc->sc_queues_lock);
2038
2039 /* Allocate IN SQ (device -> host) */
2040 in_qid = bce_vhci_alloc_qid(vhci);
2041 if (in_qid < 0) {
2042 error = ENOSPC;
2043 goto fail_sq_out_reg;
2044 }
2045 tq->sq_in = bce_alloc_sq(sc, in_qid,
2046 sizeof(struct bce_qe_submission), BCE_VHCI_TQ_EL,
2047 bce_vhci_tq_completion, tq);
2048 if (tq->sq_in == NULL) {
2049 error = ENOMEM;
2050 goto fail_sq_in_alloc;
2051 }
2052
2053 snprintf(name, sizeof(name), "VHC1-%d-%02x",
2054 dev->fw_dev_id, ep_addr | 0x80);
2055 bce_get_sq_memcfg(tq->sq_in, tq->cq, &cfg);
2056 status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc, &cfg, name, 0);
2057 if (status != 0) {
2058 device_printf(vhci->sc_dev,
2059 "failed to register IN SQ '%s': %u\n", name, status);
2060 error = EIO;
2061 goto fail_sq_in;
2062 }
2063
2064 mtx_lock(&sc->sc_queues_lock);
2065 sc->sc_queues[in_qid] = tq->sq_in;
2066 sc->sc_int_sq_list[in_qid] = tq->sq_in;
2067 mtx_unlock(&sc->sc_queues_lock);
2068
2069 /* Tell firmware to create the endpoint */
2070 memset(&cmd, 0, sizeof(cmd));
2071 cmd.cmd = BCE_VHCI_CMD_ENDPOINT_CREATE;
2072 /*
2073 * param1 = dev_id | ((ep_addr & 0x8F) << 8)
2074 * param2 = type | (interval<<8) | (maxp<<16) | (maxp_burst<<32)
2075 * Fields encode type, interval, maxpacket, and burst.
2076 */
2077 cmd.param1 = dev->fw_dev_id |
2078 ((uint32_t)(ep_addr & 0x8F) << 8);
2079 if (edesc != NULL) {
2080 uint8_t ep_type = UE_GET_XFERTYPE(edesc->bmAttributes);
2081 uint16_t maxp = UGETW(edesc->wMaxPacketSize) & 0x7FF;
2082 uint8_t mult = ((UGETW(edesc->wMaxPacketSize) >> 11) & 3) + 1;
2083 uint64_t maxp_burst = (uint64_t)mult * maxp;
2084
2085 cmd.param2 = ep_type;
2086 if (ep_type == UE_INTERRUPT || ep_type == UE_ISOCHRONOUS)
2087 cmd.param2 |= (uint64_t)(edesc->bInterval - 1) << 8;
2088 cmd.param2 |= (uint64_t)maxp << 16;
2089 cmd.param2 |= maxp_burst << 32;
2090 }
2091 /* ep0: edesc=NULL -> param2=0, firmware uses defaults for control */
2092
2093 error = bce_vhci_cmd_execute(vhci, &cmd, &reply,
2094 BCE_VHCI_CMD_TIMEOUT_SHORT);
2095 if (error != 0) {
2096 device_printf(vhci->sc_dev,
2097 "ENDPOINT_CREATE(dev=%d, ep=0x%02x) failed: %d\n",
2098 dev->fw_dev_id, ep_addr, error);
2099 goto fail_sq_in_reg;
2100 }
2101
2102 tq->active = 1;
2103 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
2104
2105 device_printf(vhci->sc_dev,
2106 "endpoint created: dev=%d ep=0x%02x\n",
2107 dev->fw_dev_id, ep_addr);
2108
2109 return (0);
2110
2111 fail_sq_in_reg:
2112 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, in_qid);
2113 mtx_lock(&sc->sc_queues_lock);
2114 sc->sc_queues[in_qid] = NULL;
2115 sc->sc_int_sq_list[in_qid] = NULL;
2116 mtx_unlock(&sc->sc_queues_lock);
2117 fail_sq_in:
2118 bce_free_sq(sc, tq->sq_in);
2119 tq->sq_in = NULL;
2120 fail_sq_in_alloc:
2121 bce_vhci_free_qid(vhci, in_qid);
2122 fail_sq_out_reg:
2123 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, out_qid);
2124 mtx_lock(&sc->sc_queues_lock);
2125 sc->sc_queues[out_qid] = NULL;
2126 sc->sc_int_sq_list[out_qid] = NULL;
2127 mtx_unlock(&sc->sc_queues_lock);
2128 fail_sq_out:
2129 bce_free_sq(sc, tq->sq_out);
2130 tq->sq_out = NULL;
2131 fail_sq_out_alloc:
2132 bce_vhci_free_qid(vhci, out_qid);
2133 fail_cq_reg:
2134 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, cq_qid);
2135 mtx_lock(&sc->sc_queues_lock);
2136 sc->sc_queues[cq_qid] = NULL;
2137 for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
2138 if (sc->sc_cq_list[i] == tq->cq) {
2139 sc->sc_cq_list[i] = NULL;
2140 break;
2141 }
2142 }
2143 mtx_unlock(&sc->sc_queues_lock);
2144 fail_cq:
2145 bce_free_cq(sc, tq->cq);
2146 tq->cq = NULL;
2147 fail_cq_alloc:
2148 bce_vhci_free_qid(vhci, cq_qid);
2149 fail_dma:
2150 bus_dmamap_unload(tq->dma_tag, tq->dma_map);
2151 fail_mem:
2152 bus_dmamem_free(tq->dma_tag, tq->dma_buf, tq->dma_map);
2153 fail_tag:
2154 bus_dma_tag_destroy(tq->dma_tag);
2155 tq->dma_tag = NULL;
2156 return (error);
2157 }
2158
2159 /*
2160 * Destroy a per-endpoint transfer queue.
2161 */
2162 static void
bce_vhci_endpoint_destroy(struct bce_vhci_softc * vhci,struct bce_vhci_device * dev,uint8_t ep_addr)2163 bce_vhci_endpoint_destroy(struct bce_vhci_softc *vhci,
2164 struct bce_vhci_device *dev, uint8_t ep_addr)
2165 {
2166 struct apple_bce_softc *sc = vhci->sc_bce;
2167 struct bce_vhci_transfer_queue *tq;
2168 struct bce_vhci_message cmd, reply;
2169 uint8_t ep_idx;
2170 int i;
2171
2172 ep_idx = bce_vhci_ep_index(ep_addr);
2173 if (ep_idx >= BCE_VHCI_MAX_ENDPOINTS)
2174 return;
2175
2176 tq = &dev->tq[ep_idx];
2177 if (tq->active == 0)
2178 return;
2179
2180 /*
2181 * Mark inactive and complete any orphaned transfers under USB_BUS_LOCK.
2182 * IRQ event handlers (find_tq) check tq->active under USB_BUS_LOCK,
2183 * so clearing it here prevents concurrent access during teardown.
2184 */
2185 USB_BUS_LOCK(&vhci->sc_bus);
2186 tq->active = 0;
2187 tq->dma_inflight = 0;
2188 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
2189 {
2190 struct usb_xfer *ax, *px, *cx;
2191
2192 ax = tq->active_xfer;
2193 px = tq->pending_xfer;
2194 cx = tq->create_xfer;
2195 tq->active_xfer = NULL;
2196 tq->pending_xfer = NULL;
2197 tq->create_xfer = NULL;
2198 tq->create_pending = 0;
2199
2200 if (ax != NULL)
2201 usbd_transfer_done(ax, USB_ERR_CANCELLED);
2202 if (px != NULL)
2203 usbd_transfer_done(px, USB_ERR_CANCELLED);
2204 if (cx != NULL)
2205 usbd_transfer_done(cx, USB_ERR_CANCELLED);
2206 }
2207 USB_BUS_UNLOCK(&vhci->sc_bus);
2208
2209 /*
2210 * Drain the reset task to ensure it is not accessing this tq's
2211 * queues concurrently. Must be done without USB_BUS_LOCK held
2212 * (taskqueue_drain may sleep).
2213 */
2214 taskqueue_drain(taskqueue_thread, &vhci->sc_reset_task);
2215
2216 /* Tell firmware to destroy the endpoint */
2217 memset(&cmd, 0, sizeof(cmd));
2218 cmd.cmd = BCE_VHCI_CMD_ENDPOINT_DESTROY;
2219 /* param1 = dev_id | ((ep_addr & 0x8F) << 8) */
2220 cmd.param1 = dev->fw_dev_id |
2221 ((uint32_t)(ep_addr & 0x8F) << 8);
2222 bce_vhci_cmd_execute(vhci, &cmd, &reply, BCE_VHCI_CMD_TIMEOUT_SHORT);
2223
2224 /* Tear down IN SQ */
2225 if (tq->sq_in != NULL) {
2226 int in_qid = tq->sq_in->qid;
2227
2228 bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, in_qid);
2229 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, in_qid);
2230 mtx_lock(&sc->sc_queues_lock);
2231 sc->sc_queues[in_qid] = NULL;
2232 sc->sc_int_sq_list[in_qid] = NULL;
2233 mtx_unlock(&sc->sc_queues_lock);
2234 bce_free_sq(sc, tq->sq_in);
2235 tq->sq_in = NULL;
2236 bce_vhci_free_qid(vhci, in_qid);
2237 }
2238
2239 /* Tear down OUT SQ */
2240 if (tq->sq_out != NULL) {
2241 int out_qid = tq->sq_out->qid;
2242
2243 bce_cmd_flush_queue(sc->sc_cmd_cmdq, sc, out_qid);
2244 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, out_qid);
2245 mtx_lock(&sc->sc_queues_lock);
2246 sc->sc_queues[out_qid] = NULL;
2247 sc->sc_int_sq_list[out_qid] = NULL;
2248 mtx_unlock(&sc->sc_queues_lock);
2249 bce_free_sq(sc, tq->sq_out);
2250 tq->sq_out = NULL;
2251 bce_vhci_free_qid(vhci, out_qid);
2252 }
2253
2254 /* Tear down CQ */
2255 if (tq->cq != NULL) {
2256 int cq_qid = tq->cq->qid;
2257
2258 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc, cq_qid);
2259 mtx_lock(&sc->sc_queues_lock);
2260 sc->sc_queues[cq_qid] = NULL;
2261 for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
2262 if (sc->sc_cq_list[i] == tq->cq) {
2263 sc->sc_cq_list[i] = NULL;
2264 break;
2265 }
2266 }
2267 mtx_unlock(&sc->sc_queues_lock);
2268 bce_free_cq(sc, tq->cq);
2269 tq->cq = NULL;
2270 bce_vhci_free_qid(vhci, cq_qid);
2271 }
2272
2273 /*
2274 * Keep DMA buffer alive: an ISR handler on a different event
2275 * SQ may have passed find_tq before we unregistered the CQ
2276 * and still references tq->dma_tag/dma_addr. The buffer is
2277 * freed in bce_vhci_tq_destroy (device_destroy / detach).
2278 */
2279
2280 /* tq->lock stays valid until device_destroy */
2281
2282 device_printf(vhci->sc_dev,
2283 "endpoint destroyed: dev=%d ep=0x%02x\n",
2284 dev->fw_dev_id, ep_addr);
2285 }
2286
2287 /*
2288 * Create a firmware device on a port and set up ep0 queues.
2289 * Called from the roothub SetPortFeature(PORT_RESET) path.
2290 *
2291 * NOTE: This runs from process context (USB explore thread) so it is
2292 * safe to sleep in bce_vhci_cmd_execute.
2293 */
2294 static int
bce_vhci_device_create(struct bce_vhci_softc * vhci,uint8_t port)2295 bce_vhci_device_create(struct bce_vhci_softc *vhci, uint8_t port)
2296 {
2297 struct bce_vhci_message cmd, reply;
2298 struct bce_vhci_device *dev;
2299 uint8_t fw_dev_id;
2300 int error, i;
2301
2302 /* Port reset */
2303 memset(&cmd, 0, sizeof(cmd));
2304 cmd.cmd = BCE_VHCI_CMD_PORT_RESET;
2305 cmd.param1 = port;
2306 cmd.param2 = 1000; /* timeout ms */
2307
2308 error = bce_vhci_cmd_execute(vhci, &cmd, &reply,
2309 BCE_VHCI_CMD_TIMEOUT_LONG);
2310 if (error != 0) {
2311 device_printf(vhci->sc_dev,
2312 "PORT_RESET(%d) failed: %d\n", port, error);
2313 return (error);
2314 }
2315
2316 device_printf(vhci->sc_dev, "port %d reset complete\n", port);
2317
2318 /* Create device */
2319 memset(&cmd, 0, sizeof(cmd));
2320 cmd.cmd = BCE_VHCI_CMD_DEVICE_CREATE;
2321 cmd.param1 = port;
2322
2323 error = bce_vhci_cmd_execute(vhci, &cmd, &reply,
2324 BCE_VHCI_CMD_TIMEOUT_SHORT);
2325 if (error != 0) {
2326 device_printf(vhci->sc_dev,
2327 "DEVICE_CREATE(port=%d) failed: %d\n", port, error);
2328 return (error);
2329 }
2330
2331 if (reply.param2 >= BCE_VHCI_MAX_DEVICES) {
2332 device_printf(vhci->sc_dev,
2333 "firmware device ID %llu out of range\n",
2334 (unsigned long long)reply.param2);
2335 /* Destroy the firmware device we just created */
2336 memset(&cmd, 0, sizeof(cmd));
2337 cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY;
2338 cmd.param1 = (uint32_t)reply.param2;
2339 bce_vhci_cmd_execute(vhci, &cmd, &reply,
2340 BCE_VHCI_CMD_TIMEOUT_SHORT);
2341 return (ERANGE);
2342 }
2343
2344 fw_dev_id = (uint8_t)reply.param2;
2345 device_printf(vhci->sc_dev,
2346 "device created: port=%d fw_dev_id=%d\n", port, fw_dev_id);
2347
2348 dev = &vhci->sc_devs[fw_dev_id];
2349 memset(dev, 0, sizeof(*dev));
2350 dev->allocated = 1;
2351 dev->fw_dev_id = fw_dev_id;
2352 dev->port = port;
2353 vhci->sc_port_to_dev[port] = fw_dev_id;
2354
2355 /* Initialize per-endpoint locks (valid for device lifetime) */
2356 {
2357 int i;
2358
2359 for (i = 0; i < BCE_VHCI_MAX_ENDPOINTS; i++)
2360 mtx_init(&dev->tq[i].lock, "bce_vhci_tq",
2361 NULL, MTX_SPIN);
2362 }
2363
2364 /* Create ep0 (control endpoint, edesc=NULL -> firmware defaults) */
2365 error = bce_vhci_endpoint_create(vhci, dev, 0x00, NULL);
2366 if (error != 0) {
2367 device_printf(vhci->sc_dev,
2368 "failed to create ep0 for dev %d: %d\n",
2369 fw_dev_id, error);
2370 /* Destroy the device */
2371 memset(&cmd, 0, sizeof(cmd));
2372 cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY;
2373 cmd.param1 = fw_dev_id;
2374 bce_vhci_cmd_execute(vhci, &cmd, &reply,
2375 BCE_VHCI_CMD_TIMEOUT_SHORT);
2376 dev->allocated = 0;
2377 for (i = 0; i < BCE_VHCI_MAX_ENDPOINTS; i++)
2378 mtx_destroy(&dev->tq[i].lock);
2379 vhci->sc_port_to_dev[port] = 0xFF;
2380 return (error);
2381 }
2382
2383 return (0);
2384 }
2385
2386 /*
2387 * Destroy a firmware device and all its endpoints.
2388 */
2389 static void
bce_vhci_device_destroy(struct bce_vhci_softc * vhci,uint8_t port)2390 bce_vhci_device_destroy(struct bce_vhci_softc *vhci, uint8_t port)
2391 {
2392 struct bce_vhci_device *dev;
2393 struct bce_vhci_message cmd, reply;
2394 uint8_t fw_dev_id;
2395 int i;
2396
2397 if (port >= BCE_VHCI_MAX_PORTS)
2398 return;
2399
2400 fw_dev_id = vhci->sc_port_to_dev[port];
2401 if (fw_dev_id >= BCE_VHCI_MAX_DEVICES)
2402 return;
2403
2404 dev = &vhci->sc_devs[fw_dev_id];
2405 if (dev->allocated == 0)
2406 return;
2407
2408 /*
2409 * Drain the create task so it does not race endpoint creation
2410 * against our teardown. Must be done before destroying
2411 * endpoints (taskqueue_drain may sleep).
2412 */
2413 taskqueue_drain(taskqueue_thread, &vhci->sc_create_task);
2414 taskqueue_drain(taskqueue_thread, &vhci->sc_fwevt_task);
2415
2416 /* Destroy all active endpoints */
2417 for (i = 0; i < BCE_VHCI_MAX_ENDPOINTS; i++) {
2418 if (dev->tq[i].active)
2419 bce_vhci_endpoint_destroy(vhci, dev,
2420 dev->tq[i].endp_addr);
2421 }
2422
2423 /* Destroy the firmware device */
2424 memset(&cmd, 0, sizeof(cmd));
2425 cmd.cmd = BCE_VHCI_CMD_DEVICE_DESTROY;
2426 cmd.param1 = fw_dev_id;
2427 bce_vhci_cmd_execute(vhci, &cmd, &reply,
2428 BCE_VHCI_CMD_TIMEOUT_SHORT);
2429
2430 /* Free deferred DMA buffers and destroy per-endpoint locks */
2431 for (i = 0; i < BCE_VHCI_MAX_ENDPOINTS; i++) {
2432 struct bce_vhci_transfer_queue *tq = &dev->tq[i];
2433
2434 if (tq->dma_tag != NULL) {
2435 bus_dmamap_unload(tq->dma_tag, tq->dma_map);
2436 bus_dmamem_free(tq->dma_tag, tq->dma_buf,
2437 tq->dma_map);
2438 bus_dma_tag_destroy(tq->dma_tag);
2439 tq->dma_tag = NULL;
2440 }
2441 mtx_destroy(&tq->lock);
2442 }
2443
2444 dev->allocated = 0;
2445 vhci->sc_port_to_dev[port] = 0xFF;
2446
2447 device_printf(vhci->sc_dev,
2448 "device destroyed: port=%d fw_dev_id=%d\n",
2449 port, fw_dev_id);
2450 }
2451
2452 /*
2453 * Find the transfer queue for a given firmware device ID and endpoint.
2454 *
2455 * No USB_BUS_LOCK needed: endpoint_destroy clears tq->active under
2456 * USB_BUS_LOCK first (preventing new find_tq matches), then sends
2457 * ENDPOINT_DESTROY synchronously, then frees SQ/DMA resources.
2458 * Callers that drop USB_BUS_LOCK before SQ operations recheck
2459 * tq->active to handle the narrow window between active=0 and
2460 * resource free. dev->allocated and tq->active are int-aligned;
2461 * reads are safe on x86 (aligned word reads are atomic).
2462 */
2463 static struct bce_vhci_transfer_queue *
bce_vhci_find_tq(struct bce_vhci_softc * vhci,uint8_t dev_id,uint8_t ep_addr)2464 bce_vhci_find_tq(struct bce_vhci_softc *vhci, uint8_t dev_id, uint8_t ep_addr)
2465 {
2466 struct bce_vhci_device *dev;
2467 uint8_t ep_idx;
2468
2469 if (dev_id >= BCE_VHCI_MAX_DEVICES)
2470 return (NULL);
2471
2472 dev = &vhci->sc_devs[dev_id];
2473 if (dev->allocated == 0)
2474 return (NULL);
2475
2476 ep_idx = bce_vhci_ep_index(ep_addr);
2477 if (ep_idx >= BCE_VHCI_MAX_ENDPOINTS)
2478 return (NULL);
2479
2480 if (dev->tq[ep_idx].active == 0)
2481 return (NULL);
2482
2483 return (&dev->tq[ep_idx]);
2484 }
2485
2486 /*
2487 * Handle TRANSFER_REQUEST from firmware.
2488 *
2489 * The firmware asks us for data by sending TRANSFER_REQUEST with:
2490 * param1 = (ep_addr << 8) | dev_id
2491 * param2 = requested byte count
2492 *
2493 * For control transfers this drives the setup/data phases.
2494 */
2495 static void
bce_vhci_handle_transfer_request(struct bce_vhci_softc * vhci,struct bce_vhci_message * msg)2496 bce_vhci_handle_transfer_request(struct bce_vhci_softc *vhci,
2497 struct bce_vhci_message *msg)
2498 {
2499 struct bce_vhci_transfer_queue *tq;
2500 struct usb_xfer *xfer;
2501 struct bce_qe_submission *s;
2502 uint8_t dev_id, ep_addr;
2503 uint32_t req_len;
2504 int bus_locked;
2505
2506 dev_id = msg->param1 & 0xFF;
2507 ep_addr = (msg->param1 >> 8) & 0xFF;
2508 req_len = (uint32_t)msg->param2;
2509
2510 tq = bce_vhci_find_tq(vhci, dev_id, ep_addr);
2511 if (tq == NULL) {
2512 device_printf(vhci->sc_dev,
2513 "TRANSFER_REQUEST for unknown dev=%d ep=0x%02x\n",
2514 dev_id, ep_addr);
2515 return;
2516 }
2517
2518 /*
2519 * Read active_xfer under USB_BUS_LOCK to serialize with pipe_close.
2520 * Called from ev_generic_completion (ithread) or from pipe_start
2521 * (already under USB_BUS_LOCK) via evt_pending replay.
2522 */
2523 bus_locked = mtx_owned(&vhci->sc_bus.bus_mtx);
2524 if (bus_locked == 0)
2525 USB_BUS_LOCK(&vhci->sc_bus);
2526 xfer = tq->active_xfer;
2527 if (xfer == NULL) {
2528 /*
2529 * Firmware sends TRANSFER_REQUEST before the USB stack
2530 * submits the xfer via pipe_start. Save the event and
2531 * replay it when pipe_start fires.
2532 */
2533 tq->evt_pending = 1;
2534 tq->evt_saved = *msg;
2535 if (bus_locked == 0)
2536 USB_BUS_UNLOCK(&vhci->sc_bus);
2537 return;
2538 }
2539 /*
2540 * For non-control endpoints (interrupt/bulk), firmware is
2541 * requesting or providing data. Submit the appropriate buffer.
2542 */
2543 if (tq->endp_addr != 0x00) {
2544 uint32_t len = req_len;
2545
2546 if (bus_locked == 0)
2547 USB_BUS_UNLOCK(&vhci->sc_bus);
2548
2549 if (len > BCE_VHCI_XFER_BUFSZ)
2550 len = BCE_VHCI_XFER_BUFSZ;
2551
2552 /*
2553 * Re-check tq->active after dropping USB_BUS_LOCK.
2554 * endpoint_destroy sets active=0 under the lock before
2555 * freeing SQ/DMA resources, so if it is clear, our
2556 * SQ pointers may be stale.
2557 */
2558 if (tq->active == 0)
2559 return;
2560
2561 if (ep_addr & 0x80) {
2562 /* IN: firmware has data for us, submit receive buf */
2563 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
2564 BUS_DMASYNC_PREREAD);
2565 mtx_lock_spin(&tq->lock);
2566 if (bce_reserve_submission(tq->sq_in) == 0) {
2567 s = bce_next_submission(tq->sq_in);
2568 s->addr = tq->dma_addr;
2569 s->length = len;
2570 s->segl_addr = 0;
2571 s->segl_length = 0;
2572 bce_submit_to_device(vhci->sc_bce,
2573 tq->sq_in);
2574 mtx_unlock_spin(&tq->lock);
2575 } else {
2576 mtx_unlock_spin(&tq->lock);
2577 /* SQ full; fail the transfer */
2578 if (bus_locked == 0)
2579 USB_BUS_LOCK(&vhci->sc_bus);
2580 if (tq->active_xfer == xfer) {
2581 tq->active_xfer = NULL;
2582 tq->dma_inflight = 0;
2583 usbd_transfer_done(xfer,
2584 USB_ERR_IOERROR);
2585 }
2586 if (bus_locked == 0)
2587 USB_BUS_UNLOCK(&vhci->sc_bus);
2588 }
2589 } else {
2590 /* OUT: firmware wants data from us */
2591 if (bus_locked == 0)
2592 USB_BUS_LOCK(&vhci->sc_bus);
2593 if (xfer == tq->active_xfer &&
2594 xfer->frlengths[0] > 0) {
2595 if (len > xfer->frlengths[0])
2596 len = xfer->frlengths[0];
2597 usbd_copy_out(&xfer->frbuffers[0], 0,
2598 tq->dma_buf, len);
2599 if (bus_locked == 0)
2600 USB_BUS_UNLOCK(&vhci->sc_bus);
2601 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
2602 BUS_DMASYNC_PREWRITE);
2603 mtx_lock_spin(&tq->lock);
2604 if (bce_reserve_submission(tq->sq_out) == 0) {
2605 s = bce_next_submission(tq->sq_out);
2606 s->addr = tq->dma_addr;
2607 s->length = len;
2608 s->segl_addr = 0;
2609 s->segl_length = 0;
2610 bce_submit_to_device(vhci->sc_bce,
2611 tq->sq_out);
2612 mtx_unlock_spin(&tq->lock);
2613 } else {
2614 mtx_unlock_spin(&tq->lock);
2615 /* SQ full; fail the transfer */
2616 if (bus_locked == 0)
2617 USB_BUS_LOCK(&vhci->sc_bus);
2618 if (tq->active_xfer == xfer) {
2619 tq->active_xfer = NULL;
2620 tq->dma_inflight = 0;
2621 usbd_transfer_done(xfer,
2622 USB_ERR_IOERROR);
2623 }
2624 if (bus_locked == 0)
2625 USB_BUS_UNLOCK(&vhci->sc_bus);
2626 }
2627 } else {
2628 /*
2629 * Zero-length OUT or cancelled xfer.
2630 * Complete immediately, then start
2631 * any pending transfer.
2632 */
2633 if (xfer == tq->active_xfer) {
2634 tq->active_xfer = NULL;
2635 tq->dma_inflight = 0;
2636 xfer->aframes = xfer->nframes;
2637
2638 if (tq->pending_xfer != NULL) {
2639 struct usb_xfer *nx;
2640
2641 nx = tq->pending_xfer;
2642 tq->pending_xfer = NULL;
2643 bce_vhci_submit_pending_out(
2644 vhci, tq, nx);
2645 }
2646
2647 usbd_transfer_done(xfer,
2648 USB_ERR_NORMAL_COMPLETION);
2649 }
2650 if (bus_locked == 0)
2651 USB_BUS_UNLOCK(&vhci->sc_bus);
2652 }
2653 }
2654 return;
2655 }
2656
2657 /*
2658 * Control endpoint (ep0) state machine.
2659 * USB_BUS_LOCK is held here, protecting tq->active_xfer,
2660 * tq->ctrl_state, and xfer validity. We drop and re-validate
2661 * only around spin-lock + DMA submission sections.
2662 */
2663 if (tq->active_xfer != xfer) {
2664 /* Transfer was cancelled while we set up; bail */
2665 if (bus_locked == 0)
2666 USB_BUS_UNLOCK(&vhci->sc_bus);
2667 return;
2668 }
2669
2670 tq->dma_inflight = 1;
2671
2672 switch (tq->ctrl_state) {
2673 case BCE_VHCI_CTRL_SETUP:
2674 {
2675 /*
2676 * Firmware wants the 8-byte setup packet.
2677 * Copy from xfer frbuffers[0] into DMA buffer and submit
2678 * on the OUT SQ.
2679 */
2680 uint32_t len;
2681
2682 len = req_len;
2683 if (len > 8)
2684 len = 8;
2685 if (len > BCE_VHCI_XFER_BUFSZ)
2686 len = BCE_VHCI_XFER_BUFSZ;
2687
2688 usbd_copy_out(&xfer->frbuffers[0], 0, tq->dma_buf, len);
2689 if (bus_locked == 0)
2690 USB_BUS_UNLOCK(&vhci->sc_bus);
2691
2692 if (tq->active == 0)
2693 return;
2694
2695 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
2696 BUS_DMASYNC_PREWRITE);
2697
2698 mtx_lock_spin(&tq->lock);
2699 if (bce_reserve_submission(tq->sq_out) != 0) {
2700 mtx_unlock_spin(&tq->lock);
2701 device_printf(vhci->sc_dev,
2702 "no OUT SQ slot for setup\n");
2703 if (bus_locked == 0)
2704 USB_BUS_LOCK(&vhci->sc_bus);
2705 if (tq->active_xfer == xfer) {
2706 tq->active_xfer = NULL;
2707 tq->dma_inflight = 0;
2708 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
2709 usbd_transfer_done(xfer, USB_ERR_IOERROR);
2710 }
2711 if (bus_locked == 0)
2712 USB_BUS_UNLOCK(&vhci->sc_bus);
2713 return;
2714 }
2715
2716 s = bce_next_submission(tq->sq_out);
2717 s->addr = tq->dma_addr;
2718 s->length = len;
2719 s->segl_addr = 0;
2720 s->segl_length = 0;
2721 bce_submit_to_device(vhci->sc_bce, tq->sq_out);
2722 mtx_unlock_spin(&tq->lock);
2723
2724 /*
2725 * Wait for OUT SQ completion (setup DMA done) before
2726 * starting the data phase. tq_completion will see
2727 * CTRL_SETUP state on ep0 OUT completion and call
2728 * data_start. Stay in CTRL_SETUP until then.
2729 */
2730 break;
2731 }
2732
2733 case BCE_VHCI_CTRL_DATA:
2734 {
2735 /*
2736 * Data phase. Direction was determined from the setup
2737 * packet bmRequestType bit 7.
2738 * USB_BUS_LOCK is held on entry (protects xfer, ctrl_state).
2739 */
2740 uint32_t len;
2741
2742 len = req_len;
2743 if (len > BCE_VHCI_XFER_BUFSZ)
2744 len = BCE_VHCI_XFER_BUFSZ;
2745 if (len > tq->ctrl_data_len)
2746 len = tq->ctrl_data_len;
2747
2748 /*
2749 * Transition to STATUS before submitting DMA so that
2750 * the SQ completion handler sees the correct state.
2751 */
2752 tq->ctrl_state = BCE_VHCI_CTRL_STATUS;
2753 if (tq->ctrl_dir == UE_DIR_OUT)
2754 tq->ctrl_actual = len;
2755
2756 if (tq->ctrl_dir == UE_DIR_IN) {
2757 /*
2758 * Device -> host: reserve msg_asynchronous FIRST,
2759 * then submit receive buffer on IN SQ.
2760 * Correct ordering prevents an orphaned SQ entry
2761 * if the msg slot is exhausted.
2762 */
2763 struct bce_vhci_message treq;
2764
2765 memset(&treq, 0, sizeof(treq));
2766 treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST;
2767 treq.param1 =
2768 ((uint32_t)tq->endp_addr << 8) | tq->dev_addr;
2769 treq.param2 = len;
2770
2771 if (bus_locked == 0)
2772 USB_BUS_UNLOCK(&vhci->sc_bus);
2773
2774 if (tq->active == 0)
2775 return;
2776
2777 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
2778 BUS_DMASYNC_PREREAD);
2779
2780 /* Reserve msg slot first */
2781 mtx_lock_spin(&vhci->sc_async_lock);
2782 if (bce_reserve_submission(
2783 vhci->msg_asynchronous.sq) != 0) {
2784 mtx_unlock_spin(&vhci->sc_async_lock);
2785 device_printf(vhci->sc_dev,
2786 "no msg_async slot for "
2787 "ctrl data IN\n");
2788 if (bus_locked == 0)
2789 USB_BUS_LOCK(&vhci->sc_bus);
2790 if (tq->active_xfer == xfer) {
2791 tq->active_xfer = NULL;
2792 tq->dma_inflight = 0;
2793 tq->ctrl_state =
2794 BCE_VHCI_CTRL_IDLE;
2795 usbd_transfer_done(xfer,
2796 USB_ERR_IOERROR);
2797 }
2798 if (bus_locked == 0)
2799 USB_BUS_UNLOCK(&vhci->sc_bus);
2800 return;
2801 }
2802 mtx_unlock_spin(&vhci->sc_async_lock);
2803
2804 /* Now reserve and submit IN SQ */
2805 mtx_lock_spin(&tq->lock);
2806 if (bce_reserve_submission(tq->sq_in) != 0) {
2807 mtx_unlock_spin(&tq->lock);
2808 /* Return the reserved msg slot */
2809 mtx_lock_spin(&vhci->sc_async_lock);
2810 atomic_add_int(&vhci->
2811 msg_asynchronous.sq->
2812 available_commands, 1);
2813 mtx_unlock_spin(&vhci->sc_async_lock);
2814 device_printf(vhci->sc_dev,
2815 "no IN SQ slot for ctrl data\n");
2816 if (bus_locked == 0)
2817 USB_BUS_LOCK(&vhci->sc_bus);
2818 if (tq->active_xfer == xfer) {
2819 tq->active_xfer = NULL;
2820 tq->dma_inflight = 0;
2821 tq->ctrl_state =
2822 BCE_VHCI_CTRL_IDLE;
2823 usbd_transfer_done(xfer,
2824 USB_ERR_IOERROR);
2825 }
2826 if (bus_locked == 0)
2827 USB_BUS_UNLOCK(&vhci->sc_bus);
2828 return;
2829 }
2830 s = bce_next_submission(tq->sq_in);
2831 s->addr = tq->dma_addr;
2832 s->length = len;
2833 s->segl_addr = 0;
2834 s->segl_length = 0;
2835 bce_submit_to_device(vhci->sc_bce, tq->sq_in);
2836 mtx_unlock_spin(&tq->lock);
2837
2838 mtx_lock_spin(&vhci->sc_async_lock);
2839 bce_vhci_msg_queue_write(vhci,
2840 &vhci->msg_asynchronous, &treq);
2841 mtx_unlock_spin(&vhci->sc_async_lock);
2842 } else {
2843 /*
2844 * Host -> device: copy data from xfer frbuffers[1]
2845 * and submit on OUT SQ. usbd_copy_out under
2846 * USB_BUS_LOCK protects xfer validity.
2847 */
2848 usbd_copy_out(&xfer->frbuffers[1], 0,
2849 tq->dma_buf, len);
2850 if (bus_locked == 0)
2851 USB_BUS_UNLOCK(&vhci->sc_bus);
2852
2853 if (tq->active == 0)
2854 return;
2855
2856 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
2857 BUS_DMASYNC_PREWRITE);
2858
2859 mtx_lock_spin(&tq->lock);
2860 if (bce_reserve_submission(tq->sq_out) != 0) {
2861 mtx_unlock_spin(&tq->lock);
2862 device_printf(vhci->sc_dev,
2863 "no OUT SQ slot for data\n");
2864 if (bus_locked == 0)
2865 USB_BUS_LOCK(&vhci->sc_bus);
2866 if (tq->active_xfer == xfer) {
2867 tq->active_xfer = NULL;
2868 tq->dma_inflight = 0;
2869 tq->ctrl_state =
2870 BCE_VHCI_CTRL_IDLE;
2871 usbd_transfer_done(xfer,
2872 USB_ERR_IOERROR);
2873 }
2874 if (bus_locked == 0)
2875 USB_BUS_UNLOCK(&vhci->sc_bus);
2876 return;
2877 }
2878
2879 s = bce_next_submission(tq->sq_out);
2880 s->addr = tq->dma_addr;
2881 s->length = len;
2882 s->segl_addr = 0;
2883 s->segl_length = 0;
2884 bce_submit_to_device(vhci->sc_bce, tq->sq_out);
2885 mtx_unlock_spin(&tq->lock);
2886 }
2887
2888 break;
2889 }
2890
2891 default:
2892 device_printf(vhci->sc_dev,
2893 "unexpected TRANSFER_REQUEST in state %d\n",
2894 tq->ctrl_state);
2895 tq->dma_inflight = 0;
2896 if (tq->active_xfer == xfer) {
2897 tq->active_xfer = NULL;
2898 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
2899 usbd_transfer_done(xfer, USB_ERR_IOERROR);
2900 }
2901 if (bus_locked == 0)
2902 USB_BUS_UNLOCK(&vhci->sc_bus);
2903 break;
2904 }
2905 }
2906
2907 /*
2908 * Complete a control transfer. Caller must hold USB_BUS_LOCK.
2909 * Maps firmware status to USB error and calls usbd_transfer_done.
2910 */
2911 static void
bce_vhci_complete_ctrl_locked(struct bce_vhci_softc * vhci,struct bce_vhci_transfer_queue * tq,struct bce_vhci_message * msg)2912 bce_vhci_complete_ctrl_locked(struct bce_vhci_softc *vhci,
2913 struct bce_vhci_transfer_queue *tq, struct bce_vhci_message *msg)
2914 {
2915 struct usb_xfer *xfer;
2916 usb_error_t usb_err;
2917
2918 xfer = tq->active_xfer;
2919 if (xfer == NULL)
2920 return;
2921
2922 /* Map firmware status to USB error */
2923 switch (msg->status) {
2924 case BCE_VHCI_SUCCESS:
2925 usb_err = USB_ERR_NORMAL_COMPLETION;
2926
2927 /*
2928 * Tell the USB stack all frames completed.
2929 * usbd_transfer_done computes
2930 * actlen = sum(frlengths[0..aframes-1]).
2931 * If aframes stays 0, actlen=0 < sumlen -> USB_ERR_SHORT_XFER.
2932 */
2933 xfer->aframes = xfer->nframes;
2934
2935 /*
2936 * If this was an IN data transfer, copy the received
2937 * data back into the xfer buffer using usbd_copy_in
2938 * (correct API for page-cache buffers).
2939 */
2940 if (tq->ctrl_dir == UE_DIR_IN && tq->ctrl_actual > 0) {
2941 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
2942 BUS_DMASYNC_POSTREAD);
2943
2944 usbd_copy_in(&xfer->frbuffers[1], 0,
2945 tq->dma_buf, tq->ctrl_actual);
2946 xfer->frlengths[1] = tq->ctrl_actual;
2947 }
2948 break;
2949 case BCE_VHCI_PIPE_STALL:
2950 usb_err = USB_ERR_STALLED;
2951 /*
2952 * Mark endpoint stalled so pipe_start will issue
2953 * ENDPOINT_RESET (0x0044) before the next transfer.
2954 */
2955 tq->stalled = 1;
2956 break;
2957 case BCE_VHCI_ABORT:
2958 usb_err = USB_ERR_CANCELLED;
2959 break;
2960 default:
2961 usb_err = USB_ERR_IOERROR;
2962 break;
2963 }
2964
2965 tq->active_xfer = NULL;
2966 tq->dma_inflight = 0;
2967 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
2968
2969 usbd_transfer_done(xfer, usb_err);
2970 }
2971
2972 /*
2973 * Handle CTRL_TRANSFER_STATUS from firmware.
2974 *
2975 * This signals the end of a control transfer.
2976 * param1 = (ep_addr << 8) | dev_id
2977 * status = BCE_VHCI_SUCCESS(1) or error code
2978 *
2979 * For IN transfers, the IN DMA completion (tq_completion) must have
2980 * set ctrl_actual before we can copy data. If the DMA completion
2981 * has not fired yet (ctrl_data_done == 0), defer this message and
2982 * let tq_completion process it when the data arrives.
2983 */
2984 static void
bce_vhci_handle_ctrl_status(struct bce_vhci_softc * vhci,struct bce_vhci_message * msg)2985 bce_vhci_handle_ctrl_status(struct bce_vhci_softc *vhci,
2986 struct bce_vhci_message *msg)
2987 {
2988 struct bce_vhci_transfer_queue *tq;
2989 uint8_t dev_id, ep_addr;
2990
2991 dev_id = msg->param1 & 0xFF;
2992 ep_addr = (msg->param1 >> 8) & 0xFF;
2993
2994 tq = bce_vhci_find_tq(vhci, dev_id, ep_addr);
2995 if (tq == NULL) {
2996 device_printf(vhci->sc_dev,
2997 "CTRL_TRANSFER_STATUS for unknown dev=%d ep=0x%02x\n",
2998 dev_id, ep_addr);
2999 return;
3000 }
3001
3002 /*
3003 * Acquire USB_BUS_LOCK before touching xfer state.
3004 * This is called from ev_generic_completion (ithread context),
3005 * so MTX_DEF is safe. Serializes with pipe_close/pipe_start.
3006 */
3007 USB_BUS_LOCK(&vhci->sc_bus);
3008
3009 if (tq->active_xfer == NULL) {
3010 USB_BUS_UNLOCK(&vhci->sc_bus);
3011 device_printf(vhci->sc_dev,
3012 "CTRL_TRANSFER_STATUS but no active xfer\n");
3013 return;
3014 }
3015
3016 /*
3017 * Defer successful completion until DMA completes.
3018 * - ctrl_state SETUP: setup packet DMA still in flight
3019 * - ctrl_data_len > 0 with !ctrl_data_done: data DMA pending
3020 * Error statuses are never deferred to avoid permanent hangs
3021 * if the DMA completion is lost due to the error.
3022 */
3023 if (msg->status == BCE_VHCI_SUCCESS &&
3024 (tq->ctrl_state == BCE_VHCI_CTRL_SETUP ||
3025 (tq->ctrl_data_len > 0 && tq->ctrl_data_done == 0))) {
3026 tq->ctrl_status_msg = *msg;
3027 tq->ctrl_status_pending = 1;
3028 USB_BUS_UNLOCK(&vhci->sc_bus);
3029 return;
3030 }
3031
3032 if (msg->status != BCE_VHCI_SUCCESS)
3033 device_printf(vhci->sc_dev,
3034 "CTRL_TRANSFER_STATUS: dev=%d ep=0x%02x status=%u\n",
3035 dev_id, ep_addr, msg->status);
3036
3037 bce_vhci_complete_ctrl_locked(vhci, tq, msg);
3038 USB_BUS_UNLOCK(&vhci->sc_bus);
3039 }
3040
3041 /*
3042 * Send a firmware event reply on msg_system.
3043 *
3044 * Firmware events on ev_commands are acknowledged by replying with
3045 * cmd | 0x8000 and a status code on msg_system (NOT msg_asynchronous,
3046 * NOT ENDPOINT_SET_STATE).
3047 */
3048 static void
bce_vhci_send_fw_event_reply(struct bce_vhci_softc * vhci,struct bce_vhci_message * req,uint16_t status)3049 bce_vhci_send_fw_event_reply(struct bce_vhci_softc *vhci,
3050 struct bce_vhci_message *req, uint16_t status)
3051 {
3052 struct bce_vhci_message resp;
3053
3054 resp.cmd = req->cmd | BCE_VHCI_CMD_REPLY_FLAG;
3055 resp.status = status;
3056 resp.param1 = req->param1;
3057 resp.param2 = 0;
3058
3059 if (bce_reserve_submission(vhci->msg_system.sq) == 0)
3060 bce_vhci_msg_queue_write(vhci, &vhci->msg_system, &resp);
3061 else
3062 device_printf(vhci->sc_dev,
3063 "failed to send FW event reply for 0x%04x\n",
3064 req->cmd);
3065 }
3066
3067 /*
3068 * Handle ENDPOINT_REQ_STATE (0x0043) from firmware.
3069 *
3070 * Called from taskqueue context (sole consumer of ev_commands).
3071 * Updates internal pause/stall state only; no messages are sent
3072 * on msg_asynchronous from here (that queue is written from ISR
3073 * context only, avoiding multi-producer races).
3074 *
3075 * The reply (cmd | 0x8000) is sent by the caller on msg_system.
3076 */
3077 static uint16_t
bce_vhci_handle_endpoint_req_state(struct bce_vhci_softc * vhci,struct bce_vhci_message * msg)3078 bce_vhci_handle_endpoint_req_state(struct bce_vhci_softc *vhci,
3079 struct bce_vhci_message *msg)
3080 {
3081 struct bce_vhci_transfer_queue *tq;
3082 uint8_t dev_id, ep_addr;
3083 uint32_t req_state;
3084
3085 dev_id = msg->param1 & 0xFF;
3086 ep_addr = (msg->param1 >> 8) & 0xFF;
3087 req_state = (uint32_t)msg->param2;
3088
3089 tq = bce_vhci_find_tq(vhci, dev_id, ep_addr);
3090 if (tq == NULL)
3091 return (BCE_VHCI_BAD_ARGUMENT);
3092
3093 /*
3094 * USB_BUS_LOCK protects paused_by, stalled, ctrl_state, and active_xfer
3095 * against concurrent access from pipe_start, pipe_close, and ISR paths.
3096 * Nesting USB_BUS_LOCK (MTX_DEF) -> tq->lock / sc_async_lock (MTX_SPIN)
3097 * is valid.
3098 */
3099 USB_BUS_LOCK(&vhci->sc_bus);
3100
3101 /* Revalidate after taking the lock; teardown may have started */
3102 if (tq->active == 0) {
3103 USB_BUS_UNLOCK(&vhci->sc_bus);
3104 return (BCE_VHCI_SUCCESS);
3105 }
3106
3107 switch (req_state) {
3108 case BCE_VHCI_ENDP_ACTIVE:
3109 {
3110 int was_paused_by_fw;
3111
3112 was_paused_by_fw =
3113 (tq->paused_by & BCE_VHCI_PAUSE_FIRMWARE) != 0;
3114 tq->paused_by &= ~BCE_VHCI_PAUSE_FIRMWARE;
3115 tq->stalled = 0;
3116 /*
3117 * Firmware flushes SQs during PAUSE, so after ACTIVE we must
3118 * re-submit the IN buffer + TRANSFER_REQUEST, but ONLY if
3119 * the endpoint was actually paused by firmware. Firmware
3120 * also sends ENDP_ACTIVE after a fresh ENDPOINT_CREATE; in
3121 * that case the create_task has already sent the initial
3122 * TRANSFER_REQUEST and a second submission here would confuse
3123 * firmware state.
3124 *
3125 * NOTE: do NOT send ENDPOINT_SET_STATE here; firmware
3126 * already knows the new state (it requested it). The event
3127 * reply from bce_vhci_send_fw_event_reply in fwevt_task is
3128 * the ack. Sending a command from within fwevt_task would
3129 * deadlock because the reply comes back on ev_commands (same
3130 * taskqueue).
3131 */
3132 if (was_paused_by_fw &&
3133 tq->ctrl_state == BCE_VHCI_CTRL_DATA &&
3134 tq->ctrl_dir == UE_DIR_IN) {
3135 /*
3136 * Control IN data phase: re-submit on
3137 * msg_asynchronous
3138 */
3139 struct bce_vhci_message treq;
3140 struct bce_qe_submission *si;
3141 uint32_t dlen;
3142
3143 dlen = tq->ctrl_data_len;
3144 if (dlen > BCE_VHCI_XFER_BUFSZ)
3145 dlen = BCE_VHCI_XFER_BUFSZ;
3146
3147 memset(&treq, 0, sizeof(treq));
3148 treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST;
3149 treq.param1 =
3150 ((uint32_t)tq->endp_addr << 8) | tq->dev_addr;
3151 treq.param2 = dlen;
3152
3153 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
3154 BUS_DMASYNC_PREREAD);
3155
3156 /*
3157 * Reserve msg slot first, then SQ --
3158 * avoids orphaned SQ entry
3159 */
3160 mtx_lock_spin(&vhci->sc_async_lock);
3161 if (bce_reserve_submission(
3162 vhci->msg_asynchronous.sq) == 0) {
3163 mtx_unlock_spin(&vhci->sc_async_lock);
3164
3165 mtx_lock_spin(&tq->lock);
3166 if (bce_reserve_submission(tq->sq_in) == 0) {
3167 si = bce_next_submission(tq->sq_in);
3168 si->addr = tq->dma_addr;
3169 si->length = dlen;
3170 si->segl_addr = 0;
3171 si->segl_length = 0;
3172 bce_submit_to_device(vhci->sc_bce,
3173 tq->sq_in);
3174 mtx_unlock_spin(&tq->lock);
3175
3176 mtx_lock_spin(&vhci->sc_async_lock);
3177 bce_vhci_msg_queue_write(vhci,
3178 &vhci->msg_asynchronous, &treq);
3179 mtx_unlock_spin(&vhci->sc_async_lock);
3180 } else {
3181 mtx_unlock_spin(&tq->lock);
3182 mtx_lock_spin(&vhci->sc_async_lock);
3183 atomic_add_int(&vhci->
3184 msg_asynchronous.sq->
3185 available_commands, 1);
3186 mtx_unlock_spin(&vhci->sc_async_lock);
3187 device_printf(vhci->sc_dev,
3188 "ctrl resume: SQ full\n");
3189 if (tq->active_xfer != NULL) {
3190 struct usb_xfer *ax;
3191 ax = tq->active_xfer;
3192 tq->active_xfer = NULL;
3193 tq->dma_inflight = 0;
3194 tq->ctrl_state =
3195 BCE_VHCI_CTRL_IDLE;
3196 usbd_transfer_done(ax,
3197 USB_ERR_IOERROR);
3198 }
3199 }
3200 } else {
3201 mtx_unlock_spin(&vhci->sc_async_lock);
3202 device_printf(vhci->sc_dev,
3203 "ctrl resume: msg full\n");
3204 if (tq->active_xfer != NULL) {
3205 struct usb_xfer *ax;
3206 ax = tq->active_xfer;
3207 tq->active_xfer = NULL;
3208 tq->dma_inflight = 0;
3209 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
3210 usbd_transfer_done(ax,
3211 USB_ERR_IOERROR);
3212 }
3213 }
3214 } else if (was_paused_by_fw &&
3215 tq->endp_addr != 0x00 && (ep_addr & UE_DIR_IN) &&
3216 tq->active_xfer != NULL) {
3217 /*
3218 * Interrupt/bulk IN: re-submit after firmware
3219 * PAUSE/ACTIVE
3220 */
3221 struct bce_vhci_message treq;
3222 struct bce_qe_submission *si;
3223 uint32_t len;
3224
3225 len = tq->active_xfer->frlengths[0];
3226 if (len > BCE_VHCI_XFER_BUFSZ)
3227 len = BCE_VHCI_XFER_BUFSZ;
3228
3229 memset(&treq, 0, sizeof(treq));
3230 treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST;
3231 treq.param1 =
3232 ((uint32_t)tq->endp_addr << 8) | tq->dev_addr;
3233 treq.param2 = len;
3234
3235 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
3236 BUS_DMASYNC_PREREAD);
3237
3238 /*
3239 * Reserve msg slot first, then SQ --
3240 * avoids orphaned SQ entry
3241 */
3242 mtx_lock_spin(&vhci->sc_async_lock);
3243 if (bce_reserve_submission(
3244 vhci->msg_asynchronous.sq) == 0) {
3245 mtx_unlock_spin(&vhci->sc_async_lock);
3246
3247 mtx_lock_spin(&tq->lock);
3248 if (bce_reserve_submission(tq->sq_in) == 0) {
3249 si = bce_next_submission(tq->sq_in);
3250 si->addr = tq->dma_addr;
3251 si->length = len;
3252 si->segl_addr = 0;
3253 si->segl_length = 0;
3254 bce_submit_to_device(vhci->sc_bce,
3255 tq->sq_in);
3256 mtx_unlock_spin(&tq->lock);
3257
3258 mtx_lock_spin(&vhci->sc_async_lock);
3259 bce_vhci_msg_queue_write(vhci,
3260 &vhci->msg_asynchronous, &treq);
3261 mtx_unlock_spin(&vhci->sc_async_lock);
3262 } else {
3263 mtx_unlock_spin(&tq->lock);
3264 mtx_lock_spin(&vhci->sc_async_lock);
3265 atomic_add_int(&vhci->
3266 msg_asynchronous.sq->
3267 available_commands, 1);
3268 mtx_unlock_spin(&vhci->sc_async_lock);
3269 device_printf(vhci->sc_dev,
3270 "IN resume: SQ full\n");
3271 if (tq->active_xfer != NULL) {
3272 struct usb_xfer *ax;
3273 ax = tq->active_xfer;
3274 tq->active_xfer = NULL;
3275 tq->dma_inflight = 0;
3276 usbd_transfer_done(ax,
3277 USB_ERR_IOERROR);
3278 }
3279 }
3280 } else {
3281 mtx_unlock_spin(&vhci->sc_async_lock);
3282 device_printf(vhci->sc_dev,
3283 "IN resume: msg full\n");
3284 if (tq->active_xfer != NULL) {
3285 struct usb_xfer *ax;
3286 ax = tq->active_xfer;
3287 tq->active_xfer = NULL;
3288 tq->dma_inflight = 0;
3289 usbd_transfer_done(ax,
3290 USB_ERR_IOERROR);
3291 }
3292 }
3293 }
3294 } /* end ENDP_ACTIVE scope */
3295 break;
3296 case BCE_VHCI_ENDP_PAUSED:
3297 tq->paused_by |= BCE_VHCI_PAUSE_FIRMWARE;
3298 /*
3299 * Do NOT send ENDPOINT_SET_STATE; same deadlock
3300 * reason as ACTIVE above. The event reply is the ack.
3301 */
3302 /*
3303 * Flush pending SQ submissions after PAUSE.
3304 * Without this, our pre-submitted IN buffer stays in
3305 * firmware's view and confuses the re-submit after RESUME.
3306 *
3307 * bce_cmd_flush_queue sleeps (waits on semaphore), so we
3308 * must drop USB_BUS_LOCK before calling it. QIDs are
3309 * snapshotted under the lock; endpoint_destroy also takes
3310 * the lock before starting teardown, so the qids remain
3311 * valid with firmware while we flush. A double-flush
3312 * (here + endpoint_destroy) is harmless.
3313 */
3314 if (tq->active) {
3315 struct apple_bce_softc *sc = vhci->sc_bce;
3316 int sq_in_qid = (tq->sq_in != NULL) ?
3317 tq->sq_in->qid : -1;
3318 int sq_out_qid = (tq->sq_out != NULL) ?
3319 tq->sq_out->qid : -1;
3320
3321 USB_BUS_UNLOCK(&vhci->sc_bus);
3322 if (sq_in_qid >= 0)
3323 bce_cmd_flush_queue(sc->sc_cmd_cmdq,
3324 sc, sq_in_qid);
3325 if (sq_out_qid >= 0)
3326 bce_cmd_flush_queue(sc->sc_cmd_cmdq,
3327 sc, sq_out_qid);
3328 USB_BUS_LOCK(&vhci->sc_bus);
3329 }
3330 break;
3331 default:
3332 USB_BUS_UNLOCK(&vhci->sc_bus);
3333 return (BCE_VHCI_BAD_ARGUMENT);
3334 }
3335
3336 USB_BUS_UNLOCK(&vhci->sc_bus);
3337 return (BCE_VHCI_SUCCESS);
3338 }
3339
3340 /*
3341 * Handle unsolicited ENDPOINT_SET_STATE (0x0042) from firmware.
3342 *
3343 * Firmware notifies us of a state change it initiated (e.g., stall
3344 * after a protocol error).
3345 *
3346 * param1 = (ep_addr << 8) | dev_id
3347 * param2 = new_state
3348 */
3349 static uint16_t
bce_vhci_handle_endpoint_set_state(struct bce_vhci_softc * vhci,struct bce_vhci_message * msg)3350 bce_vhci_handle_endpoint_set_state(struct bce_vhci_softc *vhci,
3351 struct bce_vhci_message *msg)
3352 {
3353 struct bce_vhci_transfer_queue *tq;
3354 uint8_t dev_id, ep_addr;
3355 uint32_t new_state;
3356
3357 dev_id = msg->param1 & 0xFF;
3358 ep_addr = (msg->param1 >> 8) & 0xFF;
3359 new_state = (uint32_t)msg->param2;
3360
3361 device_printf(vhci->sc_dev,
3362 "ENDPOINT_SET_STATE: dev=%d ep=0x%02x state=%u\n",
3363 dev_id, ep_addr, new_state);
3364
3365 tq = bce_vhci_find_tq(vhci, dev_id, ep_addr);
3366 if (tq == NULL)
3367 return (BCE_VHCI_BAD_ARGUMENT);
3368
3369 switch (new_state) {
3370 case BCE_VHCI_ENDP_STALLED:
3371 /*
3372 * USB_BUS_LOCK protects stalled against concurrent access
3373 * from pipe_start, pipe_close, and the endpoint_req_state path.
3374 */
3375 USB_BUS_LOCK(&vhci->sc_bus);
3376 tq->stalled = 1;
3377 USB_BUS_UNLOCK(&vhci->sc_bus);
3378 /*
3379 * Do not touch active_xfer here; this runs from
3380 * taskqueue while ISR may be using it. The stall
3381 * will be reported via CTRL_TRANSFER_STATUS(STALL)
3382 * from firmware on the ISR path.
3383 */
3384 return (BCE_VHCI_SUCCESS);
3385 default:
3386 return (BCE_VHCI_BAD_ARGUMENT);
3387 }
3388 }
3389
3390 /*
3391 * QID bitmap allocator (internal, caller must hold sc_queues_lock).
3392 * BCE_MAX_QUEUE_COUNT = 256 = 8 * 32 bits.
3393 * Bit set means QID is in use.
3394 */
3395 static int
bce_vhci_alloc_qid_locked(struct bce_vhci_softc * vhci)3396 bce_vhci_alloc_qid_locked(struct bce_vhci_softc *vhci)
3397 {
3398 struct apple_bce_softc *sc __unused = vhci->sc_bce;
3399 int i, bit;
3400
3401 mtx_assert(&sc->sc_queues_lock, MA_OWNED);
3402 for (i = 0; i < 8; i++) {
3403 if (vhci->sc_qid_bitmap[i] == 0xFFFFFFFF)
3404 continue;
3405 bit = ffs(~vhci->sc_qid_bitmap[i]) - 1;
3406 vhci->sc_qid_bitmap[i] |= (1u << bit);
3407 return (i * 32 + bit);
3408 }
3409 return (-1);
3410 }
3411
3412 static int
bce_vhci_alloc_qid(struct bce_vhci_softc * vhci)3413 bce_vhci_alloc_qid(struct bce_vhci_softc *vhci)
3414 {
3415 struct apple_bce_softc *sc = vhci->sc_bce;
3416 int qid;
3417
3418 mtx_lock(&sc->sc_queues_lock);
3419 qid = bce_vhci_alloc_qid_locked(vhci);
3420 mtx_unlock(&sc->sc_queues_lock);
3421 return (qid);
3422 }
3423
3424 /*
3425 * Allocate next queue ID pair (CQ + SQ).
3426 * Returns the CQ qid; SQ qid = CQ qid + 1.
3427 * Finds two consecutive free bits in the bitmap.
3428 */
3429 static int
bce_vhci_alloc_qid_pair(struct bce_vhci_softc * vhci)3430 bce_vhci_alloc_qid_pair(struct bce_vhci_softc *vhci)
3431 {
3432 struct apple_bce_softc *sc = vhci->sc_bce;
3433 int qid, j;
3434
3435 mtx_lock(&sc->sc_queues_lock);
3436
3437 /* Find first free QID and check the next one is also free */
3438 qid = bce_vhci_alloc_qid_locked(vhci);
3439 if (qid < 0) {
3440 mtx_unlock(&sc->sc_queues_lock);
3441 return (-1);
3442 }
3443 if (qid + 1 >= BCE_MAX_QUEUE_COUNT) {
3444 /* Free the one we just allocated */
3445 vhci->sc_qid_bitmap[qid / 32] &= ~(1u << (qid % 32));
3446 mtx_unlock(&sc->sc_queues_lock);
3447 return (-1);
3448 }
3449 /* Check next QID is free */
3450 if (vhci->sc_qid_bitmap[(qid + 1) / 32] & (1u << ((qid + 1) % 32))) {
3451 /* Next is taken; free qid and search for consecutive pair */
3452 vhci->sc_qid_bitmap[qid / 32] &= ~(1u << (qid % 32));
3453 /* Brute-force search for consecutive pair */
3454 for (j = BCE_QUEUE_USER_MIN; j < BCE_MAX_QUEUE_COUNT - 1; j++) {
3455 uint32_t w0 = vhci->sc_qid_bitmap[j / 32];
3456 uint32_t w1 = vhci->sc_qid_bitmap[(j + 1) / 32];
3457 int b0 = j % 32;
3458 int b1 = (j + 1) % 32;
3459
3460 if ((w0 & (1u << b0)) == 0 &&
3461 (w1 & (1u << b1)) == 0) {
3462 vhci->sc_qid_bitmap[j / 32] |= (1u << b0);
3463 vhci->sc_qid_bitmap[(j + 1) / 32] |=
3464 (1u << b1);
3465 mtx_unlock(&sc->sc_queues_lock);
3466 return (j);
3467 }
3468 }
3469 mtx_unlock(&sc->sc_queues_lock);
3470 return (-1);
3471 }
3472 /* Next QID is free; allocate it */
3473 vhci->sc_qid_bitmap[(qid + 1) / 32] |= (1u << ((qid + 1) % 32));
3474 mtx_unlock(&sc->sc_queues_lock);
3475 return (qid);
3476 }
3477
3478 static void
bce_vhci_free_qid(struct bce_vhci_softc * vhci,int qid)3479 bce_vhci_free_qid(struct bce_vhci_softc *vhci, int qid)
3480 {
3481 struct apple_bce_softc *sc = vhci->sc_bce;
3482
3483 if (qid < 0 || qid >= BCE_MAX_QUEUE_COUNT)
3484 return;
3485 mtx_lock(&sc->sc_queues_lock);
3486 vhci->sc_qid_bitmap[qid / 32] &= ~(1u << (qid % 32));
3487 mtx_unlock(&sc->sc_queues_lock);
3488 }
3489
3490 static int
bce_vhci_create_queues(struct bce_vhci_softc * vhci)3491 bce_vhci_create_queues(struct bce_vhci_softc *vhci)
3492 {
3493 struct apple_bce_softc *sc = vhci->sc_bce;
3494 struct bce_queue_memcfg cfg;
3495 uint32_t status;
3496 int error, qid_pair, q, i;
3497
3498 /* Initialize QID bitmap: mark QIDs 0..BCE_QUEUE_USER_MIN-1 as used */
3499 memset(vhci->sc_qid_bitmap, 0, sizeof(vhci->sc_qid_bitmap));
3500 for (q = 0; q < BCE_QUEUE_USER_MIN; q++)
3501 vhci->sc_qid_bitmap[q / 32] |= (1u << (q % 32));
3502
3503 /* Initialize command queue locks early (destroy_queues expects them) */
3504 sx_init(&vhci->cmd.exec_lock, "bce_vhci_cmdex");
3505 mtx_init(&vhci->cmd.lock, "bce_vhci_cmd", NULL, MTX_SPIN);
3506 sema_init(&vhci->cmd.completion, 0, "bce_vhci_cmd");
3507 vhci->cmd.pending = 0;
3508
3509 /*
3510 * Create 5 message queues (host -> device).
3511 * Each gets its own CQ + SQ pair.
3512 */
3513 #define CREATE_MSG_QUEUE(field, name) \
3514 do { \
3515 qid_pair = bce_vhci_alloc_qid_pair(vhci); \
3516 if (qid_pair < 0) { \
3517 error = ENOMEM; \
3518 device_printf(vhci->sc_dev, \
3519 "queue IDs exhausted for %s\n", name); \
3520 goto fail; \
3521 } \
3522 error = bce_vhci_msg_queue_create(vhci, &vhci->field, \
3523 name, qid_pair, qid_pair + 1, \
3524 bce_vhci_msg_queue_completion, &vhci->field); \
3525 if (error != 0) { \
3526 device_printf(vhci->sc_dev, \
3527 "failed to create %s: %d\n", name, error); \
3528 goto fail; \
3529 } \
3530 } while (0)
3531
3532 CREATE_MSG_QUEUE(msg_commands, "VHC1HostCommands");
3533 CREATE_MSG_QUEUE(msg_system, "VHC1HostSystemEvents");
3534 CREATE_MSG_QUEUE(msg_isochronous, "VHC1HostIsochronousEvents");
3535 CREATE_MSG_QUEUE(msg_interrupt, "VHC1HostInterruptEvents");
3536 CREATE_MSG_QUEUE(msg_asynchronous, "VHC1HostAsynchronousEvents");
3537 #undef CREATE_MSG_QUEUE
3538
3539 /*
3540 * Create shared event CQ (one CQ for all 5 event queues).
3541 */
3542 {
3543 int ev_cq_qid = bce_vhci_alloc_qid(vhci);
3544
3545 if (ev_cq_qid < 0) {
3546 device_printf(vhci->sc_dev,
3547 "queue IDs exhausted for event CQ\n");
3548 error = ENOMEM;
3549 goto fail;
3550 }
3551
3552 vhci->ev_cq = bce_alloc_cq(sc, ev_cq_qid,
3553 BCE_VHCI_EVT_QUEUE_EL);
3554 if (vhci->ev_cq == NULL) {
3555 bce_vhci_free_qid(vhci, ev_cq_qid);
3556 error = ENOMEM;
3557 goto fail;
3558 }
3559
3560 bce_get_cq_memcfg(vhci->ev_cq, &cfg);
3561 cfg.vector_or_cq = 4;
3562 status = bce_cmd_register_queue(sc->sc_cmd_cmdq, sc,
3563 &cfg, NULL, 0);
3564 if (status != 0) {
3565 device_printf(vhci->sc_dev,
3566 "failed to register event CQ: %u\n", status);
3567 bce_free_cq(sc, vhci->ev_cq);
3568 vhci->ev_cq = NULL;
3569 bce_vhci_free_qid(vhci, ev_cq_qid);
3570 error = EIO;
3571 goto fail;
3572 }
3573
3574 mtx_lock(&sc->sc_queues_lock);
3575 sc->sc_queues[ev_cq_qid] = vhci->ev_cq;
3576 for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
3577 if (sc->sc_cq_list[i] == NULL) {
3578 sc->sc_cq_list[i] = vhci->ev_cq;
3579 break;
3580 }
3581 }
3582 if (i == BCE_MAX_CQ_COUNT) {
3583 sc->sc_queues[ev_cq_qid] = NULL;
3584 mtx_unlock(&sc->sc_queues_lock);
3585 device_printf(vhci->sc_dev,
3586 "CQ list full for event CQ\n");
3587 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc,
3588 ev_cq_qid);
3589 bce_free_cq(sc, vhci->ev_cq);
3590 vhci->ev_cq = NULL;
3591 bce_vhci_free_qid(vhci, ev_cq_qid);
3592 error = ENOSPC;
3593 goto fail;
3594 }
3595 mtx_unlock(&sc->sc_queues_lock);
3596 }
3597
3598 /*
3599 * Create 5 event queues (device -> host), all sharing ev_cq.
3600 */
3601 #define CREATE_EVT_QUEUE(field, name, fn) \
3602 do { \
3603 int eq_qid = bce_vhci_alloc_qid(vhci); \
3604 if (eq_qid < 0) { \
3605 device_printf(vhci->sc_dev, \
3606 "queue IDs exhausted for %s\n", name); \
3607 error = ENOMEM; \
3608 goto fail; \
3609 } \
3610 error = bce_vhci_evt_queue_create(vhci, &vhci->field, \
3611 name, eq_qid, fn); \
3612 if (error != 0) { \
3613 device_printf(vhci->sc_dev, \
3614 "failed to create %s: %d\n", name, error); \
3615 bce_vhci_free_qid(vhci, eq_qid); \
3616 goto fail; \
3617 } \
3618 } while (0)
3619
3620 CREATE_EVT_QUEUE(ev_commands, "VHC1FirmwareCommands",
3621 bce_vhci_ev_cmd_completion);
3622 CREATE_EVT_QUEUE(ev_system, "VHC1FirmwareSystemEvents",
3623 bce_vhci_ev_system_completion);
3624 CREATE_EVT_QUEUE(ev_isochronous, "VHC1FirmwareIsochronousEvents",
3625 bce_vhci_ev_generic_completion);
3626 CREATE_EVT_QUEUE(ev_interrupt, "VHC1FirmwareInterruptEvents",
3627 bce_vhci_ev_generic_completion);
3628 CREATE_EVT_QUEUE(ev_asynchronous, "VHC1FirmwareAsynchronousEvents",
3629 bce_vhci_ev_generic_completion);
3630 #undef CREATE_EVT_QUEUE
3631
3632 /* Wire command queue to its message queue */
3633 vhci->cmd.msg = &vhci->msg_commands;
3634
3635 device_printf(vhci->sc_dev, "VHCI queues created\n");
3636 return (0);
3637
3638 fail:
3639 bce_vhci_destroy_queues(vhci);
3640 return (error);
3641 }
3642
3643 static void
bce_vhci_destroy_queues(struct bce_vhci_softc * vhci)3644 bce_vhci_destroy_queues(struct bce_vhci_softc *vhci)
3645 {
3646 struct apple_bce_softc *sc = vhci->sc_bce;
3647 int i;
3648
3649 /* Destroy event queues first (they may deliver cmd replies) */
3650 bce_vhci_evt_queue_destroy(vhci, &vhci->ev_asynchronous);
3651 bce_vhci_evt_queue_destroy(vhci, &vhci->ev_interrupt);
3652 bce_vhci_evt_queue_destroy(vhci, &vhci->ev_isochronous);
3653 bce_vhci_evt_queue_destroy(vhci, &vhci->ev_system);
3654 bce_vhci_evt_queue_destroy(vhci, &vhci->ev_commands);
3655
3656 /* Destroy command queue state after events are drained */
3657 sema_destroy(&vhci->cmd.completion);
3658 if (mtx_initialized(&vhci->cmd.lock))
3659 mtx_destroy(&vhci->cmd.lock);
3660 sx_destroy(&vhci->cmd.exec_lock);
3661
3662 /* Destroy shared event CQ */
3663 if (vhci->ev_cq != NULL) {
3664 bce_cmd_unregister_queue(sc->sc_cmd_cmdq, sc,
3665 vhci->ev_cq->qid);
3666 mtx_lock(&sc->sc_queues_lock);
3667 sc->sc_queues[vhci->ev_cq->qid] = NULL;
3668 for (i = 0; i < BCE_MAX_CQ_COUNT; i++) {
3669 if (sc->sc_cq_list[i] == vhci->ev_cq) {
3670 sc->sc_cq_list[i] = NULL;
3671 break;
3672 }
3673 }
3674 mtx_unlock(&sc->sc_queues_lock);
3675 bce_free_cq(sc, vhci->ev_cq);
3676 vhci->ev_cq = NULL;
3677 }
3678
3679 /* Destroy message queues */
3680 bce_vhci_msg_queue_destroy(vhci, &vhci->msg_asynchronous);
3681 bce_vhci_msg_queue_destroy(vhci, &vhci->msg_interrupt);
3682 bce_vhci_msg_queue_destroy(vhci, &vhci->msg_isochronous);
3683 bce_vhci_msg_queue_destroy(vhci, &vhci->msg_system);
3684 bce_vhci_msg_queue_destroy(vhci, &vhci->msg_commands);
3685 }
3686
3687 static int
bce_vhci_start_controller(struct bce_vhci_softc * vhci)3688 bce_vhci_start_controller(struct bce_vhci_softc *vhci)
3689 {
3690 struct bce_vhci_message cmd, reply;
3691 uint16_t port_mask;
3692 uint8_t port_count;
3693 uint32_t port_status;
3694 int error;
3695 int i;
3696
3697 /*
3698 * CONTROLLER_ENABLE: param1 = 0x7100 | bus_number(1)
3699 * Reply param2 = port bitmask
3700 */
3701 memset(&cmd, 0, sizeof(cmd));
3702 cmd.cmd = BCE_VHCI_CMD_CONTROLLER_ENABLE;
3703 cmd.param1 = 0x7100 | 1;
3704
3705 error = bce_vhci_cmd_execute(vhci, &cmd, &reply,
3706 BCE_VHCI_CMD_TIMEOUT_LONG);
3707 if (error != 0) {
3708 device_printf(vhci->sc_dev,
3709 "CONTROLLER_ENABLE failed: %d\n", error);
3710 return (error);
3711 }
3712
3713 port_mask = (uint16_t)reply.param2;
3714 vhci->sc_port_mask = port_mask;
3715
3716 /* Count ports from mask */
3717 port_count = 0;
3718 for (i = 0; i < BCE_VHCI_MAX_PORTS; i++) {
3719 if (port_mask & (1u << i))
3720 port_count = i + 1;
3721 }
3722 vhci->sc_port_count = port_count;
3723
3724 device_printf(vhci->sc_dev,
3725 "controller enabled: port_mask=0x%x, %d ports\n",
3726 port_mask, port_count);
3727
3728 /*
3729 * CONTROLLER_START
3730 */
3731 memset(&cmd, 0, sizeof(cmd));
3732 cmd.cmd = BCE_VHCI_CMD_CONTROLLER_START;
3733
3734 error = bce_vhci_cmd_execute(vhci, &cmd, &reply,
3735 BCE_VHCI_CMD_TIMEOUT_LONG);
3736 if (error != 0) {
3737 device_printf(vhci->sc_dev,
3738 "CONTROLLER_START failed: %d\n", error);
3739 /* Disable the controller we just enabled */
3740 memset(&cmd, 0, sizeof(cmd));
3741 cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE;
3742 bce_vhci_cmd_execute(vhci, &cmd, &reply,
3743 BCE_VHCI_CMD_TIMEOUT_LONG);
3744 return (error);
3745 }
3746
3747 vhci->sc_started = 1;
3748
3749 /*
3750 * Power on each port and read initial status.
3751 */
3752 for (i = 0; i < port_count; i++) {
3753 if ((port_mask & (1u << i)) == 0)
3754 continue;
3755
3756 memset(&cmd, 0, sizeof(cmd));
3757 cmd.cmd = BCE_VHCI_CMD_PORT_POWER_ON;
3758 cmd.param1 = i;
3759
3760 error = bce_vhci_cmd_execute(vhci, &cmd, &reply,
3761 BCE_VHCI_CMD_TIMEOUT_SHORT);
3762 if (error != 0) {
3763 device_printf(vhci->sc_dev,
3764 "PORT_POWER_ON(%d) failed: %d\n", i, error);
3765 continue;
3766 }
3767 vhci->sc_port_power[i] = 1;
3768
3769 /* Read initial port status */
3770 memset(&cmd, 0, sizeof(cmd));
3771 cmd.cmd = BCE_VHCI_CMD_PORT_STATUS;
3772 cmd.param1 = i;
3773
3774 error = bce_vhci_cmd_execute(vhci, &cmd, &reply,
3775 BCE_VHCI_CMD_TIMEOUT_SHORT);
3776 if (error != 0) {
3777 device_printf(vhci->sc_dev,
3778 "PORT_STATUS(%d) failed: %d\n", i, error);
3779 continue;
3780 }
3781
3782 port_status = (uint32_t)reply.param2;
3783 device_printf(vhci->sc_dev,
3784 "port %d: raw_status=0x%x\n", i, port_status);
3785
3786 /*
3787 * Translate firmware port status to USB status bits.
3788 */
3789 vhci->sc_port_status[i] = UPS_PORT_POWER;
3790 if (port_status & BCE_VHCI_PORT_ENABLED)
3791 vhci->sc_port_status[i] |=
3792 UPS_PORT_ENABLED | UPS_HIGH_SPEED;
3793 if (port_status & BCE_VHCI_PORT_CONNECTED)
3794 vhci->sc_port_status[i] |=
3795 UPS_CURRENT_CONNECT_STATUS;
3796 if (port_status & BCE_VHCI_PORT_SUSPENDED)
3797 vhci->sc_port_status[i] |= UPS_SUSPEND;
3798 if (port_status & BCE_VHCI_PORT_OVERCURRENT)
3799 vhci->sc_port_status[i] |=
3800 UPS_OVERCURRENT_INDICATOR;
3801
3802 if (vhci->sc_port_status[i] & UPS_CURRENT_CONNECT_STATUS)
3803 vhci->sc_port_change[i] |= UPS_C_CONNECT_STATUS;
3804 }
3805
3806 return (0);
3807 }
3808
3809 static usb_error_t
bce_vhci_roothub_exec(struct usb_device * udev,struct usb_device_request * req,const void ** pptr,uint16_t * plength)3810 bce_vhci_roothub_exec(struct usb_device *udev,
3811 struct usb_device_request *req, const void **pptr, uint16_t *plength)
3812 {
3813 struct bce_vhci_softc *vhci;
3814 const void *ptr;
3815 uint16_t len;
3816 uint16_t value, index;
3817 usb_error_t err;
3818
3819 vhci = (struct bce_vhci_softc *)udev->bus;
3820 USB_BUS_LOCK_ASSERT(&vhci->sc_bus, MA_OWNED);
3821
3822 ptr = NULL;
3823 len = 0;
3824 err = 0;
3825
3826 value = UGETW(req->wValue);
3827 index = UGETW(req->wIndex);
3828
3829 switch (req->bRequest) {
3830 case UR_CLEAR_FEATURE:
3831 switch (req->bmRequestType) {
3832 case UT_WRITE_CLASS_OTHER:
3833 /* ClearPortFeature */
3834 if (index < 1 || index > vhci->sc_port_count) {
3835 err = USB_ERR_IOERROR;
3836 break;
3837 }
3838 switch (value) {
3839 case UHF_C_PORT_CONNECTION:
3840 vhci->sc_port_change[index - 1] &=
3841 ~UPS_C_CONNECT_STATUS;
3842 break;
3843 case UHF_C_PORT_ENABLE:
3844 vhci->sc_port_change[index - 1] &=
3845 ~UPS_C_PORT_ENABLED;
3846 break;
3847 case UHF_C_PORT_RESET:
3848 vhci->sc_port_change[index - 1] &=
3849 ~UPS_C_PORT_RESET;
3850 break;
3851 case UHF_C_PORT_OVER_CURRENT:
3852 vhci->sc_port_change[index - 1] &=
3853 ~UPS_C_OVERCURRENT_INDICATOR;
3854 break;
3855 case UHF_C_PORT_SUSPEND:
3856 vhci->sc_port_change[index - 1] &=
3857 ~UPS_C_SUSPEND;
3858 break;
3859 case UHF_PORT_ENABLE:
3860 vhci->sc_port_status[index - 1] &=
3861 ~UPS_PORT_ENABLED;
3862 break;
3863 case UHF_PORT_SUSPEND:
3864 vhci->sc_port_status[index - 1] &=
3865 ~UPS_SUSPEND;
3866 break;
3867 case UHF_PORT_POWER:
3868 if (vhci->sc_port_power[index - 1]) {
3869 struct bce_vhci_message cmd_pw;
3870 struct bce_vhci_message reply_pw;
3871 int pw_port = index - 1;
3872
3873 memset(&cmd_pw, 0, sizeof(cmd_pw));
3874 cmd_pw.cmd =
3875 BCE_VHCI_CMD_PORT_POWER_OFF;
3876 cmd_pw.param1 = pw_port;
3877
3878 USB_BUS_UNLOCK(&vhci->sc_bus);
3879 if (bce_vhci_cmd_execute(vhci, &cmd_pw,
3880 &reply_pw,
3881 BCE_VHCI_CMD_TIMEOUT_SHORT) != 0) {
3882 device_printf(vhci->sc_dev,
3883 "PORT_POWER_OFF(%d)"
3884 " failed\n", pw_port);
3885 USB_BUS_LOCK(&vhci->sc_bus);
3886 err = USB_ERR_IOERROR;
3887 break;
3888 }
3889 USB_BUS_LOCK(&vhci->sc_bus);
3890 vhci->sc_port_power[pw_port] = 0;
3891 vhci->sc_port_status[pw_port] = 0;
3892 }
3893 break;
3894 default:
3895 err = USB_ERR_IOERROR;
3896 break;
3897 }
3898 break;
3899 default:
3900 err = USB_ERR_IOERROR;
3901 break;
3902 }
3903 break;
3904
3905 case UR_GET_DESCRIPTOR:
3906 if (req->bmRequestType == UT_READ_CLASS_DEVICE) {
3907 /* Hub descriptor (USB 2.0) */
3908 struct usb_hub_descriptor hd;
3909 uint8_t nports = vhci->sc_port_count;
3910 uint8_t padsz = (nports + 7) / 8;
3911
3912 memset(&hd, 0, sizeof(hd));
3913 hd.bDescLength = 7 + 2 * padsz;
3914 hd.bDescriptorType = UDESC_HUB;
3915 hd.bNbrPorts = nports;
3916 USETW(hd.wHubCharacteristics,
3917 UHD_PWR_INDIVIDUAL);
3918 hd.bPwrOn2PwrGood = 50;
3919 len = hd.bDescLength;
3920 if (len > sizeof(vhci->sc_hub_idata))
3921 len = sizeof(vhci->sc_hub_idata);
3922 memcpy(vhci->sc_hub_idata, &hd, len);
3923 ptr = vhci->sc_hub_idata;
3924 break;
3925 }
3926 switch (value >> 8) {
3927 case UDESC_DEVICE:
3928 if ((value & 0xff) != 0) {
3929 err = USB_ERR_IOERROR;
3930 break;
3931 }
3932 len = sizeof(bce_vhci_devd);
3933 ptr = &bce_vhci_devd;
3934 break;
3935 case UDESC_DEVICE_QUALIFIER:
3936 if ((value & 0xff) != 0) {
3937 err = USB_ERR_IOERROR;
3938 break;
3939 }
3940 len = sizeof(bce_vhci_odevd);
3941 ptr = &bce_vhci_odevd;
3942 break;
3943 case UDESC_CONFIG:
3944 if ((value & 0xff) != 0) {
3945 err = USB_ERR_IOERROR;
3946 break;
3947 }
3948 len = sizeof(bce_vhci_confd);
3949 ptr = bce_vhci_confd;
3950 break;
3951 case UDESC_STRING:
3952 switch (value & 0xff) {
3953 case 0: /* Language */
3954 ptr = "\x04\x03\x09\x04";
3955 len = 4;
3956 break;
3957 case 1: /* Vendor */
3958 ptr = "\x0c\x03\x41\x00\x70\x00\x70\x00"
3959 "\x6c\x00\x65\x00";
3960 len = 12;
3961 break;
3962 case 2: /* Product */
3963 ptr = "\x1a\x03\x54\x00\x32\x00\x20\x00"
3964 "\x42\x00\x43\x00\x45\x00\x20\x00"
3965 "\x56\x00\x48\x00\x43\x00\x49\x00";
3966 len = 26;
3967 break;
3968 default:
3969 err = USB_ERR_IOERROR;
3970 break;
3971 }
3972 break;
3973 default:
3974 err = USB_ERR_IOERROR;
3975 break;
3976 }
3977 break;
3978
3979 case UR_GET_INTERFACE:
3980 len = 1;
3981 ptr = "\x00"; /* alt setting 0 */
3982 break;
3983
3984 case UR_GET_STATUS:
3985 switch (req->bmRequestType) {
3986 case UT_READ_CLASS_OTHER:
3987 {
3988 /* GetPortStatus */
3989 struct usb_port_status ps;
3990 uint16_t port;
3991
3992 if (index < 1 || index > vhci->sc_port_count) {
3993 err = USB_ERR_IOERROR;
3994 break;
3995 }
3996 port = index - 1;
3997
3998 USETW(ps.wPortStatus,
3999 vhci->sc_port_status[port]);
4000 USETW(ps.wPortChange,
4001 vhci->sc_port_change[port]);
4002
4003 len = sizeof(ps);
4004 memcpy(&vhci->sc_hub_idata, &ps, sizeof(ps));
4005 ptr = &vhci->sc_hub_idata;
4006 break;
4007 }
4008 case UT_READ_CLASS_DEVICE:
4009 {
4010 /* GetHubStatus */
4011 len = 4;
4012 ptr = "\x00\x00\x00\x00";
4013 break;
4014 }
4015 case UT_READ_DEVICE:
4016 {
4017 len = 2;
4018 ptr = "\x01\x00"; /* self-powered */
4019 break;
4020 }
4021 default:
4022 err = USB_ERR_IOERROR;
4023 break;
4024 }
4025 break;
4026
4027 case UR_SET_ADDRESS:
4028 if (value >= BCE_VHCI_MAX_DEVICES) {
4029 err = USB_ERR_IOERROR;
4030 break;
4031 }
4032 break;
4033
4034 case UR_SET_CONFIG:
4035 case UR_SET_INTERFACE:
4036 break;
4037
4038 case UR_SET_FEATURE:
4039 switch (req->bmRequestType) {
4040 case UT_WRITE_CLASS_OTHER:
4041 /* SetPortFeature */
4042 if (index < 1 || index > vhci->sc_port_count) {
4043 err = USB_ERR_IOERROR;
4044 break;
4045 }
4046 switch (value) {
4047 case UHF_PORT_POWER:
4048 if (vhci->sc_port_power[index - 1] == 0) {
4049 struct bce_vhci_message cmd_pw;
4050 struct bce_vhci_message reply_pw;
4051 int pw_port = index - 1;
4052 int pw_err;
4053
4054 memset(&cmd_pw, 0, sizeof(cmd_pw));
4055 cmd_pw.cmd = BCE_VHCI_CMD_PORT_POWER_ON;
4056 cmd_pw.param1 = pw_port;
4057
4058 USB_BUS_UNLOCK(&vhci->sc_bus);
4059 pw_err = bce_vhci_cmd_execute(vhci,
4060 &cmd_pw, &reply_pw,
4061 BCE_VHCI_CMD_TIMEOUT_SHORT);
4062 USB_BUS_LOCK(&vhci->sc_bus);
4063 if (pw_err != 0) {
4064 device_printf(vhci->sc_dev,
4065 "PORT_POWER_ON(%d)"
4066 " failed\n", pw_port);
4067 err = USB_ERR_IOERROR;
4068 } else {
4069 vhci->sc_port_power[pw_port] =
4070 1;
4071 vhci->sc_port_status[pw_port] |=
4072 UPS_PORT_POWER;
4073 }
4074 }
4075 break;
4076 case UHF_PORT_RESET:
4077 {
4078 int reset_err;
4079
4080 vhci->sc_port_status[index - 1] |=
4081 UPS_RESET;
4082
4083 /*
4084 * Drop bus lock for firmware I/O.
4085 * Explore thread is single-threaded
4086 * so this is safe.
4087 */
4088 USB_BUS_UNLOCK(&vhci->sc_bus);
4089
4090 /* Destroy existing device before re-creating */
4091 bce_vhci_device_destroy(vhci, index - 1);
4092
4093 reset_err = bce_vhci_device_create(vhci,
4094 index - 1);
4095 USB_BUS_LOCK(&vhci->sc_bus);
4096
4097 vhci->sc_port_status[index - 1] &=
4098 ~UPS_RESET;
4099 if (reset_err == 0) {
4100 vhci->sc_port_status[index - 1] |=
4101 UPS_PORT_ENABLED |
4102 UPS_HIGH_SPEED;
4103 vhci->sc_port_change[index - 1] |=
4104 UPS_C_PORT_RESET;
4105 } else {
4106 device_printf(vhci->sc_dev,
4107 "port %d reset failed: %d\n",
4108 index, reset_err);
4109 err = USB_ERR_IOERROR;
4110 }
4111 break;
4112 }
4113 case UHF_PORT_ENABLE:
4114 vhci->sc_port_status[index - 1] |=
4115 UPS_PORT_ENABLED;
4116 break;
4117 case UHF_PORT_SUSPEND:
4118 vhci->sc_port_status[index - 1] |=
4119 UPS_SUSPEND;
4120 break;
4121 default:
4122 err = USB_ERR_IOERROR;
4123 break;
4124 }
4125 break;
4126 default:
4127 err = USB_ERR_IOERROR;
4128 break;
4129 }
4130 break;
4131
4132 case UR_GET_CONFIG:
4133 len = 1;
4134 ptr = "\x01"; /* config 1 */
4135 break;
4136
4137 default:
4138 err = USB_ERR_IOERROR;
4139 break;
4140 }
4141
4142 if (err == 0) {
4143 if (pptr != NULL)
4144 *pptr = ptr;
4145 if (plength != NULL)
4146 *plength = len;
4147 }
4148 return (err);
4149 }
4150
4151 static void
bce_vhci_endpoint_init(struct usb_device * udev,struct usb_endpoint_descriptor * edesc,struct usb_endpoint * ep)4152 bce_vhci_endpoint_init(struct usb_device *udev,
4153 struct usb_endpoint_descriptor *edesc, struct usb_endpoint *ep)
4154 {
4155
4156 ep->methods = &bce_vhci_pipe_methods;
4157 }
4158
4159 static void
bce_vhci_xfer_setup(struct usb_setup_params * parm)4160 bce_vhci_xfer_setup(struct usb_setup_params *parm)
4161 {
4162 struct usb_xfer *xfer = parm->curr_xfer;
4163
4164 parm->hc_max_packet_size = 0x400; /* 1024 */
4165 parm->hc_max_packet_count = 1;
4166 parm->hc_max_frame_size = BCE_VHCI_XFER_BUFSZ;
4167
4168 usbd_transfer_setup_sub(parm);
4169
4170 if (parm->err)
4171 return;
4172
4173 /* No HCD-specific TD/QH structures needed */
4174 xfer->flags_int.bdma_enable = 0;
4175 }
4176
4177 static void
bce_vhci_xfer_unsetup(struct usb_xfer * xfer)4178 bce_vhci_xfer_unsetup(struct usb_xfer *xfer)
4179 {
4180 /* Nothing to free */
4181 }
4182
4183 static void
bce_vhci_get_dma_delay(struct usb_device * udev,uint32_t * pus)4184 bce_vhci_get_dma_delay(struct usb_device *udev, uint32_t *pus)
4185 {
4186
4187 *pus = 0; /* No hardware DMA delay */
4188 }
4189
4190 static void
bce_vhci_pipe_open(struct usb_xfer * xfer)4191 bce_vhci_pipe_open(struct usb_xfer *xfer)
4192 {
4193 /* Nothing to do; endpoint resources managed elsewhere */
4194 }
4195
4196 static void
bce_vhci_pipe_close(struct usb_xfer * xfer)4197 bce_vhci_pipe_close(struct usb_xfer *xfer)
4198 {
4199 struct bce_vhci_softc *vhci;
4200 int i;
4201
4202 vhci = (struct bce_vhci_softc *)xfer->xroot->bus;
4203
4204 /*
4205 * If this xfer is the active transfer on any endpoint,
4206 * clear it. We do not flush the firmware SQ here because
4207 * USB_BUS_LOCK is held (cannot sleep). Stale SQ completions
4208 * are discarded in tq_completion (active_xfer == NULL check).
4209 */
4210 for (i = 0; i < BCE_VHCI_MAX_DEVICES; i++) {
4211 struct bce_vhci_device *dev = &vhci->sc_devs[i];
4212 int j;
4213
4214 if (dev->allocated == 0)
4215 continue;
4216 for (j = 0; j < BCE_VHCI_MAX_ENDPOINTS; j++) {
4217 struct bce_vhci_transfer_queue *tq = &dev->tq[j];
4218
4219 if (tq->active_xfer == xfer) {
4220 tq->active_xfer = NULL;
4221 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
4222 tq->ctrl_data_done = 0;
4223 tq->ctrl_status_pending = 0;
4224 tq->evt_pending = 0;
4225 }
4226 if (tq->pending_xfer == xfer)
4227 tq->pending_xfer = NULL;
4228 if (tq->create_xfer == xfer) {
4229 tq->create_xfer = NULL;
4230 tq->create_pending = 0;
4231 }
4232 }
4233 }
4234
4235 /* Cancel any pending transfer */
4236 if (xfer->flags_int.transferring) {
4237 usbd_transfer_done(xfer, USB_ERR_CANCELLED);
4238 }
4239 }
4240
4241 static void
bce_vhci_pipe_enter(struct usb_xfer * xfer)4242 bce_vhci_pipe_enter(struct usb_xfer *xfer)
4243 {
4244 /* Called before start, can validate */
4245 }
4246
4247 /*
4248 * Start a transfer.
4249 *
4250 * Called with USB_BUS_LOCK held. For control transfers, we parse the
4251 * setup packet, record the direction and data length, set the
4252 * endpoint state machine to SETUP, and wait for firmware
4253 * TRANSFER_REQUEST events to drive the transfer forward.
4254 *
4255 * For interrupt/bulk, we STALL for now (not yet implemented).
4256 */
4257 static void
bce_vhci_pipe_start(struct usb_xfer * xfer)4258 bce_vhci_pipe_start(struct usb_xfer *xfer)
4259 {
4260 struct bce_vhci_softc *vhci;
4261 struct bce_vhci_device *dev;
4262 struct bce_vhci_transfer_queue *tq;
4263 struct usb_device_request setup;
4264 uint8_t xfer_type;
4265 uint8_t ep_addr;
4266
4267 vhci = (struct bce_vhci_softc *)xfer->xroot->bus;
4268 xfer_type = xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE;
4269 ep_addr = xfer->endpointno;
4270
4271 if (xfer_type == UE_INTERRUPT || xfer_type == UE_BULK) {
4272 struct usb_device *udev = xfer->xroot->udev;
4273 struct bce_vhci_message treq;
4274 uint8_t port, fw_dev_id, ep_idx;
4275 uint32_t len;
4276
4277 port = udev->port_no;
4278 if (port < 1 || port > vhci->sc_port_count) {
4279 usbd_transfer_done(xfer, USB_ERR_STALLED);
4280 return;
4281 }
4282
4283 fw_dev_id = vhci->sc_port_to_dev[port - 1];
4284 if (fw_dev_id >= BCE_VHCI_MAX_DEVICES) {
4285 usbd_transfer_done(xfer, USB_ERR_STALLED);
4286 return;
4287 }
4288
4289 dev = &vhci->sc_devs[fw_dev_id];
4290 if (dev->allocated == 0) {
4291 usbd_transfer_done(xfer, USB_ERR_STALLED);
4292 return;
4293 }
4294
4295 ep_idx = bce_vhci_ep_index(ep_addr);
4296 tq = &dev->tq[ep_idx];
4297
4298 /* Create endpoint with firmware if not yet active */
4299 if (tq->active == 0) {
4300 /*
4301 * Cannot sleep here: we may be in a USB callback
4302 * (e.g. usbhid_intr_in_callback) holding a
4303 * non-sleepable lock. Defer to taskqueue_thread
4304 * and return STALLED so the USB stack retries.
4305 */
4306 if (vhci->sc_detaching == 0 &&
4307 tq->create_xfer == NULL) {
4308 tq->endp_addr = ep_addr;
4309 tq->dev_addr = fw_dev_id;
4310 tq->create_pending = 1;
4311 tq->create_edesc = xfer->endpoint->edesc;
4312 /* held; task submits it */
4313 tq->create_xfer = xfer;
4314 taskqueue_enqueue(taskqueue_thread,
4315 &vhci->sc_create_task);
4316 } else {
4317 usbd_transfer_done(xfer, USB_ERR_STALLED);
4318 }
4319 return;
4320 }
4321
4322 if (tq->active_xfer != NULL || tq->dma_inflight != 0) {
4323 /*
4324 * Pipeline: queue xfer for when active_xfer finishes.
4325 * Also queue if old DMA is still inflight (pipe_close
4326 * cleared active_xfer but completion not yet seen).
4327 * Returning STALLED triggers stall recovery
4328 * (CLEAR_FEATURE loop); hold the xfer instead.
4329 */
4330 if (tq->pending_xfer == NULL)
4331 tq->pending_xfer = xfer;
4332 else
4333 usbd_transfer_done(xfer, USB_ERR_CANCELLED);
4334 return;
4335 }
4336
4337 tq->active_xfer = xfer;
4338 tq->dma_inflight = 1;
4339
4340 /*
4341 * If firmware already sent a TRANSFER_REQUEST before
4342 * the USB stack called pipe_start, replay it now
4343 * instead of sending a duplicate host request.
4344 */
4345 if (tq->evt_pending) {
4346 tq->evt_pending = 0;
4347 bce_vhci_handle_transfer_request(vhci,
4348 &tq->evt_saved);
4349 return;
4350 }
4351
4352 if (ep_addr & UE_DIR_IN) {
4353 /* IN transfer: reserve msg first, then SQ */
4354 struct bce_qe_submission *si;
4355
4356 len = xfer->frlengths[0];
4357 if (len > BCE_VHCI_XFER_BUFSZ)
4358 len = BCE_VHCI_XFER_BUFSZ;
4359
4360 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
4361 BUS_DMASYNC_PREREAD);
4362
4363 memset(&treq, 0, sizeof(treq));
4364 treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST;
4365 treq.param1 = (ep_addr << 8) | tq->dev_addr;
4366 treq.param2 = len;
4367
4368 /* Reserve msg slot first */
4369 mtx_lock_spin(&vhci->sc_async_lock);
4370 if (bce_reserve_submission(
4371 vhci->msg_asynchronous.sq) != 0) {
4372 mtx_unlock_spin(&vhci->sc_async_lock);
4373 tq->active_xfer = NULL;
4374 tq->dma_inflight = 0;
4375 usbd_transfer_done(xfer, USB_ERR_IOERROR);
4376 return;
4377 }
4378 mtx_unlock_spin(&vhci->sc_async_lock);
4379
4380 /* Then reserve SQ slot */
4381 mtx_lock_spin(&tq->lock);
4382 if (bce_reserve_submission(tq->sq_in) != 0) {
4383 mtx_unlock_spin(&tq->lock);
4384 /* Restore msg slot reserved but won't use */
4385 mtx_lock_spin(&vhci->sc_async_lock);
4386 atomic_add_int(&vhci->
4387 msg_asynchronous.sq->
4388 available_commands, 1);
4389 mtx_unlock_spin(&vhci->sc_async_lock);
4390 tq->active_xfer = NULL;
4391 tq->dma_inflight = 0;
4392 usbd_transfer_done(xfer, USB_ERR_IOERROR);
4393 return;
4394 }
4395
4396 si = bce_next_submission(tq->sq_in);
4397 si->addr = tq->dma_addr;
4398 si->length = len;
4399 si->segl_addr = 0;
4400 si->segl_length = 0;
4401 bce_submit_to_device(vhci->sc_bce, tq->sq_in);
4402 mtx_unlock_spin(&tq->lock);
4403
4404 mtx_lock_spin(&vhci->sc_async_lock);
4405 bce_vhci_msg_queue_write(vhci,
4406 &vhci->msg_asynchronous, &treq);
4407 mtx_unlock_spin(&vhci->sc_async_lock);
4408 } else {
4409 /* OUT transfer: reserve msg first, then SQ */
4410 struct bce_qe_submission *so;
4411
4412 len = xfer->frlengths[0];
4413 if (len > BCE_VHCI_XFER_BUFSZ)
4414 len = BCE_VHCI_XFER_BUFSZ;
4415
4416 usbd_copy_out(&xfer->frbuffers[0], 0,
4417 tq->dma_buf, len);
4418 bus_dmamap_sync(tq->dma_tag, tq->dma_map,
4419 BUS_DMASYNC_PREWRITE);
4420
4421 memset(&treq, 0, sizeof(treq));
4422 treq.cmd = BCE_VHCI_CMD_TRANSFER_REQUEST;
4423 treq.param1 = (ep_addr << 8) | tq->dev_addr;
4424 treq.param2 = len;
4425
4426 /* Reserve msg slot first */
4427 mtx_lock_spin(&vhci->sc_async_lock);
4428 if (bce_reserve_submission(
4429 vhci->msg_asynchronous.sq) != 0) {
4430 mtx_unlock_spin(&vhci->sc_async_lock);
4431 tq->active_xfer = NULL;
4432 tq->dma_inflight = 0;
4433 usbd_transfer_done(xfer, USB_ERR_IOERROR);
4434 return;
4435 }
4436 mtx_unlock_spin(&vhci->sc_async_lock);
4437
4438 /* Then reserve SQ slot */
4439 mtx_lock_spin(&tq->lock);
4440 if (bce_reserve_submission(tq->sq_out) == 0) {
4441 so = bce_next_submission(tq->sq_out);
4442 so->addr = tq->dma_addr;
4443 so->length = len;
4444 so->segl_addr = 0;
4445 so->segl_length = 0;
4446 bce_submit_to_device(vhci->sc_bce,
4447 tq->sq_out);
4448 } else {
4449 mtx_unlock_spin(&tq->lock);
4450 /* Restore msg slot reserved but won't use */
4451 mtx_lock_spin(&vhci->sc_async_lock);
4452 atomic_add_int(&vhci->
4453 msg_asynchronous.sq->
4454 available_commands, 1);
4455 mtx_unlock_spin(&vhci->sc_async_lock);
4456 tq->active_xfer = NULL;
4457 tq->dma_inflight = 0;
4458 usbd_transfer_done(xfer, USB_ERR_IOERROR);
4459 return;
4460 }
4461 mtx_unlock_spin(&tq->lock);
4462
4463 mtx_lock_spin(&vhci->sc_async_lock);
4464 bce_vhci_msg_queue_write(vhci,
4465 &vhci->msg_asynchronous, &treq);
4466 mtx_unlock_spin(&vhci->sc_async_lock);
4467 }
4468 return;
4469 }
4470
4471 if (xfer_type != UE_CONTROL) {
4472 device_printf(vhci->sc_dev,
4473 "xfer start ep=0x%02x type=%d (not supported)\n",
4474 ep_addr, xfer_type);
4475 usbd_transfer_done(xfer, USB_ERR_STALLED);
4476 return;
4477 }
4478
4479 /*
4480 * Control transfer on ep0. Map the USB device's port number
4481 * to the firmware device ID. The USB stack's port_no comes
4482 * from our root hub, so it maps directly to our port index.
4483 *
4484 * For the root hub itself, the USB stack handles it via
4485 * roothub_exec, so we should never see it here.
4486 */
4487 {
4488 struct usb_device *udev = xfer->xroot->udev;
4489 uint8_t port, fw_dev_id;
4490
4491 port = udev->port_no;
4492 if (port < 1 || port > vhci->sc_port_count) {
4493 device_printf(vhci->sc_dev,
4494 "control xfer: invalid port %d\n", port);
4495 usbd_transfer_done(xfer, USB_ERR_STALLED);
4496 return;
4497 }
4498
4499 fw_dev_id = vhci->sc_port_to_dev[port - 1];
4500 if (fw_dev_id >= BCE_VHCI_MAX_DEVICES) {
4501 device_printf(vhci->sc_dev,
4502 "control xfer: no firmware device for "
4503 "port %d\n", port);
4504 usbd_transfer_done(xfer, USB_ERR_STALLED);
4505 return;
4506 }
4507
4508 dev = &vhci->sc_devs[fw_dev_id];
4509 if (dev->allocated == 0 || dev->tq[0].active == 0) {
4510 device_printf(vhci->sc_dev,
4511 "control xfer: device %d ep0 not ready\n",
4512 fw_dev_id);
4513 usbd_transfer_done(xfer, USB_ERR_STALLED);
4514 return;
4515 }
4516
4517 tq = &dev->tq[0];
4518 }
4519
4520 /*
4521 * If the endpoint is stalled from a previous transfer, we need
4522 * ENDPOINT_RESET (0x0044) before the next transfer can succeed.
4523 * We cannot sleep in pipe_start (USB device mutex held), so
4524 * the reset runs asynchronously on taskqueue_thread.
4525 *
4526 * Return USB_ERR_STALLED to the USB stack so it retries; the
4527 * retry will succeed once sc_reset_task clears tq->stalled.
4528 */
4529 if (tq->stalled) {
4530 device_printf(vhci->sc_dev,
4531 "control xfer: ep0 stalled, scheduling ENDPOINT_RESET "
4532 "(dev=%d)\n", tq->dev_addr);
4533 if (vhci->sc_detaching == 0)
4534 taskqueue_enqueue(taskqueue_thread,
4535 &vhci->sc_reset_task);
4536 usbd_transfer_done(xfer, USB_ERR_STALLED);
4537 return;
4538 }
4539
4540 if (tq->active_xfer != NULL || tq->dma_inflight != 0) {
4541 device_printf(vhci->sc_dev,
4542 "control xfer: ep0 busy (dev=%d)\n", tq->dev_addr);
4543 usbd_transfer_done(xfer, USB_ERR_STALLED);
4544 return;
4545 }
4546
4547 /* Read the 8-byte setup packet from frbuffers[0] */
4548 usbd_copy_out(&xfer->frbuffers[0], 0, &setup, sizeof(setup));
4549
4550 /* Determine data direction and length from setup packet */
4551 tq->ctrl_dir = (setup.bmRequestType & UT_READ) ?
4552 UE_DIR_IN : UE_DIR_OUT;
4553 tq->ctrl_data_len = UGETW(setup.wLength);
4554 tq->ctrl_actual = 0;
4555 tq->ctrl_data_done = 0;
4556 tq->ctrl_status_pending = 0;
4557 tq->active_xfer = xfer;
4558 tq->ctrl_state = BCE_VHCI_CTRL_SETUP;
4559
4560
4561 /*
4562 * SET_ADDRESS is handled by firmware via DEVICE_CREATE --
4563 * complete immediately without forwarding to firmware.
4564 */
4565 if (setup.bRequest == UR_SET_ADDRESS) {
4566 tq->ctrl_state = BCE_VHCI_CTRL_IDLE;
4567 tq->active_xfer = NULL;
4568 xfer->aframes = xfer->nframes;
4569 usbd_transfer_done(xfer, USB_ERR_NORMAL_COMPLETION);
4570 return;
4571 }
4572
4573 /*
4574 * Check for a deferred TRANSFER_REQUEST that arrived before
4575 * pipe_start. If one is pending, replay it now.
4576 */
4577 if (tq->evt_pending) {
4578 tq->evt_pending = 0;
4579 bce_vhci_handle_transfer_request(vhci, &tq->evt_saved);
4580 }
4581 }
4582
4583 /*
4584 * DMA tag callback (required by usb_bus_mem_alloc_all)
4585 */
4586
4587 static void
bce_vhci_iterate_hw_softc(struct usb_bus * bus,usb_bus_mem_sub_cb_t * cb)4588 bce_vhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb)
4589 {
4590 /* No hardware-specific DMA pages needed */
4591 }
4592
4593 static int
bce_vhci_probe(device_t dev)4594 bce_vhci_probe(device_t dev)
4595 {
4596
4597 device_set_desc(dev, "Apple T2 BCE Virtual USB Host Controller");
4598 return (BUS_PROBE_DEFAULT);
4599 }
4600
4601 static int
bce_vhci_attach_dev(device_t dev)4602 bce_vhci_attach_dev(device_t dev)
4603 {
4604 struct bce_vhci_softc *vhci;
4605 struct apple_bce_softc *bce;
4606 int err;
4607
4608 vhci = device_get_softc(dev);
4609 bce = device_get_softc(device_get_parent(dev));
4610 if (bce == NULL) {
4611 device_printf(dev, "no BCE parent\n");
4612 return (ENXIO);
4613 }
4614
4615 vhci->sc_dev = dev;
4616 vhci->sc_bce = bce;
4617
4618 /* Sanity check parent state */
4619 if (bce->sc_cmd_cmdq == NULL || bce->sc_dma_tag == NULL) {
4620 device_printf(dev,
4621 "BCE parent not ready (cmdq=%p dma_tag=%p)\n",
4622 bce->sc_cmd_cmdq, bce->sc_dma_tag);
4623 return (ENXIO);
4624 }
4625
4626 mtx_init(&vhci->sc_async_lock, "bce_vhci_async", NULL, MTX_SPIN);
4627 mtx_init(&vhci->sc_fwevt_lock, "bce_vhci_fwevt", NULL, MTX_SPIN);
4628
4629 /* Initialize device state */
4630 memset(vhci->sc_devs, 0, sizeof(vhci->sc_devs));
4631 memset(vhci->sc_port_to_dev, 0xFF, sizeof(vhci->sc_port_to_dev));
4632
4633 TASK_INIT(&vhci->sc_fwevt_task, 0, bce_vhci_fwevt_task, vhci);
4634 TASK_INIT(&vhci->sc_reset_task, 0, bce_vhci_reset_task, vhci);
4635 TASK_INIT(&vhci->sc_create_task, 0, bce_vhci_create_task, vhci);
4636 TASK_INIT(&vhci->sc_port_chg_task, 0, bce_vhci_port_chg_task, vhci);
4637
4638 /*
4639 * Initialize USB bus early so bus_mtx is valid before
4640 * firmware events can call USB_BUS_LOCK.
4641 */
4642 vhci->sc_bus.parent = dev;
4643 vhci->sc_bus.devices = vhci->sc_devices;
4644 vhci->sc_bus.devices_max = BCE_VHCI_MAX_DEVICES;
4645 vhci->sc_bus.dma_bits = 32;
4646 vhci->sc_bus.usbrev = USB_REV_2_0;
4647 vhci->sc_bus.methods = &bce_vhci_bus_methods;
4648
4649 err = usb_bus_mem_alloc_all(&vhci->sc_bus,
4650 USB_GET_DMA_TAG(dev), &bce_vhci_iterate_hw_softc);
4651 if (err != 0) {
4652 device_printf(dev, "usb_bus_mem_alloc_all failed: %d\n", err);
4653 goto fail;
4654 }
4655
4656 /*
4657 * Create BCE message/event queues for VHCI communication.
4658 */
4659 err = bce_vhci_create_queues(vhci);
4660 if (err != 0) {
4661 device_printf(dev, "failed to create VHCI queues: %d\n", err);
4662 goto fail_mem;
4663 }
4664
4665 /*
4666 * Start controller: ENABLE -> discover ports -> START -> power on.
4667 */
4668 err = bce_vhci_start_controller(vhci);
4669 if (err != 0) {
4670 device_printf(dev,
4671 "failed to start VHCI controller: %d\n", err);
4672 goto fail_queues;
4673 }
4674
4675 /* Create usbus child */
4676 vhci->sc_bus.bdev = device_add_child(dev, "usbus", DEVICE_UNIT_ANY);
4677 if (vhci->sc_bus.bdev == NULL) {
4678 device_printf(dev, "failed to add usbus child\n");
4679 err = ENOMEM;
4680 goto fail_ctrl;
4681 }
4682 device_set_ivars(vhci->sc_bus.bdev, &vhci->sc_bus);
4683
4684 err = device_probe_and_attach(vhci->sc_bus.bdev);
4685 if (err != 0) {
4686 device_printf(dev, "usbus attach failed: %d\n", err);
4687 goto fail_child;
4688 }
4689
4690 device_printf(dev, "BCE VHCI attached, %d ports\n",
4691 vhci->sc_port_count);
4692 return (0);
4693
4694 fail_child:
4695 device_delete_child(dev, vhci->sc_bus.bdev);
4696 fail_ctrl:
4697 /* Disable controller before tearing down queues */
4698 if (vhci->sc_started && vhci->msg_commands.sq != NULL) {
4699 struct bce_vhci_message cmd_dis, reply_dis;
4700
4701 memset(&cmd_dis, 0, sizeof(cmd_dis));
4702 cmd_dis.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE;
4703 bce_vhci_cmd_execute(vhci, &cmd_dis, &reply_dis,
4704 BCE_VHCI_CMD_TIMEOUT_LONG);
4705 vhci->sc_started = 0;
4706 }
4707 fail_queues:
4708 vhci->sc_detaching = 1;
4709 taskqueue_drain(taskqueue_thread, &vhci->sc_reset_task);
4710 taskqueue_drain(taskqueue_thread, &vhci->sc_create_task);
4711 taskqueue_drain(taskqueue_thread, &vhci->sc_port_chg_task);
4712 taskqueue_drain(taskqueue_thread, &vhci->sc_fwevt_task);
4713 bce_vhci_destroy_queues(vhci);
4714 fail_mem:
4715 usb_bus_mem_free_all(&vhci->sc_bus, &bce_vhci_iterate_hw_softc);
4716 fail:
4717 mtx_destroy(&vhci->sc_fwevt_lock);
4718 mtx_destroy(&vhci->sc_async_lock);
4719 return (err);
4720 }
4721
4722 static int
bce_vhci_detach_dev(device_t dev)4723 bce_vhci_detach_dev(device_t dev)
4724 {
4725 struct bce_vhci_softc *vhci;
4726
4727 vhci = device_get_softc(dev);
4728
4729 /* Stop deferred work before tearing down USB child */
4730 vhci->sc_detaching = 1;
4731
4732 /* Detach usbus child */
4733 if (vhci->sc_bus.bdev != NULL) {
4734 int err;
4735
4736 err = device_delete_children(dev);
4737 if (err != 0) {
4738 vhci->sc_detaching = 0;
4739 return (err);
4740 }
4741 vhci->sc_bus.bdev = NULL;
4742 }
4743
4744 /*
4745 * Drain tasks that may reference tq state before
4746 * destroying endpoints.
4747 */
4748 taskqueue_drain(taskqueue_thread, &vhci->sc_reset_task);
4749 taskqueue_drain(taskqueue_thread, &vhci->sc_create_task);
4750 taskqueue_drain(taskqueue_thread, &vhci->sc_port_chg_task);
4751
4752 /* Destroy all firmware devices and their endpoints */
4753 {
4754 int i;
4755
4756 for (i = 0; i < BCE_VHCI_MAX_PORTS; i++)
4757 bce_vhci_device_destroy(vhci, i);
4758 }
4759
4760 /* Send CONTROLLER_DISABLE if we started */
4761 if (vhci->sc_started && vhci->msg_commands.sq != NULL) {
4762 struct bce_vhci_message cmd, reply;
4763
4764 memset(&cmd, 0, sizeof(cmd));
4765 cmd.cmd = BCE_VHCI_CMD_CONTROLLER_DISABLE;
4766 bce_vhci_cmd_execute(vhci, &cmd, &reply,
4767 BCE_VHCI_CMD_TIMEOUT_LONG);
4768 vhci->sc_started = 0;
4769 }
4770
4771 /* Drain firmware event task after command completion */
4772 taskqueue_drain(taskqueue_thread, &vhci->sc_fwevt_task);
4773
4774 /* Tear down VHCI queues (unregisters from IRQ dispatch first) */
4775 bce_vhci_destroy_queues(vhci);
4776
4777 usb_bus_mem_free_all(&vhci->sc_bus, &bce_vhci_iterate_hw_softc);
4778 mtx_destroy(&vhci->sc_fwevt_lock);
4779 mtx_destroy(&vhci->sc_async_lock);
4780
4781 return (0);
4782 }
4783
4784 int
bce_vhci_attach(struct apple_bce_softc * sc)4785 bce_vhci_attach(struct apple_bce_softc *sc)
4786 {
4787 device_t vhci_dev;
4788
4789 vhci_dev = device_add_child(sc->sc_dev, "bce_vhci", DEVICE_UNIT_ANY);
4790 if (vhci_dev == NULL) {
4791 device_printf(sc->sc_dev,
4792 "failed to add bce_vhci child\n");
4793 return (ENOMEM);
4794 }
4795 sc->sc_vhci_dev = vhci_dev;
4796 device_set_ivars(vhci_dev, sc);
4797
4798 if (device_probe_and_attach(vhci_dev) != 0) {
4799 device_delete_child(sc->sc_dev, vhci_dev);
4800 sc->sc_vhci_dev = NULL;
4801 return (ENXIO);
4802 }
4803 return (0);
4804 }
4805
4806 int
bce_vhci_detach(struct apple_bce_softc * sc)4807 bce_vhci_detach(struct apple_bce_softc *sc)
4808 {
4809 device_t vhci_dev;
4810 int error;
4811
4812 vhci_dev = sc->sc_vhci_dev;
4813 if (vhci_dev == NULL)
4814 return (0);
4815
4816 error = device_delete_child(sc->sc_dev, vhci_dev);
4817 if (error == 0)
4818 sc->sc_vhci_dev = NULL;
4819
4820 return (error);
4821 }
4822