145 	struct bce_queue_sq *sq;  in bce_handle_cq_completions()  local
157 			sq = (struct bce_queue_sq *)sc->sc_queues[e->qid];  in bce_handle_cq_completions()
158 			if (sq != NULL &&  in bce_handle_cq_completions()
159 			    e->completion_index < sq->el_count) {  in bce_handle_cq_completions()
160 				sq->completion_data[e->completion_index].status =  in bce_handle_cq_completions()
162 				sq->completion_data[e->completion_index].data_size =  in bce_handle_cq_completions()
164 				sq->completion_data[e->completion_index].result =  in bce_handle_cq_completions()
167 				sq->completion_tail =  in bce_handle_cq_completions()
168 				    (e->completion_index + 1) % sq->el_count;  in bce_handle_cq_completions()
169 				sq->has_pending = 1;  in bce_handle_cq_completions()
185 		sq = sc->sc_int_sq_list[i];  in bce_handle_cq_completions()
186 		if (sq != NULL && sq->has_pending) {  in bce_handle_cq_completions()
187 			sq->has_pending = 0;  in bce_handle_cq_completions()
188 			if (sq->completion != NULL)  in bce_handle_cq_completions()
189 				sq->completion(sq);  in bce_handle_cq_completions()
201 	struct bce_queue_sq *sq;  in bce_alloc_sq()  local
205 	sq = malloc(sizeof(*sq), M_BCE, M_WAITOK | M_ZERO);  in bce_alloc_sq()
206 	sq->qid = qid;  in bce_alloc_sq()
207 	sq->el_size = el_size;  in bce_alloc_sq()
208 	sq->el_count = el_count;  in bce_alloc_sq()
209 	sq->completion = compl;  in bce_alloc_sq()
210 	sq->userdata = userdata;  in bce_alloc_sq()
211 	sq->available_commands = el_count - 1;  in bce_alloc_sq()
220 	    &sq->dma_tag);  in bce_alloc_sq()
222 		free(sq, M_BCE);  in bce_alloc_sq()
226 	error = bus_dmamem_alloc(sq->dma_tag, &sq->data,  in bce_alloc_sq()
227 	    BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_COHERENT, &sq->dma_map);  in bce_alloc_sq()
229 		bus_dma_tag_destroy(sq->dma_tag);  in bce_alloc_sq()
230 		free(sq, M_BCE);  in bce_alloc_sq()
234 	error = bus_dmamap_load(sq->dma_tag, sq->dma_map, sq->data,  in bce_alloc_sq()
237 		bus_dmamem_free(sq->dma_tag, sq->data, sq->dma_map);  in bce_alloc_sq()
238 		bus_dma_tag_destroy(sq->dma_tag);  in bce_alloc_sq()
239 		free(sq, M_BCE);  in bce_alloc_sq()
242 	sq->dma_addr = cb.addr;  in bce_alloc_sq()
244 	sq->completion_data = malloc(sizeof(*sq->completion_data) * el_count,  in bce_alloc_sq()
247 	return (sq);  in bce_alloc_sq()
251 bce_free_sq(struct apple_bce_softc *sc, struct bce_queue_sq *sq)  in bce_free_sq()  argument
253 	if (sq == NULL)  in bce_free_sq()
255 	free(sq->completion_data, M_BCE);  in bce_free_sq()
256 	bus_dmamap_unload(sq->dma_tag, sq->dma_map);  in bce_free_sq()
257 	bus_dmamem_free(sq->dma_tag, sq->data, sq->dma_map);  in bce_free_sq()
258 	bus_dma_tag_destroy(sq->dma_tag);  in bce_free_sq()
259 	free(sq, M_BCE);  in bce_free_sq()
263 bce_get_sq_memcfg(struct bce_queue_sq *sq, struct bce_queue_cq *cq,  in bce_get_sq_memcfg()  argument
266 	cfg->qid = (uint16_t)sq->qid;  in bce_get_sq_memcfg()
267 	cfg->el_count = (uint16_t)sq->el_count;  in bce_get_sq_memcfg()
270 	cfg->addr = sq->dma_addr;  in bce_get_sq_memcfg()
271 	cfg->length = sq->el_count * sq->el_size;  in bce_get_sq_memcfg()
275 bce_reserve_submission(struct bce_queue_sq *sq)  in bce_reserve_submission()  argument
280 		old = atomic_load_int(&sq->available_commands);  in bce_reserve_submission()
284 	} while (!atomic_cmpset_int(&sq->available_commands, old, new));  in bce_reserve_submission()
290 bce_next_submission(struct bce_queue_sq *sq)  in bce_next_submission()  argument
294 	ret = bce_sq_element(sq, sq->tail);  in bce_next_submission()
295 	sq->tail = (sq->tail + 1) % sq->el_count;  in bce_next_submission()
300 bce_submit_to_device(struct apple_bce_softc *sc, struct bce_queue_sq *sq)  in bce_submit_to_device()  argument
302 	bus_dmamap_sync(sq->dma_tag, sq->dma_map, BUS_DMASYNC_PREWRITE);  in bce_submit_to_device()
303 	bus_write_4(sc->sc_bar2, BCE_REG_DOORBELL_BASE + sq->qid * 4,  in bce_submit_to_device()
304 	    sq->tail);  in bce_submit_to_device()
308 bce_notify_submission_complete(struct bce_queue_sq *sq)  in bce_notify_submission_complete()  argument
310 	sq->head = (sq->head + 1) % sq->el_count;  in bce_notify_submission_complete()
311 	atomic_add_int(&sq->available_commands, 1);  in bce_notify_submission_complete()
317 static void	bce_cmdq_completion(struct bce_queue_sq *sq);
320 bce_alloc_cmdq(struct apple_bce_softc *sc, struct bce_queue_sq *sq)  in bce_alloc_cmdq()  argument
325 	cmdq->sq = sq;  in bce_alloc_cmdq()
327 	cmdq->tres = malloc(sizeof(void *) * sq->el_count, M_BCE,  in bce_alloc_cmdq()
331 	sq->completion = bce_cmdq_completion;  in bce_alloc_cmdq()
332 	sq->userdata = cmdq;  in bce_alloc_cmdq()
351 bce_cmdq_completion(struct bce_queue_sq *sq)  in bce_cmdq_completion()  argument
353 	struct bce_queue_cmdq *cmdq = sq->userdata;  in bce_cmdq_completion()
357 	while (sq->completion_cidx != sq->completion_tail) {  in bce_cmdq_completion()
360 		cd = &sq->completion_data[sq->completion_cidx];  in bce_cmdq_completion()
361 		res = cmdq->tres[sq->completion_cidx];  in bce_cmdq_completion()
366 			cmdq->tres[sq->completion_cidx] = NULL;  in bce_cmdq_completion()
368 		sq->completion_cidx = (sq->completion_cidx + 1) %  in bce_cmdq_completion()
369 		    sq->el_count;  in bce_cmdq_completion()
370 		bce_notify_submission_complete(sq);  in bce_cmdq_completion()
387 	if (bce_reserve_submission(cmdq->sq) != 0) {  in bce_cmd_register_queue()
393 	slot = cmdq->sq->tail;  in bce_cmd_register_queue()
395 	cmd = bce_next_submission(cmdq->sq);  in bce_cmd_register_queue()
411 	bce_submit_to_device(sc, cmdq->sq);  in bce_cmd_register_queue()
433 	if (bce_reserve_submission(cmdq->sq) != 0) {  in bce_cmd_unregister_queue()
439 	slot = cmdq->sq->tail;  in bce_cmd_unregister_queue()
441 	cmd = bce_next_submission(cmdq->sq);  in bce_cmd_unregister_queue()
447 	bce_submit_to_device(sc, cmdq->sq);  in bce_cmd_unregister_queue()
469 	if (bce_reserve_submission(cmdq->sq) != 0) {  in bce_cmd_flush_queue()
475 	slot = cmdq->sq->tail;  in bce_cmd_flush_queue()
477 	cmd = bce_next_submission(cmdq->sq);  in bce_cmd_flush_queue()
483 	bce_submit_to_device(sc, cmdq->sq);  in bce_cmd_flush_queue()