219 static void gsi_trans_map(struct gsi_trans *trans, u32 index)  in gsi_trans_map()  argument
221 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_map()
224 	index += trans->used_count - 1;  in gsi_trans_map()
227 	channel->trans_info.map[index % channel->tre_ring.count] = trans;  in gsi_trans_map()
250 	return &trans_info->trans[trans_id %= channel->tre_count];  in gsi_channel_trans_complete()
254 static void gsi_trans_move_committed(struct gsi_trans *trans)  in gsi_trans_move_committed()  argument
256 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_move_committed()
264 static void gsi_trans_move_pending(struct gsi_trans *trans)  in gsi_trans_move_pending()  argument
266 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_move_pending()
268 	u16 trans_index = trans - &trans_info->trans[0];  in gsi_trans_move_pending()
277 void gsi_trans_move_complete(struct gsi_trans *trans)  in gsi_trans_move_complete()  argument
279 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_move_complete()
281 	u16 trans_index = trans - trans_info->trans;  in gsi_trans_move_complete()
291 void gsi_trans_move_polled(struct gsi_trans *trans)  in gsi_trans_move_polled()  argument
293 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in gsi_trans_move_polled()
341 	struct gsi_trans *trans;  in gsi_channel_trans_alloc()  local
354 	trans = &trans_info->trans[trans_index];  in gsi_channel_trans_alloc()
355 	memset(trans, 0, sizeof(*trans));  in gsi_channel_trans_alloc()
358 	trans->gsi = gsi;  in gsi_channel_trans_alloc()
359 	trans->channel_id = channel_id;  in gsi_channel_trans_alloc()
360 	trans->rsvd_count = tre_count;  in gsi_channel_trans_alloc()
361 	init_completion(&trans->completion);  in gsi_channel_trans_alloc()
364 	trans->sgl = gsi_trans_pool_alloc(&trans_info->sg_pool, tre_count);  in gsi_channel_trans_alloc()
365 	sg_init_marker(trans->sgl, tre_count);  in gsi_channel_trans_alloc()
367 	trans->direction = direction;  in gsi_channel_trans_alloc()
368 	refcount_set(&trans->refcount, 1);  in gsi_channel_trans_alloc()
373 	return trans;  in gsi_channel_trans_alloc()
377 void gsi_trans_free(struct gsi_trans *trans)  in gsi_trans_free()  argument
381 	if (!refcount_dec_and_test(&trans->refcount))  in gsi_trans_free()
387 	trans_info = &trans->gsi->channel[trans->channel_id].trans_info;  in gsi_trans_free()
388 	if (!trans->used_count) {  in gsi_trans_free()
394 		ipa_gsi_trans_release(trans);  in gsi_trans_free()
403 	gsi_trans_tre_release(trans_info, trans->rsvd_count);  in gsi_trans_free()
407 void gsi_trans_cmd_add(struct gsi_trans *trans, void *buf, u32 size,  in gsi_trans_cmd_add()  argument
410 	u32 which = trans->used_count++;  in gsi_trans_cmd_add()
413 	WARN_ON(which >= trans->rsvd_count);  in gsi_trans_cmd_add()
429 	sg = &trans->sgl[which];  in gsi_trans_cmd_add()
434 	trans->cmd_opcode[which] = opcode;  in gsi_trans_cmd_add()
438 int gsi_trans_page_add(struct gsi_trans *trans, struct page *page, u32 size,  in gsi_trans_page_add()  argument
441 	struct scatterlist *sg = &trans->sgl[0];  in gsi_trans_page_add()
444 	if (WARN_ON(trans->rsvd_count != 1))  in gsi_trans_page_add()
446 	if (WARN_ON(trans->used_count))  in gsi_trans_page_add()
450 	ret = dma_map_sg(trans->gsi->dev, sg, 1, trans->direction);  in gsi_trans_page_add()
454 	trans->used_count++;	/* Transaction now owns the (DMA mapped) page */  in gsi_trans_page_add()
460 int gsi_trans_skb_add(struct gsi_trans *trans, struct sk_buff *skb)  in gsi_trans_skb_add()  argument
462 	struct scatterlist *sg = &trans->sgl[0];  in gsi_trans_skb_add()
466 	if (WARN_ON(trans->rsvd_count != 1))  in gsi_trans_skb_add()
468 	if (WARN_ON(trans->used_count))  in gsi_trans_skb_add()
477 	ret = dma_map_sg(trans->gsi->dev, sg, used_count, trans->direction);  in gsi_trans_skb_add()
482 	trans->used_count += used_count;  in gsi_trans_skb_add()
545 static void __gsi_trans_commit(struct gsi_trans *trans, bool ring_db)  in __gsi_trans_commit()  argument
547 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];  in __gsi_trans_commit()
558 	WARN_ON(!trans->used_count);  in __gsi_trans_commit()
565 	cmd_opcode = channel->command ? &trans->cmd_opcode[0] : NULL;  in __gsi_trans_commit()
568 	for_each_sg(trans->sgl, sg, trans->used_count, i) {  in __gsi_trans_commit()
569 		bool last_tre = i == trans->used_count - 1;  in __gsi_trans_commit()
583 	gsi_trans_map(trans, tre_ring->index);  in __gsi_trans_commit()
585 	tre_ring->index += trans->used_count;  in __gsi_trans_commit()
587 	trans->len = byte_count;  in __gsi_trans_commit()
589 		gsi_trans_tx_committed(trans);  in __gsi_trans_commit()
591 	gsi_trans_move_committed(trans);  in __gsi_trans_commit()
597 			gsi_trans_tx_queued(trans);  in __gsi_trans_commit()
598 		gsi_trans_move_pending(trans);  in __gsi_trans_commit()
604 void gsi_trans_commit(struct gsi_trans *trans, bool ring_db)  in gsi_trans_commit()  argument
606 	if (trans->used_count)  in gsi_trans_commit()
607 		__gsi_trans_commit(trans, ring_db);  in gsi_trans_commit()
609 		gsi_trans_free(trans);  in gsi_trans_commit()
613 void gsi_trans_commit_wait(struct gsi_trans *trans)  in gsi_trans_commit_wait()  argument
615 	if (!trans->used_count)  in gsi_trans_commit_wait()
618 	refcount_inc(&trans->refcount);  in gsi_trans_commit_wait()
620 	__gsi_trans_commit(trans, true);  in gsi_trans_commit_wait()
622 	wait_for_completion(&trans->completion);  in gsi_trans_commit_wait()
625 	gsi_trans_free(trans);  in gsi_trans_commit_wait()
629 void gsi_trans_complete(struct gsi_trans *trans)  in gsi_trans_complete()  argument
632 	if (trans->direction != DMA_NONE)  in gsi_trans_complete()
633 		dma_unmap_sg(trans->gsi->dev, trans->sgl, trans->used_count,  in gsi_trans_complete()
634 			     trans->direction);  in gsi_trans_complete()
636 	ipa_gsi_trans_complete(trans);  in gsi_trans_complete()
638 	complete(&trans->completion);  in gsi_trans_complete()
640 	gsi_trans_free(trans);  in gsi_trans_complete()
657 		struct gsi_trans *trans;  in gsi_channel_trans_cancel_pending()  local
659 		trans = &trans_info->trans[trans_id % channel->tre_count];  in gsi_channel_trans_cancel_pending()
660 		trans->cancelled = true;  in gsi_channel_trans_cancel_pending()
733 	trans_info->trans = kcalloc(tre_count, sizeof(*trans_info->trans),  in gsi_channel_trans_init()
735 	if (!trans_info->trans)  in gsi_channel_trans_init()
774 	kfree(trans_info->trans);  in gsi_channel_trans_init()
788 	kfree(trans_info->trans);  in gsi_channel_trans_exit()