312 static int ath12k_ce_rx_buf_enqueue_pipe(struct ath12k_ce_pipe *pipe,  in ath12k_ce_rx_buf_enqueue_pipe()  argument
315 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_rx_buf_enqueue_pipe()
316 	struct ath12k_ce_ring *ring = pipe->dest_ring;  in ath12k_ce_rx_buf_enqueue_pipe()
350 	pipe->rx_buf_needed--;  in ath12k_ce_rx_buf_enqueue_pipe()
361 static int ath12k_ce_rx_post_pipe(struct ath12k_ce_pipe *pipe)  in ath12k_ce_rx_post_pipe()  argument
363 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_rx_post_pipe()
368 	if (!(pipe->dest_ring || pipe->status_ring))  in ath12k_ce_rx_post_pipe()
372 	while (pipe->rx_buf_needed) {  in ath12k_ce_rx_post_pipe()
373 		skb = dev_alloc_skb(pipe->buf_sz);  in ath12k_ce_rx_post_pipe()
393 		ret = ath12k_ce_rx_buf_enqueue_pipe(pipe, skb, paddr);  in ath12k_ce_rx_post_pipe()
409 static int ath12k_ce_completed_recv_next(struct ath12k_ce_pipe *pipe,  in ath12k_ce_completed_recv_next()  argument
412 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_completed_recv_next()
421 	sw_index = pipe->dest_ring->sw_index;  in ath12k_ce_completed_recv_next()
422 	nentries_mask = pipe->dest_ring->nentries_mask;  in ath12k_ce_completed_recv_next()
424 	srng = &ab->hal.srng_list[pipe->status_ring->hal_ring_id];  in ath12k_ce_completed_recv_next()
438 	*skb = pipe->dest_ring->skb[sw_index];  in ath12k_ce_completed_recv_next()
439 	pipe->dest_ring->skb[sw_index] = NULL;  in ath12k_ce_completed_recv_next()
442 	pipe->dest_ring->sw_index = sw_index;  in ath12k_ce_completed_recv_next()
444 	pipe->rx_buf_needed++;  in ath12k_ce_completed_recv_next()
455 static void ath12k_ce_recv_process_cb(struct ath12k_ce_pipe *pipe)  in ath12k_ce_recv_process_cb()  argument
457 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_recv_process_cb()
464 	while (ath12k_ce_completed_recv_next(pipe, &skb, &nbytes) == 0) {  in ath12k_ce_recv_process_cb()
482 			   pipe->pipe_num, skb->len);  in ath12k_ce_recv_process_cb()
483 		pipe->recv_cb(ab, skb);  in ath12k_ce_recv_process_cb()
486 	ret = ath12k_ce_rx_post_pipe(pipe);  in ath12k_ce_recv_process_cb()
489 			    pipe->pipe_num, ret);  in ath12k_ce_recv_process_cb()
495 static struct sk_buff *ath12k_ce_completed_send_next(struct ath12k_ce_pipe *pipe)  in ath12k_ce_completed_send_next()  argument
497 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_completed_send_next()
506 	sw_index = pipe->src_ring->sw_index;  in ath12k_ce_completed_send_next()
507 	nentries_mask = pipe->src_ring->nentries_mask;  in ath12k_ce_completed_send_next()
509 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath12k_ce_completed_send_next()
521 	skb = pipe->src_ring->skb[sw_index];  in ath12k_ce_completed_send_next()
523 	pipe->src_ring->skb[sw_index] = NULL;  in ath12k_ce_completed_send_next()
526 	pipe->src_ring->sw_index = sw_index;  in ath12k_ce_completed_send_next()
536 static void ath12k_ce_send_done_cb(struct ath12k_ce_pipe *pipe)  in ath12k_ce_send_done_cb()  argument
538 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_send_done_cb()
541 	while (!IS_ERR(skb = ath12k_ce_completed_send_next(pipe))) {  in ath12k_ce_send_done_cb()
668 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath12k_ce_alloc_pipe()  local
674 	pipe->attr_flags = attr->flags;  in ath12k_ce_alloc_pipe()
677 		pipe->send_cb = ath12k_ce_send_done_cb;  in ath12k_ce_alloc_pipe()
683 		pipe->src_ring = ring;  in ath12k_ce_alloc_pipe()
687 		pipe->recv_cb = attr->recv_cb;  in ath12k_ce_alloc_pipe()
693 		pipe->dest_ring = ring;  in ath12k_ce_alloc_pipe()
699 		pipe->status_ring = ring;  in ath12k_ce_alloc_pipe()
707 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath12k_ce_per_engine_service()  local
709 	if (pipe->send_cb)  in ath12k_ce_per_engine_service()
710 		pipe->send_cb(pipe);  in ath12k_ce_per_engine_service()
712 	if (pipe->recv_cb)  in ath12k_ce_per_engine_service()
713 		ath12k_ce_recv_process_cb(pipe);  in ath12k_ce_per_engine_service()
718 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath12k_ce_poll_send_completed()  local
720 	if ((pipe->attr_flags & CE_ATTR_DIS_INTR) && pipe->send_cb)  in ath12k_ce_poll_send_completed()
721 		pipe->send_cb(pipe);  in ath12k_ce_poll_send_completed()
727 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath12k_ce_send()  local
740 	if (pipe->attr_flags & CE_ATTR_DIS_INTR) {  in ath12k_ce_send()
742 		write_index = pipe->src_ring->write_index;  in ath12k_ce_send()
744 		sw_index = pipe->src_ring->sw_index;  in ath12k_ce_send()
749 			num_used = pipe->src_ring->nentries - sw_index +  in ath12k_ce_send()
755 			ath12k_ce_poll_send_completed(ab, pipe->pipe_num);  in ath12k_ce_send()
763 	write_index = pipe->src_ring->write_index;  in ath12k_ce_send()
764 	nentries_mask = pipe->src_ring->nentries_mask;  in ath12k_ce_send()
766 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath12k_ce_send()
785 	if (pipe->attr_flags & CE_ATTR_BYTE_SWAP_DATA)  in ath12k_ce_send()
791 	pipe->src_ring->skb[write_index] = skb;  in ath12k_ce_send()
792 	pipe->src_ring->write_index = CE_RING_IDX_INCR(nentries_mask,  in ath12k_ce_send()
805 static void ath12k_ce_rx_pipe_cleanup(struct ath12k_ce_pipe *pipe)  in ath12k_ce_rx_pipe_cleanup()  argument
807 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_rx_pipe_cleanup()
808 	struct ath12k_ce_ring *ring = pipe->dest_ring;  in ath12k_ce_rx_pipe_cleanup()
812 	if (!(ring && pipe->buf_sz))  in ath12k_ce_rx_pipe_cleanup()
829 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_cleanup_pipes()  local
833 		pipe = &ab->ce.ce_pipe[pipe_num];  in ath12k_ce_cleanup_pipes()
834 		ath12k_ce_rx_pipe_cleanup(pipe);  in ath12k_ce_cleanup_pipes()
845 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_rx_post_buf()  local
850 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_rx_post_buf()
851 		ret = ath12k_ce_rx_post_pipe(pipe);  in ath12k_ce_rx_post_buf()
913 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_init_pipes()  local
921 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_init_pipes()
923 		if (pipe->src_ring) {  in ath12k_ce_init_pipes()
924 			ret = ath12k_ce_init_ring(ab, pipe->src_ring, i,  in ath12k_ce_init_pipes()
933 			pipe->src_ring->write_index = 0;  in ath12k_ce_init_pipes()
934 			pipe->src_ring->sw_index = 0;  in ath12k_ce_init_pipes()
937 		if (pipe->dest_ring) {  in ath12k_ce_init_pipes()
938 			ret = ath12k_ce_init_ring(ab, pipe->dest_ring, i,  in ath12k_ce_init_pipes()
947 			pipe->rx_buf_needed = pipe->dest_ring->nentries ?  in ath12k_ce_init_pipes()
948 					      pipe->dest_ring->nentries - 2 : 0;  in ath12k_ce_init_pipes()
950 			pipe->dest_ring->write_index = 0;  in ath12k_ce_init_pipes()
951 			pipe->dest_ring->sw_index = 0;  in ath12k_ce_init_pipes()
954 		if (pipe->status_ring) {  in ath12k_ce_init_pipes()
955 			ret = ath12k_ce_init_ring(ab, pipe->status_ring, i,  in ath12k_ce_init_pipes()
964 			pipe->status_ring->write_index = 0;  in ath12k_ce_init_pipes()
965 			pipe->status_ring->sw_index = 0;  in ath12k_ce_init_pipes()
974 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_free_pipes()  local
979 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_free_pipes()
981 		if (pipe->src_ring) {  in ath12k_ce_free_pipes()
984 					  pipe->src_ring->nentries * desc_sz +  in ath12k_ce_free_pipes()
986 					  pipe->src_ring->base_addr_owner_space,  in ath12k_ce_free_pipes()
987 					  pipe->src_ring->base_addr_ce_space);  in ath12k_ce_free_pipes()
988 			kfree(pipe->src_ring);  in ath12k_ce_free_pipes()
989 			pipe->src_ring = NULL;  in ath12k_ce_free_pipes()
992 		if (pipe->dest_ring) {  in ath12k_ce_free_pipes()
995 					  pipe->dest_ring->nentries * desc_sz +  in ath12k_ce_free_pipes()
997 					  pipe->dest_ring->base_addr_owner_space,  in ath12k_ce_free_pipes()
998 					  pipe->dest_ring->base_addr_ce_space);  in ath12k_ce_free_pipes()
999 			kfree(pipe->dest_ring);  in ath12k_ce_free_pipes()
1000 			pipe->dest_ring = NULL;  in ath12k_ce_free_pipes()
1003 		if (pipe->status_ring) {  in ath12k_ce_free_pipes()
1007 					  pipe->status_ring->nentries * desc_sz +  in ath12k_ce_free_pipes()
1009 					  pipe->status_ring->base_addr_owner_space,  in ath12k_ce_free_pipes()
1010 					  pipe->status_ring->base_addr_ce_space);  in ath12k_ce_free_pipes()
1011 			kfree(pipe->status_ring);  in ath12k_ce_free_pipes()
1012 			pipe->status_ring = NULL;  in ath12k_ce_free_pipes()
1019 	struct ath12k_ce_pipe *pipe;  in ath12k_ce_alloc_pipes()  local
1028 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_alloc_pipes()
1029 		pipe->pipe_num = i;  in ath12k_ce_alloc_pipes()
1030 		pipe->ab = ab;  in ath12k_ce_alloc_pipes()
1031 		pipe->buf_sz = attr->src_sz_max;  in ath12k_ce_alloc_pipes()