315 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_rx_buf_enqueue_pipe()  local
323 	lockdep_assert_held(&ab->ce.ce_lock);  in ath12k_ce_rx_buf_enqueue_pipe()
327 	srng = &ab->hal.srng_list[ring->hal_ring_id];  in ath12k_ce_rx_buf_enqueue_pipe()
331 	ath12k_hal_srng_access_begin(ab, srng);  in ath12k_ce_rx_buf_enqueue_pipe()
333 	if (unlikely(ath12k_hal_srng_src_num_free(ab, srng, false) < 1)) {  in ath12k_ce_rx_buf_enqueue_pipe()
338 	desc = ath12k_hal_srng_src_get_next_entry(ab, srng);  in ath12k_ce_rx_buf_enqueue_pipe()
354 	ath12k_hal_srng_access_end(ab, srng);  in ath12k_ce_rx_buf_enqueue_pipe()
363 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_rx_post_pipe()  local
371 	spin_lock_bh(&ab->ce.ce_lock);  in ath12k_ce_rx_post_pipe()
381 		paddr = dma_map_single(ab->dev, skb->data,  in ath12k_ce_rx_post_pipe()
384 		if (unlikely(dma_mapping_error(ab->dev, paddr))) {  in ath12k_ce_rx_post_pipe()
385 			ath12k_warn(ab, "failed to dma map ce rx buf\n");  in ath12k_ce_rx_post_pipe()
395 			ath12k_warn(ab, "failed to enqueue rx buf: %d\n", ret);  in ath12k_ce_rx_post_pipe()
396 			dma_unmap_single(ab->dev, paddr,  in ath12k_ce_rx_post_pipe()
405 	spin_unlock_bh(&ab->ce.ce_lock);  in ath12k_ce_rx_post_pipe()
412 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_completed_recv_next()  local
419 	spin_lock_bh(&ab->ce.ce_lock);  in ath12k_ce_completed_recv_next()
424 	srng = &ab->hal.srng_list[pipe->status_ring->hal_ring_id];  in ath12k_ce_completed_recv_next()
428 	ath12k_hal_srng_access_begin(ab, srng);  in ath12k_ce_completed_recv_next()
430 	desc = ath12k_hal_srng_dst_get_next_entry(ab, srng);  in ath12k_ce_completed_recv_next()
446 	ath12k_hal_srng_access_end(ab, srng);  in ath12k_ce_completed_recv_next()
450 	spin_unlock_bh(&ab->ce.ce_lock);  in ath12k_ce_completed_recv_next()
457 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_recv_process_cb()  local
466 		dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(skb)->paddr,  in ath12k_ce_recv_process_cb()
470 			ath12k_warn(ab, "unexpected rx length (nbytes %d, max %d)",  in ath12k_ce_recv_process_cb()
481 		ath12k_dbg(ab, ATH12K_DBG_AHB, "rx ce pipe %d len %d\n",  in ath12k_ce_recv_process_cb()
483 		pipe->recv_cb(ab, skb);  in ath12k_ce_recv_process_cb()
488 		ath12k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",  in ath12k_ce_recv_process_cb()
490 		mod_timer(&ab->rx_replenish_retry,  in ath12k_ce_recv_process_cb()
497 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_completed_send_next()  local
504 	spin_lock_bh(&ab->ce.ce_lock);  in ath12k_ce_completed_send_next()
509 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath12k_ce_completed_send_next()
513 	ath12k_hal_srng_access_begin(ab, srng);  in ath12k_ce_completed_send_next()
515 	desc = ath12k_hal_srng_src_reap_next(ab, srng);  in ath12k_ce_completed_send_next()
531 	spin_unlock_bh(&ab->ce.ce_lock);  in ath12k_ce_completed_send_next()
538 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_send_done_cb()  local
545 		dma_unmap_single(ab->dev, ATH12K_SKB_CB(skb)->paddr, skb->len,  in ath12k_ce_send_done_cb()
551 static void ath12k_ce_srng_msi_ring_params_setup(struct ath12k_base *ab, u32 ce_id,  in ath12k_ce_srng_msi_ring_params_setup()  argument
561 	ret = ath12k_hif_get_user_msi_vector(ab, "CE",  in ath12k_ce_srng_msi_ring_params_setup()
568 	ath12k_hif_get_msi_address(ab, &addr_lo, &addr_hi);  in ath12k_ce_srng_msi_ring_params_setup()
569 	ath12k_hif_get_ce_msi_idx(ab, ce_id, &msi_data_idx);  in ath12k_ce_srng_msi_ring_params_setup()
577 static int ath12k_ce_init_ring(struct ath12k_base *ab,  in ath12k_ce_init_ring()  argument
588 	if (!(CE_ATTR_DIS_INTR & ab->hw_params->host_ce_config[ce_id].flags))  in ath12k_ce_init_ring()
589 		ath12k_ce_srng_msi_ring_params_setup(ab, ce_id, &params);  in ath12k_ce_init_ring()
593 		if (!(CE_ATTR_DIS_INTR & ab->hw_params->host_ce_config[ce_id].flags))  in ath12k_ce_init_ring()
597 		params.max_buffer_len = ab->hw_params->host_ce_config[ce_id].src_sz_max;  in ath12k_ce_init_ring()
598 		if (!(ab->hw_params->host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {  in ath12k_ce_init_ring()
605 		if (!(ab->hw_params->host_ce_config[ce_id].flags & CE_ATTR_DIS_INTR)) {  in ath12k_ce_init_ring()
611 		ath12k_warn(ab, "Invalid CE ring type %d\n", type);  in ath12k_ce_init_ring()
617 	ret = ath12k_hal_srng_setup(ab, type, ce_id, 0, &params);  in ath12k_ce_init_ring()
619 		ath12k_warn(ab, "failed to setup srng: %d ring_id %d\n",  in ath12k_ce_init_ring()
630 ath12k_ce_alloc_ring(struct ath12k_base *ab, int nentries, int desc_sz)  in ath12k_ce_alloc_ring()  argument
646 		dma_alloc_coherent(ab->dev,  in ath12k_ce_alloc_ring()
666 static int ath12k_ce_alloc_pipe(struct ath12k_base *ab, int ce_id)  in ath12k_ce_alloc_pipe()  argument
668 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath12k_ce_alloc_pipe()
669 	const struct ce_attr *attr = &ab->hw_params->host_ce_config[ce_id];  in ath12k_ce_alloc_pipe()
680 		ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz);  in ath12k_ce_alloc_pipe()
690 		ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz);  in ath12k_ce_alloc_pipe()
696 		ring = ath12k_ce_alloc_ring(ab, nentries, desc_sz);  in ath12k_ce_alloc_pipe()
705 void ath12k_ce_per_engine_service(struct ath12k_base *ab, u16 ce_id)  in ath12k_ce_per_engine_service()  argument
707 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id];  in ath12k_ce_per_engine_service()
716 void ath12k_ce_poll_send_completed(struct ath12k_base *ab, u8 pipe_id)  in ath12k_ce_poll_send_completed()  argument
718 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath12k_ce_poll_send_completed()
724 int ath12k_ce_send(struct ath12k_base *ab, struct sk_buff *skb, u8 pipe_id,  in ath12k_ce_send()  argument
727 	struct ath12k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id];  in ath12k_ce_send()
741 		spin_lock_bh(&ab->ce.ce_lock);  in ath12k_ce_send()
752 		spin_unlock_bh(&ab->ce.ce_lock);  in ath12k_ce_send()
755 			ath12k_ce_poll_send_completed(ab, pipe->pipe_num);  in ath12k_ce_send()
758 	if (test_bit(ATH12K_FLAG_CRASH_FLUSH, &ab->dev_flags))  in ath12k_ce_send()
761 	spin_lock_bh(&ab->ce.ce_lock);  in ath12k_ce_send()
766 	srng = &ab->hal.srng_list[pipe->src_ring->hal_ring_id];  in ath12k_ce_send()
770 	ath12k_hal_srng_access_begin(ab, srng);  in ath12k_ce_send()
772 	if (unlikely(ath12k_hal_srng_src_num_free(ab, srng, false) < 1)) {  in ath12k_ce_send()
773 		ath12k_hal_srng_access_end(ab, srng);  in ath12k_ce_send()
778 	desc = ath12k_hal_srng_src_get_next_reaped(ab, srng);  in ath12k_ce_send()
780 		ath12k_hal_srng_access_end(ab, srng);  in ath12k_ce_send()
795 	ath12k_hal_srng_access_end(ab, srng);  in ath12k_ce_send()
800 	spin_unlock_bh(&ab->ce.ce_lock);  in ath12k_ce_send()
807 	struct ath12k_base *ab = pipe->ab;  in ath12k_ce_rx_pipe_cleanup()  local
821 		dma_unmap_single(ab->dev, ATH12K_SKB_RXCB(skb)->paddr,  in ath12k_ce_rx_pipe_cleanup()
827 void ath12k_ce_cleanup_pipes(struct ath12k_base *ab)  in ath12k_ce_cleanup_pipes()  argument
832 	for (pipe_num = 0; pipe_num < ab->hw_params->ce_count; pipe_num++) {  in ath12k_ce_cleanup_pipes()
833 		pipe = &ab->ce.ce_pipe[pipe_num];  in ath12k_ce_cleanup_pipes()
837 		ath12k_ce_poll_send_completed(ab, pipe_num);  in ath12k_ce_cleanup_pipes()
843 void ath12k_ce_rx_post_buf(struct ath12k_base *ab)  in ath12k_ce_rx_post_buf()  argument
849 	for (i = 0; i < ab->hw_params->ce_count; i++) {  in ath12k_ce_rx_post_buf()
850 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_rx_post_buf()
856 			ath12k_warn(ab, "failed to post rx buf to pipe: %d err: %d\n",  in ath12k_ce_rx_post_buf()
858 			mod_timer(&ab->rx_replenish_retry,  in ath12k_ce_rx_post_buf()
868 	struct ath12k_base *ab = timer_container_of(ab, t, rx_replenish_retry);  in ath12k_ce_rx_replenish_retry()  local
870 	ath12k_ce_rx_post_buf(ab);  in ath12k_ce_rx_replenish_retry()
873 static void ath12k_ce_shadow_config(struct ath12k_base *ab)  in ath12k_ce_shadow_config()  argument
877 	for (i = 0; i < ab->hw_params->ce_count; i++) {  in ath12k_ce_shadow_config()
878 		if (ab->hw_params->host_ce_config[i].src_nentries)  in ath12k_ce_shadow_config()
879 			ath12k_hal_srng_update_shadow_config(ab, HAL_CE_SRC, i);  in ath12k_ce_shadow_config()
881 		if (ab->hw_params->host_ce_config[i].dest_nentries) {  in ath12k_ce_shadow_config()
882 			ath12k_hal_srng_update_shadow_config(ab, HAL_CE_DST, i);  in ath12k_ce_shadow_config()
883 			ath12k_hal_srng_update_shadow_config(ab, HAL_CE_DST_STATUS, i);  in ath12k_ce_shadow_config()
888 void ath12k_ce_get_shadow_config(struct ath12k_base *ab,  in ath12k_ce_get_shadow_config()  argument
891 	if (!ab->hw_params->supports_shadow_regs)  in ath12k_ce_get_shadow_config()
894 	ath12k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);  in ath12k_ce_get_shadow_config()
904 	ath12k_hal_srng_shadow_config(ab);  in ath12k_ce_get_shadow_config()
905 	ath12k_ce_shadow_config(ab);  in ath12k_ce_get_shadow_config()
908 	ath12k_hal_srng_get_shadow_config(ab, shadow_cfg, shadow_cfg_len);  in ath12k_ce_get_shadow_config()
911 int ath12k_ce_init_pipes(struct ath12k_base *ab)  in ath12k_ce_init_pipes()  argument
917 	ath12k_ce_get_shadow_config(ab, &ab->qmi.ce_cfg.shadow_reg_v3,  in ath12k_ce_init_pipes()
918 				    &ab->qmi.ce_cfg.shadow_reg_v3_len);  in ath12k_ce_init_pipes()
920 	for (i = 0; i < ab->hw_params->ce_count; i++) {  in ath12k_ce_init_pipes()
921 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_init_pipes()
924 			ret = ath12k_ce_init_ring(ab, pipe->src_ring, i,  in ath12k_ce_init_pipes()
927 				ath12k_warn(ab, "failed to init src ring: %d\n",  in ath12k_ce_init_pipes()
938 			ret = ath12k_ce_init_ring(ab, pipe->dest_ring, i,  in ath12k_ce_init_pipes()
941 				ath12k_warn(ab, "failed to init dest ring: %d\n",  in ath12k_ce_init_pipes()
955 			ret = ath12k_ce_init_ring(ab, pipe->status_ring, i,  in ath12k_ce_init_pipes()
958 				ath12k_warn(ab, "failed to init dest status ing: %d\n",  in ath12k_ce_init_pipes()
972 void ath12k_ce_free_pipes(struct ath12k_base *ab)  in ath12k_ce_free_pipes()  argument
978 	for (i = 0; i < ab->hw_params->ce_count; i++) {  in ath12k_ce_free_pipes()
979 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_free_pipes()
983 			dma_free_coherent(ab->dev,  in ath12k_ce_free_pipes()
994 			dma_free_coherent(ab->dev,  in ath12k_ce_free_pipes()
1006 			dma_free_coherent(ab->dev,  in ath12k_ce_free_pipes()
1017 int ath12k_ce_alloc_pipes(struct ath12k_base *ab)  in ath12k_ce_alloc_pipes()  argument
1024 	spin_lock_init(&ab->ce.ce_lock);  in ath12k_ce_alloc_pipes()
1026 	for (i = 0; i < ab->hw_params->ce_count; i++) {  in ath12k_ce_alloc_pipes()
1027 		attr = &ab->hw_params->host_ce_config[i];  in ath12k_ce_alloc_pipes()
1028 		pipe = &ab->ce.ce_pipe[i];  in ath12k_ce_alloc_pipes()
1030 		pipe->ab = ab;  in ath12k_ce_alloc_pipes()
1033 		ret = ath12k_ce_alloc_pipe(ab, i);  in ath12k_ce_alloc_pipes()
1036 			ath12k_ce_free_pipes(ab);  in ath12k_ce_alloc_pipes()
1044 int ath12k_ce_get_attr_flags(struct ath12k_base *ab, int ce_id)  in ath12k_ce_get_attr_flags()  argument
1046 	if (ce_id >= ab->hw_params->ce_count)  in ath12k_ce_get_attr_flags()
1049 	return ab->hw_params->host_ce_config[ce_id].flags;  in ath12k_ce_get_attr_flags()