Lines Matching +full:- +full:ar
1 // SPDX-License-Identifier: ISC
3 * Copyright (c) 2005-2011 Atheros Communications Inc.
4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
23 * an address, length, and meta-data.
38 * There are several "contexts" managed by this layer -- more, it
39 * may seem -- than should be needed. These are provided mainly for
41 * implementation. There are per-CopyEngine recv, send, and watermark
45 * also a per-transfer context supplied by the caller when a buffer
47 * These per-transfer contexts are echoed back to the caller when
51 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar, in shadow_sr_wr_ind_addr() argument
54 u32 ce_id = ce_state->id; in shadow_sr_wr_ind_addr()
74 ath10k_warn(ar, "invalid CE id: %d", ce_id); in shadow_sr_wr_ind_addr()
84 return ((offset << addr_map->lsb) & addr_map->mask); in ath10k_set_ring_byte()
87 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset) in ath10k_ce_read32() argument
89 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_read32()
91 return ce->bus_ops->read32(ar, offset); in ath10k_ce_read32()
94 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value) in ath10k_ce_write32() argument
96 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_write32()
98 ce->bus_ops->write32(ar, offset, value); in ath10k_ce_write32()
101 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, in ath10k_ce_dest_ring_write_index_set() argument
105 ath10k_ce_write32(ar, ce_ctrl_addr + in ath10k_ce_dest_ring_write_index_set()
106 ar->hw_ce_regs->dst_wr_index_addr, n); in ath10k_ce_dest_ring_write_index_set()
109 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, in ath10k_ce_dest_ring_write_index_get() argument
112 return ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_dest_ring_write_index_get()
113 ar->hw_ce_regs->dst_wr_index_addr); in ath10k_ce_dest_ring_write_index_get()
116 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, in ath10k_ce_src_ring_write_index_set() argument
120 ath10k_ce_write32(ar, ce_ctrl_addr + in ath10k_ce_src_ring_write_index_set()
121 ar->hw_ce_regs->sr_wr_index_addr, n); in ath10k_ce_src_ring_write_index_set()
124 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, in ath10k_ce_src_ring_write_index_get() argument
127 return ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_src_ring_write_index_get()
128 ar->hw_ce_regs->sr_wr_index_addr); in ath10k_ce_src_ring_write_index_get()
131 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar, in ath10k_ce_src_ring_read_index_from_ddr() argument
134 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_src_ring_read_index_from_ddr()
136 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK; in ath10k_ce_src_ring_read_index_from_ddr()
139 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, in ath10k_ce_src_ring_read_index_get() argument
142 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_src_ring_read_index_get()
144 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in ath10k_ce_src_ring_read_index_get()
147 if (ar->hw_params.rri_on_ddr && in ath10k_ce_src_ring_read_index_get()
148 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) in ath10k_ce_src_ring_read_index_get()
149 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id); in ath10k_ce_src_ring_read_index_get()
151 index = ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_src_ring_read_index_get()
152 ar->hw_ce_regs->current_srri_addr); in ath10k_ce_src_ring_read_index_get()
158 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar, in ath10k_ce_shadow_src_ring_write_index_set() argument
162 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value); in ath10k_ce_shadow_src_ring_write_index_set()
165 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, in ath10k_ce_src_ring_base_addr_set() argument
169 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_src_ring_base_addr_set()
170 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in ath10k_ce_src_ring_base_addr_set()
171 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id); in ath10k_ce_src_ring_base_addr_set()
174 ath10k_ce_write32(ar, ce_ctrl_addr + in ath10k_ce_src_ring_base_addr_set()
175 ar->hw_ce_regs->sr_base_addr_lo, addr_lo); in ath10k_ce_src_ring_base_addr_set()
177 if (ce_state->ops->ce_set_src_ring_base_addr_hi) { in ath10k_ce_src_ring_base_addr_set()
178 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr, in ath10k_ce_src_ring_base_addr_set()
183 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar, in ath10k_ce_set_src_ring_base_addr_hi() argument
189 ath10k_ce_write32(ar, ce_ctrl_addr + in ath10k_ce_set_src_ring_base_addr_hi()
190 ar->hw_ce_regs->sr_base_addr_hi, addr_hi); in ath10k_ce_set_src_ring_base_addr_hi()
193 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, in ath10k_ce_src_ring_size_set() argument
197 ath10k_ce_write32(ar, ce_ctrl_addr + in ath10k_ce_src_ring_size_set()
198 ar->hw_ce_regs->sr_size_addr, n); in ath10k_ce_src_ring_size_set()
201 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, in ath10k_ce_src_ring_dmax_set() argument
205 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; in ath10k_ce_src_ring_dmax_set()
207 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_src_ring_dmax_set()
208 ctrl_regs->addr); in ath10k_ce_src_ring_dmax_set()
210 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, in ath10k_ce_src_ring_dmax_set()
211 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) | in ath10k_ce_src_ring_dmax_set()
212 ath10k_set_ring_byte(n, ctrl_regs->dmax)); in ath10k_ce_src_ring_dmax_set()
215 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, in ath10k_ce_src_ring_byte_swap_set() argument
219 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; in ath10k_ce_src_ring_byte_swap_set()
221 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_src_ring_byte_swap_set()
222 ctrl_regs->addr); in ath10k_ce_src_ring_byte_swap_set()
224 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, in ath10k_ce_src_ring_byte_swap_set()
225 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) | in ath10k_ce_src_ring_byte_swap_set()
226 ath10k_set_ring_byte(n, ctrl_regs->src_ring)); in ath10k_ce_src_ring_byte_swap_set()
229 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, in ath10k_ce_dest_ring_byte_swap_set() argument
233 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; in ath10k_ce_dest_ring_byte_swap_set()
235 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_dest_ring_byte_swap_set()
236 ctrl_regs->addr); in ath10k_ce_dest_ring_byte_swap_set()
238 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, in ath10k_ce_dest_ring_byte_swap_set()
239 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) | in ath10k_ce_dest_ring_byte_swap_set()
240 ath10k_set_ring_byte(n, ctrl_regs->dst_ring)); in ath10k_ce_dest_ring_byte_swap_set()
244 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id) in ath10k_ce_dest_ring_read_index_from_ddr() argument
246 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_dest_ring_read_index_from_ddr()
248 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) & in ath10k_ce_dest_ring_read_index_from_ddr()
252 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, in ath10k_ce_dest_ring_read_index_get() argument
255 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_dest_ring_read_index_get()
257 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in ath10k_ce_dest_ring_read_index_get()
260 if (ar->hw_params.rri_on_ddr && in ath10k_ce_dest_ring_read_index_get()
261 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) in ath10k_ce_dest_ring_read_index_get()
262 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id); in ath10k_ce_dest_ring_read_index_get()
264 index = ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_dest_ring_read_index_get()
265 ar->hw_ce_regs->current_drri_addr); in ath10k_ce_dest_ring_read_index_get()
270 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, in ath10k_ce_dest_ring_base_addr_set() argument
274 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_dest_ring_base_addr_set()
275 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in ath10k_ce_dest_ring_base_addr_set()
276 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id); in ath10k_ce_dest_ring_base_addr_set()
279 ath10k_ce_write32(ar, ce_ctrl_addr + in ath10k_ce_dest_ring_base_addr_set()
280 ar->hw_ce_regs->dr_base_addr_lo, addr_lo); in ath10k_ce_dest_ring_base_addr_set()
282 if (ce_state->ops->ce_set_dest_ring_base_addr_hi) { in ath10k_ce_dest_ring_base_addr_set()
283 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr, in ath10k_ce_dest_ring_base_addr_set()
288 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar, in ath10k_ce_set_dest_ring_base_addr_hi() argument
295 reg_value = ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_set_dest_ring_base_addr_hi()
296 ar->hw_ce_regs->dr_base_addr_hi); in ath10k_ce_set_dest_ring_base_addr_hi()
299 ath10k_ce_write32(ar, ce_ctrl_addr + in ath10k_ce_set_dest_ring_base_addr_hi()
300 ar->hw_ce_regs->dr_base_addr_hi, reg_value); in ath10k_ce_set_dest_ring_base_addr_hi()
303 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, in ath10k_ce_dest_ring_size_set() argument
307 ath10k_ce_write32(ar, ce_ctrl_addr + in ath10k_ce_dest_ring_size_set()
308 ar->hw_ce_regs->dr_size_addr, n); in ath10k_ce_dest_ring_size_set()
311 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, in ath10k_ce_src_ring_highmark_set() argument
315 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; in ath10k_ce_src_ring_highmark_set()
316 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); in ath10k_ce_src_ring_highmark_set()
318 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, in ath10k_ce_src_ring_highmark_set()
319 (addr & ~(srcr_wm->wm_high->mask)) | in ath10k_ce_src_ring_highmark_set()
320 (ath10k_set_ring_byte(n, srcr_wm->wm_high))); in ath10k_ce_src_ring_highmark_set()
323 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, in ath10k_ce_src_ring_lowmark_set() argument
327 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; in ath10k_ce_src_ring_lowmark_set()
328 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); in ath10k_ce_src_ring_lowmark_set()
330 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, in ath10k_ce_src_ring_lowmark_set()
331 (addr & ~(srcr_wm->wm_low->mask)) | in ath10k_ce_src_ring_lowmark_set()
332 (ath10k_set_ring_byte(n, srcr_wm->wm_low))); in ath10k_ce_src_ring_lowmark_set()
335 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, in ath10k_ce_dest_ring_highmark_set() argument
339 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; in ath10k_ce_dest_ring_highmark_set()
340 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); in ath10k_ce_dest_ring_highmark_set()
342 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, in ath10k_ce_dest_ring_highmark_set()
343 (addr & ~(dstr_wm->wm_high->mask)) | in ath10k_ce_dest_ring_highmark_set()
344 (ath10k_set_ring_byte(n, dstr_wm->wm_high))); in ath10k_ce_dest_ring_highmark_set()
347 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, in ath10k_ce_dest_ring_lowmark_set() argument
351 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; in ath10k_ce_dest_ring_lowmark_set()
352 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); in ath10k_ce_dest_ring_lowmark_set()
354 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, in ath10k_ce_dest_ring_lowmark_set()
355 (addr & ~(dstr_wm->wm_low->mask)) | in ath10k_ce_dest_ring_lowmark_set()
356 (ath10k_set_ring_byte(n, dstr_wm->wm_low))); in ath10k_ce_dest_ring_lowmark_set()
359 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, in ath10k_ce_copy_complete_inter_enable() argument
362 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; in ath10k_ce_copy_complete_inter_enable()
364 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_copy_complete_inter_enable()
365 ar->hw_ce_regs->host_ie_addr); in ath10k_ce_copy_complete_inter_enable()
367 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, in ath10k_ce_copy_complete_inter_enable()
368 host_ie_addr | host_ie->copy_complete->mask); in ath10k_ce_copy_complete_inter_enable()
371 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, in ath10k_ce_copy_complete_intr_disable() argument
374 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; in ath10k_ce_copy_complete_intr_disable()
376 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_copy_complete_intr_disable()
377 ar->hw_ce_regs->host_ie_addr); in ath10k_ce_copy_complete_intr_disable()
379 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, in ath10k_ce_copy_complete_intr_disable()
380 host_ie_addr & ~(host_ie->copy_complete->mask)); in ath10k_ce_copy_complete_intr_disable()
383 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, in ath10k_ce_watermark_intr_disable() argument
386 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; in ath10k_ce_watermark_intr_disable()
388 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + in ath10k_ce_watermark_intr_disable()
389 ar->hw_ce_regs->host_ie_addr); in ath10k_ce_watermark_intr_disable()
391 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, in ath10k_ce_watermark_intr_disable()
392 host_ie_addr & ~(wm_regs->wm_mask)); in ath10k_ce_watermark_intr_disable()
395 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, in ath10k_ce_error_intr_disable() argument
398 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; in ath10k_ce_error_intr_disable()
400 u32 misc_ie_addr = ath10k_ce_read32(ar, in ath10k_ce_error_intr_disable()
401 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr); in ath10k_ce_error_intr_disable()
403 ath10k_ce_write32(ar, in ath10k_ce_error_intr_disable()
404 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, in ath10k_ce_error_intr_disable()
405 misc_ie_addr & ~(misc_regs->err_mask)); in ath10k_ce_error_intr_disable()
408 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, in ath10k_ce_engine_int_status_clear() argument
412 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; in ath10k_ce_engine_int_status_clear()
414 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask); in ath10k_ce_engine_int_status_clear()
428 struct ath10k *ar = ce_state->ar; in _ath10k_ce_send_nolock() local
429 struct ath10k_ce_ring *src_ring = ce_state->src_ring; in _ath10k_ce_send_nolock()
431 unsigned int nentries_mask = src_ring->nentries_mask; in _ath10k_ce_send_nolock()
432 unsigned int sw_index = src_ring->sw_index; in _ath10k_ce_send_nolock()
433 unsigned int write_index = src_ring->write_index; in _ath10k_ce_send_nolock()
434 u32 ctrl_addr = ce_state->ctrl_addr; in _ath10k_ce_send_nolock()
438 if (nbytes > ce_state->src_sz_max) in _ath10k_ce_send_nolock()
439 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", in _ath10k_ce_send_nolock()
440 __func__, nbytes, ce_state->src_sz_max); in _ath10k_ce_send_nolock()
443 write_index, sw_index - 1) <= 0)) { in _ath10k_ce_send_nolock()
444 ret = -ENOSR; in _ath10k_ce_send_nolock()
448 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, in _ath10k_ce_send_nolock()
464 src_ring->per_transfer_context[write_index] = per_transfer_context; in _ath10k_ce_send_nolock()
471 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); in _ath10k_ce_send_nolock()
473 src_ring->write_index = write_index; in _ath10k_ce_send_nolock()
485 struct ath10k *ar = ce_state->ar; in _ath10k_ce_send_nolock_64() local
486 struct ath10k_ce_ring *src_ring = ce_state->src_ring; in _ath10k_ce_send_nolock_64()
488 unsigned int nentries_mask = src_ring->nentries_mask; in _ath10k_ce_send_nolock_64()
490 unsigned int write_index = src_ring->write_index; in _ath10k_ce_send_nolock_64()
491 u32 ctrl_addr = ce_state->ctrl_addr; in _ath10k_ce_send_nolock_64()
496 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags)) in _ath10k_ce_send_nolock_64()
497 return -ESHUTDOWN; in _ath10k_ce_send_nolock_64()
499 if (nbytes > ce_state->src_sz_max) in _ath10k_ce_send_nolock_64()
500 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", in _ath10k_ce_send_nolock_64()
501 __func__, nbytes, ce_state->src_sz_max); in _ath10k_ce_send_nolock_64()
503 if (ar->hw_params.rri_on_ddr) in _ath10k_ce_send_nolock_64()
504 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id); in _ath10k_ce_send_nolock_64()
506 sw_index = src_ring->sw_index; in _ath10k_ce_send_nolock_64()
509 write_index, sw_index - 1) <= 0)) { in _ath10k_ce_send_nolock_64()
510 ret = -ENOSR; in _ath10k_ce_send_nolock_64()
514 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, in _ath10k_ce_send_nolock_64()
540 src_ring->per_transfer_context[write_index] = per_transfer_context; in _ath10k_ce_send_nolock_64()
546 if (ar->hw_params.shadow_reg_support) in _ath10k_ce_send_nolock_64()
547 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state, in _ath10k_ce_send_nolock_64()
550 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, in _ath10k_ce_send_nolock_64()
554 src_ring->write_index = write_index; in _ath10k_ce_send_nolock_64()
566 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context, in ath10k_ce_send_nolock()
573 struct ath10k *ar = pipe->ar; in __ath10k_ce_send_revert() local
574 struct ath10k_ce *ce = ath10k_ce_priv(ar); in __ath10k_ce_send_revert()
575 struct ath10k_ce_ring *src_ring = pipe->src_ring; in __ath10k_ce_send_revert()
576 u32 ctrl_addr = pipe->ctrl_addr; in __ath10k_ce_send_revert()
578 lockdep_assert_held(&ce->ce_lock); in __ath10k_ce_send_revert()
582 * scatter-gather transfer (before index register is updated) in __ath10k_ce_send_revert()
585 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) in __ath10k_ce_send_revert()
588 if (WARN_ON_ONCE(src_ring->write_index == in __ath10k_ce_send_revert()
589 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) in __ath10k_ce_send_revert()
592 src_ring->write_index--; in __ath10k_ce_send_revert()
593 src_ring->write_index &= src_ring->nentries_mask; in __ath10k_ce_send_revert()
595 src_ring->per_transfer_context[src_ring->write_index] = NULL; in __ath10k_ce_send_revert()
606 struct ath10k *ar = ce_state->ar; in ath10k_ce_send() local
607 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_send()
610 spin_lock_bh(&ce->ce_lock); in ath10k_ce_send()
613 spin_unlock_bh(&ce->ce_lock); in ath10k_ce_send()
621 struct ath10k *ar = pipe->ar; in ath10k_ce_num_free_src_entries() local
622 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_num_free_src_entries()
625 spin_lock_bh(&ce->ce_lock); in ath10k_ce_num_free_src_entries()
626 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, in ath10k_ce_num_free_src_entries()
627 pipe->src_ring->write_index, in ath10k_ce_num_free_src_entries()
628 pipe->src_ring->sw_index - 1); in ath10k_ce_num_free_src_entries()
629 spin_unlock_bh(&ce->ce_lock); in ath10k_ce_num_free_src_entries()
637 struct ath10k *ar = pipe->ar; in __ath10k_ce_rx_num_free_bufs() local
638 struct ath10k_ce *ce = ath10k_ce_priv(ar); in __ath10k_ce_rx_num_free_bufs()
639 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; in __ath10k_ce_rx_num_free_bufs()
640 unsigned int nentries_mask = dest_ring->nentries_mask; in __ath10k_ce_rx_num_free_bufs()
641 unsigned int write_index = dest_ring->write_index; in __ath10k_ce_rx_num_free_bufs()
642 unsigned int sw_index = dest_ring->sw_index; in __ath10k_ce_rx_num_free_bufs()
644 lockdep_assert_held(&ce->ce_lock); in __ath10k_ce_rx_num_free_bufs()
646 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); in __ath10k_ce_rx_num_free_bufs()
653 struct ath10k *ar = pipe->ar; in __ath10k_ce_rx_post_buf() local
654 struct ath10k_ce *ce = ath10k_ce_priv(ar); in __ath10k_ce_rx_post_buf()
655 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; in __ath10k_ce_rx_post_buf()
656 unsigned int nentries_mask = dest_ring->nentries_mask; in __ath10k_ce_rx_post_buf()
657 unsigned int write_index = dest_ring->write_index; in __ath10k_ce_rx_post_buf()
658 unsigned int sw_index = dest_ring->sw_index; in __ath10k_ce_rx_post_buf()
659 struct ce_desc *base = dest_ring->base_addr_owner_space; in __ath10k_ce_rx_post_buf()
661 u32 ctrl_addr = pipe->ctrl_addr; in __ath10k_ce_rx_post_buf()
663 lockdep_assert_held(&ce->ce_lock); in __ath10k_ce_rx_post_buf()
665 if ((pipe->id != 5) && in __ath10k_ce_rx_post_buf()
666 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) in __ath10k_ce_rx_post_buf()
667 return -ENOSPC; in __ath10k_ce_rx_post_buf()
669 desc->addr = __cpu_to_le32(paddr); in __ath10k_ce_rx_post_buf()
670 desc->nbytes = 0; in __ath10k_ce_rx_post_buf()
672 dest_ring->per_transfer_context[write_index] = ctx; in __ath10k_ce_rx_post_buf()
674 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); in __ath10k_ce_rx_post_buf()
675 dest_ring->write_index = write_index; in __ath10k_ce_rx_post_buf()
684 struct ath10k *ar = pipe->ar; in __ath10k_ce_rx_post_buf_64() local
685 struct ath10k_ce *ce = ath10k_ce_priv(ar); in __ath10k_ce_rx_post_buf_64()
686 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; in __ath10k_ce_rx_post_buf_64()
687 unsigned int nentries_mask = dest_ring->nentries_mask; in __ath10k_ce_rx_post_buf_64()
688 unsigned int write_index = dest_ring->write_index; in __ath10k_ce_rx_post_buf_64()
689 unsigned int sw_index = dest_ring->sw_index; in __ath10k_ce_rx_post_buf_64()
690 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; in __ath10k_ce_rx_post_buf_64()
693 u32 ctrl_addr = pipe->ctrl_addr; in __ath10k_ce_rx_post_buf_64()
695 lockdep_assert_held(&ce->ce_lock); in __ath10k_ce_rx_post_buf_64()
697 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) in __ath10k_ce_rx_post_buf_64()
698 return -ENOSPC; in __ath10k_ce_rx_post_buf_64()
700 desc->addr = __cpu_to_le64(paddr); in __ath10k_ce_rx_post_buf_64()
701 desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK); in __ath10k_ce_rx_post_buf_64()
703 desc->nbytes = 0; in __ath10k_ce_rx_post_buf_64()
705 dest_ring->per_transfer_context[write_index] = ctx; in __ath10k_ce_rx_post_buf_64()
707 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); in __ath10k_ce_rx_post_buf_64()
708 dest_ring->write_index = write_index; in __ath10k_ce_rx_post_buf_64()
715 struct ath10k *ar = pipe->ar; in ath10k_ce_rx_update_write_idx() local
716 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; in ath10k_ce_rx_update_write_idx()
717 unsigned int nentries_mask = dest_ring->nentries_mask; in ath10k_ce_rx_update_write_idx()
718 unsigned int write_index = dest_ring->write_index; in ath10k_ce_rx_update_write_idx()
719 u32 ctrl_addr = pipe->ctrl_addr; in ath10k_ce_rx_update_write_idx()
720 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); in ath10k_ce_rx_update_write_idx()
725 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index) in ath10k_ce_rx_update_write_idx()
726 nentries -= 1; in ath10k_ce_rx_update_write_idx()
729 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); in ath10k_ce_rx_update_write_idx()
730 dest_ring->write_index = write_index; in ath10k_ce_rx_update_write_idx()
737 struct ath10k *ar = pipe->ar; in ath10k_ce_rx_post_buf() local
738 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_rx_post_buf()
741 spin_lock_bh(&ce->ce_lock); in ath10k_ce_rx_post_buf()
742 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr); in ath10k_ce_rx_post_buf()
743 spin_unlock_bh(&ce->ce_lock); in ath10k_ce_rx_post_buf()
758 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; in _ath10k_ce_completed_recv_next_nolock()
759 unsigned int nentries_mask = dest_ring->nentries_mask; in _ath10k_ce_completed_recv_next_nolock()
760 unsigned int sw_index = dest_ring->sw_index; in _ath10k_ce_completed_recv_next_nolock()
762 struct ce_desc *base = dest_ring->base_addr_owner_space; in _ath10k_ce_completed_recv_next_nolock()
778 return -EIO; in _ath10k_ce_completed_recv_next_nolock()
781 desc->nbytes = 0; in _ath10k_ce_completed_recv_next_nolock()
788 dest_ring->per_transfer_context[sw_index]; in _ath10k_ce_completed_recv_next_nolock()
793 if (ce_state->id != 5) in _ath10k_ce_completed_recv_next_nolock()
794 dest_ring->per_transfer_context[sw_index] = NULL; in _ath10k_ce_completed_recv_next_nolock()
798 dest_ring->sw_index = sw_index; in _ath10k_ce_completed_recv_next_nolock()
808 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; in _ath10k_ce_completed_recv_next_nolock_64()
809 unsigned int nentries_mask = dest_ring->nentries_mask; in _ath10k_ce_completed_recv_next_nolock_64()
810 unsigned int sw_index = dest_ring->sw_index; in _ath10k_ce_completed_recv_next_nolock_64()
811 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; in _ath10k_ce_completed_recv_next_nolock_64()
827 return -EIO; in _ath10k_ce_completed_recv_next_nolock_64()
830 desc->nbytes = 0; in _ath10k_ce_completed_recv_next_nolock_64()
837 dest_ring->per_transfer_context[sw_index]; in _ath10k_ce_completed_recv_next_nolock_64()
842 if (ce_state->id != 5) in _ath10k_ce_completed_recv_next_nolock_64()
843 dest_ring->per_transfer_context[sw_index] = NULL; in _ath10k_ce_completed_recv_next_nolock_64()
847 dest_ring->sw_index = sw_index; in _ath10k_ce_completed_recv_next_nolock_64()
856 return ce_state->ops->ce_completed_recv_next_nolock(ce_state, in ath10k_ce_completed_recv_next_nolock()
866 struct ath10k *ar = ce_state->ar; in ath10k_ce_completed_recv_next() local
867 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_completed_recv_next()
870 spin_lock_bh(&ce->ce_lock); in ath10k_ce_completed_recv_next()
871 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state, in ath10k_ce_completed_recv_next()
875 spin_unlock_bh(&ce->ce_lock); in ath10k_ce_completed_recv_next()
890 struct ath10k *ar; in _ath10k_ce_revoke_recv_next() local
893 dest_ring = ce_state->dest_ring; in _ath10k_ce_revoke_recv_next()
896 return -EIO; in _ath10k_ce_revoke_recv_next()
898 ar = ce_state->ar; in _ath10k_ce_revoke_recv_next()
899 ce = ath10k_ce_priv(ar); in _ath10k_ce_revoke_recv_next()
901 spin_lock_bh(&ce->ce_lock); in _ath10k_ce_revoke_recv_next()
903 nentries_mask = dest_ring->nentries_mask; in _ath10k_ce_revoke_recv_next()
904 sw_index = dest_ring->sw_index; in _ath10k_ce_revoke_recv_next()
905 write_index = dest_ring->write_index; in _ath10k_ce_revoke_recv_next()
907 struct ce_desc *base = dest_ring->base_addr_owner_space; in _ath10k_ce_revoke_recv_next()
911 *bufferp = __le32_to_cpu(desc->addr); in _ath10k_ce_revoke_recv_next()
915 dest_ring->per_transfer_context[sw_index]; in _ath10k_ce_revoke_recv_next()
918 dest_ring->per_transfer_context[sw_index] = NULL; in _ath10k_ce_revoke_recv_next()
919 desc->nbytes = 0; in _ath10k_ce_revoke_recv_next()
923 dest_ring->sw_index = sw_index; in _ath10k_ce_revoke_recv_next()
926 ret = -EIO; in _ath10k_ce_revoke_recv_next()
929 spin_unlock_bh(&ce->ce_lock); in _ath10k_ce_revoke_recv_next()
943 struct ath10k *ar; in _ath10k_ce_revoke_recv_next_64() local
946 dest_ring = ce_state->dest_ring; in _ath10k_ce_revoke_recv_next_64()
949 return -EIO; in _ath10k_ce_revoke_recv_next_64()
951 ar = ce_state->ar; in _ath10k_ce_revoke_recv_next_64()
952 ce = ath10k_ce_priv(ar); in _ath10k_ce_revoke_recv_next_64()
954 spin_lock_bh(&ce->ce_lock); in _ath10k_ce_revoke_recv_next_64()
956 nentries_mask = dest_ring->nentries_mask; in _ath10k_ce_revoke_recv_next_64()
957 sw_index = dest_ring->sw_index; in _ath10k_ce_revoke_recv_next_64()
958 write_index = dest_ring->write_index; in _ath10k_ce_revoke_recv_next_64()
960 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; in _ath10k_ce_revoke_recv_next_64()
965 *bufferp = __le64_to_cpu(desc->addr); in _ath10k_ce_revoke_recv_next_64()
969 dest_ring->per_transfer_context[sw_index]; in _ath10k_ce_revoke_recv_next_64()
972 dest_ring->per_transfer_context[sw_index] = NULL; in _ath10k_ce_revoke_recv_next_64()
973 desc->nbytes = 0; in _ath10k_ce_revoke_recv_next_64()
977 dest_ring->sw_index = sw_index; in _ath10k_ce_revoke_recv_next_64()
980 ret = -EIO; in _ath10k_ce_revoke_recv_next_64()
983 spin_unlock_bh(&ce->ce_lock); in _ath10k_ce_revoke_recv_next_64()
992 return ce_state->ops->ce_revoke_recv_next(ce_state, in ath10k_ce_revoke_recv_next()
1005 struct ath10k_ce_ring *src_ring = ce_state->src_ring; in _ath10k_ce_completed_send_next_nolock()
1006 u32 ctrl_addr = ce_state->ctrl_addr; in _ath10k_ce_completed_send_next_nolock()
1007 struct ath10k *ar = ce_state->ar; in _ath10k_ce_completed_send_next_nolock() local
1008 unsigned int nentries_mask = src_ring->nentries_mask; in _ath10k_ce_completed_send_next_nolock()
1009 unsigned int sw_index = src_ring->sw_index; in _ath10k_ce_completed_send_next_nolock()
1013 if (src_ring->hw_index == sw_index) { in _ath10k_ce_completed_send_next_nolock()
1022 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); in _ath10k_ce_completed_send_next_nolock()
1024 return -ENODEV; in _ath10k_ce_completed_send_next_nolock()
1027 src_ring->hw_index = read_index; in _ath10k_ce_completed_send_next_nolock()
1030 if (ar->hw_params.rri_on_ddr) in _ath10k_ce_completed_send_next_nolock()
1031 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); in _ath10k_ce_completed_send_next_nolock()
1033 read_index = src_ring->hw_index; in _ath10k_ce_completed_send_next_nolock()
1036 return -EIO; in _ath10k_ce_completed_send_next_nolock()
1040 src_ring->per_transfer_context[sw_index]; in _ath10k_ce_completed_send_next_nolock()
1043 src_ring->per_transfer_context[sw_index] = NULL; in _ath10k_ce_completed_send_next_nolock()
1044 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, in _ath10k_ce_completed_send_next_nolock()
1046 desc->nbytes = 0; in _ath10k_ce_completed_send_next_nolock()
1050 src_ring->sw_index = sw_index; in _ath10k_ce_completed_send_next_nolock()
1058 struct ath10k_ce_ring *src_ring = ce_state->src_ring; in _ath10k_ce_completed_send_next_nolock_64()
1059 u32 ctrl_addr = ce_state->ctrl_addr; in _ath10k_ce_completed_send_next_nolock_64()
1060 struct ath10k *ar = ce_state->ar; in _ath10k_ce_completed_send_next_nolock_64() local
1061 unsigned int nentries_mask = src_ring->nentries_mask; in _ath10k_ce_completed_send_next_nolock_64()
1062 unsigned int sw_index = src_ring->sw_index; in _ath10k_ce_completed_send_next_nolock_64()
1066 if (src_ring->hw_index == sw_index) { in _ath10k_ce_completed_send_next_nolock_64()
1075 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); in _ath10k_ce_completed_send_next_nolock_64()
1077 return -ENODEV; in _ath10k_ce_completed_send_next_nolock_64()
1080 src_ring->hw_index = read_index; in _ath10k_ce_completed_send_next_nolock_64()
1083 if (ar->hw_params.rri_on_ddr) in _ath10k_ce_completed_send_next_nolock_64()
1084 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); in _ath10k_ce_completed_send_next_nolock_64()
1086 read_index = src_ring->hw_index; in _ath10k_ce_completed_send_next_nolock_64()
1089 return -EIO; in _ath10k_ce_completed_send_next_nolock_64()
1093 src_ring->per_transfer_context[sw_index]; in _ath10k_ce_completed_send_next_nolock_64()
1096 src_ring->per_transfer_context[sw_index] = NULL; in _ath10k_ce_completed_send_next_nolock_64()
1097 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, in _ath10k_ce_completed_send_next_nolock_64()
1099 desc->nbytes = 0; in _ath10k_ce_completed_send_next_nolock_64()
1103 src_ring->sw_index = sw_index; in _ath10k_ce_completed_send_next_nolock_64()
1111 return ce_state->ops->ce_completed_send_next_nolock(ce_state, in ath10k_ce_completed_send_next_nolock()
1116 static void ath10k_ce_extract_desc_data(struct ath10k *ar, in ath10k_ce_extract_desc_data() argument
1123 struct ce_desc *base = src_ring->base_addr_owner_space; in ath10k_ce_extract_desc_data()
1127 *bufferp = __le32_to_cpu(desc->addr); in ath10k_ce_extract_desc_data()
1128 *nbytesp = __le16_to_cpu(desc->nbytes); in ath10k_ce_extract_desc_data()
1129 *transfer_idp = MS(__le16_to_cpu(desc->flags), in ath10k_ce_extract_desc_data()
1133 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar, in ath10k_ce_extract_desc_data_64() argument
1140 struct ce_desc_64 *base = src_ring->base_addr_owner_space; in ath10k_ce_extract_desc_data_64()
1145 *bufferp = __le64_to_cpu(desc->addr); in ath10k_ce_extract_desc_data_64()
1146 *nbytesp = __le16_to_cpu(desc->nbytes); in ath10k_ce_extract_desc_data_64()
1147 *transfer_idp = MS(__le16_to_cpu(desc->flags), in ath10k_ce_extract_desc_data_64()
1163 struct ath10k *ar; in ath10k_ce_cancel_send_next() local
1166 src_ring = ce_state->src_ring; in ath10k_ce_cancel_send_next()
1169 return -EIO; in ath10k_ce_cancel_send_next()
1171 ar = ce_state->ar; in ath10k_ce_cancel_send_next()
1172 ce = ath10k_ce_priv(ar); in ath10k_ce_cancel_send_next()
1174 spin_lock_bh(&ce->ce_lock); in ath10k_ce_cancel_send_next()
1176 nentries_mask = src_ring->nentries_mask; in ath10k_ce_cancel_send_next()
1177 sw_index = src_ring->sw_index; in ath10k_ce_cancel_send_next()
1178 write_index = src_ring->write_index; in ath10k_ce_cancel_send_next()
1181 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index, in ath10k_ce_cancel_send_next()
1187 src_ring->per_transfer_context[sw_index]; in ath10k_ce_cancel_send_next()
1190 src_ring->per_transfer_context[sw_index] = NULL; in ath10k_ce_cancel_send_next()
1194 src_ring->sw_index = sw_index; in ath10k_ce_cancel_send_next()
1197 ret = -EIO; in ath10k_ce_cancel_send_next()
1200 spin_unlock_bh(&ce->ce_lock); in ath10k_ce_cancel_send_next()
1209 struct ath10k *ar = ce_state->ar; in ath10k_ce_completed_send_next() local
1210 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_completed_send_next()
1213 spin_lock_bh(&ce->ce_lock); in ath10k_ce_completed_send_next()
1216 spin_unlock_bh(&ce->ce_lock); in ath10k_ce_completed_send_next()
1223 * Guts of interrupt handler for per-engine interrupts on a particular CE.
1228 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) in ath10k_ce_per_engine_service() argument
1230 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_per_engine_service()
1231 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in ath10k_ce_per_engine_service()
1232 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; in ath10k_ce_per_engine_service()
1233 u32 ctrl_addr = ce_state->ctrl_addr; in ath10k_ce_per_engine_service()
1245 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, in ath10k_ce_per_engine_service()
1246 wm_regs->cc_mask | wm_regs->wm_mask); in ath10k_ce_per_engine_service()
1248 if (ce_state->recv_cb) in ath10k_ce_per_engine_service()
1249 ce_state->recv_cb(ce_state); in ath10k_ce_per_engine_service()
1251 if (ce_state->send_cb) in ath10k_ce_per_engine_service()
1252 ce_state->send_cb(ce_state); in ath10k_ce_per_engine_service()
1257 * Handler for per-engine interrupts on ALL active CEs.
1262 void ath10k_ce_per_engine_service_any(struct ath10k *ar) in ath10k_ce_per_engine_service_any() argument
1267 intr_summary = ath10k_ce_interrupt_summary(ar); in ath10k_ce_per_engine_service_any()
1276 ath10k_ce_per_engine_service(ar, ce_id); in ath10k_ce_per_engine_service_any()
1290 u32 ctrl_addr = ce_state->ctrl_addr; in ath10k_ce_per_engine_handler_adjust()
1291 struct ath10k *ar = ce_state->ar; in ath10k_ce_per_engine_handler_adjust() local
1292 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; in ath10k_ce_per_engine_handler_adjust()
1295 (ce_state->send_cb || ce_state->recv_cb)) in ath10k_ce_per_engine_handler_adjust()
1296 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); in ath10k_ce_per_engine_handler_adjust()
1298 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); in ath10k_ce_per_engine_handler_adjust()
1300 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); in ath10k_ce_per_engine_handler_adjust()
1303 void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id) in ath10k_ce_disable_interrupt() argument
1305 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_disable_interrupt()
1309 ce_state = &ce->ce_states[ce_id]; in ath10k_ce_disable_interrupt()
1310 if (ce_state->attr_flags & CE_ATTR_POLL) in ath10k_ce_disable_interrupt()
1313 ctrl_addr = ath10k_ce_base_address(ar, ce_id); in ath10k_ce_disable_interrupt()
1315 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); in ath10k_ce_disable_interrupt()
1316 ath10k_ce_error_intr_disable(ar, ctrl_addr); in ath10k_ce_disable_interrupt()
1317 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); in ath10k_ce_disable_interrupt()
1321 void ath10k_ce_disable_interrupts(struct ath10k *ar) in ath10k_ce_disable_interrupts() argument
1326 ath10k_ce_disable_interrupt(ar, ce_id); in ath10k_ce_disable_interrupts()
1330 void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id) in ath10k_ce_enable_interrupt() argument
1332 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_enable_interrupt()
1335 ce_state = &ce->ce_states[ce_id]; in ath10k_ce_enable_interrupt()
1336 if (ce_state->attr_flags & CE_ATTR_POLL) in ath10k_ce_enable_interrupt()
1343 void ath10k_ce_enable_interrupts(struct ath10k *ar) in ath10k_ce_enable_interrupts() argument
1351 ath10k_ce_enable_interrupt(ar, ce_id); in ath10k_ce_enable_interrupts()
1355 static int ath10k_ce_init_src_ring(struct ath10k *ar, in ath10k_ce_init_src_ring() argument
1359 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_init_src_ring()
1360 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in ath10k_ce_init_src_ring()
1361 struct ath10k_ce_ring *src_ring = ce_state->src_ring; in ath10k_ce_init_src_ring()
1362 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); in ath10k_ce_init_src_ring()
1364 nentries = roundup_pow_of_two(attr->src_nentries); in ath10k_ce_init_src_ring()
1366 if (ar->hw_params.target_64bit) in ath10k_ce_init_src_ring()
1367 memset(src_ring->base_addr_owner_space, 0, in ath10k_ce_init_src_ring()
1370 memset(src_ring->base_addr_owner_space, 0, in ath10k_ce_init_src_ring()
1373 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); in ath10k_ce_init_src_ring()
1374 src_ring->sw_index &= src_ring->nentries_mask; in ath10k_ce_init_src_ring()
1375 src_ring->hw_index = src_ring->sw_index; in ath10k_ce_init_src_ring()
1377 src_ring->write_index = in ath10k_ce_init_src_ring()
1378 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); in ath10k_ce_init_src_ring()
1379 src_ring->write_index &= src_ring->nentries_mask; in ath10k_ce_init_src_ring()
1381 ath10k_ce_src_ring_base_addr_set(ar, ce_id, in ath10k_ce_init_src_ring()
1382 src_ring->base_addr_ce_space); in ath10k_ce_init_src_ring()
1383 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); in ath10k_ce_init_src_ring()
1384 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); in ath10k_ce_init_src_ring()
1385 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); in ath10k_ce_init_src_ring()
1386 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); in ath10k_ce_init_src_ring()
1387 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); in ath10k_ce_init_src_ring()
1389 ath10k_dbg(ar, ATH10K_DBG_BOOT, in ath10k_ce_init_src_ring()
1391 ce_id, nentries, src_ring->base_addr_owner_space); in ath10k_ce_init_src_ring()
1396 static int ath10k_ce_init_dest_ring(struct ath10k *ar, in ath10k_ce_init_dest_ring() argument
1400 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_init_dest_ring()
1401 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in ath10k_ce_init_dest_ring()
1402 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; in ath10k_ce_init_dest_ring()
1403 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); in ath10k_ce_init_dest_ring()
1405 nentries = roundup_pow_of_two(attr->dest_nentries); in ath10k_ce_init_dest_ring()
1407 if (ar->hw_params.target_64bit) in ath10k_ce_init_dest_ring()
1408 memset(dest_ring->base_addr_owner_space, 0, in ath10k_ce_init_dest_ring()
1411 memset(dest_ring->base_addr_owner_space, 0, in ath10k_ce_init_dest_ring()
1414 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); in ath10k_ce_init_dest_ring()
1415 dest_ring->sw_index &= dest_ring->nentries_mask; in ath10k_ce_init_dest_ring()
1416 dest_ring->write_index = in ath10k_ce_init_dest_ring()
1417 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); in ath10k_ce_init_dest_ring()
1418 dest_ring->write_index &= dest_ring->nentries_mask; in ath10k_ce_init_dest_ring()
1420 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, in ath10k_ce_init_dest_ring()
1421 dest_ring->base_addr_ce_space); in ath10k_ce_init_dest_ring()
1422 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); in ath10k_ce_init_dest_ring()
1423 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); in ath10k_ce_init_dest_ring()
1424 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); in ath10k_ce_init_dest_ring()
1425 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); in ath10k_ce_init_dest_ring()
1427 ath10k_dbg(ar, ATH10K_DBG_BOOT, in ath10k_ce_init_dest_ring()
1429 ce_id, nentries, dest_ring->base_addr_owner_space); in ath10k_ce_init_dest_ring()
1434 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar, in ath10k_ce_alloc_shadow_base() argument
1438 src_ring->shadow_base_unaligned = kcalloc(nentries, in ath10k_ce_alloc_shadow_base()
1441 if (!src_ring->shadow_base_unaligned) in ath10k_ce_alloc_shadow_base()
1442 return -ENOMEM; in ath10k_ce_alloc_shadow_base()
1444 src_ring->shadow_base = (struct ce_desc_64 *) in ath10k_ce_alloc_shadow_base()
1445 PTR_ALIGN(src_ring->shadow_base_unaligned, in ath10k_ce_alloc_shadow_base()
1451 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, in ath10k_ce_alloc_src_ring() argument
1455 u32 nentries = attr->src_nentries; in ath10k_ce_alloc_src_ring()
1464 return ERR_PTR(-ENOMEM); in ath10k_ce_alloc_src_ring()
1466 src_ring->nentries = nentries; in ath10k_ce_alloc_src_ring()
1467 src_ring->nentries_mask = nentries - 1; in ath10k_ce_alloc_src_ring()
1473 src_ring->base_addr_owner_space_unaligned = in ath10k_ce_alloc_src_ring()
1474 dma_alloc_coherent(ar->dev, in ath10k_ce_alloc_src_ring()
1478 if (!src_ring->base_addr_owner_space_unaligned) { in ath10k_ce_alloc_src_ring()
1480 return ERR_PTR(-ENOMEM); in ath10k_ce_alloc_src_ring()
1483 src_ring->base_addr_ce_space_unaligned = base_addr; in ath10k_ce_alloc_src_ring()
1485 src_ring->base_addr_owner_space = in ath10k_ce_alloc_src_ring()
1486 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, in ath10k_ce_alloc_src_ring()
1488 src_ring->base_addr_ce_space = in ath10k_ce_alloc_src_ring()
1489 ALIGN(src_ring->base_addr_ce_space_unaligned, in ath10k_ce_alloc_src_ring()
1492 if (ar->hw_params.shadow_reg_support) { in ath10k_ce_alloc_src_ring()
1493 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); in ath10k_ce_alloc_src_ring()
1495 dma_free_coherent(ar->dev, in ath10k_ce_alloc_src_ring()
1498 src_ring->base_addr_owner_space_unaligned, in ath10k_ce_alloc_src_ring()
1509 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id, in ath10k_ce_alloc_src_ring_64() argument
1513 u32 nentries = attr->src_nentries; in ath10k_ce_alloc_src_ring_64()
1522 return ERR_PTR(-ENOMEM); in ath10k_ce_alloc_src_ring_64()
1524 src_ring->nentries = nentries; in ath10k_ce_alloc_src_ring_64()
1525 src_ring->nentries_mask = nentries - 1; in ath10k_ce_alloc_src_ring_64()
1530 src_ring->base_addr_owner_space_unaligned = in ath10k_ce_alloc_src_ring_64()
1531 dma_alloc_coherent(ar->dev, in ath10k_ce_alloc_src_ring_64()
1535 if (!src_ring->base_addr_owner_space_unaligned) { in ath10k_ce_alloc_src_ring_64()
1537 return ERR_PTR(-ENOMEM); in ath10k_ce_alloc_src_ring_64()
1540 src_ring->base_addr_ce_space_unaligned = base_addr; in ath10k_ce_alloc_src_ring_64()
1542 src_ring->base_addr_owner_space = in ath10k_ce_alloc_src_ring_64()
1543 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, in ath10k_ce_alloc_src_ring_64()
1545 src_ring->base_addr_ce_space = in ath10k_ce_alloc_src_ring_64()
1546 ALIGN(src_ring->base_addr_ce_space_unaligned, in ath10k_ce_alloc_src_ring_64()
1549 if (ar->hw_params.shadow_reg_support) { in ath10k_ce_alloc_src_ring_64()
1550 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); in ath10k_ce_alloc_src_ring_64()
1552 dma_free_coherent(ar->dev, in ath10k_ce_alloc_src_ring_64()
1555 src_ring->base_addr_owner_space_unaligned, in ath10k_ce_alloc_src_ring_64()
1566 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, in ath10k_ce_alloc_dest_ring() argument
1573 nentries = roundup_pow_of_two(attr->dest_nentries); in ath10k_ce_alloc_dest_ring()
1578 return ERR_PTR(-ENOMEM); in ath10k_ce_alloc_dest_ring()
1580 dest_ring->nentries = nentries; in ath10k_ce_alloc_dest_ring()
1581 dest_ring->nentries_mask = nentries - 1; in ath10k_ce_alloc_dest_ring()
1587 dest_ring->base_addr_owner_space_unaligned = in ath10k_ce_alloc_dest_ring()
1588 dma_alloc_coherent(ar->dev, in ath10k_ce_alloc_dest_ring()
1592 if (!dest_ring->base_addr_owner_space_unaligned) { in ath10k_ce_alloc_dest_ring()
1594 return ERR_PTR(-ENOMEM); in ath10k_ce_alloc_dest_ring()
1597 dest_ring->base_addr_ce_space_unaligned = base_addr; in ath10k_ce_alloc_dest_ring()
1599 dest_ring->base_addr_owner_space = in ath10k_ce_alloc_dest_ring()
1600 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, in ath10k_ce_alloc_dest_ring()
1602 dest_ring->base_addr_ce_space = in ath10k_ce_alloc_dest_ring()
1603 ALIGN(dest_ring->base_addr_ce_space_unaligned, in ath10k_ce_alloc_dest_ring()
1610 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id, in ath10k_ce_alloc_dest_ring_64() argument
1617 nentries = roundup_pow_of_two(attr->dest_nentries); in ath10k_ce_alloc_dest_ring_64()
1622 return ERR_PTR(-ENOMEM); in ath10k_ce_alloc_dest_ring_64()
1624 dest_ring->nentries = nentries; in ath10k_ce_alloc_dest_ring_64()
1625 dest_ring->nentries_mask = nentries - 1; in ath10k_ce_alloc_dest_ring_64()
1630 dest_ring->base_addr_owner_space_unaligned = in ath10k_ce_alloc_dest_ring_64()
1631 dma_alloc_coherent(ar->dev, in ath10k_ce_alloc_dest_ring_64()
1635 if (!dest_ring->base_addr_owner_space_unaligned) { in ath10k_ce_alloc_dest_ring_64()
1637 return ERR_PTR(-ENOMEM); in ath10k_ce_alloc_dest_ring_64()
1640 dest_ring->base_addr_ce_space_unaligned = base_addr; in ath10k_ce_alloc_dest_ring_64()
1645 dest_ring->base_addr_owner_space = in ath10k_ce_alloc_dest_ring_64()
1646 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, in ath10k_ce_alloc_dest_ring_64()
1648 dest_ring->base_addr_ce_space = in ath10k_ce_alloc_dest_ring_64()
1649 ALIGN(dest_ring->base_addr_ce_space_unaligned, in ath10k_ce_alloc_dest_ring_64()
1656 * Initialize a Copy Engine based on caller-supplied attributes.
1662 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, in ath10k_ce_init_pipe() argument
1667 if (attr->src_nentries) { in ath10k_ce_init_pipe()
1668 ret = ath10k_ce_init_src_ring(ar, ce_id, attr); in ath10k_ce_init_pipe()
1670 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n", in ath10k_ce_init_pipe()
1676 if (attr->dest_nentries) { in ath10k_ce_init_pipe()
1677 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); in ath10k_ce_init_pipe()
1679 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n", in ath10k_ce_init_pipe()
1689 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) in ath10k_ce_deinit_src_ring() argument
1691 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); in ath10k_ce_deinit_src_ring()
1693 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0); in ath10k_ce_deinit_src_ring()
1694 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0); in ath10k_ce_deinit_src_ring()
1695 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0); in ath10k_ce_deinit_src_ring()
1696 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0); in ath10k_ce_deinit_src_ring()
1699 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) in ath10k_ce_deinit_dest_ring() argument
1701 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); in ath10k_ce_deinit_dest_ring()
1703 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0); in ath10k_ce_deinit_dest_ring()
1704 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0); in ath10k_ce_deinit_dest_ring()
1705 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0); in ath10k_ce_deinit_dest_ring()
1708 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) in ath10k_ce_deinit_pipe() argument
1710 ath10k_ce_deinit_src_ring(ar, ce_id); in ath10k_ce_deinit_pipe()
1711 ath10k_ce_deinit_dest_ring(ar, ce_id); in ath10k_ce_deinit_pipe()
1715 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) in _ath10k_ce_free_pipe() argument
1717 struct ath10k_ce *ce = ath10k_ce_priv(ar); in _ath10k_ce_free_pipe()
1718 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in _ath10k_ce_free_pipe()
1720 if (ce_state->src_ring) { in _ath10k_ce_free_pipe()
1721 if (ar->hw_params.shadow_reg_support) in _ath10k_ce_free_pipe()
1722 kfree(ce_state->src_ring->shadow_base_unaligned); in _ath10k_ce_free_pipe()
1723 dma_free_coherent(ar->dev, in _ath10k_ce_free_pipe()
1724 (ce_state->src_ring->nentries * in _ath10k_ce_free_pipe()
1727 ce_state->src_ring->base_addr_owner_space, in _ath10k_ce_free_pipe()
1728 ce_state->src_ring->base_addr_ce_space); in _ath10k_ce_free_pipe()
1729 kfree(ce_state->src_ring); in _ath10k_ce_free_pipe()
1732 if (ce_state->dest_ring) { in _ath10k_ce_free_pipe()
1733 dma_free_coherent(ar->dev, in _ath10k_ce_free_pipe()
1734 (ce_state->dest_ring->nentries * in _ath10k_ce_free_pipe()
1737 ce_state->dest_ring->base_addr_owner_space, in _ath10k_ce_free_pipe()
1738 ce_state->dest_ring->base_addr_ce_space); in _ath10k_ce_free_pipe()
1739 kfree(ce_state->dest_ring); in _ath10k_ce_free_pipe()
1742 ce_state->src_ring = NULL; in _ath10k_ce_free_pipe()
1743 ce_state->dest_ring = NULL; in _ath10k_ce_free_pipe()
1746 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id) in _ath10k_ce_free_pipe_64() argument
1748 struct ath10k_ce *ce = ath10k_ce_priv(ar); in _ath10k_ce_free_pipe_64()
1749 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in _ath10k_ce_free_pipe_64()
1751 if (ce_state->src_ring) { in _ath10k_ce_free_pipe_64()
1752 if (ar->hw_params.shadow_reg_support) in _ath10k_ce_free_pipe_64()
1753 kfree(ce_state->src_ring->shadow_base_unaligned); in _ath10k_ce_free_pipe_64()
1754 dma_free_coherent(ar->dev, in _ath10k_ce_free_pipe_64()
1755 (ce_state->src_ring->nentries * in _ath10k_ce_free_pipe_64()
1758 ce_state->src_ring->base_addr_owner_space, in _ath10k_ce_free_pipe_64()
1759 ce_state->src_ring->base_addr_ce_space); in _ath10k_ce_free_pipe_64()
1760 kfree(ce_state->src_ring); in _ath10k_ce_free_pipe_64()
1763 if (ce_state->dest_ring) { in _ath10k_ce_free_pipe_64()
1764 dma_free_coherent(ar->dev, in _ath10k_ce_free_pipe_64()
1765 (ce_state->dest_ring->nentries * in _ath10k_ce_free_pipe_64()
1768 ce_state->dest_ring->base_addr_owner_space, in _ath10k_ce_free_pipe_64()
1769 ce_state->dest_ring->base_addr_ce_space); in _ath10k_ce_free_pipe_64()
1770 kfree(ce_state->dest_ring); in _ath10k_ce_free_pipe_64()
1773 ce_state->src_ring = NULL; in _ath10k_ce_free_pipe_64()
1774 ce_state->dest_ring = NULL; in _ath10k_ce_free_pipe_64()
1777 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) in ath10k_ce_free_pipe() argument
1779 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_free_pipe()
1780 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in ath10k_ce_free_pipe()
1782 ce_state->ops->ce_free_pipe(ar, ce_id); in ath10k_ce_free_pipe()
1786 void ath10k_ce_dump_registers(struct ath10k *ar, in ath10k_ce_dump_registers() argument
1789 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_dump_registers()
1793 lockdep_assert_held(&ar->dump_mutex); in ath10k_ce_dump_registers()
1795 ath10k_err(ar, "Copy Engine register dump:\n"); in ath10k_ce_dump_registers()
1797 spin_lock_bh(&ce->ce_lock); in ath10k_ce_dump_registers()
1799 addr = ath10k_ce_base_address(ar, id); in ath10k_ce_dump_registers()
1803 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr)); in ath10k_ce_dump_registers()
1805 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr)); in ath10k_ce_dump_registers()
1807 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr)); in ath10k_ce_dump_registers()
1809 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr)); in ath10k_ce_dump_registers()
1812 crash_data->ce_crash_data[id] = ce_data; in ath10k_ce_dump_registers()
1815 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id, in ath10k_ce_dump_registers()
1817 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u\n", id, in ath10k_ce_dump_registers()
1826 spin_unlock_bh(&ce->ce_lock); in ath10k_ce_dump_registers()
1859 static void ath10k_ce_set_ops(struct ath10k *ar, in ath10k_ce_set_ops() argument
1862 switch (ar->hw_rev) { in ath10k_ce_set_ops()
1864 ce_state->ops = &ce_64_ops; in ath10k_ce_set_ops()
1867 ce_state->ops = &ce_ops; in ath10k_ce_set_ops()
1872 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, in ath10k_ce_alloc_pipe() argument
1875 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_alloc_pipe()
1876 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; in ath10k_ce_alloc_pipe()
1879 ath10k_ce_set_ops(ar, ce_state); in ath10k_ce_alloc_pipe()
1886 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); in ath10k_ce_alloc_pipe()
1888 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); in ath10k_ce_alloc_pipe()
1890 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); in ath10k_ce_alloc_pipe()
1892 ce_state->ar = ar; in ath10k_ce_alloc_pipe()
1893 ce_state->id = ce_id; in ath10k_ce_alloc_pipe()
1894 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); in ath10k_ce_alloc_pipe()
1895 ce_state->attr_flags = attr->flags; in ath10k_ce_alloc_pipe()
1896 ce_state->src_sz_max = attr->src_sz_max; in ath10k_ce_alloc_pipe()
1898 if (attr->src_nentries) in ath10k_ce_alloc_pipe()
1899 ce_state->send_cb = attr->send_cb; in ath10k_ce_alloc_pipe()
1901 if (attr->dest_nentries) in ath10k_ce_alloc_pipe()
1902 ce_state->recv_cb = attr->recv_cb; in ath10k_ce_alloc_pipe()
1904 if (attr->src_nentries) { in ath10k_ce_alloc_pipe()
1905 ce_state->src_ring = in ath10k_ce_alloc_pipe()
1906 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr); in ath10k_ce_alloc_pipe()
1907 if (IS_ERR(ce_state->src_ring)) { in ath10k_ce_alloc_pipe()
1908 ret = PTR_ERR(ce_state->src_ring); in ath10k_ce_alloc_pipe()
1909 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n", in ath10k_ce_alloc_pipe()
1911 ce_state->src_ring = NULL; in ath10k_ce_alloc_pipe()
1916 if (attr->dest_nentries) { in ath10k_ce_alloc_pipe()
1917 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar, in ath10k_ce_alloc_pipe()
1920 if (IS_ERR(ce_state->dest_ring)) { in ath10k_ce_alloc_pipe()
1921 ret = PTR_ERR(ce_state->dest_ring); in ath10k_ce_alloc_pipe()
1922 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n", in ath10k_ce_alloc_pipe()
1924 ce_state->dest_ring = NULL; in ath10k_ce_alloc_pipe()
1933 void ath10k_ce_alloc_rri(struct ath10k *ar) in ath10k_ce_alloc_rri() argument
1939 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_alloc_rri()
1941 ce->vaddr_rri = dma_alloc_coherent(ar->dev, in ath10k_ce_alloc_rri()
1943 &ce->paddr_rri, GFP_KERNEL); in ath10k_ce_alloc_rri()
1945 if (!ce->vaddr_rri) in ath10k_ce_alloc_rri()
1948 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low, in ath10k_ce_alloc_rri()
1949 lower_32_bits(ce->paddr_rri)); in ath10k_ce_alloc_rri()
1950 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high, in ath10k_ce_alloc_rri()
1951 (upper_32_bits(ce->paddr_rri) & in ath10k_ce_alloc_rri()
1955 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr; in ath10k_ce_alloc_rri()
1956 ce_base_addr = ath10k_ce_base_address(ar, i); in ath10k_ce_alloc_rri()
1957 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs); in ath10k_ce_alloc_rri()
1958 value |= ar->hw_ce_regs->upd->mask; in ath10k_ce_alloc_rri()
1959 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value); in ath10k_ce_alloc_rri()
1964 void ath10k_ce_free_rri(struct ath10k *ar) in ath10k_ce_free_rri() argument
1966 struct ath10k_ce *ce = ath10k_ce_priv(ar); in ath10k_ce_free_rri()
1968 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)), in ath10k_ce_free_rri()
1969 ce->vaddr_rri, in ath10k_ce_free_rri()
1970 ce->paddr_rri); in ath10k_ce_free_rri()