xref: /linux/drivers/net/wireless/ath/ath10k/ce.c (revision 507e190946297c34a27d9366b0661d5e506fdd03)
1 /*
2  * Copyright (c) 2005-2011 Atheros Communications Inc.
3  * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
4  *
5  * Permission to use, copy, modify, and/or distribute this software for any
6  * purpose with or without fee is hereby granted, provided that the above
7  * copyright notice and this permission notice appear in all copies.
8  *
9  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16  */
17 
18 #include "hif.h"
19 #include "pci.h"
20 #include "ce.h"
21 #include "debug.h"
22 
23 /*
24  * Support for Copy Engine hardware, which is mainly used for
25  * communication between Host and Target over a PCIe interconnect.
26  */
27 
28 /*
29  * A single CopyEngine (CE) comprises two "rings":
30  *   a source ring
31  *   a destination ring
32  *
33  * Each ring consists of a number of descriptors which specify
34  * an address, length, and meta-data.
35  *
36  * Typically, one side of the PCIe interconnect (Host or Target)
37  * controls one ring and the other side controls the other ring.
38  * The source side chooses when to initiate a transfer and it
39  * chooses what to send (buffer address, length). The destination
40  * side keeps a supply of "anonymous receive buffers" available and
41  * it handles incoming data as it arrives (when the destination
42  * receives an interrupt).
43  *
44  * The sender may send a simple buffer (address/length) or it may
45  * send a small list of buffers.  When a small list is sent, hardware
46  * "gathers" these and they end up in a single destination buffer
47  * with a single interrupt.
48  *
49  * There are several "contexts" managed by this layer -- more, it
50  * may seem -- than should be needed. These are provided mainly for
51  * maximum flexibility and especially to facilitate a simpler HIF
52  * implementation. There are per-CopyEngine recv, send, and watermark
53  * contexts. These are supplied by the caller when a recv, send,
54  * or watermark handler is established and they are echoed back to
55  * the caller when the respective callbacks are invoked. There is
56  * also a per-transfer context supplied by the caller when a buffer
57  * (or sendlist) is sent and when a buffer is enqueued for recv.
58  * These per-transfer contexts are echoed back to the caller when
59  * the buffer is sent/received.
60  */
61 
62 static inline unsigned int
63 ath10k_set_ring_byte(unsigned int offset,
64 		     struct ath10k_hw_ce_regs_addr_map *addr_map)
65 {
66 	return ((offset << addr_map->lsb) & addr_map->mask);
67 }
68 
69 static inline unsigned int
70 ath10k_get_ring_byte(unsigned int offset,
71 		     struct ath10k_hw_ce_regs_addr_map *addr_map)
72 {
73 	return ((offset & addr_map->mask) >> (addr_map->lsb));
74 }
75 
76 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
77 						       u32 ce_ctrl_addr,
78 						       unsigned int n)
79 {
80 	ath10k_pci_write32(ar, ce_ctrl_addr +
81 			ar->hw_ce_regs->dst_wr_index_addr, n);
82 }
83 
84 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
85 						      u32 ce_ctrl_addr)
86 {
87 	return ath10k_pci_read32(ar, ce_ctrl_addr +
88 			ar->hw_ce_regs->dst_wr_index_addr);
89 }
90 
91 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
92 						      u32 ce_ctrl_addr,
93 						      unsigned int n)
94 {
95 	ath10k_pci_write32(ar, ce_ctrl_addr +
96 			ar->hw_ce_regs->sr_wr_index_addr, n);
97 }
98 
99 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
100 						     u32 ce_ctrl_addr)
101 {
102 	return ath10k_pci_read32(ar, ce_ctrl_addr +
103 			ar->hw_ce_regs->sr_wr_index_addr);
104 }
105 
106 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
107 						    u32 ce_ctrl_addr)
108 {
109 	return ath10k_pci_read32(ar, ce_ctrl_addr +
110 			ar->hw_ce_regs->current_srri_addr);
111 }
112 
113 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
114 						    u32 ce_ctrl_addr,
115 						    unsigned int addr)
116 {
117 	ath10k_pci_write32(ar, ce_ctrl_addr +
118 			ar->hw_ce_regs->sr_base_addr, addr);
119 }
120 
121 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
122 					       u32 ce_ctrl_addr,
123 					       unsigned int n)
124 {
125 	ath10k_pci_write32(ar, ce_ctrl_addr +
126 			ar->hw_ce_regs->sr_size_addr, n);
127 }
128 
129 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
130 					       u32 ce_ctrl_addr,
131 					       unsigned int n)
132 {
133 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
134 	u32 ctrl1_addr = ath10k_pci_read32(ar,
135 					   ce_ctrl_addr + ctrl_regs->addr);
136 
137 	ath10k_pci_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
138 			   (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
139 			   ath10k_set_ring_byte(n, ctrl_regs->dmax));
140 }
141 
142 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
143 						    u32 ce_ctrl_addr,
144 						    unsigned int n)
145 {
146 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
147 	u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + ctrl_regs->addr);
148 
149 	ath10k_pci_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
150 			   (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
151 			   ath10k_set_ring_byte(n, ctrl_regs->src_ring));
152 }
153 
154 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
155 						     u32 ce_ctrl_addr,
156 						     unsigned int n)
157 {
158 	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
159 	u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + ctrl_regs->addr);
160 
161 	ath10k_pci_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
162 			   (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
163 			   ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
164 }
165 
166 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
167 						     u32 ce_ctrl_addr)
168 {
169 	return ath10k_pci_read32(ar, ce_ctrl_addr +
170 				 ar->hw_ce_regs->current_drri_addr);
171 }
172 
173 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
174 						     u32 ce_ctrl_addr,
175 						     u32 addr)
176 {
177 	ath10k_pci_write32(ar, ce_ctrl_addr +
178 			ar->hw_ce_regs->dr_base_addr, addr);
179 }
180 
181 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
182 						u32 ce_ctrl_addr,
183 						unsigned int n)
184 {
185 	ath10k_pci_write32(ar, ce_ctrl_addr +
186 			ar->hw_ce_regs->dr_size_addr, n);
187 }
188 
189 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
190 						   u32 ce_ctrl_addr,
191 						   unsigned int n)
192 {
193 	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
194 	u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + srcr_wm->addr);
195 
196 	ath10k_pci_write32(ar, ce_ctrl_addr + srcr_wm->addr,
197 			   (addr & ~(srcr_wm->wm_high->mask)) |
198 			   (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
199 }
200 
201 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
202 						  u32 ce_ctrl_addr,
203 						  unsigned int n)
204 {
205 	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
206 	u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + srcr_wm->addr);
207 
208 	ath10k_pci_write32(ar, ce_ctrl_addr + srcr_wm->addr,
209 			   (addr & ~(srcr_wm->wm_low->mask)) |
210 			   (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
211 }
212 
213 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
214 						    u32 ce_ctrl_addr,
215 						    unsigned int n)
216 {
217 	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
218 	u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + dstr_wm->addr);
219 
220 	ath10k_pci_write32(ar, ce_ctrl_addr + dstr_wm->addr,
221 			   (addr & ~(dstr_wm->wm_high->mask)) |
222 			   (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
223 }
224 
225 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
226 						   u32 ce_ctrl_addr,
227 						   unsigned int n)
228 {
229 	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
230 	u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + dstr_wm->addr);
231 
232 	ath10k_pci_write32(ar, ce_ctrl_addr + dstr_wm->addr,
233 			   (addr & ~(dstr_wm->wm_low->mask)) |
234 			   (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
235 }
236 
237 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
238 							u32 ce_ctrl_addr)
239 {
240 	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
241 	u32 host_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
242 					ar->hw_ce_regs->host_ie_addr);
243 
244 	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
245 			   host_ie_addr | host_ie->copy_complete->mask);
246 }
247 
248 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
249 							u32 ce_ctrl_addr)
250 {
251 	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
252 	u32 host_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
253 					ar->hw_ce_regs->host_ie_addr);
254 
255 	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
256 			   host_ie_addr & ~(host_ie->copy_complete->mask));
257 }
258 
259 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
260 						    u32 ce_ctrl_addr)
261 {
262 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
263 	u32 host_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
264 					ar->hw_ce_regs->host_ie_addr);
265 
266 	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
267 			   host_ie_addr & ~(wm_regs->wm_mask));
268 }
269 
270 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
271 					       u32 ce_ctrl_addr)
272 {
273 	struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
274 	u32 misc_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
275 					ar->hw_ce_regs->misc_ie_addr);
276 
277 	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
278 			   misc_ie_addr | misc_regs->err_mask);
279 }
280 
281 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
282 						u32 ce_ctrl_addr)
283 {
284 	struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
285 	u32 misc_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr +
286 					ar->hw_ce_regs->misc_ie_addr);
287 
288 	ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
289 			   misc_ie_addr & ~(misc_regs->err_mask));
290 }
291 
292 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
293 						     u32 ce_ctrl_addr,
294 						     unsigned int mask)
295 {
296 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
297 
298 	ath10k_pci_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
299 }
300 
301 /*
302  * Guts of ath10k_ce_send.
303  * The caller takes responsibility for any needed locking.
304  */
305 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
306 			  void *per_transfer_context,
307 			  u32 buffer,
308 			  unsigned int nbytes,
309 			  unsigned int transfer_id,
310 			  unsigned int flags)
311 {
312 	struct ath10k *ar = ce_state->ar;
313 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
314 	struct ce_desc *desc, sdesc;
315 	unsigned int nentries_mask = src_ring->nentries_mask;
316 	unsigned int sw_index = src_ring->sw_index;
317 	unsigned int write_index = src_ring->write_index;
318 	u32 ctrl_addr = ce_state->ctrl_addr;
319 	u32 desc_flags = 0;
320 	int ret = 0;
321 
322 	if (nbytes > ce_state->src_sz_max)
323 		ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
324 			    __func__, nbytes, ce_state->src_sz_max);
325 
326 	if (unlikely(CE_RING_DELTA(nentries_mask,
327 				   write_index, sw_index - 1) <= 0)) {
328 		ret = -ENOSR;
329 		goto exit;
330 	}
331 
332 	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
333 				   write_index);
334 
335 	desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
336 
337 	if (flags & CE_SEND_FLAG_GATHER)
338 		desc_flags |= CE_DESC_FLAGS_GATHER;
339 	if (flags & CE_SEND_FLAG_BYTE_SWAP)
340 		desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
341 
342 	sdesc.addr   = __cpu_to_le32(buffer);
343 	sdesc.nbytes = __cpu_to_le16(nbytes);
344 	sdesc.flags  = __cpu_to_le16(desc_flags);
345 
346 	*desc = sdesc;
347 
348 	src_ring->per_transfer_context[write_index] = per_transfer_context;
349 
350 	/* Update Source Ring Write Index */
351 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
352 
353 	/* WORKAROUND */
354 	if (!(flags & CE_SEND_FLAG_GATHER))
355 		ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
356 
357 	src_ring->write_index = write_index;
358 exit:
359 	return ret;
360 }
361 
362 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
363 {
364 	struct ath10k *ar = pipe->ar;
365 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
366 	struct ath10k_ce_ring *src_ring = pipe->src_ring;
367 	u32 ctrl_addr = pipe->ctrl_addr;
368 
369 	lockdep_assert_held(&ar_pci->ce_lock);
370 
371 	/*
372 	 * This function must be called only if there is an incomplete
373 	 * scatter-gather transfer (before index register is updated)
374 	 * that needs to be cleaned up.
375 	 */
376 	if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
377 		return;
378 
379 	if (WARN_ON_ONCE(src_ring->write_index ==
380 			 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
381 		return;
382 
383 	src_ring->write_index--;
384 	src_ring->write_index &= src_ring->nentries_mask;
385 
386 	src_ring->per_transfer_context[src_ring->write_index] = NULL;
387 }
388 
389 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
390 		   void *per_transfer_context,
391 		   u32 buffer,
392 		   unsigned int nbytes,
393 		   unsigned int transfer_id,
394 		   unsigned int flags)
395 {
396 	struct ath10k *ar = ce_state->ar;
397 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
398 	int ret;
399 
400 	spin_lock_bh(&ar_pci->ce_lock);
401 	ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
402 				    buffer, nbytes, transfer_id, flags);
403 	spin_unlock_bh(&ar_pci->ce_lock);
404 
405 	return ret;
406 }
407 
408 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
409 {
410 	struct ath10k *ar = pipe->ar;
411 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
412 	int delta;
413 
414 	spin_lock_bh(&ar_pci->ce_lock);
415 	delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
416 			      pipe->src_ring->write_index,
417 			      pipe->src_ring->sw_index - 1);
418 	spin_unlock_bh(&ar_pci->ce_lock);
419 
420 	return delta;
421 }
422 
423 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
424 {
425 	struct ath10k *ar = pipe->ar;
426 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
427 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
428 	unsigned int nentries_mask = dest_ring->nentries_mask;
429 	unsigned int write_index = dest_ring->write_index;
430 	unsigned int sw_index = dest_ring->sw_index;
431 
432 	lockdep_assert_held(&ar_pci->ce_lock);
433 
434 	return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
435 }
436 
437 int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
438 {
439 	struct ath10k *ar = pipe->ar;
440 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
441 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
442 	unsigned int nentries_mask = dest_ring->nentries_mask;
443 	unsigned int write_index = dest_ring->write_index;
444 	unsigned int sw_index = dest_ring->sw_index;
445 	struct ce_desc *base = dest_ring->base_addr_owner_space;
446 	struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
447 	u32 ctrl_addr = pipe->ctrl_addr;
448 
449 	lockdep_assert_held(&ar_pci->ce_lock);
450 
451 	if ((pipe->id != 5) &&
452 	    CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
453 		return -ENOSPC;
454 
455 	desc->addr = __cpu_to_le32(paddr);
456 	desc->nbytes = 0;
457 
458 	dest_ring->per_transfer_context[write_index] = ctx;
459 	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
460 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
461 	dest_ring->write_index = write_index;
462 
463 	return 0;
464 }
465 
466 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
467 {
468 	struct ath10k *ar = pipe->ar;
469 	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
470 	unsigned int nentries_mask = dest_ring->nentries_mask;
471 	unsigned int write_index = dest_ring->write_index;
472 	u32 ctrl_addr = pipe->ctrl_addr;
473 	u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
474 
475 	/* Prevent CE ring stuck issue that will occur when ring is full.
476 	 * Make sure that write index is 1 less than read index.
477 	 */
478 	if ((cur_write_idx + nentries)  == dest_ring->sw_index)
479 		nentries -= 1;
480 
481 	write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
482 	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
483 	dest_ring->write_index = write_index;
484 }
485 
486 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr)
487 {
488 	struct ath10k *ar = pipe->ar;
489 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
490 	int ret;
491 
492 	spin_lock_bh(&ar_pci->ce_lock);
493 	ret = __ath10k_ce_rx_post_buf(pipe, ctx, paddr);
494 	spin_unlock_bh(&ar_pci->ce_lock);
495 
496 	return ret;
497 }
498 
499 /*
500  * Guts of ath10k_ce_completed_recv_next.
501  * The caller takes responsibility for any necessary locking.
502  */
503 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
504 					 void **per_transfer_contextp,
505 					 unsigned int *nbytesp)
506 {
507 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
508 	unsigned int nentries_mask = dest_ring->nentries_mask;
509 	unsigned int sw_index = dest_ring->sw_index;
510 
511 	struct ce_desc *base = dest_ring->base_addr_owner_space;
512 	struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
513 	struct ce_desc sdesc;
514 	u16 nbytes;
515 
516 	/* Copy in one go for performance reasons */
517 	sdesc = *desc;
518 
519 	nbytes = __le16_to_cpu(sdesc.nbytes);
520 	if (nbytes == 0) {
521 		/*
522 		 * This closes a relatively unusual race where the Host
523 		 * sees the updated DRRI before the update to the
524 		 * corresponding descriptor has completed. We treat this
525 		 * as a descriptor that is not yet done.
526 		 */
527 		return -EIO;
528 	}
529 
530 	desc->nbytes = 0;
531 
532 	/* Return data from completed destination descriptor */
533 	*nbytesp = nbytes;
534 
535 	if (per_transfer_contextp)
536 		*per_transfer_contextp =
537 			dest_ring->per_transfer_context[sw_index];
538 
539 	/* Copy engine 5 (HTT Rx) will reuse the same transfer context.
540 	 * So update transfer context all CEs except CE5.
541 	 */
542 	if (ce_state->id != 5)
543 		dest_ring->per_transfer_context[sw_index] = NULL;
544 
545 	/* Update sw_index */
546 	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
547 	dest_ring->sw_index = sw_index;
548 
549 	return 0;
550 }
551 
552 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
553 				  void **per_transfer_contextp,
554 				  unsigned int *nbytesp)
555 {
556 	struct ath10k *ar = ce_state->ar;
557 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
558 	int ret;
559 
560 	spin_lock_bh(&ar_pci->ce_lock);
561 	ret = ath10k_ce_completed_recv_next_nolock(ce_state,
562 						   per_transfer_contextp,
563 						   nbytesp);
564 	spin_unlock_bh(&ar_pci->ce_lock);
565 
566 	return ret;
567 }
568 
569 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
570 			       void **per_transfer_contextp,
571 			       u32 *bufferp)
572 {
573 	struct ath10k_ce_ring *dest_ring;
574 	unsigned int nentries_mask;
575 	unsigned int sw_index;
576 	unsigned int write_index;
577 	int ret;
578 	struct ath10k *ar;
579 	struct ath10k_pci *ar_pci;
580 
581 	dest_ring = ce_state->dest_ring;
582 
583 	if (!dest_ring)
584 		return -EIO;
585 
586 	ar = ce_state->ar;
587 	ar_pci = ath10k_pci_priv(ar);
588 
589 	spin_lock_bh(&ar_pci->ce_lock);
590 
591 	nentries_mask = dest_ring->nentries_mask;
592 	sw_index = dest_ring->sw_index;
593 	write_index = dest_ring->write_index;
594 	if (write_index != sw_index) {
595 		struct ce_desc *base = dest_ring->base_addr_owner_space;
596 		struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
597 
598 		/* Return data from completed destination descriptor */
599 		*bufferp = __le32_to_cpu(desc->addr);
600 
601 		if (per_transfer_contextp)
602 			*per_transfer_contextp =
603 				dest_ring->per_transfer_context[sw_index];
604 
605 		/* sanity */
606 		dest_ring->per_transfer_context[sw_index] = NULL;
607 		desc->nbytes = 0;
608 
609 		/* Update sw_index */
610 		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
611 		dest_ring->sw_index = sw_index;
612 		ret = 0;
613 	} else {
614 		ret = -EIO;
615 	}
616 
617 	spin_unlock_bh(&ar_pci->ce_lock);
618 
619 	return ret;
620 }
621 
622 /*
623  * Guts of ath10k_ce_completed_send_next.
624  * The caller takes responsibility for any necessary locking.
625  */
626 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
627 					 void **per_transfer_contextp)
628 {
629 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
630 	u32 ctrl_addr = ce_state->ctrl_addr;
631 	struct ath10k *ar = ce_state->ar;
632 	unsigned int nentries_mask = src_ring->nentries_mask;
633 	unsigned int sw_index = src_ring->sw_index;
634 	unsigned int read_index;
635 	struct ce_desc *desc;
636 
637 	if (src_ring->hw_index == sw_index) {
638 		/*
639 		 * The SW completion index has caught up with the cached
640 		 * version of the HW completion index.
641 		 * Update the cached HW completion index to see whether
642 		 * the SW has really caught up to the HW, or if the cached
643 		 * value of the HW index has become stale.
644 		 */
645 
646 		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
647 		if (read_index == 0xffffffff)
648 			return -ENODEV;
649 
650 		read_index &= nentries_mask;
651 		src_ring->hw_index = read_index;
652 	}
653 
654 	read_index = src_ring->hw_index;
655 
656 	if (read_index == sw_index)
657 		return -EIO;
658 
659 	if (per_transfer_contextp)
660 		*per_transfer_contextp =
661 			src_ring->per_transfer_context[sw_index];
662 
663 	/* sanity */
664 	src_ring->per_transfer_context[sw_index] = NULL;
665 	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
666 				   sw_index);
667 	desc->nbytes = 0;
668 
669 	/* Update sw_index */
670 	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
671 	src_ring->sw_index = sw_index;
672 
673 	return 0;
674 }
675 
676 /* NB: Modeled after ath10k_ce_completed_send_next */
677 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
678 			       void **per_transfer_contextp,
679 			       u32 *bufferp,
680 			       unsigned int *nbytesp,
681 			       unsigned int *transfer_idp)
682 {
683 	struct ath10k_ce_ring *src_ring;
684 	unsigned int nentries_mask;
685 	unsigned int sw_index;
686 	unsigned int write_index;
687 	int ret;
688 	struct ath10k *ar;
689 	struct ath10k_pci *ar_pci;
690 
691 	src_ring = ce_state->src_ring;
692 
693 	if (!src_ring)
694 		return -EIO;
695 
696 	ar = ce_state->ar;
697 	ar_pci = ath10k_pci_priv(ar);
698 
699 	spin_lock_bh(&ar_pci->ce_lock);
700 
701 	nentries_mask = src_ring->nentries_mask;
702 	sw_index = src_ring->sw_index;
703 	write_index = src_ring->write_index;
704 
705 	if (write_index != sw_index) {
706 		struct ce_desc *base = src_ring->base_addr_owner_space;
707 		struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
708 
709 		/* Return data from completed source descriptor */
710 		*bufferp = __le32_to_cpu(desc->addr);
711 		*nbytesp = __le16_to_cpu(desc->nbytes);
712 		*transfer_idp = MS(__le16_to_cpu(desc->flags),
713 						CE_DESC_FLAGS_META_DATA);
714 
715 		if (per_transfer_contextp)
716 			*per_transfer_contextp =
717 				src_ring->per_transfer_context[sw_index];
718 
719 		/* sanity */
720 		src_ring->per_transfer_context[sw_index] = NULL;
721 
722 		/* Update sw_index */
723 		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
724 		src_ring->sw_index = sw_index;
725 		ret = 0;
726 	} else {
727 		ret = -EIO;
728 	}
729 
730 	spin_unlock_bh(&ar_pci->ce_lock);
731 
732 	return ret;
733 }
734 
735 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
736 				  void **per_transfer_contextp)
737 {
738 	struct ath10k *ar = ce_state->ar;
739 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
740 	int ret;
741 
742 	spin_lock_bh(&ar_pci->ce_lock);
743 	ret = ath10k_ce_completed_send_next_nolock(ce_state,
744 						   per_transfer_contextp);
745 	spin_unlock_bh(&ar_pci->ce_lock);
746 
747 	return ret;
748 }
749 
750 /*
751  * Guts of interrupt handler for per-engine interrupts on a particular CE.
752  *
753  * Invokes registered callbacks for recv_complete,
754  * send_complete, and watermarks.
755  */
756 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
757 {
758 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
759 	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
760 	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
761 	u32 ctrl_addr = ce_state->ctrl_addr;
762 
763 	spin_lock_bh(&ar_pci->ce_lock);
764 
765 	/* Clear the copy-complete interrupts that will be handled here. */
766 	ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->cc_mask);
767 
768 	spin_unlock_bh(&ar_pci->ce_lock);
769 
770 	if (ce_state->recv_cb)
771 		ce_state->recv_cb(ce_state);
772 
773 	if (ce_state->send_cb)
774 		ce_state->send_cb(ce_state);
775 
776 	spin_lock_bh(&ar_pci->ce_lock);
777 
778 	/*
779 	 * Misc CE interrupts are not being handled, but still need
780 	 * to be cleared.
781 	 */
782 	ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask);
783 
784 	spin_unlock_bh(&ar_pci->ce_lock);
785 }
786 
787 /*
788  * Handler for per-engine interrupts on ALL active CEs.
789  * This is used in cases where the system is sharing a
790  * single interrput for all CEs
791  */
792 
793 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
794 {
795 	int ce_id;
796 	u32 intr_summary;
797 
798 	intr_summary = CE_INTERRUPT_SUMMARY(ar);
799 
800 	for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
801 		if (intr_summary & (1 << ce_id))
802 			intr_summary &= ~(1 << ce_id);
803 		else
804 			/* no intr pending on this CE */
805 			continue;
806 
807 		ath10k_ce_per_engine_service(ar, ce_id);
808 	}
809 }
810 
811 /*
812  * Adjust interrupts for the copy complete handler.
813  * If it's needed for either send or recv, then unmask
814  * this interrupt; otherwise, mask it.
815  *
816  * Called with ce_lock held.
817  */
818 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
819 {
820 	u32 ctrl_addr = ce_state->ctrl_addr;
821 	struct ath10k *ar = ce_state->ar;
822 	bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
823 
824 	if ((!disable_copy_compl_intr) &&
825 	    (ce_state->send_cb || ce_state->recv_cb))
826 		ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
827 	else
828 		ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
829 
830 	ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
831 }
832 
833 int ath10k_ce_disable_interrupts(struct ath10k *ar)
834 {
835 	int ce_id;
836 
837 	for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
838 		u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
839 
840 		ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
841 		ath10k_ce_error_intr_disable(ar, ctrl_addr);
842 		ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
843 	}
844 
845 	return 0;
846 }
847 
848 void ath10k_ce_enable_interrupts(struct ath10k *ar)
849 {
850 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
851 	int ce_id;
852 
853 	/* Skip the last copy engine, CE7 the diagnostic window, as that
854 	 * uses polling and isn't initialized for interrupts.
855 	 */
856 	for (ce_id = 0; ce_id < CE_COUNT - 1; ce_id++)
857 		ath10k_ce_per_engine_handler_adjust(&ar_pci->ce_states[ce_id]);
858 }
859 
860 static int ath10k_ce_init_src_ring(struct ath10k *ar,
861 				   unsigned int ce_id,
862 				   const struct ce_attr *attr)
863 {
864 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
865 	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
866 	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
867 	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
868 
869 	nentries = roundup_pow_of_two(attr->src_nentries);
870 
871 	memset(src_ring->base_addr_owner_space, 0,
872 	       nentries * sizeof(struct ce_desc));
873 
874 	src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
875 	src_ring->sw_index &= src_ring->nentries_mask;
876 	src_ring->hw_index = src_ring->sw_index;
877 
878 	src_ring->write_index =
879 		ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
880 	src_ring->write_index &= src_ring->nentries_mask;
881 
882 	ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr,
883 					 src_ring->base_addr_ce_space);
884 	ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
885 	ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
886 	ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
887 	ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
888 	ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
889 
890 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
891 		   "boot init ce src ring id %d entries %d base_addr %pK\n",
892 		   ce_id, nentries, src_ring->base_addr_owner_space);
893 
894 	return 0;
895 }
896 
897 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
898 				    unsigned int ce_id,
899 				    const struct ce_attr *attr)
900 {
901 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
902 	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
903 	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
904 	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
905 
906 	nentries = roundup_pow_of_two(attr->dest_nentries);
907 
908 	memset(dest_ring->base_addr_owner_space, 0,
909 	       nentries * sizeof(struct ce_desc));
910 
911 	dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
912 	dest_ring->sw_index &= dest_ring->nentries_mask;
913 	dest_ring->write_index =
914 		ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
915 	dest_ring->write_index &= dest_ring->nentries_mask;
916 
917 	ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr,
918 					  dest_ring->base_addr_ce_space);
919 	ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
920 	ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
921 	ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
922 	ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
923 
924 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
925 		   "boot ce dest ring id %d entries %d base_addr %pK\n",
926 		   ce_id, nentries, dest_ring->base_addr_owner_space);
927 
928 	return 0;
929 }
930 
931 static struct ath10k_ce_ring *
932 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
933 			 const struct ce_attr *attr)
934 {
935 	struct ath10k_ce_ring *src_ring;
936 	u32 nentries = attr->src_nentries;
937 	dma_addr_t base_addr;
938 
939 	nentries = roundup_pow_of_two(nentries);
940 
941 	src_ring = kzalloc(sizeof(*src_ring) +
942 			   (nentries *
943 			    sizeof(*src_ring->per_transfer_context)),
944 			   GFP_KERNEL);
945 	if (src_ring == NULL)
946 		return ERR_PTR(-ENOMEM);
947 
948 	src_ring->nentries = nentries;
949 	src_ring->nentries_mask = nentries - 1;
950 
951 	/*
952 	 * Legacy platforms that do not support cache
953 	 * coherent DMA are unsupported
954 	 */
955 	src_ring->base_addr_owner_space_unaligned =
956 		dma_alloc_coherent(ar->dev,
957 				   (nentries * sizeof(struct ce_desc) +
958 				    CE_DESC_RING_ALIGN),
959 				   &base_addr, GFP_KERNEL);
960 	if (!src_ring->base_addr_owner_space_unaligned) {
961 		kfree(src_ring);
962 		return ERR_PTR(-ENOMEM);
963 	}
964 
965 	src_ring->base_addr_ce_space_unaligned = base_addr;
966 
967 	src_ring->base_addr_owner_space = PTR_ALIGN(
968 			src_ring->base_addr_owner_space_unaligned,
969 			CE_DESC_RING_ALIGN);
970 	src_ring->base_addr_ce_space = ALIGN(
971 			src_ring->base_addr_ce_space_unaligned,
972 			CE_DESC_RING_ALIGN);
973 
974 	return src_ring;
975 }
976 
977 static struct ath10k_ce_ring *
978 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
979 			  const struct ce_attr *attr)
980 {
981 	struct ath10k_ce_ring *dest_ring;
982 	u32 nentries;
983 	dma_addr_t base_addr;
984 
985 	nentries = roundup_pow_of_two(attr->dest_nentries);
986 
987 	dest_ring = kzalloc(sizeof(*dest_ring) +
988 			    (nentries *
989 			     sizeof(*dest_ring->per_transfer_context)),
990 			    GFP_KERNEL);
991 	if (dest_ring == NULL)
992 		return ERR_PTR(-ENOMEM);
993 
994 	dest_ring->nentries = nentries;
995 	dest_ring->nentries_mask = nentries - 1;
996 
997 	/*
998 	 * Legacy platforms that do not support cache
999 	 * coherent DMA are unsupported
1000 	 */
1001 	dest_ring->base_addr_owner_space_unaligned =
1002 		dma_zalloc_coherent(ar->dev,
1003 				    (nentries * sizeof(struct ce_desc) +
1004 				     CE_DESC_RING_ALIGN),
1005 				    &base_addr, GFP_KERNEL);
1006 	if (!dest_ring->base_addr_owner_space_unaligned) {
1007 		kfree(dest_ring);
1008 		return ERR_PTR(-ENOMEM);
1009 	}
1010 
1011 	dest_ring->base_addr_ce_space_unaligned = base_addr;
1012 
1013 	dest_ring->base_addr_owner_space = PTR_ALIGN(
1014 			dest_ring->base_addr_owner_space_unaligned,
1015 			CE_DESC_RING_ALIGN);
1016 	dest_ring->base_addr_ce_space = ALIGN(
1017 			dest_ring->base_addr_ce_space_unaligned,
1018 			CE_DESC_RING_ALIGN);
1019 
1020 	return dest_ring;
1021 }
1022 
1023 /*
1024  * Initialize a Copy Engine based on caller-supplied attributes.
1025  * This may be called once to initialize both source and destination
1026  * rings or it may be called twice for separate source and destination
1027  * initialization. It may be that only one side or the other is
1028  * initialized by software/firmware.
1029  */
1030 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1031 			const struct ce_attr *attr)
1032 {
1033 	int ret;
1034 
1035 	if (attr->src_nentries) {
1036 		ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1037 		if (ret) {
1038 			ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1039 				   ce_id, ret);
1040 			return ret;
1041 		}
1042 	}
1043 
1044 	if (attr->dest_nentries) {
1045 		ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1046 		if (ret) {
1047 			ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1048 				   ce_id, ret);
1049 			return ret;
1050 		}
1051 	}
1052 
1053 	return 0;
1054 }
1055 
1056 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1057 {
1058 	u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1059 
1060 	ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0);
1061 	ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1062 	ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1063 	ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1064 }
1065 
1066 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1067 {
1068 	u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1069 
1070 	ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0);
1071 	ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1072 	ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1073 }
1074 
1075 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1076 {
1077 	ath10k_ce_deinit_src_ring(ar, ce_id);
1078 	ath10k_ce_deinit_dest_ring(ar, ce_id);
1079 }
1080 
1081 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1082 			 const struct ce_attr *attr)
1083 {
1084 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1085 	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
1086 	int ret;
1087 
1088 	/*
1089 	 * Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1090 	 * additional TX locking checks.
1091 	 *
1092 	 * For the lack of a better place do the check here.
1093 	 */
1094 	BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1095 		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1096 	BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1097 		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1098 	BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1099 		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1100 
1101 	ce_state->ar = ar;
1102 	ce_state->id = ce_id;
1103 	ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1104 	ce_state->attr_flags = attr->flags;
1105 	ce_state->src_sz_max = attr->src_sz_max;
1106 
1107 	if (attr->src_nentries)
1108 		ce_state->send_cb = attr->send_cb;
1109 
1110 	if (attr->dest_nentries)
1111 		ce_state->recv_cb = attr->recv_cb;
1112 
1113 	if (attr->src_nentries) {
1114 		ce_state->src_ring = ath10k_ce_alloc_src_ring(ar, ce_id, attr);
1115 		if (IS_ERR(ce_state->src_ring)) {
1116 			ret = PTR_ERR(ce_state->src_ring);
1117 			ath10k_err(ar, "failed to allocate copy engine source ring %d: %d\n",
1118 				   ce_id, ret);
1119 			ce_state->src_ring = NULL;
1120 			return ret;
1121 		}
1122 	}
1123 
1124 	if (attr->dest_nentries) {
1125 		ce_state->dest_ring = ath10k_ce_alloc_dest_ring(ar, ce_id,
1126 								attr);
1127 		if (IS_ERR(ce_state->dest_ring)) {
1128 			ret = PTR_ERR(ce_state->dest_ring);
1129 			ath10k_err(ar, "failed to allocate copy engine destination ring %d: %d\n",
1130 				   ce_id, ret);
1131 			ce_state->dest_ring = NULL;
1132 			return ret;
1133 		}
1134 	}
1135 
1136 	return 0;
1137 }
1138 
1139 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1140 {
1141 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1142 	struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
1143 
1144 	if (ce_state->src_ring) {
1145 		dma_free_coherent(ar->dev,
1146 				  (ce_state->src_ring->nentries *
1147 				   sizeof(struct ce_desc) +
1148 				   CE_DESC_RING_ALIGN),
1149 				  ce_state->src_ring->base_addr_owner_space,
1150 				  ce_state->src_ring->base_addr_ce_space);
1151 		kfree(ce_state->src_ring);
1152 	}
1153 
1154 	if (ce_state->dest_ring) {
1155 		dma_free_coherent(ar->dev,
1156 				  (ce_state->dest_ring->nentries *
1157 				   sizeof(struct ce_desc) +
1158 				   CE_DESC_RING_ALIGN),
1159 				  ce_state->dest_ring->base_addr_owner_space,
1160 				  ce_state->dest_ring->base_addr_ce_space);
1161 		kfree(ce_state->dest_ring);
1162 	}
1163 
1164 	ce_state->src_ring = NULL;
1165 	ce_state->dest_ring = NULL;
1166 }
1167 
1168 void ath10k_ce_dump_registers(struct ath10k *ar,
1169 			      struct ath10k_fw_crash_data *crash_data)
1170 {
1171 	struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1172 	struct ath10k_ce_crash_data ce;
1173 	u32 addr, id;
1174 
1175 	lockdep_assert_held(&ar->data_lock);
1176 
1177 	ath10k_err(ar, "Copy Engine register dump:\n");
1178 
1179 	spin_lock_bh(&ar_pci->ce_lock);
1180 	for (id = 0; id < CE_COUNT; id++) {
1181 		addr = ath10k_ce_base_address(ar, id);
1182 		ce.base_addr = cpu_to_le32(addr);
1183 
1184 		ce.src_wr_idx =
1185 			cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1186 		ce.src_r_idx =
1187 			cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1188 		ce.dst_wr_idx =
1189 			cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1190 		ce.dst_r_idx =
1191 			cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1192 
1193 		if (crash_data)
1194 			crash_data->ce_crash_data[id] = ce;
1195 
1196 		ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1197 			   le32_to_cpu(ce.base_addr),
1198 			   le32_to_cpu(ce.src_wr_idx),
1199 			   le32_to_cpu(ce.src_r_idx),
1200 			   le32_to_cpu(ce.dst_wr_idx),
1201 			   le32_to_cpu(ce.dst_r_idx));
1202 	}
1203 
1204 	spin_unlock_bh(&ar_pci->ce_lock);
1205 }
1206