1 /* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18 #include "hif.h" 19 #include "pci.h" 20 #include "ce.h" 21 #include "debug.h" 22 23 /* 24 * Support for Copy Engine hardware, which is mainly used for 25 * communication between Host and Target over a PCIe interconnect. 26 */ 27 28 /* 29 * A single CopyEngine (CE) comprises two "rings": 30 * a source ring 31 * a destination ring 32 * 33 * Each ring consists of a number of descriptors which specify 34 * an address, length, and meta-data. 35 * 36 * Typically, one side of the PCIe interconnect (Host or Target) 37 * controls one ring and the other side controls the other ring. 38 * The source side chooses when to initiate a transfer and it 39 * chooses what to send (buffer address, length). The destination 40 * side keeps a supply of "anonymous receive buffers" available and 41 * it handles incoming data as it arrives (when the destination 42 * receives an interrupt). 43 * 44 * The sender may send a simple buffer (address/length) or it may 45 * send a small list of buffers. When a small list is sent, hardware 46 * "gathers" these and they end up in a single destination buffer 47 * with a single interrupt. 48 * 49 * There are several "contexts" managed by this layer -- more, it 50 * may seem -- than should be needed. These are provided mainly for 51 * maximum flexibility and especially to facilitate a simpler HIF 52 * implementation. There are per-CopyEngine recv, send, and watermark 53 * contexts. These are supplied by the caller when a recv, send, 54 * or watermark handler is established and they are echoed back to 55 * the caller when the respective callbacks are invoked. There is 56 * also a per-transfer context supplied by the caller when a buffer 57 * (or sendlist) is sent and when a buffer is enqueued for recv. 58 * These per-transfer contexts are echoed back to the caller when 59 * the buffer is sent/received. 60 */ 61 62 static inline unsigned int 63 ath10k_set_ring_byte(unsigned int offset, 64 struct ath10k_hw_ce_regs_addr_map *addr_map) 65 { 66 return ((offset << addr_map->lsb) & addr_map->mask); 67 } 68 69 static inline unsigned int 70 ath10k_get_ring_byte(unsigned int offset, 71 struct ath10k_hw_ce_regs_addr_map *addr_map) 72 { 73 return ((offset & addr_map->mask) >> (addr_map->lsb)); 74 } 75 76 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 77 u32 ce_ctrl_addr, 78 unsigned int n) 79 { 80 ath10k_pci_write32(ar, ce_ctrl_addr + 81 ar->hw_ce_regs->dst_wr_index_addr, n); 82 } 83 84 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 85 u32 ce_ctrl_addr) 86 { 87 return ath10k_pci_read32(ar, ce_ctrl_addr + 88 ar->hw_ce_regs->dst_wr_index_addr); 89 } 90 91 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 92 u32 ce_ctrl_addr, 93 unsigned int n) 94 { 95 ath10k_pci_write32(ar, ce_ctrl_addr + 96 ar->hw_ce_regs->sr_wr_index_addr, n); 97 } 98 99 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 100 u32 ce_ctrl_addr) 101 { 102 return ath10k_pci_read32(ar, ce_ctrl_addr + 103 ar->hw_ce_regs->sr_wr_index_addr); 104 } 105 106 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 107 u32 ce_ctrl_addr) 108 { 109 return ath10k_pci_read32(ar, ce_ctrl_addr + 110 ar->hw_ce_regs->current_srri_addr); 111 } 112 113 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 114 u32 ce_ctrl_addr, 115 unsigned int addr) 116 { 117 ath10k_pci_write32(ar, ce_ctrl_addr + 118 ar->hw_ce_regs->sr_base_addr, addr); 119 } 120 121 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 122 u32 ce_ctrl_addr, 123 unsigned int n) 124 { 125 ath10k_pci_write32(ar, ce_ctrl_addr + 126 ar->hw_ce_regs->sr_size_addr, n); 127 } 128 129 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 130 u32 ce_ctrl_addr, 131 unsigned int n) 132 { 133 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 134 u32 ctrl1_addr = ath10k_pci_read32(ar, 135 ce_ctrl_addr + ctrl_regs->addr); 136 137 ath10k_pci_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 138 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) | 139 ath10k_set_ring_byte(n, ctrl_regs->dmax)); 140 } 141 142 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 143 u32 ce_ctrl_addr, 144 unsigned int n) 145 { 146 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 147 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + ctrl_regs->addr); 148 149 ath10k_pci_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 150 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) | 151 ath10k_set_ring_byte(n, ctrl_regs->src_ring)); 152 } 153 154 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 155 u32 ce_ctrl_addr, 156 unsigned int n) 157 { 158 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 159 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + ctrl_regs->addr); 160 161 ath10k_pci_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 162 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) | 163 ath10k_set_ring_byte(n, ctrl_regs->dst_ring)); 164 } 165 166 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 167 u32 ce_ctrl_addr) 168 { 169 return ath10k_pci_read32(ar, ce_ctrl_addr + 170 ar->hw_ce_regs->current_drri_addr); 171 } 172 173 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 174 u32 ce_ctrl_addr, 175 u32 addr) 176 { 177 ath10k_pci_write32(ar, ce_ctrl_addr + 178 ar->hw_ce_regs->dr_base_addr, addr); 179 } 180 181 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 182 u32 ce_ctrl_addr, 183 unsigned int n) 184 { 185 ath10k_pci_write32(ar, ce_ctrl_addr + 186 ar->hw_ce_regs->dr_size_addr, n); 187 } 188 189 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 190 u32 ce_ctrl_addr, 191 unsigned int n) 192 { 193 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 194 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + srcr_wm->addr); 195 196 ath10k_pci_write32(ar, ce_ctrl_addr + srcr_wm->addr, 197 (addr & ~(srcr_wm->wm_high->mask)) | 198 (ath10k_set_ring_byte(n, srcr_wm->wm_high))); 199 } 200 201 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 202 u32 ce_ctrl_addr, 203 unsigned int n) 204 { 205 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 206 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + srcr_wm->addr); 207 208 ath10k_pci_write32(ar, ce_ctrl_addr + srcr_wm->addr, 209 (addr & ~(srcr_wm->wm_low->mask)) | 210 (ath10k_set_ring_byte(n, srcr_wm->wm_low))); 211 } 212 213 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 214 u32 ce_ctrl_addr, 215 unsigned int n) 216 { 217 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 218 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + dstr_wm->addr); 219 220 ath10k_pci_write32(ar, ce_ctrl_addr + dstr_wm->addr, 221 (addr & ~(dstr_wm->wm_high->mask)) | 222 (ath10k_set_ring_byte(n, dstr_wm->wm_high))); 223 } 224 225 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 226 u32 ce_ctrl_addr, 227 unsigned int n) 228 { 229 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 230 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + dstr_wm->addr); 231 232 ath10k_pci_write32(ar, ce_ctrl_addr + dstr_wm->addr, 233 (addr & ~(dstr_wm->wm_low->mask)) | 234 (ath10k_set_ring_byte(n, dstr_wm->wm_low))); 235 } 236 237 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 238 u32 ce_ctrl_addr) 239 { 240 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 241 u32 host_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr + 242 ar->hw_ce_regs->host_ie_addr); 243 244 ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 245 host_ie_addr | host_ie->copy_complete->mask); 246 } 247 248 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 249 u32 ce_ctrl_addr) 250 { 251 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 252 u32 host_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr + 253 ar->hw_ce_regs->host_ie_addr); 254 255 ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 256 host_ie_addr & ~(host_ie->copy_complete->mask)); 257 } 258 259 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 260 u32 ce_ctrl_addr) 261 { 262 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 263 u32 host_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr + 264 ar->hw_ce_regs->host_ie_addr); 265 266 ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 267 host_ie_addr & ~(wm_regs->wm_mask)); 268 } 269 270 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 271 u32 ce_ctrl_addr) 272 { 273 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 274 u32 misc_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr + 275 ar->hw_ce_regs->misc_ie_addr); 276 277 ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 278 misc_ie_addr | misc_regs->err_mask); 279 } 280 281 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, 282 u32 ce_ctrl_addr) 283 { 284 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 285 u32 misc_ie_addr = ath10k_pci_read32(ar, ce_ctrl_addr + 286 ar->hw_ce_regs->misc_ie_addr); 287 288 ath10k_pci_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 289 misc_ie_addr & ~(misc_regs->err_mask)); 290 } 291 292 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 293 u32 ce_ctrl_addr, 294 unsigned int mask) 295 { 296 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 297 298 ath10k_pci_write32(ar, ce_ctrl_addr + wm_regs->addr, mask); 299 } 300 301 /* 302 * Guts of ath10k_ce_send. 303 * The caller takes responsibility for any needed locking. 304 */ 305 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 306 void *per_transfer_context, 307 u32 buffer, 308 unsigned int nbytes, 309 unsigned int transfer_id, 310 unsigned int flags) 311 { 312 struct ath10k *ar = ce_state->ar; 313 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 314 struct ce_desc *desc, sdesc; 315 unsigned int nentries_mask = src_ring->nentries_mask; 316 unsigned int sw_index = src_ring->sw_index; 317 unsigned int write_index = src_ring->write_index; 318 u32 ctrl_addr = ce_state->ctrl_addr; 319 u32 desc_flags = 0; 320 int ret = 0; 321 322 if (nbytes > ce_state->src_sz_max) 323 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 324 __func__, nbytes, ce_state->src_sz_max); 325 326 if (unlikely(CE_RING_DELTA(nentries_mask, 327 write_index, sw_index - 1) <= 0)) { 328 ret = -ENOSR; 329 goto exit; 330 } 331 332 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 333 write_index); 334 335 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 336 337 if (flags & CE_SEND_FLAG_GATHER) 338 desc_flags |= CE_DESC_FLAGS_GATHER; 339 if (flags & CE_SEND_FLAG_BYTE_SWAP) 340 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 341 342 sdesc.addr = __cpu_to_le32(buffer); 343 sdesc.nbytes = __cpu_to_le16(nbytes); 344 sdesc.flags = __cpu_to_le16(desc_flags); 345 346 *desc = sdesc; 347 348 src_ring->per_transfer_context[write_index] = per_transfer_context; 349 350 /* Update Source Ring Write Index */ 351 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 352 353 /* WORKAROUND */ 354 if (!(flags & CE_SEND_FLAG_GATHER)) 355 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 356 357 src_ring->write_index = write_index; 358 exit: 359 return ret; 360 } 361 362 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe) 363 { 364 struct ath10k *ar = pipe->ar; 365 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 366 struct ath10k_ce_ring *src_ring = pipe->src_ring; 367 u32 ctrl_addr = pipe->ctrl_addr; 368 369 lockdep_assert_held(&ar_pci->ce_lock); 370 371 /* 372 * This function must be called only if there is an incomplete 373 * scatter-gather transfer (before index register is updated) 374 * that needs to be cleaned up. 375 */ 376 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) 377 return; 378 379 if (WARN_ON_ONCE(src_ring->write_index == 380 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) 381 return; 382 383 src_ring->write_index--; 384 src_ring->write_index &= src_ring->nentries_mask; 385 386 src_ring->per_transfer_context[src_ring->write_index] = NULL; 387 } 388 389 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, 390 void *per_transfer_context, 391 u32 buffer, 392 unsigned int nbytes, 393 unsigned int transfer_id, 394 unsigned int flags) 395 { 396 struct ath10k *ar = ce_state->ar; 397 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 398 int ret; 399 400 spin_lock_bh(&ar_pci->ce_lock); 401 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 402 buffer, nbytes, transfer_id, flags); 403 spin_unlock_bh(&ar_pci->ce_lock); 404 405 return ret; 406 } 407 408 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe) 409 { 410 struct ath10k *ar = pipe->ar; 411 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 412 int delta; 413 414 spin_lock_bh(&ar_pci->ce_lock); 415 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, 416 pipe->src_ring->write_index, 417 pipe->src_ring->sw_index - 1); 418 spin_unlock_bh(&ar_pci->ce_lock); 419 420 return delta; 421 } 422 423 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe) 424 { 425 struct ath10k *ar = pipe->ar; 426 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 427 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 428 unsigned int nentries_mask = dest_ring->nentries_mask; 429 unsigned int write_index = dest_ring->write_index; 430 unsigned int sw_index = dest_ring->sw_index; 431 432 lockdep_assert_held(&ar_pci->ce_lock); 433 434 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 435 } 436 437 int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr) 438 { 439 struct ath10k *ar = pipe->ar; 440 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 441 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 442 unsigned int nentries_mask = dest_ring->nentries_mask; 443 unsigned int write_index = dest_ring->write_index; 444 unsigned int sw_index = dest_ring->sw_index; 445 struct ce_desc *base = dest_ring->base_addr_owner_space; 446 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 447 u32 ctrl_addr = pipe->ctrl_addr; 448 449 lockdep_assert_held(&ar_pci->ce_lock); 450 451 if ((pipe->id != 5) && 452 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 453 return -ENOSPC; 454 455 desc->addr = __cpu_to_le32(paddr); 456 desc->nbytes = 0; 457 458 dest_ring->per_transfer_context[write_index] = ctx; 459 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 460 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 461 dest_ring->write_index = write_index; 462 463 return 0; 464 } 465 466 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries) 467 { 468 struct ath10k *ar = pipe->ar; 469 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 470 unsigned int nentries_mask = dest_ring->nentries_mask; 471 unsigned int write_index = dest_ring->write_index; 472 u32 ctrl_addr = pipe->ctrl_addr; 473 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 474 475 /* Prevent CE ring stuck issue that will occur when ring is full. 476 * Make sure that write index is 1 less than read index. 477 */ 478 if ((cur_write_idx + nentries) == dest_ring->sw_index) 479 nentries -= 1; 480 481 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries); 482 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 483 dest_ring->write_index = write_index; 484 } 485 486 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr) 487 { 488 struct ath10k *ar = pipe->ar; 489 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 490 int ret; 491 492 spin_lock_bh(&ar_pci->ce_lock); 493 ret = __ath10k_ce_rx_post_buf(pipe, ctx, paddr); 494 spin_unlock_bh(&ar_pci->ce_lock); 495 496 return ret; 497 } 498 499 /* 500 * Guts of ath10k_ce_completed_recv_next. 501 * The caller takes responsibility for any necessary locking. 502 */ 503 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 504 void **per_transfer_contextp, 505 unsigned int *nbytesp) 506 { 507 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 508 unsigned int nentries_mask = dest_ring->nentries_mask; 509 unsigned int sw_index = dest_ring->sw_index; 510 511 struct ce_desc *base = dest_ring->base_addr_owner_space; 512 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 513 struct ce_desc sdesc; 514 u16 nbytes; 515 516 /* Copy in one go for performance reasons */ 517 sdesc = *desc; 518 519 nbytes = __le16_to_cpu(sdesc.nbytes); 520 if (nbytes == 0) { 521 /* 522 * This closes a relatively unusual race where the Host 523 * sees the updated DRRI before the update to the 524 * corresponding descriptor has completed. We treat this 525 * as a descriptor that is not yet done. 526 */ 527 return -EIO; 528 } 529 530 desc->nbytes = 0; 531 532 /* Return data from completed destination descriptor */ 533 *nbytesp = nbytes; 534 535 if (per_transfer_contextp) 536 *per_transfer_contextp = 537 dest_ring->per_transfer_context[sw_index]; 538 539 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 540 * So update transfer context all CEs except CE5. 541 */ 542 if (ce_state->id != 5) 543 dest_ring->per_transfer_context[sw_index] = NULL; 544 545 /* Update sw_index */ 546 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 547 dest_ring->sw_index = sw_index; 548 549 return 0; 550 } 551 552 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, 553 void **per_transfer_contextp, 554 unsigned int *nbytesp) 555 { 556 struct ath10k *ar = ce_state->ar; 557 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 558 int ret; 559 560 spin_lock_bh(&ar_pci->ce_lock); 561 ret = ath10k_ce_completed_recv_next_nolock(ce_state, 562 per_transfer_contextp, 563 nbytesp); 564 spin_unlock_bh(&ar_pci->ce_lock); 565 566 return ret; 567 } 568 569 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 570 void **per_transfer_contextp, 571 u32 *bufferp) 572 { 573 struct ath10k_ce_ring *dest_ring; 574 unsigned int nentries_mask; 575 unsigned int sw_index; 576 unsigned int write_index; 577 int ret; 578 struct ath10k *ar; 579 struct ath10k_pci *ar_pci; 580 581 dest_ring = ce_state->dest_ring; 582 583 if (!dest_ring) 584 return -EIO; 585 586 ar = ce_state->ar; 587 ar_pci = ath10k_pci_priv(ar); 588 589 spin_lock_bh(&ar_pci->ce_lock); 590 591 nentries_mask = dest_ring->nentries_mask; 592 sw_index = dest_ring->sw_index; 593 write_index = dest_ring->write_index; 594 if (write_index != sw_index) { 595 struct ce_desc *base = dest_ring->base_addr_owner_space; 596 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 597 598 /* Return data from completed destination descriptor */ 599 *bufferp = __le32_to_cpu(desc->addr); 600 601 if (per_transfer_contextp) 602 *per_transfer_contextp = 603 dest_ring->per_transfer_context[sw_index]; 604 605 /* sanity */ 606 dest_ring->per_transfer_context[sw_index] = NULL; 607 desc->nbytes = 0; 608 609 /* Update sw_index */ 610 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 611 dest_ring->sw_index = sw_index; 612 ret = 0; 613 } else { 614 ret = -EIO; 615 } 616 617 spin_unlock_bh(&ar_pci->ce_lock); 618 619 return ret; 620 } 621 622 /* 623 * Guts of ath10k_ce_completed_send_next. 624 * The caller takes responsibility for any necessary locking. 625 */ 626 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 627 void **per_transfer_contextp) 628 { 629 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 630 u32 ctrl_addr = ce_state->ctrl_addr; 631 struct ath10k *ar = ce_state->ar; 632 unsigned int nentries_mask = src_ring->nentries_mask; 633 unsigned int sw_index = src_ring->sw_index; 634 unsigned int read_index; 635 struct ce_desc *desc; 636 637 if (src_ring->hw_index == sw_index) { 638 /* 639 * The SW completion index has caught up with the cached 640 * version of the HW completion index. 641 * Update the cached HW completion index to see whether 642 * the SW has really caught up to the HW, or if the cached 643 * value of the HW index has become stale. 644 */ 645 646 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 647 if (read_index == 0xffffffff) 648 return -ENODEV; 649 650 read_index &= nentries_mask; 651 src_ring->hw_index = read_index; 652 } 653 654 read_index = src_ring->hw_index; 655 656 if (read_index == sw_index) 657 return -EIO; 658 659 if (per_transfer_contextp) 660 *per_transfer_contextp = 661 src_ring->per_transfer_context[sw_index]; 662 663 /* sanity */ 664 src_ring->per_transfer_context[sw_index] = NULL; 665 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 666 sw_index); 667 desc->nbytes = 0; 668 669 /* Update sw_index */ 670 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 671 src_ring->sw_index = sw_index; 672 673 return 0; 674 } 675 676 /* NB: Modeled after ath10k_ce_completed_send_next */ 677 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, 678 void **per_transfer_contextp, 679 u32 *bufferp, 680 unsigned int *nbytesp, 681 unsigned int *transfer_idp) 682 { 683 struct ath10k_ce_ring *src_ring; 684 unsigned int nentries_mask; 685 unsigned int sw_index; 686 unsigned int write_index; 687 int ret; 688 struct ath10k *ar; 689 struct ath10k_pci *ar_pci; 690 691 src_ring = ce_state->src_ring; 692 693 if (!src_ring) 694 return -EIO; 695 696 ar = ce_state->ar; 697 ar_pci = ath10k_pci_priv(ar); 698 699 spin_lock_bh(&ar_pci->ce_lock); 700 701 nentries_mask = src_ring->nentries_mask; 702 sw_index = src_ring->sw_index; 703 write_index = src_ring->write_index; 704 705 if (write_index != sw_index) { 706 struct ce_desc *base = src_ring->base_addr_owner_space; 707 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 708 709 /* Return data from completed source descriptor */ 710 *bufferp = __le32_to_cpu(desc->addr); 711 *nbytesp = __le16_to_cpu(desc->nbytes); 712 *transfer_idp = MS(__le16_to_cpu(desc->flags), 713 CE_DESC_FLAGS_META_DATA); 714 715 if (per_transfer_contextp) 716 *per_transfer_contextp = 717 src_ring->per_transfer_context[sw_index]; 718 719 /* sanity */ 720 src_ring->per_transfer_context[sw_index] = NULL; 721 722 /* Update sw_index */ 723 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 724 src_ring->sw_index = sw_index; 725 ret = 0; 726 } else { 727 ret = -EIO; 728 } 729 730 spin_unlock_bh(&ar_pci->ce_lock); 731 732 return ret; 733 } 734 735 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, 736 void **per_transfer_contextp) 737 { 738 struct ath10k *ar = ce_state->ar; 739 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 740 int ret; 741 742 spin_lock_bh(&ar_pci->ce_lock); 743 ret = ath10k_ce_completed_send_next_nolock(ce_state, 744 per_transfer_contextp); 745 spin_unlock_bh(&ar_pci->ce_lock); 746 747 return ret; 748 } 749 750 /* 751 * Guts of interrupt handler for per-engine interrupts on a particular CE. 752 * 753 * Invokes registered callbacks for recv_complete, 754 * send_complete, and watermarks. 755 */ 756 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 757 { 758 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 759 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 760 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 761 u32 ctrl_addr = ce_state->ctrl_addr; 762 763 spin_lock_bh(&ar_pci->ce_lock); 764 765 /* Clear the copy-complete interrupts that will be handled here. */ 766 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->cc_mask); 767 768 spin_unlock_bh(&ar_pci->ce_lock); 769 770 if (ce_state->recv_cb) 771 ce_state->recv_cb(ce_state); 772 773 if (ce_state->send_cb) 774 ce_state->send_cb(ce_state); 775 776 spin_lock_bh(&ar_pci->ce_lock); 777 778 /* 779 * Misc CE interrupts are not being handled, but still need 780 * to be cleared. 781 */ 782 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask); 783 784 spin_unlock_bh(&ar_pci->ce_lock); 785 } 786 787 /* 788 * Handler for per-engine interrupts on ALL active CEs. 789 * This is used in cases where the system is sharing a 790 * single interrput for all CEs 791 */ 792 793 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 794 { 795 int ce_id; 796 u32 intr_summary; 797 798 intr_summary = CE_INTERRUPT_SUMMARY(ar); 799 800 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) { 801 if (intr_summary & (1 << ce_id)) 802 intr_summary &= ~(1 << ce_id); 803 else 804 /* no intr pending on this CE */ 805 continue; 806 807 ath10k_ce_per_engine_service(ar, ce_id); 808 } 809 } 810 811 /* 812 * Adjust interrupts for the copy complete handler. 813 * If it's needed for either send or recv, then unmask 814 * this interrupt; otherwise, mask it. 815 * 816 * Called with ce_lock held. 817 */ 818 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state) 819 { 820 u32 ctrl_addr = ce_state->ctrl_addr; 821 struct ath10k *ar = ce_state->ar; 822 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; 823 824 if ((!disable_copy_compl_intr) && 825 (ce_state->send_cb || ce_state->recv_cb)) 826 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 827 else 828 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 829 830 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 831 } 832 833 int ath10k_ce_disable_interrupts(struct ath10k *ar) 834 { 835 int ce_id; 836 837 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) { 838 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 839 840 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 841 ath10k_ce_error_intr_disable(ar, ctrl_addr); 842 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 843 } 844 845 return 0; 846 } 847 848 void ath10k_ce_enable_interrupts(struct ath10k *ar) 849 { 850 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 851 int ce_id; 852 853 /* Skip the last copy engine, CE7 the diagnostic window, as that 854 * uses polling and isn't initialized for interrupts. 855 */ 856 for (ce_id = 0; ce_id < CE_COUNT - 1; ce_id++) 857 ath10k_ce_per_engine_handler_adjust(&ar_pci->ce_states[ce_id]); 858 } 859 860 static int ath10k_ce_init_src_ring(struct ath10k *ar, 861 unsigned int ce_id, 862 const struct ce_attr *attr) 863 { 864 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 865 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 866 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 867 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 868 869 nentries = roundup_pow_of_two(attr->src_nentries); 870 871 memset(src_ring->base_addr_owner_space, 0, 872 nentries * sizeof(struct ce_desc)); 873 874 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 875 src_ring->sw_index &= src_ring->nentries_mask; 876 src_ring->hw_index = src_ring->sw_index; 877 878 src_ring->write_index = 879 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 880 src_ring->write_index &= src_ring->nentries_mask; 881 882 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 883 src_ring->base_addr_ce_space); 884 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 885 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 886 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 887 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 888 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 889 890 ath10k_dbg(ar, ATH10K_DBG_BOOT, 891 "boot init ce src ring id %d entries %d base_addr %pK\n", 892 ce_id, nentries, src_ring->base_addr_owner_space); 893 894 return 0; 895 } 896 897 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 898 unsigned int ce_id, 899 const struct ce_attr *attr) 900 { 901 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 902 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 903 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 904 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 905 906 nentries = roundup_pow_of_two(attr->dest_nentries); 907 908 memset(dest_ring->base_addr_owner_space, 0, 909 nentries * sizeof(struct ce_desc)); 910 911 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 912 dest_ring->sw_index &= dest_ring->nentries_mask; 913 dest_ring->write_index = 914 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 915 dest_ring->write_index &= dest_ring->nentries_mask; 916 917 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 918 dest_ring->base_addr_ce_space); 919 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 920 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 921 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 922 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 923 924 ath10k_dbg(ar, ATH10K_DBG_BOOT, 925 "boot ce dest ring id %d entries %d base_addr %pK\n", 926 ce_id, nentries, dest_ring->base_addr_owner_space); 927 928 return 0; 929 } 930 931 static struct ath10k_ce_ring * 932 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, 933 const struct ce_attr *attr) 934 { 935 struct ath10k_ce_ring *src_ring; 936 u32 nentries = attr->src_nentries; 937 dma_addr_t base_addr; 938 939 nentries = roundup_pow_of_two(nentries); 940 941 src_ring = kzalloc(sizeof(*src_ring) + 942 (nentries * 943 sizeof(*src_ring->per_transfer_context)), 944 GFP_KERNEL); 945 if (src_ring == NULL) 946 return ERR_PTR(-ENOMEM); 947 948 src_ring->nentries = nentries; 949 src_ring->nentries_mask = nentries - 1; 950 951 /* 952 * Legacy platforms that do not support cache 953 * coherent DMA are unsupported 954 */ 955 src_ring->base_addr_owner_space_unaligned = 956 dma_alloc_coherent(ar->dev, 957 (nentries * sizeof(struct ce_desc) + 958 CE_DESC_RING_ALIGN), 959 &base_addr, GFP_KERNEL); 960 if (!src_ring->base_addr_owner_space_unaligned) { 961 kfree(src_ring); 962 return ERR_PTR(-ENOMEM); 963 } 964 965 src_ring->base_addr_ce_space_unaligned = base_addr; 966 967 src_ring->base_addr_owner_space = PTR_ALIGN( 968 src_ring->base_addr_owner_space_unaligned, 969 CE_DESC_RING_ALIGN); 970 src_ring->base_addr_ce_space = ALIGN( 971 src_ring->base_addr_ce_space_unaligned, 972 CE_DESC_RING_ALIGN); 973 974 return src_ring; 975 } 976 977 static struct ath10k_ce_ring * 978 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, 979 const struct ce_attr *attr) 980 { 981 struct ath10k_ce_ring *dest_ring; 982 u32 nentries; 983 dma_addr_t base_addr; 984 985 nentries = roundup_pow_of_two(attr->dest_nentries); 986 987 dest_ring = kzalloc(sizeof(*dest_ring) + 988 (nentries * 989 sizeof(*dest_ring->per_transfer_context)), 990 GFP_KERNEL); 991 if (dest_ring == NULL) 992 return ERR_PTR(-ENOMEM); 993 994 dest_ring->nentries = nentries; 995 dest_ring->nentries_mask = nentries - 1; 996 997 /* 998 * Legacy platforms that do not support cache 999 * coherent DMA are unsupported 1000 */ 1001 dest_ring->base_addr_owner_space_unaligned = 1002 dma_zalloc_coherent(ar->dev, 1003 (nentries * sizeof(struct ce_desc) + 1004 CE_DESC_RING_ALIGN), 1005 &base_addr, GFP_KERNEL); 1006 if (!dest_ring->base_addr_owner_space_unaligned) { 1007 kfree(dest_ring); 1008 return ERR_PTR(-ENOMEM); 1009 } 1010 1011 dest_ring->base_addr_ce_space_unaligned = base_addr; 1012 1013 dest_ring->base_addr_owner_space = PTR_ALIGN( 1014 dest_ring->base_addr_owner_space_unaligned, 1015 CE_DESC_RING_ALIGN); 1016 dest_ring->base_addr_ce_space = ALIGN( 1017 dest_ring->base_addr_ce_space_unaligned, 1018 CE_DESC_RING_ALIGN); 1019 1020 return dest_ring; 1021 } 1022 1023 /* 1024 * Initialize a Copy Engine based on caller-supplied attributes. 1025 * This may be called once to initialize both source and destination 1026 * rings or it may be called twice for separate source and destination 1027 * initialization. It may be that only one side or the other is 1028 * initialized by software/firmware. 1029 */ 1030 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, 1031 const struct ce_attr *attr) 1032 { 1033 int ret; 1034 1035 if (attr->src_nentries) { 1036 ret = ath10k_ce_init_src_ring(ar, ce_id, attr); 1037 if (ret) { 1038 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n", 1039 ce_id, ret); 1040 return ret; 1041 } 1042 } 1043 1044 if (attr->dest_nentries) { 1045 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); 1046 if (ret) { 1047 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n", 1048 ce_id, ret); 1049 return ret; 1050 } 1051 } 1052 1053 return 0; 1054 } 1055 1056 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) 1057 { 1058 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1059 1060 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0); 1061 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0); 1062 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0); 1063 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0); 1064 } 1065 1066 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) 1067 { 1068 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1069 1070 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0); 1071 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0); 1072 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0); 1073 } 1074 1075 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) 1076 { 1077 ath10k_ce_deinit_src_ring(ar, ce_id); 1078 ath10k_ce_deinit_dest_ring(ar, ce_id); 1079 } 1080 1081 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, 1082 const struct ce_attr *attr) 1083 { 1084 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1085 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 1086 int ret; 1087 1088 /* 1089 * Make sure there's enough CE ringbuffer entries for HTT TX to avoid 1090 * additional TX locking checks. 1091 * 1092 * For the lack of a better place do the check here. 1093 */ 1094 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC > 1095 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1096 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC > 1097 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1098 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC > 1099 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1100 1101 ce_state->ar = ar; 1102 ce_state->id = ce_id; 1103 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1104 ce_state->attr_flags = attr->flags; 1105 ce_state->src_sz_max = attr->src_sz_max; 1106 1107 if (attr->src_nentries) 1108 ce_state->send_cb = attr->send_cb; 1109 1110 if (attr->dest_nentries) 1111 ce_state->recv_cb = attr->recv_cb; 1112 1113 if (attr->src_nentries) { 1114 ce_state->src_ring = ath10k_ce_alloc_src_ring(ar, ce_id, attr); 1115 if (IS_ERR(ce_state->src_ring)) { 1116 ret = PTR_ERR(ce_state->src_ring); 1117 ath10k_err(ar, "failed to allocate copy engine source ring %d: %d\n", 1118 ce_id, ret); 1119 ce_state->src_ring = NULL; 1120 return ret; 1121 } 1122 } 1123 1124 if (attr->dest_nentries) { 1125 ce_state->dest_ring = ath10k_ce_alloc_dest_ring(ar, ce_id, 1126 attr); 1127 if (IS_ERR(ce_state->dest_ring)) { 1128 ret = PTR_ERR(ce_state->dest_ring); 1129 ath10k_err(ar, "failed to allocate copy engine destination ring %d: %d\n", 1130 ce_id, ret); 1131 ce_state->dest_ring = NULL; 1132 return ret; 1133 } 1134 } 1135 1136 return 0; 1137 } 1138 1139 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1140 { 1141 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1142 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 1143 1144 if (ce_state->src_ring) { 1145 dma_free_coherent(ar->dev, 1146 (ce_state->src_ring->nentries * 1147 sizeof(struct ce_desc) + 1148 CE_DESC_RING_ALIGN), 1149 ce_state->src_ring->base_addr_owner_space, 1150 ce_state->src_ring->base_addr_ce_space); 1151 kfree(ce_state->src_ring); 1152 } 1153 1154 if (ce_state->dest_ring) { 1155 dma_free_coherent(ar->dev, 1156 (ce_state->dest_ring->nentries * 1157 sizeof(struct ce_desc) + 1158 CE_DESC_RING_ALIGN), 1159 ce_state->dest_ring->base_addr_owner_space, 1160 ce_state->dest_ring->base_addr_ce_space); 1161 kfree(ce_state->dest_ring); 1162 } 1163 1164 ce_state->src_ring = NULL; 1165 ce_state->dest_ring = NULL; 1166 } 1167 1168 void ath10k_ce_dump_registers(struct ath10k *ar, 1169 struct ath10k_fw_crash_data *crash_data) 1170 { 1171 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1172 struct ath10k_ce_crash_data ce; 1173 u32 addr, id; 1174 1175 lockdep_assert_held(&ar->data_lock); 1176 1177 ath10k_err(ar, "Copy Engine register dump:\n"); 1178 1179 spin_lock_bh(&ar_pci->ce_lock); 1180 for (id = 0; id < CE_COUNT; id++) { 1181 addr = ath10k_ce_base_address(ar, id); 1182 ce.base_addr = cpu_to_le32(addr); 1183 1184 ce.src_wr_idx = 1185 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr)); 1186 ce.src_r_idx = 1187 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr)); 1188 ce.dst_wr_idx = 1189 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr)); 1190 ce.dst_r_idx = 1191 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr)); 1192 1193 if (crash_data) 1194 crash_data->ce_crash_data[id] = ce; 1195 1196 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id, 1197 le32_to_cpu(ce.base_addr), 1198 le32_to_cpu(ce.src_wr_idx), 1199 le32_to_cpu(ce.src_r_idx), 1200 le32_to_cpu(ce.dst_wr_idx), 1201 le32_to_cpu(ce.dst_r_idx)); 1202 } 1203 1204 spin_unlock_bh(&ar_pci->ce_lock); 1205 } 1206