1 /* 2 * Copyright (c) 2005-2011 Atheros Communications Inc. 3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18 #include "hif.h" 19 #include "pci.h" 20 #include "ce.h" 21 #include "debug.h" 22 23 /* 24 * Support for Copy Engine hardware, which is mainly used for 25 * communication between Host and Target over a PCIe interconnect. 26 */ 27 28 /* 29 * A single CopyEngine (CE) comprises two "rings": 30 * a source ring 31 * a destination ring 32 * 33 * Each ring consists of a number of descriptors which specify 34 * an address, length, and meta-data. 35 * 36 * Typically, one side of the PCIe interconnect (Host or Target) 37 * controls one ring and the other side controls the other ring. 38 * The source side chooses when to initiate a transfer and it 39 * chooses what to send (buffer address, length). The destination 40 * side keeps a supply of "anonymous receive buffers" available and 41 * it handles incoming data as it arrives (when the destination 42 * receives an interrupt). 43 * 44 * The sender may send a simple buffer (address/length) or it may 45 * send a small list of buffers. When a small list is sent, hardware 46 * "gathers" these and they end up in a single destination buffer 47 * with a single interrupt. 48 * 49 * There are several "contexts" managed by this layer -- more, it 50 * may seem -- than should be needed. These are provided mainly for 51 * maximum flexibility and especially to facilitate a simpler HIF 52 * implementation. There are per-CopyEngine recv, send, and watermark 53 * contexts. These are supplied by the caller when a recv, send, 54 * or watermark handler is established and they are echoed back to 55 * the caller when the respective callbacks are invoked. There is 56 * also a per-transfer context supplied by the caller when a buffer 57 * (or sendlist) is sent and when a buffer is enqueued for recv. 58 * These per-transfer contexts are echoed back to the caller when 59 * the buffer is sent/received. 60 */ 61 62 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 63 u32 ce_ctrl_addr, 64 unsigned int n) 65 { 66 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n); 67 } 68 69 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 70 u32 ce_ctrl_addr) 71 { 72 return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS); 73 } 74 75 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 76 u32 ce_ctrl_addr, 77 unsigned int n) 78 { 79 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n); 80 } 81 82 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 83 u32 ce_ctrl_addr) 84 { 85 return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS); 86 } 87 88 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 89 u32 ce_ctrl_addr) 90 { 91 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS); 92 } 93 94 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 95 u32 ce_ctrl_addr, 96 unsigned int addr) 97 { 98 ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr); 99 } 100 101 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 102 u32 ce_ctrl_addr, 103 unsigned int n) 104 { 105 ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n); 106 } 107 108 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 109 u32 ce_ctrl_addr, 110 unsigned int n) 111 { 112 u32 ctrl1_addr = ath10k_pci_read32((ar), 113 (ce_ctrl_addr) + CE_CTRL1_ADDRESS); 114 115 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 116 (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) | 117 CE_CTRL1_DMAX_LENGTH_SET(n)); 118 } 119 120 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 121 u32 ce_ctrl_addr, 122 unsigned int n) 123 { 124 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 125 126 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 127 (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) | 128 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n)); 129 } 130 131 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 132 u32 ce_ctrl_addr, 133 unsigned int n) 134 { 135 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS); 136 137 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS, 138 (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) | 139 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n)); 140 } 141 142 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 143 u32 ce_ctrl_addr) 144 { 145 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS); 146 } 147 148 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 149 u32 ce_ctrl_addr, 150 u32 addr) 151 { 152 ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr); 153 } 154 155 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 156 u32 ce_ctrl_addr, 157 unsigned int n) 158 { 159 ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n); 160 } 161 162 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 163 u32 ce_ctrl_addr, 164 unsigned int n) 165 { 166 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 167 168 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 169 (addr & ~SRC_WATERMARK_HIGH_MASK) | 170 SRC_WATERMARK_HIGH_SET(n)); 171 } 172 173 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 174 u32 ce_ctrl_addr, 175 unsigned int n) 176 { 177 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS); 178 179 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS, 180 (addr & ~SRC_WATERMARK_LOW_MASK) | 181 SRC_WATERMARK_LOW_SET(n)); 182 } 183 184 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 185 u32 ce_ctrl_addr, 186 unsigned int n) 187 { 188 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 189 190 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 191 (addr & ~DST_WATERMARK_HIGH_MASK) | 192 DST_WATERMARK_HIGH_SET(n)); 193 } 194 195 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 196 u32 ce_ctrl_addr, 197 unsigned int n) 198 { 199 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS); 200 201 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS, 202 (addr & ~DST_WATERMARK_LOW_MASK) | 203 DST_WATERMARK_LOW_SET(n)); 204 } 205 206 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 207 u32 ce_ctrl_addr) 208 { 209 u32 host_ie_addr = ath10k_pci_read32(ar, 210 ce_ctrl_addr + HOST_IE_ADDRESS); 211 212 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 213 host_ie_addr | HOST_IE_COPY_COMPLETE_MASK); 214 } 215 216 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 217 u32 ce_ctrl_addr) 218 { 219 u32 host_ie_addr = ath10k_pci_read32(ar, 220 ce_ctrl_addr + HOST_IE_ADDRESS); 221 222 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 223 host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK); 224 } 225 226 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 227 u32 ce_ctrl_addr) 228 { 229 u32 host_ie_addr = ath10k_pci_read32(ar, 230 ce_ctrl_addr + HOST_IE_ADDRESS); 231 232 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS, 233 host_ie_addr & ~CE_WATERMARK_MASK); 234 } 235 236 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar, 237 u32 ce_ctrl_addr) 238 { 239 u32 misc_ie_addr = ath10k_pci_read32(ar, 240 ce_ctrl_addr + MISC_IE_ADDRESS); 241 242 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS, 243 misc_ie_addr | CE_ERROR_MASK); 244 } 245 246 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, 247 u32 ce_ctrl_addr) 248 { 249 u32 misc_ie_addr = ath10k_pci_read32(ar, 250 ce_ctrl_addr + MISC_IE_ADDRESS); 251 252 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS, 253 misc_ie_addr & ~CE_ERROR_MASK); 254 } 255 256 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 257 u32 ce_ctrl_addr, 258 unsigned int mask) 259 { 260 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask); 261 } 262 263 /* 264 * Guts of ath10k_ce_send. 265 * The caller takes responsibility for any needed locking. 266 */ 267 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 268 void *per_transfer_context, 269 u32 buffer, 270 unsigned int nbytes, 271 unsigned int transfer_id, 272 unsigned int flags) 273 { 274 struct ath10k *ar = ce_state->ar; 275 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 276 struct ce_desc *desc, sdesc; 277 unsigned int nentries_mask = src_ring->nentries_mask; 278 unsigned int sw_index = src_ring->sw_index; 279 unsigned int write_index = src_ring->write_index; 280 u32 ctrl_addr = ce_state->ctrl_addr; 281 u32 desc_flags = 0; 282 int ret = 0; 283 284 if (nbytes > ce_state->src_sz_max) 285 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 286 __func__, nbytes, ce_state->src_sz_max); 287 288 if (unlikely(CE_RING_DELTA(nentries_mask, 289 write_index, sw_index - 1) <= 0)) { 290 ret = -ENOSR; 291 goto exit; 292 } 293 294 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 295 write_index); 296 297 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 298 299 if (flags & CE_SEND_FLAG_GATHER) 300 desc_flags |= CE_DESC_FLAGS_GATHER; 301 if (flags & CE_SEND_FLAG_BYTE_SWAP) 302 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 303 304 sdesc.addr = __cpu_to_le32(buffer); 305 sdesc.nbytes = __cpu_to_le16(nbytes); 306 sdesc.flags = __cpu_to_le16(desc_flags); 307 308 *desc = sdesc; 309 310 src_ring->per_transfer_context[write_index] = per_transfer_context; 311 312 /* Update Source Ring Write Index */ 313 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 314 315 /* WORKAROUND */ 316 if (!(flags & CE_SEND_FLAG_GATHER)) 317 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 318 319 src_ring->write_index = write_index; 320 exit: 321 return ret; 322 } 323 324 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe) 325 { 326 struct ath10k *ar = pipe->ar; 327 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 328 struct ath10k_ce_ring *src_ring = pipe->src_ring; 329 u32 ctrl_addr = pipe->ctrl_addr; 330 331 lockdep_assert_held(&ar_pci->ce_lock); 332 333 /* 334 * This function must be called only if there is an incomplete 335 * scatter-gather transfer (before index register is updated) 336 * that needs to be cleaned up. 337 */ 338 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) 339 return; 340 341 if (WARN_ON_ONCE(src_ring->write_index == 342 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) 343 return; 344 345 src_ring->write_index--; 346 src_ring->write_index &= src_ring->nentries_mask; 347 348 src_ring->per_transfer_context[src_ring->write_index] = NULL; 349 } 350 351 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, 352 void *per_transfer_context, 353 u32 buffer, 354 unsigned int nbytes, 355 unsigned int transfer_id, 356 unsigned int flags) 357 { 358 struct ath10k *ar = ce_state->ar; 359 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 360 int ret; 361 362 spin_lock_bh(&ar_pci->ce_lock); 363 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 364 buffer, nbytes, transfer_id, flags); 365 spin_unlock_bh(&ar_pci->ce_lock); 366 367 return ret; 368 } 369 370 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe) 371 { 372 struct ath10k *ar = pipe->ar; 373 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 374 int delta; 375 376 spin_lock_bh(&ar_pci->ce_lock); 377 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, 378 pipe->src_ring->write_index, 379 pipe->src_ring->sw_index - 1); 380 spin_unlock_bh(&ar_pci->ce_lock); 381 382 return delta; 383 } 384 385 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe) 386 { 387 struct ath10k *ar = pipe->ar; 388 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 389 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 390 unsigned int nentries_mask = dest_ring->nentries_mask; 391 unsigned int write_index = dest_ring->write_index; 392 unsigned int sw_index = dest_ring->sw_index; 393 394 lockdep_assert_held(&ar_pci->ce_lock); 395 396 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 397 } 398 399 int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr) 400 { 401 struct ath10k *ar = pipe->ar; 402 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 403 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 404 unsigned int nentries_mask = dest_ring->nentries_mask; 405 unsigned int write_index = dest_ring->write_index; 406 unsigned int sw_index = dest_ring->sw_index; 407 struct ce_desc *base = dest_ring->base_addr_owner_space; 408 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 409 u32 ctrl_addr = pipe->ctrl_addr; 410 411 lockdep_assert_held(&ar_pci->ce_lock); 412 413 if ((pipe->id != 5) && 414 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 415 return -ENOSPC; 416 417 desc->addr = __cpu_to_le32(paddr); 418 desc->nbytes = 0; 419 420 dest_ring->per_transfer_context[write_index] = ctx; 421 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 422 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 423 dest_ring->write_index = write_index; 424 425 return 0; 426 } 427 428 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries) 429 { 430 struct ath10k *ar = pipe->ar; 431 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 432 unsigned int nentries_mask = dest_ring->nentries_mask; 433 unsigned int write_index = dest_ring->write_index; 434 u32 ctrl_addr = pipe->ctrl_addr; 435 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 436 437 /* Prevent CE ring stuck issue that will occur when ring is full. 438 * Make sure that write index is 1 less than read index. 439 */ 440 if ((cur_write_idx + nentries) == dest_ring->sw_index) 441 nentries -= 1; 442 443 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries); 444 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 445 dest_ring->write_index = write_index; 446 } 447 448 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, u32 paddr) 449 { 450 struct ath10k *ar = pipe->ar; 451 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 452 int ret; 453 454 spin_lock_bh(&ar_pci->ce_lock); 455 ret = __ath10k_ce_rx_post_buf(pipe, ctx, paddr); 456 spin_unlock_bh(&ar_pci->ce_lock); 457 458 return ret; 459 } 460 461 /* 462 * Guts of ath10k_ce_completed_recv_next. 463 * The caller takes responsibility for any necessary locking. 464 */ 465 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 466 void **per_transfer_contextp, 467 unsigned int *nbytesp) 468 { 469 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 470 unsigned int nentries_mask = dest_ring->nentries_mask; 471 unsigned int sw_index = dest_ring->sw_index; 472 473 struct ce_desc *base = dest_ring->base_addr_owner_space; 474 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 475 struct ce_desc sdesc; 476 u16 nbytes; 477 478 /* Copy in one go for performance reasons */ 479 sdesc = *desc; 480 481 nbytes = __le16_to_cpu(sdesc.nbytes); 482 if (nbytes == 0) { 483 /* 484 * This closes a relatively unusual race where the Host 485 * sees the updated DRRI before the update to the 486 * corresponding descriptor has completed. We treat this 487 * as a descriptor that is not yet done. 488 */ 489 return -EIO; 490 } 491 492 desc->nbytes = 0; 493 494 /* Return data from completed destination descriptor */ 495 *nbytesp = nbytes; 496 497 if (per_transfer_contextp) 498 *per_transfer_contextp = 499 dest_ring->per_transfer_context[sw_index]; 500 501 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 502 * So update transfer context all CEs except CE5. 503 */ 504 if (ce_state->id != 5) 505 dest_ring->per_transfer_context[sw_index] = NULL; 506 507 /* Update sw_index */ 508 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 509 dest_ring->sw_index = sw_index; 510 511 return 0; 512 } 513 514 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, 515 void **per_transfer_contextp, 516 unsigned int *nbytesp) 517 { 518 struct ath10k *ar = ce_state->ar; 519 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 520 int ret; 521 522 spin_lock_bh(&ar_pci->ce_lock); 523 ret = ath10k_ce_completed_recv_next_nolock(ce_state, 524 per_transfer_contextp, 525 nbytesp); 526 spin_unlock_bh(&ar_pci->ce_lock); 527 528 return ret; 529 } 530 531 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 532 void **per_transfer_contextp, 533 u32 *bufferp) 534 { 535 struct ath10k_ce_ring *dest_ring; 536 unsigned int nentries_mask; 537 unsigned int sw_index; 538 unsigned int write_index; 539 int ret; 540 struct ath10k *ar; 541 struct ath10k_pci *ar_pci; 542 543 dest_ring = ce_state->dest_ring; 544 545 if (!dest_ring) 546 return -EIO; 547 548 ar = ce_state->ar; 549 ar_pci = ath10k_pci_priv(ar); 550 551 spin_lock_bh(&ar_pci->ce_lock); 552 553 nentries_mask = dest_ring->nentries_mask; 554 sw_index = dest_ring->sw_index; 555 write_index = dest_ring->write_index; 556 if (write_index != sw_index) { 557 struct ce_desc *base = dest_ring->base_addr_owner_space; 558 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 559 560 /* Return data from completed destination descriptor */ 561 *bufferp = __le32_to_cpu(desc->addr); 562 563 if (per_transfer_contextp) 564 *per_transfer_contextp = 565 dest_ring->per_transfer_context[sw_index]; 566 567 /* sanity */ 568 dest_ring->per_transfer_context[sw_index] = NULL; 569 desc->nbytes = 0; 570 571 /* Update sw_index */ 572 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 573 dest_ring->sw_index = sw_index; 574 ret = 0; 575 } else { 576 ret = -EIO; 577 } 578 579 spin_unlock_bh(&ar_pci->ce_lock); 580 581 return ret; 582 } 583 584 /* 585 * Guts of ath10k_ce_completed_send_next. 586 * The caller takes responsibility for any necessary locking. 587 */ 588 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 589 void **per_transfer_contextp) 590 { 591 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 592 u32 ctrl_addr = ce_state->ctrl_addr; 593 struct ath10k *ar = ce_state->ar; 594 unsigned int nentries_mask = src_ring->nentries_mask; 595 unsigned int sw_index = src_ring->sw_index; 596 unsigned int read_index; 597 598 if (src_ring->hw_index == sw_index) { 599 /* 600 * The SW completion index has caught up with the cached 601 * version of the HW completion index. 602 * Update the cached HW completion index to see whether 603 * the SW has really caught up to the HW, or if the cached 604 * value of the HW index has become stale. 605 */ 606 607 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 608 if (read_index == 0xffffffff) 609 return -ENODEV; 610 611 read_index &= nentries_mask; 612 src_ring->hw_index = read_index; 613 } 614 615 read_index = src_ring->hw_index; 616 617 if (read_index == sw_index) 618 return -EIO; 619 620 if (per_transfer_contextp) 621 *per_transfer_contextp = 622 src_ring->per_transfer_context[sw_index]; 623 624 /* sanity */ 625 src_ring->per_transfer_context[sw_index] = NULL; 626 627 /* Update sw_index */ 628 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 629 src_ring->sw_index = sw_index; 630 631 return 0; 632 } 633 634 /* NB: Modeled after ath10k_ce_completed_send_next */ 635 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, 636 void **per_transfer_contextp, 637 u32 *bufferp, 638 unsigned int *nbytesp, 639 unsigned int *transfer_idp) 640 { 641 struct ath10k_ce_ring *src_ring; 642 unsigned int nentries_mask; 643 unsigned int sw_index; 644 unsigned int write_index; 645 int ret; 646 struct ath10k *ar; 647 struct ath10k_pci *ar_pci; 648 649 src_ring = ce_state->src_ring; 650 651 if (!src_ring) 652 return -EIO; 653 654 ar = ce_state->ar; 655 ar_pci = ath10k_pci_priv(ar); 656 657 spin_lock_bh(&ar_pci->ce_lock); 658 659 nentries_mask = src_ring->nentries_mask; 660 sw_index = src_ring->sw_index; 661 write_index = src_ring->write_index; 662 663 if (write_index != sw_index) { 664 struct ce_desc *base = src_ring->base_addr_owner_space; 665 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 666 667 /* Return data from completed source descriptor */ 668 *bufferp = __le32_to_cpu(desc->addr); 669 *nbytesp = __le16_to_cpu(desc->nbytes); 670 *transfer_idp = MS(__le16_to_cpu(desc->flags), 671 CE_DESC_FLAGS_META_DATA); 672 673 if (per_transfer_contextp) 674 *per_transfer_contextp = 675 src_ring->per_transfer_context[sw_index]; 676 677 /* sanity */ 678 src_ring->per_transfer_context[sw_index] = NULL; 679 680 /* Update sw_index */ 681 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 682 src_ring->sw_index = sw_index; 683 ret = 0; 684 } else { 685 ret = -EIO; 686 } 687 688 spin_unlock_bh(&ar_pci->ce_lock); 689 690 return ret; 691 } 692 693 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, 694 void **per_transfer_contextp) 695 { 696 struct ath10k *ar = ce_state->ar; 697 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 698 int ret; 699 700 spin_lock_bh(&ar_pci->ce_lock); 701 ret = ath10k_ce_completed_send_next_nolock(ce_state, 702 per_transfer_contextp); 703 spin_unlock_bh(&ar_pci->ce_lock); 704 705 return ret; 706 } 707 708 /* 709 * Guts of interrupt handler for per-engine interrupts on a particular CE. 710 * 711 * Invokes registered callbacks for recv_complete, 712 * send_complete, and watermarks. 713 */ 714 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 715 { 716 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 717 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 718 u32 ctrl_addr = ce_state->ctrl_addr; 719 720 spin_lock_bh(&ar_pci->ce_lock); 721 722 /* Clear the copy-complete interrupts that will be handled here. */ 723 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 724 HOST_IS_COPY_COMPLETE_MASK); 725 726 spin_unlock_bh(&ar_pci->ce_lock); 727 728 if (ce_state->recv_cb) 729 ce_state->recv_cb(ce_state); 730 731 if (ce_state->send_cb) 732 ce_state->send_cb(ce_state); 733 734 spin_lock_bh(&ar_pci->ce_lock); 735 736 /* 737 * Misc CE interrupts are not being handled, but still need 738 * to be cleared. 739 */ 740 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK); 741 742 spin_unlock_bh(&ar_pci->ce_lock); 743 } 744 745 /* 746 * Handler for per-engine interrupts on ALL active CEs. 747 * This is used in cases where the system is sharing a 748 * single interrput for all CEs 749 */ 750 751 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 752 { 753 int ce_id; 754 u32 intr_summary; 755 756 intr_summary = CE_INTERRUPT_SUMMARY(ar); 757 758 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) { 759 if (intr_summary & (1 << ce_id)) 760 intr_summary &= ~(1 << ce_id); 761 else 762 /* no intr pending on this CE */ 763 continue; 764 765 ath10k_ce_per_engine_service(ar, ce_id); 766 } 767 } 768 769 /* 770 * Adjust interrupts for the copy complete handler. 771 * If it's needed for either send or recv, then unmask 772 * this interrupt; otherwise, mask it. 773 * 774 * Called with ce_lock held. 775 */ 776 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state) 777 { 778 u32 ctrl_addr = ce_state->ctrl_addr; 779 struct ath10k *ar = ce_state->ar; 780 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; 781 782 if ((!disable_copy_compl_intr) && 783 (ce_state->send_cb || ce_state->recv_cb)) 784 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 785 else 786 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 787 788 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 789 } 790 791 int ath10k_ce_disable_interrupts(struct ath10k *ar) 792 { 793 int ce_id; 794 795 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) { 796 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 797 798 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 799 ath10k_ce_error_intr_disable(ar, ctrl_addr); 800 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 801 } 802 803 return 0; 804 } 805 806 void ath10k_ce_enable_interrupts(struct ath10k *ar) 807 { 808 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 809 int ce_id; 810 811 /* Skip the last copy engine, CE7 the diagnostic window, as that 812 * uses polling and isn't initialized for interrupts. 813 */ 814 for (ce_id = 0; ce_id < CE_COUNT - 1; ce_id++) 815 ath10k_ce_per_engine_handler_adjust(&ar_pci->ce_states[ce_id]); 816 } 817 818 static int ath10k_ce_init_src_ring(struct ath10k *ar, 819 unsigned int ce_id, 820 const struct ce_attr *attr) 821 { 822 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 823 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 824 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 825 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 826 827 nentries = roundup_pow_of_two(attr->src_nentries); 828 829 memset(src_ring->base_addr_owner_space, 0, 830 nentries * sizeof(struct ce_desc)); 831 832 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 833 src_ring->sw_index &= src_ring->nentries_mask; 834 src_ring->hw_index = src_ring->sw_index; 835 836 src_ring->write_index = 837 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 838 src_ring->write_index &= src_ring->nentries_mask; 839 840 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 841 src_ring->base_addr_ce_space); 842 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 843 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 844 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 845 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 846 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 847 848 ath10k_dbg(ar, ATH10K_DBG_BOOT, 849 "boot init ce src ring id %d entries %d base_addr %pK\n", 850 ce_id, nentries, src_ring->base_addr_owner_space); 851 852 return 0; 853 } 854 855 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 856 unsigned int ce_id, 857 const struct ce_attr *attr) 858 { 859 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 860 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 861 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 862 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 863 864 nentries = roundup_pow_of_two(attr->dest_nentries); 865 866 memset(dest_ring->base_addr_owner_space, 0, 867 nentries * sizeof(struct ce_desc)); 868 869 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 870 dest_ring->sw_index &= dest_ring->nentries_mask; 871 dest_ring->write_index = 872 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 873 dest_ring->write_index &= dest_ring->nentries_mask; 874 875 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 876 dest_ring->base_addr_ce_space); 877 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 878 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 879 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 880 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 881 882 ath10k_dbg(ar, ATH10K_DBG_BOOT, 883 "boot ce dest ring id %d entries %d base_addr %pK\n", 884 ce_id, nentries, dest_ring->base_addr_owner_space); 885 886 return 0; 887 } 888 889 static struct ath10k_ce_ring * 890 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, 891 const struct ce_attr *attr) 892 { 893 struct ath10k_ce_ring *src_ring; 894 u32 nentries = attr->src_nentries; 895 dma_addr_t base_addr; 896 897 nentries = roundup_pow_of_two(nentries); 898 899 src_ring = kzalloc(sizeof(*src_ring) + 900 (nentries * 901 sizeof(*src_ring->per_transfer_context)), 902 GFP_KERNEL); 903 if (src_ring == NULL) 904 return ERR_PTR(-ENOMEM); 905 906 src_ring->nentries = nentries; 907 src_ring->nentries_mask = nentries - 1; 908 909 /* 910 * Legacy platforms that do not support cache 911 * coherent DMA are unsupported 912 */ 913 src_ring->base_addr_owner_space_unaligned = 914 dma_alloc_coherent(ar->dev, 915 (nentries * sizeof(struct ce_desc) + 916 CE_DESC_RING_ALIGN), 917 &base_addr, GFP_KERNEL); 918 if (!src_ring->base_addr_owner_space_unaligned) { 919 kfree(src_ring); 920 return ERR_PTR(-ENOMEM); 921 } 922 923 src_ring->base_addr_ce_space_unaligned = base_addr; 924 925 src_ring->base_addr_owner_space = PTR_ALIGN( 926 src_ring->base_addr_owner_space_unaligned, 927 CE_DESC_RING_ALIGN); 928 src_ring->base_addr_ce_space = ALIGN( 929 src_ring->base_addr_ce_space_unaligned, 930 CE_DESC_RING_ALIGN); 931 932 return src_ring; 933 } 934 935 static struct ath10k_ce_ring * 936 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, 937 const struct ce_attr *attr) 938 { 939 struct ath10k_ce_ring *dest_ring; 940 u32 nentries; 941 dma_addr_t base_addr; 942 943 nentries = roundup_pow_of_two(attr->dest_nentries); 944 945 dest_ring = kzalloc(sizeof(*dest_ring) + 946 (nentries * 947 sizeof(*dest_ring->per_transfer_context)), 948 GFP_KERNEL); 949 if (dest_ring == NULL) 950 return ERR_PTR(-ENOMEM); 951 952 dest_ring->nentries = nentries; 953 dest_ring->nentries_mask = nentries - 1; 954 955 /* 956 * Legacy platforms that do not support cache 957 * coherent DMA are unsupported 958 */ 959 dest_ring->base_addr_owner_space_unaligned = 960 dma_zalloc_coherent(ar->dev, 961 (nentries * sizeof(struct ce_desc) + 962 CE_DESC_RING_ALIGN), 963 &base_addr, GFP_KERNEL); 964 if (!dest_ring->base_addr_owner_space_unaligned) { 965 kfree(dest_ring); 966 return ERR_PTR(-ENOMEM); 967 } 968 969 dest_ring->base_addr_ce_space_unaligned = base_addr; 970 971 dest_ring->base_addr_owner_space = PTR_ALIGN( 972 dest_ring->base_addr_owner_space_unaligned, 973 CE_DESC_RING_ALIGN); 974 dest_ring->base_addr_ce_space = ALIGN( 975 dest_ring->base_addr_ce_space_unaligned, 976 CE_DESC_RING_ALIGN); 977 978 return dest_ring; 979 } 980 981 /* 982 * Initialize a Copy Engine based on caller-supplied attributes. 983 * This may be called once to initialize both source and destination 984 * rings or it may be called twice for separate source and destination 985 * initialization. It may be that only one side or the other is 986 * initialized by software/firmware. 987 */ 988 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, 989 const struct ce_attr *attr) 990 { 991 int ret; 992 993 if (attr->src_nentries) { 994 ret = ath10k_ce_init_src_ring(ar, ce_id, attr); 995 if (ret) { 996 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n", 997 ce_id, ret); 998 return ret; 999 } 1000 } 1001 1002 if (attr->dest_nentries) { 1003 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); 1004 if (ret) { 1005 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n", 1006 ce_id, ret); 1007 return ret; 1008 } 1009 } 1010 1011 return 0; 1012 } 1013 1014 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) 1015 { 1016 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1017 1018 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr, 0); 1019 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0); 1020 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0); 1021 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0); 1022 } 1023 1024 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) 1025 { 1026 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1027 1028 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr, 0); 1029 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0); 1030 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0); 1031 } 1032 1033 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) 1034 { 1035 ath10k_ce_deinit_src_ring(ar, ce_id); 1036 ath10k_ce_deinit_dest_ring(ar, ce_id); 1037 } 1038 1039 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, 1040 const struct ce_attr *attr) 1041 { 1042 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1043 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 1044 int ret; 1045 1046 /* 1047 * Make sure there's enough CE ringbuffer entries for HTT TX to avoid 1048 * additional TX locking checks. 1049 * 1050 * For the lack of a better place do the check here. 1051 */ 1052 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC > 1053 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1054 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC > 1055 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1056 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC > 1057 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1058 1059 ce_state->ar = ar; 1060 ce_state->id = ce_id; 1061 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1062 ce_state->attr_flags = attr->flags; 1063 ce_state->src_sz_max = attr->src_sz_max; 1064 1065 if (attr->src_nentries) 1066 ce_state->send_cb = attr->send_cb; 1067 1068 if (attr->dest_nentries) 1069 ce_state->recv_cb = attr->recv_cb; 1070 1071 if (attr->src_nentries) { 1072 ce_state->src_ring = ath10k_ce_alloc_src_ring(ar, ce_id, attr); 1073 if (IS_ERR(ce_state->src_ring)) { 1074 ret = PTR_ERR(ce_state->src_ring); 1075 ath10k_err(ar, "failed to allocate copy engine source ring %d: %d\n", 1076 ce_id, ret); 1077 ce_state->src_ring = NULL; 1078 return ret; 1079 } 1080 } 1081 1082 if (attr->dest_nentries) { 1083 ce_state->dest_ring = ath10k_ce_alloc_dest_ring(ar, ce_id, 1084 attr); 1085 if (IS_ERR(ce_state->dest_ring)) { 1086 ret = PTR_ERR(ce_state->dest_ring); 1087 ath10k_err(ar, "failed to allocate copy engine destination ring %d: %d\n", 1088 ce_id, ret); 1089 ce_state->dest_ring = NULL; 1090 return ret; 1091 } 1092 } 1093 1094 return 0; 1095 } 1096 1097 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1098 { 1099 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1100 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id]; 1101 1102 if (ce_state->src_ring) { 1103 dma_free_coherent(ar->dev, 1104 (ce_state->src_ring->nentries * 1105 sizeof(struct ce_desc) + 1106 CE_DESC_RING_ALIGN), 1107 ce_state->src_ring->base_addr_owner_space, 1108 ce_state->src_ring->base_addr_ce_space); 1109 kfree(ce_state->src_ring); 1110 } 1111 1112 if (ce_state->dest_ring) { 1113 dma_free_coherent(ar->dev, 1114 (ce_state->dest_ring->nentries * 1115 sizeof(struct ce_desc) + 1116 CE_DESC_RING_ALIGN), 1117 ce_state->dest_ring->base_addr_owner_space, 1118 ce_state->dest_ring->base_addr_ce_space); 1119 kfree(ce_state->dest_ring); 1120 } 1121 1122 ce_state->src_ring = NULL; 1123 ce_state->dest_ring = NULL; 1124 } 1125 1126 void ath10k_ce_dump_registers(struct ath10k *ar, 1127 struct ath10k_fw_crash_data *crash_data) 1128 { 1129 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar); 1130 struct ath10k_ce_crash_data ce; 1131 u32 addr, id; 1132 1133 lockdep_assert_held(&ar->data_lock); 1134 1135 ath10k_err(ar, "Copy Engine register dump:\n"); 1136 1137 spin_lock_bh(&ar_pci->ce_lock); 1138 for (id = 0; id < CE_COUNT; id++) { 1139 addr = ath10k_ce_base_address(ar, id); 1140 ce.base_addr = cpu_to_le32(addr); 1141 1142 ce.src_wr_idx = 1143 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr)); 1144 ce.src_r_idx = 1145 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr)); 1146 ce.dst_wr_idx = 1147 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr)); 1148 ce.dst_r_idx = 1149 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr)); 1150 1151 if (crash_data) 1152 crash_data->ce_crash_data[id] = ce; 1153 1154 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id, 1155 le32_to_cpu(ce.base_addr), 1156 le32_to_cpu(ce.src_wr_idx), 1157 le32_to_cpu(ce.src_r_idx), 1158 le32_to_cpu(ce.dst_wr_idx), 1159 le32_to_cpu(ce.dst_r_idx)); 1160 } 1161 1162 spin_unlock_bh(&ar_pci->ce_lock); 1163 } 1164