1 // SPDX-License-Identifier: ISC 2 /* 3 * Copyright (c) 2005-2011 Atheros Communications Inc. 4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. 5 * Copyright (c) 2018 The Linux Foundation. All rights reserved. 6 * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. 7 */ 8 9 #include <linux/export.h> 10 #include "hif.h" 11 #include "ce.h" 12 #include "debug.h" 13 14 /* 15 * Support for Copy Engine hardware, which is mainly used for 16 * communication between Host and Target over a PCIe interconnect. 17 */ 18 19 /* 20 * A single CopyEngine (CE) comprises two "rings": 21 * a source ring 22 * a destination ring 23 * 24 * Each ring consists of a number of descriptors which specify 25 * an address, length, and meta-data. 26 * 27 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target) 28 * controls one ring and the other side controls the other ring. 29 * The source side chooses when to initiate a transfer and it 30 * chooses what to send (buffer address, length). The destination 31 * side keeps a supply of "anonymous receive buffers" available and 32 * it handles incoming data as it arrives (when the destination 33 * receives an interrupt). 34 * 35 * The sender may send a simple buffer (address/length) or it may 36 * send a small list of buffers. When a small list is sent, hardware 37 * "gathers" these and they end up in a single destination buffer 38 * with a single interrupt. 39 * 40 * There are several "contexts" managed by this layer -- more, it 41 * may seem -- than should be needed. These are provided mainly for 42 * maximum flexibility and especially to facilitate a simpler HIF 43 * implementation. There are per-CopyEngine recv, send, and watermark 44 * contexts. These are supplied by the caller when a recv, send, 45 * or watermark handler is established and they are echoed back to 46 * the caller when the respective callbacks are invoked. There is 47 * also a per-transfer context supplied by the caller when a buffer 48 * (or sendlist) is sent and when a buffer is enqueued for recv. 49 * These per-transfer contexts are echoed back to the caller when 50 * the buffer is sent/received. 51 */ 52 53 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar, 54 struct ath10k_ce_pipe *ce_state) 55 { 56 u32 ce_id = ce_state->id; 57 u32 addr = 0; 58 59 switch (ce_id) { 60 case 0: 61 addr = 0x00032000; 62 break; 63 case 3: 64 addr = 0x0003200C; 65 break; 66 case 4: 67 addr = 0x00032010; 68 break; 69 case 5: 70 addr = 0x00032014; 71 break; 72 case 7: 73 addr = 0x0003201C; 74 break; 75 default: 76 ath10k_warn(ar, "invalid CE id: %d", ce_id); 77 break; 78 } 79 return addr; 80 } 81 82 static inline unsigned int 83 ath10k_set_ring_byte(unsigned int offset, 84 const struct ath10k_hw_ce_regs_addr_map *addr_map) 85 { 86 return ((offset << addr_map->lsb) & addr_map->mask); 87 } 88 89 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset) 90 { 91 struct ath10k_ce *ce = ath10k_ce_priv(ar); 92 93 return ce->bus_ops->read32(ar, offset); 94 } 95 96 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value) 97 { 98 struct ath10k_ce *ce = ath10k_ce_priv(ar); 99 100 ce->bus_ops->write32(ar, offset, value); 101 } 102 103 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 104 u32 ce_ctrl_addr, 105 unsigned int n) 106 { 107 ath10k_ce_write32(ar, ce_ctrl_addr + 108 ar->hw_ce_regs->dst_wr_index_addr, n); 109 } 110 111 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 112 u32 ce_ctrl_addr) 113 { 114 return ath10k_ce_read32(ar, ce_ctrl_addr + 115 ar->hw_ce_regs->dst_wr_index_addr); 116 } 117 118 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 119 u32 ce_ctrl_addr, 120 unsigned int n) 121 { 122 ath10k_ce_write32(ar, ce_ctrl_addr + 123 ar->hw_ce_regs->sr_wr_index_addr, n); 124 } 125 126 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 127 u32 ce_ctrl_addr) 128 { 129 return ath10k_ce_read32(ar, ce_ctrl_addr + 130 ar->hw_ce_regs->sr_wr_index_addr); 131 } 132 133 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar, 134 u32 ce_id) 135 { 136 struct ath10k_ce *ce = ath10k_ce_priv(ar); 137 138 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK; 139 } 140 141 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 142 u32 ce_ctrl_addr) 143 { 144 struct ath10k_ce *ce = ath10k_ce_priv(ar); 145 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); 146 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 147 u32 index; 148 149 if (ar->hw_params.rri_on_ddr && 150 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) 151 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id); 152 else 153 index = ath10k_ce_read32(ar, ce_ctrl_addr + 154 ar->hw_ce_regs->current_srri_addr); 155 156 return index; 157 } 158 159 static inline void 160 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar, 161 struct ath10k_ce_pipe *ce_state, 162 unsigned int value) 163 { 164 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value); 165 } 166 167 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 168 u32 ce_id, 169 u64 addr) 170 { 171 struct ath10k_ce *ce = ath10k_ce_priv(ar); 172 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 173 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id); 174 u32 addr_lo = lower_32_bits(addr); 175 176 ath10k_ce_write32(ar, ce_ctrl_addr + 177 ar->hw_ce_regs->sr_base_addr_lo, addr_lo); 178 179 if (ce_state->ops->ce_set_src_ring_base_addr_hi) { 180 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr, 181 addr); 182 } 183 } 184 185 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar, 186 u32 ce_ctrl_addr, 187 u64 addr) 188 { 189 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK; 190 191 ath10k_ce_write32(ar, ce_ctrl_addr + 192 ar->hw_ce_regs->sr_base_addr_hi, addr_hi); 193 } 194 195 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 196 u32 ce_ctrl_addr, 197 unsigned int n) 198 { 199 ath10k_ce_write32(ar, ce_ctrl_addr + 200 ar->hw_ce_regs->sr_size_addr, n); 201 } 202 203 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 204 u32 ce_ctrl_addr, 205 unsigned int n) 206 { 207 const struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 208 209 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 210 ctrl_regs->addr); 211 212 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 213 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) | 214 ath10k_set_ring_byte(n, ctrl_regs->dmax)); 215 } 216 217 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 218 u32 ce_ctrl_addr, 219 unsigned int n) 220 { 221 const struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 222 223 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 224 ctrl_regs->addr); 225 226 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 227 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) | 228 ath10k_set_ring_byte(n, ctrl_regs->src_ring)); 229 } 230 231 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 232 u32 ce_ctrl_addr, 233 unsigned int n) 234 { 235 const struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 236 237 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 238 ctrl_regs->addr); 239 240 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 241 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) | 242 ath10k_set_ring_byte(n, ctrl_regs->dst_ring)); 243 } 244 245 static inline 246 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id) 247 { 248 struct ath10k_ce *ce = ath10k_ce_priv(ar); 249 250 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) & 251 CE_DDR_RRI_MASK; 252 } 253 254 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 255 u32 ce_ctrl_addr) 256 { 257 struct ath10k_ce *ce = ath10k_ce_priv(ar); 258 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); 259 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 260 u32 index; 261 262 if (ar->hw_params.rri_on_ddr && 263 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) 264 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id); 265 else 266 index = ath10k_ce_read32(ar, ce_ctrl_addr + 267 ar->hw_ce_regs->current_drri_addr); 268 269 return index; 270 } 271 272 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 273 u32 ce_id, 274 u64 addr) 275 { 276 struct ath10k_ce *ce = ath10k_ce_priv(ar); 277 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 278 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id); 279 u32 addr_lo = lower_32_bits(addr); 280 281 ath10k_ce_write32(ar, ce_ctrl_addr + 282 ar->hw_ce_regs->dr_base_addr_lo, addr_lo); 283 284 if (ce_state->ops->ce_set_dest_ring_base_addr_hi) { 285 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr, 286 addr); 287 } 288 } 289 290 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar, 291 u32 ce_ctrl_addr, 292 u64 addr) 293 { 294 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK; 295 u32 reg_value; 296 297 reg_value = ath10k_ce_read32(ar, ce_ctrl_addr + 298 ar->hw_ce_regs->dr_base_addr_hi); 299 reg_value &= ~CE_DESC_ADDR_HI_MASK; 300 reg_value |= addr_hi; 301 ath10k_ce_write32(ar, ce_ctrl_addr + 302 ar->hw_ce_regs->dr_base_addr_hi, reg_value); 303 } 304 305 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 306 u32 ce_ctrl_addr, 307 unsigned int n) 308 { 309 ath10k_ce_write32(ar, ce_ctrl_addr + 310 ar->hw_ce_regs->dr_size_addr, n); 311 } 312 313 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 314 u32 ce_ctrl_addr, 315 unsigned int n) 316 { 317 const struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 318 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 319 320 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 321 (addr & ~(srcr_wm->wm_high->mask)) | 322 (ath10k_set_ring_byte(n, srcr_wm->wm_high))); 323 } 324 325 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 326 u32 ce_ctrl_addr, 327 unsigned int n) 328 { 329 const struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 330 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 331 332 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 333 (addr & ~(srcr_wm->wm_low->mask)) | 334 (ath10k_set_ring_byte(n, srcr_wm->wm_low))); 335 } 336 337 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 338 u32 ce_ctrl_addr, 339 unsigned int n) 340 { 341 const struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 342 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 343 344 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 345 (addr & ~(dstr_wm->wm_high->mask)) | 346 (ath10k_set_ring_byte(n, dstr_wm->wm_high))); 347 } 348 349 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 350 u32 ce_ctrl_addr, 351 unsigned int n) 352 { 353 const struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 354 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 355 356 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 357 (addr & ~(dstr_wm->wm_low->mask)) | 358 (ath10k_set_ring_byte(n, dstr_wm->wm_low))); 359 } 360 361 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 362 u32 ce_ctrl_addr) 363 { 364 const struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 365 366 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 367 ar->hw_ce_regs->host_ie_addr); 368 369 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 370 host_ie_addr | host_ie->copy_complete->mask); 371 } 372 373 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 374 u32 ce_ctrl_addr) 375 { 376 const struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 377 378 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 379 ar->hw_ce_regs->host_ie_addr); 380 381 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 382 host_ie_addr & ~(host_ie->copy_complete->mask)); 383 } 384 385 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 386 u32 ce_ctrl_addr) 387 { 388 const struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 389 390 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 391 ar->hw_ce_regs->host_ie_addr); 392 393 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 394 host_ie_addr & ~(wm_regs->wm_mask)); 395 } 396 397 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, 398 u32 ce_ctrl_addr) 399 { 400 const struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 401 402 u32 misc_ie_addr = ath10k_ce_read32(ar, 403 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr); 404 405 ath10k_ce_write32(ar, 406 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 407 misc_ie_addr & ~(misc_regs->err_mask)); 408 } 409 410 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 411 u32 ce_ctrl_addr, 412 unsigned int mask) 413 { 414 const struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 415 416 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask); 417 } 418 419 /* 420 * Guts of ath10k_ce_send. 421 * The caller takes responsibility for any needed locking. 422 */ 423 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 424 void *per_transfer_context, 425 dma_addr_t buffer, 426 unsigned int nbytes, 427 unsigned int transfer_id, 428 unsigned int flags) 429 { 430 struct ath10k *ar = ce_state->ar; 431 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 432 struct ce_desc *desc, sdesc; 433 unsigned int nentries_mask = src_ring->nentries_mask; 434 unsigned int sw_index = src_ring->sw_index; 435 unsigned int write_index = src_ring->write_index; 436 u32 ctrl_addr = ce_state->ctrl_addr; 437 u32 desc_flags = 0; 438 int ret = 0; 439 440 if (nbytes > ce_state->src_sz_max) 441 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 442 __func__, nbytes, ce_state->src_sz_max); 443 444 if (unlikely(CE_RING_DELTA(nentries_mask, 445 write_index, sw_index - 1) <= 0)) { 446 ret = -ENOSR; 447 goto exit; 448 } 449 450 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 451 write_index); 452 453 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 454 455 if (flags & CE_SEND_FLAG_GATHER) 456 desc_flags |= CE_DESC_FLAGS_GATHER; 457 if (flags & CE_SEND_FLAG_BYTE_SWAP) 458 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 459 460 sdesc.addr = __cpu_to_le32(buffer); 461 sdesc.nbytes = __cpu_to_le16(nbytes); 462 sdesc.flags = __cpu_to_le16(desc_flags); 463 464 *desc = sdesc; 465 466 src_ring->per_transfer_context[write_index] = per_transfer_context; 467 468 /* Update Source Ring Write Index */ 469 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 470 471 /* WORKAROUND */ 472 if (!(flags & CE_SEND_FLAG_GATHER)) 473 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 474 475 src_ring->write_index = write_index; 476 exit: 477 return ret; 478 } 479 480 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state, 481 void *per_transfer_context, 482 dma_addr_t buffer, 483 unsigned int nbytes, 484 unsigned int transfer_id, 485 unsigned int flags) 486 { 487 struct ath10k *ar = ce_state->ar; 488 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 489 struct ce_desc_64 *desc, sdesc; 490 unsigned int nentries_mask = src_ring->nentries_mask; 491 unsigned int sw_index; 492 unsigned int write_index = src_ring->write_index; 493 u32 ctrl_addr = ce_state->ctrl_addr; 494 __le32 *addr; 495 u32 desc_flags = 0; 496 int ret = 0; 497 498 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags)) 499 return -ESHUTDOWN; 500 501 if (nbytes > ce_state->src_sz_max) 502 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 503 __func__, nbytes, ce_state->src_sz_max); 504 505 if (ar->hw_params.rri_on_ddr) 506 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id); 507 else 508 sw_index = src_ring->sw_index; 509 510 if (unlikely(CE_RING_DELTA(nentries_mask, 511 write_index, sw_index - 1) <= 0)) { 512 ret = -ENOSR; 513 goto exit; 514 } 515 516 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, 517 write_index); 518 519 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 520 521 if (flags & CE_SEND_FLAG_GATHER) 522 desc_flags |= CE_DESC_FLAGS_GATHER; 523 524 if (flags & CE_SEND_FLAG_BYTE_SWAP) 525 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 526 527 addr = (__le32 *)&sdesc.addr; 528 529 flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK; 530 addr[0] = __cpu_to_le32(buffer); 531 addr[1] = __cpu_to_le32(flags); 532 if (flags & CE_SEND_FLAG_GATHER) 533 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER); 534 else 535 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER)); 536 537 sdesc.nbytes = __cpu_to_le16(nbytes); 538 sdesc.flags = __cpu_to_le16(desc_flags); 539 540 *desc = sdesc; 541 542 src_ring->per_transfer_context[write_index] = per_transfer_context; 543 544 /* Update Source Ring Write Index */ 545 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 546 547 if (!(flags & CE_SEND_FLAG_GATHER)) { 548 if (ar->hw_params.shadow_reg_support) 549 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state, 550 write_index); 551 else 552 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, 553 write_index); 554 } 555 556 src_ring->write_index = write_index; 557 exit: 558 return ret; 559 } 560 561 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 562 void *per_transfer_context, 563 dma_addr_t buffer, 564 unsigned int nbytes, 565 unsigned int transfer_id, 566 unsigned int flags) 567 { 568 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context, 569 buffer, nbytes, transfer_id, flags); 570 } 571 EXPORT_SYMBOL(ath10k_ce_send_nolock); 572 573 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe) 574 { 575 struct ath10k *ar = pipe->ar; 576 struct ath10k_ce *ce = ath10k_ce_priv(ar); 577 struct ath10k_ce_ring *src_ring = pipe->src_ring; 578 u32 ctrl_addr = pipe->ctrl_addr; 579 580 lockdep_assert_held(&ce->ce_lock); 581 582 /* 583 * This function must be called only if there is an incomplete 584 * scatter-gather transfer (before index register is updated) 585 * that needs to be cleaned up. 586 */ 587 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) 588 return; 589 590 if (WARN_ON_ONCE(src_ring->write_index == 591 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) 592 return; 593 594 src_ring->write_index--; 595 src_ring->write_index &= src_ring->nentries_mask; 596 597 src_ring->per_transfer_context[src_ring->write_index] = NULL; 598 } 599 EXPORT_SYMBOL(__ath10k_ce_send_revert); 600 601 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, 602 void *per_transfer_context, 603 dma_addr_t buffer, 604 unsigned int nbytes, 605 unsigned int transfer_id, 606 unsigned int flags) 607 { 608 struct ath10k *ar = ce_state->ar; 609 struct ath10k_ce *ce = ath10k_ce_priv(ar); 610 int ret; 611 612 spin_lock_bh(&ce->ce_lock); 613 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 614 buffer, nbytes, transfer_id, flags); 615 spin_unlock_bh(&ce->ce_lock); 616 617 return ret; 618 } 619 EXPORT_SYMBOL(ath10k_ce_send); 620 621 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe) 622 { 623 struct ath10k *ar = pipe->ar; 624 struct ath10k_ce *ce = ath10k_ce_priv(ar); 625 int delta; 626 627 spin_lock_bh(&ce->ce_lock); 628 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, 629 pipe->src_ring->write_index, 630 pipe->src_ring->sw_index - 1); 631 spin_unlock_bh(&ce->ce_lock); 632 633 return delta; 634 } 635 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries); 636 637 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe) 638 { 639 struct ath10k *ar = pipe->ar; 640 struct ath10k_ce *ce = ath10k_ce_priv(ar); 641 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 642 unsigned int nentries_mask = dest_ring->nentries_mask; 643 unsigned int write_index = dest_ring->write_index; 644 unsigned int sw_index = dest_ring->sw_index; 645 646 lockdep_assert_held(&ce->ce_lock); 647 648 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 649 } 650 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs); 651 652 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 653 dma_addr_t paddr) 654 { 655 struct ath10k *ar = pipe->ar; 656 struct ath10k_ce *ce = ath10k_ce_priv(ar); 657 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 658 unsigned int nentries_mask = dest_ring->nentries_mask; 659 unsigned int write_index = dest_ring->write_index; 660 unsigned int sw_index = dest_ring->sw_index; 661 struct ce_desc *base = dest_ring->base_addr_owner_space; 662 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 663 u32 ctrl_addr = pipe->ctrl_addr; 664 665 lockdep_assert_held(&ce->ce_lock); 666 667 if ((pipe->id != 5) && 668 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 669 return -ENOSPC; 670 671 desc->addr = __cpu_to_le32(paddr); 672 desc->nbytes = 0; 673 674 dest_ring->per_transfer_context[write_index] = ctx; 675 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 676 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 677 dest_ring->write_index = write_index; 678 679 return 0; 680 } 681 682 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe, 683 void *ctx, 684 dma_addr_t paddr) 685 { 686 struct ath10k *ar = pipe->ar; 687 struct ath10k_ce *ce = ath10k_ce_priv(ar); 688 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 689 unsigned int nentries_mask = dest_ring->nentries_mask; 690 unsigned int write_index = dest_ring->write_index; 691 unsigned int sw_index = dest_ring->sw_index; 692 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 693 struct ce_desc_64 *desc = 694 CE_DEST_RING_TO_DESC_64(base, write_index); 695 u32 ctrl_addr = pipe->ctrl_addr; 696 697 lockdep_assert_held(&ce->ce_lock); 698 699 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 700 return -ENOSPC; 701 702 desc->addr = __cpu_to_le64(paddr); 703 desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK); 704 705 desc->nbytes = 0; 706 707 dest_ring->per_transfer_context[write_index] = ctx; 708 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 709 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 710 dest_ring->write_index = write_index; 711 712 return 0; 713 } 714 715 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries) 716 { 717 struct ath10k *ar = pipe->ar; 718 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 719 unsigned int nentries_mask = dest_ring->nentries_mask; 720 unsigned int write_index = dest_ring->write_index; 721 u32 ctrl_addr = pipe->ctrl_addr; 722 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 723 724 /* Prevent CE ring stuck issue that will occur when ring is full. 725 * Make sure that write index is 1 less than read index. 726 */ 727 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index) 728 nentries -= 1; 729 730 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries); 731 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 732 dest_ring->write_index = write_index; 733 } 734 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx); 735 736 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 737 dma_addr_t paddr) 738 { 739 struct ath10k *ar = pipe->ar; 740 struct ath10k_ce *ce = ath10k_ce_priv(ar); 741 int ret; 742 743 spin_lock_bh(&ce->ce_lock); 744 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr); 745 spin_unlock_bh(&ce->ce_lock); 746 747 return ret; 748 } 749 EXPORT_SYMBOL(ath10k_ce_rx_post_buf); 750 751 /* 752 * Guts of ath10k_ce_completed_recv_next. 753 * The caller takes responsibility for any necessary locking. 754 */ 755 static int 756 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 757 void **per_transfer_contextp, 758 unsigned int *nbytesp) 759 { 760 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 761 unsigned int nentries_mask = dest_ring->nentries_mask; 762 unsigned int sw_index = dest_ring->sw_index; 763 764 struct ce_desc *base = dest_ring->base_addr_owner_space; 765 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 766 struct ce_desc sdesc; 767 u16 nbytes; 768 769 /* Copy in one go for performance reasons */ 770 sdesc = *desc; 771 772 nbytes = __le16_to_cpu(sdesc.nbytes); 773 if (nbytes == 0) { 774 /* 775 * This closes a relatively unusual race where the Host 776 * sees the updated DRRI before the update to the 777 * corresponding descriptor has completed. We treat this 778 * as a descriptor that is not yet done. 779 */ 780 return -EIO; 781 } 782 783 desc->nbytes = 0; 784 785 /* Return data from completed destination descriptor */ 786 *nbytesp = nbytes; 787 788 if (per_transfer_contextp) 789 *per_transfer_contextp = 790 dest_ring->per_transfer_context[sw_index]; 791 792 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 793 * So update transfer context all CEs except CE5. 794 */ 795 if (ce_state->id != 5) 796 dest_ring->per_transfer_context[sw_index] = NULL; 797 798 /* Update sw_index */ 799 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 800 dest_ring->sw_index = sw_index; 801 802 return 0; 803 } 804 805 static int 806 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state, 807 void **per_transfer_contextp, 808 unsigned int *nbytesp) 809 { 810 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 811 unsigned int nentries_mask = dest_ring->nentries_mask; 812 unsigned int sw_index = dest_ring->sw_index; 813 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 814 struct ce_desc_64 *desc = 815 CE_DEST_RING_TO_DESC_64(base, sw_index); 816 struct ce_desc_64 sdesc; 817 u16 nbytes; 818 819 /* Copy in one go for performance reasons */ 820 sdesc = *desc; 821 822 nbytes = __le16_to_cpu(sdesc.nbytes); 823 if (nbytes == 0) { 824 /* This closes a relatively unusual race where the Host 825 * sees the updated DRRI before the update to the 826 * corresponding descriptor has completed. We treat this 827 * as a descriptor that is not yet done. 828 */ 829 return -EIO; 830 } 831 832 desc->nbytes = 0; 833 834 /* Return data from completed destination descriptor */ 835 *nbytesp = nbytes; 836 837 if (per_transfer_contextp) 838 *per_transfer_contextp = 839 dest_ring->per_transfer_context[sw_index]; 840 841 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 842 * So update transfer context all CEs except CE5. 843 */ 844 if (ce_state->id != 5) 845 dest_ring->per_transfer_context[sw_index] = NULL; 846 847 /* Update sw_index */ 848 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 849 dest_ring->sw_index = sw_index; 850 851 return 0; 852 } 853 854 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 855 void **per_transfer_ctx, 856 unsigned int *nbytesp) 857 { 858 return ce_state->ops->ce_completed_recv_next_nolock(ce_state, 859 per_transfer_ctx, 860 nbytesp); 861 } 862 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock); 863 864 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, 865 void **per_transfer_contextp, 866 unsigned int *nbytesp) 867 { 868 struct ath10k *ar = ce_state->ar; 869 struct ath10k_ce *ce = ath10k_ce_priv(ar); 870 int ret; 871 872 spin_lock_bh(&ce->ce_lock); 873 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state, 874 per_transfer_contextp, 875 nbytesp); 876 877 spin_unlock_bh(&ce->ce_lock); 878 879 return ret; 880 } 881 EXPORT_SYMBOL(ath10k_ce_completed_recv_next); 882 883 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 884 void **per_transfer_contextp, 885 dma_addr_t *bufferp) 886 { 887 struct ath10k_ce_ring *dest_ring; 888 unsigned int nentries_mask; 889 unsigned int sw_index; 890 unsigned int write_index; 891 int ret; 892 struct ath10k *ar; 893 struct ath10k_ce *ce; 894 895 dest_ring = ce_state->dest_ring; 896 897 if (!dest_ring) 898 return -EIO; 899 900 ar = ce_state->ar; 901 ce = ath10k_ce_priv(ar); 902 903 spin_lock_bh(&ce->ce_lock); 904 905 nentries_mask = dest_ring->nentries_mask; 906 sw_index = dest_ring->sw_index; 907 write_index = dest_ring->write_index; 908 if (write_index != sw_index) { 909 struct ce_desc *base = dest_ring->base_addr_owner_space; 910 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 911 912 /* Return data from completed destination descriptor */ 913 *bufferp = __le32_to_cpu(desc->addr); 914 915 if (per_transfer_contextp) 916 *per_transfer_contextp = 917 dest_ring->per_transfer_context[sw_index]; 918 919 /* sanity */ 920 dest_ring->per_transfer_context[sw_index] = NULL; 921 desc->nbytes = 0; 922 923 /* Update sw_index */ 924 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 925 dest_ring->sw_index = sw_index; 926 ret = 0; 927 } else { 928 ret = -EIO; 929 } 930 931 spin_unlock_bh(&ce->ce_lock); 932 933 return ret; 934 } 935 936 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state, 937 void **per_transfer_contextp, 938 dma_addr_t *bufferp) 939 { 940 struct ath10k_ce_ring *dest_ring; 941 unsigned int nentries_mask; 942 unsigned int sw_index; 943 unsigned int write_index; 944 int ret; 945 struct ath10k *ar; 946 struct ath10k_ce *ce; 947 948 dest_ring = ce_state->dest_ring; 949 950 if (!dest_ring) 951 return -EIO; 952 953 ar = ce_state->ar; 954 ce = ath10k_ce_priv(ar); 955 956 spin_lock_bh(&ce->ce_lock); 957 958 nentries_mask = dest_ring->nentries_mask; 959 sw_index = dest_ring->sw_index; 960 write_index = dest_ring->write_index; 961 if (write_index != sw_index) { 962 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 963 struct ce_desc_64 *desc = 964 CE_DEST_RING_TO_DESC_64(base, sw_index); 965 966 /* Return data from completed destination descriptor */ 967 *bufferp = __le64_to_cpu(desc->addr); 968 969 if (per_transfer_contextp) 970 *per_transfer_contextp = 971 dest_ring->per_transfer_context[sw_index]; 972 973 /* sanity */ 974 dest_ring->per_transfer_context[sw_index] = NULL; 975 desc->nbytes = 0; 976 977 /* Update sw_index */ 978 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 979 dest_ring->sw_index = sw_index; 980 ret = 0; 981 } else { 982 ret = -EIO; 983 } 984 985 spin_unlock_bh(&ce->ce_lock); 986 987 return ret; 988 } 989 990 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 991 void **per_transfer_contextp, 992 dma_addr_t *bufferp) 993 { 994 return ce_state->ops->ce_revoke_recv_next(ce_state, 995 per_transfer_contextp, 996 bufferp); 997 } 998 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next); 999 1000 /* 1001 * Guts of ath10k_ce_completed_send_next. 1002 * The caller takes responsibility for any necessary locking. 1003 */ 1004 static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 1005 void **per_transfer_contextp) 1006 { 1007 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1008 u32 ctrl_addr = ce_state->ctrl_addr; 1009 struct ath10k *ar = ce_state->ar; 1010 unsigned int nentries_mask = src_ring->nentries_mask; 1011 unsigned int sw_index = src_ring->sw_index; 1012 unsigned int read_index; 1013 struct ce_desc *desc; 1014 1015 if (src_ring->hw_index == sw_index) { 1016 /* 1017 * The SW completion index has caught up with the cached 1018 * version of the HW completion index. 1019 * Update the cached HW completion index to see whether 1020 * the SW has really caught up to the HW, or if the cached 1021 * value of the HW index has become stale. 1022 */ 1023 1024 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1025 if (read_index == 0xffffffff) 1026 return -ENODEV; 1027 1028 read_index &= nentries_mask; 1029 src_ring->hw_index = read_index; 1030 } 1031 1032 if (ar->hw_params.rri_on_ddr) 1033 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1034 else 1035 read_index = src_ring->hw_index; 1036 1037 if (read_index == sw_index) 1038 return -EIO; 1039 1040 if (per_transfer_contextp) 1041 *per_transfer_contextp = 1042 src_ring->per_transfer_context[sw_index]; 1043 1044 /* sanity */ 1045 src_ring->per_transfer_context[sw_index] = NULL; 1046 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 1047 sw_index); 1048 desc->nbytes = 0; 1049 1050 /* Update sw_index */ 1051 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1052 src_ring->sw_index = sw_index; 1053 1054 return 0; 1055 } 1056 1057 static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state, 1058 void **per_transfer_contextp) 1059 { 1060 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1061 u32 ctrl_addr = ce_state->ctrl_addr; 1062 struct ath10k *ar = ce_state->ar; 1063 unsigned int nentries_mask = src_ring->nentries_mask; 1064 unsigned int sw_index = src_ring->sw_index; 1065 unsigned int read_index; 1066 struct ce_desc_64 *desc; 1067 1068 if (src_ring->hw_index == sw_index) { 1069 /* 1070 * The SW completion index has caught up with the cached 1071 * version of the HW completion index. 1072 * Update the cached HW completion index to see whether 1073 * the SW has really caught up to the HW, or if the cached 1074 * value of the HW index has become stale. 1075 */ 1076 1077 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1078 if (read_index == 0xffffffff) 1079 return -ENODEV; 1080 1081 read_index &= nentries_mask; 1082 src_ring->hw_index = read_index; 1083 } 1084 1085 if (ar->hw_params.rri_on_ddr) 1086 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1087 else 1088 read_index = src_ring->hw_index; 1089 1090 if (read_index == sw_index) 1091 return -EIO; 1092 1093 if (per_transfer_contextp) 1094 *per_transfer_contextp = 1095 src_ring->per_transfer_context[sw_index]; 1096 1097 /* sanity */ 1098 src_ring->per_transfer_context[sw_index] = NULL; 1099 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, 1100 sw_index); 1101 desc->nbytes = 0; 1102 1103 /* Update sw_index */ 1104 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1105 src_ring->sw_index = sw_index; 1106 1107 return 0; 1108 } 1109 1110 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 1111 void **per_transfer_contextp) 1112 { 1113 return ce_state->ops->ce_completed_send_next_nolock(ce_state, 1114 per_transfer_contextp); 1115 } 1116 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock); 1117 1118 static void ath10k_ce_extract_desc_data(struct ath10k *ar, 1119 struct ath10k_ce_ring *src_ring, 1120 u32 sw_index, 1121 dma_addr_t *bufferp, 1122 u32 *nbytesp, 1123 u32 *transfer_idp) 1124 { 1125 struct ce_desc *base = src_ring->base_addr_owner_space; 1126 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 1127 1128 /* Return data from completed source descriptor */ 1129 *bufferp = __le32_to_cpu(desc->addr); 1130 *nbytesp = __le16_to_cpu(desc->nbytes); 1131 *transfer_idp = MS(__le16_to_cpu(desc->flags), 1132 CE_DESC_FLAGS_META_DATA); 1133 } 1134 1135 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar, 1136 struct ath10k_ce_ring *src_ring, 1137 u32 sw_index, 1138 dma_addr_t *bufferp, 1139 u32 *nbytesp, 1140 u32 *transfer_idp) 1141 { 1142 struct ce_desc_64 *base = src_ring->base_addr_owner_space; 1143 struct ce_desc_64 *desc = 1144 CE_SRC_RING_TO_DESC_64(base, sw_index); 1145 1146 /* Return data from completed source descriptor */ 1147 *bufferp = __le64_to_cpu(desc->addr); 1148 *nbytesp = __le16_to_cpu(desc->nbytes); 1149 *transfer_idp = MS(__le16_to_cpu(desc->flags), 1150 CE_DESC_FLAGS_META_DATA); 1151 } 1152 1153 /* NB: Modeled after ath10k_ce_completed_send_next */ 1154 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, 1155 void **per_transfer_contextp, 1156 dma_addr_t *bufferp, 1157 unsigned int *nbytesp, 1158 unsigned int *transfer_idp) 1159 { 1160 struct ath10k_ce_ring *src_ring; 1161 unsigned int nentries_mask; 1162 unsigned int sw_index; 1163 unsigned int write_index; 1164 int ret; 1165 struct ath10k *ar; 1166 struct ath10k_ce *ce; 1167 1168 src_ring = ce_state->src_ring; 1169 1170 if (!src_ring) 1171 return -EIO; 1172 1173 ar = ce_state->ar; 1174 ce = ath10k_ce_priv(ar); 1175 1176 spin_lock_bh(&ce->ce_lock); 1177 1178 nentries_mask = src_ring->nentries_mask; 1179 sw_index = src_ring->sw_index; 1180 write_index = src_ring->write_index; 1181 1182 if (write_index != sw_index) { 1183 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index, 1184 bufferp, nbytesp, 1185 transfer_idp); 1186 1187 if (per_transfer_contextp) 1188 *per_transfer_contextp = 1189 src_ring->per_transfer_context[sw_index]; 1190 1191 /* sanity */ 1192 src_ring->per_transfer_context[sw_index] = NULL; 1193 1194 /* Update sw_index */ 1195 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1196 src_ring->sw_index = sw_index; 1197 ret = 0; 1198 } else { 1199 ret = -EIO; 1200 } 1201 1202 spin_unlock_bh(&ce->ce_lock); 1203 1204 return ret; 1205 } 1206 EXPORT_SYMBOL(ath10k_ce_cancel_send_next); 1207 1208 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, 1209 void **per_transfer_contextp) 1210 { 1211 struct ath10k *ar = ce_state->ar; 1212 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1213 int ret; 1214 1215 spin_lock_bh(&ce->ce_lock); 1216 ret = ath10k_ce_completed_send_next_nolock(ce_state, 1217 per_transfer_contextp); 1218 spin_unlock_bh(&ce->ce_lock); 1219 1220 return ret; 1221 } 1222 EXPORT_SYMBOL(ath10k_ce_completed_send_next); 1223 1224 /* 1225 * Guts of interrupt handler for per-engine interrupts on a particular CE. 1226 * 1227 * Invokes registered callbacks for recv_complete, 1228 * send_complete, and watermarks. 1229 */ 1230 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 1231 { 1232 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1233 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1234 const struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 1235 u32 ctrl_addr = ce_state->ctrl_addr; 1236 1237 /* 1238 * Clear before handling 1239 * 1240 * Misc CE interrupts are not being handled, but still need 1241 * to be cleared. 1242 * 1243 * NOTE: When the last copy engine interrupt is cleared the 1244 * hardware will go to sleep. Once this happens any access to 1245 * the CE registers can cause a hardware fault. 1246 */ 1247 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 1248 wm_regs->cc_mask | wm_regs->wm_mask); 1249 1250 if (ce_state->recv_cb) 1251 ce_state->recv_cb(ce_state); 1252 1253 if (ce_state->send_cb) 1254 ce_state->send_cb(ce_state); 1255 } 1256 EXPORT_SYMBOL(ath10k_ce_per_engine_service); 1257 1258 /* 1259 * Handler for per-engine interrupts on ALL active CEs. 1260 * This is used in cases where the system is sharing a 1261 * single interrupt for all CEs 1262 */ 1263 1264 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 1265 { 1266 int ce_id; 1267 u32 intr_summary; 1268 1269 intr_summary = ath10k_ce_interrupt_summary(ar); 1270 1271 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) { 1272 if (intr_summary & (1 << ce_id)) 1273 intr_summary &= ~(1 << ce_id); 1274 else 1275 /* no intr pending on this CE */ 1276 continue; 1277 1278 ath10k_ce_per_engine_service(ar, ce_id); 1279 } 1280 } 1281 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any); 1282 1283 /* 1284 * Adjust interrupts for the copy complete handler. 1285 * If it's needed for either send or recv, then unmask 1286 * this interrupt; otherwise, mask it. 1287 * 1288 * Called with ce_lock held. 1289 */ 1290 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state) 1291 { 1292 u32 ctrl_addr = ce_state->ctrl_addr; 1293 struct ath10k *ar = ce_state->ar; 1294 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; 1295 1296 if ((!disable_copy_compl_intr) && 1297 (ce_state->send_cb || ce_state->recv_cb)) 1298 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 1299 else 1300 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1301 1302 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1303 } 1304 1305 void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id) 1306 { 1307 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1308 struct ath10k_ce_pipe *ce_state; 1309 u32 ctrl_addr; 1310 1311 ce_state = &ce->ce_states[ce_id]; 1312 if (ce_state->attr_flags & CE_ATTR_POLL) 1313 return; 1314 1315 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1316 1317 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1318 ath10k_ce_error_intr_disable(ar, ctrl_addr); 1319 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1320 } 1321 EXPORT_SYMBOL(ath10k_ce_disable_interrupt); 1322 1323 void ath10k_ce_disable_interrupts(struct ath10k *ar) 1324 { 1325 int ce_id; 1326 1327 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) 1328 ath10k_ce_disable_interrupt(ar, ce_id); 1329 } 1330 EXPORT_SYMBOL(ath10k_ce_disable_interrupts); 1331 1332 void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id) 1333 { 1334 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1335 struct ath10k_ce_pipe *ce_state; 1336 1337 ce_state = &ce->ce_states[ce_id]; 1338 if (ce_state->attr_flags & CE_ATTR_POLL) 1339 return; 1340 1341 ath10k_ce_per_engine_handler_adjust(ce_state); 1342 } 1343 EXPORT_SYMBOL(ath10k_ce_enable_interrupt); 1344 1345 void ath10k_ce_enable_interrupts(struct ath10k *ar) 1346 { 1347 int ce_id; 1348 1349 /* Enable interrupts for copy engine that 1350 * are not using polling mode. 1351 */ 1352 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) 1353 ath10k_ce_enable_interrupt(ar, ce_id); 1354 } 1355 EXPORT_SYMBOL(ath10k_ce_enable_interrupts); 1356 1357 static int ath10k_ce_init_src_ring(struct ath10k *ar, 1358 unsigned int ce_id, 1359 const struct ce_attr *attr) 1360 { 1361 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1362 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1363 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1364 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1365 1366 nentries = roundup_pow_of_two(attr->src_nentries); 1367 1368 if (ar->hw_params.target_64bit) 1369 memset(src_ring->base_addr_owner_space, 0, 1370 nentries * sizeof(struct ce_desc_64)); 1371 else 1372 memset(src_ring->base_addr_owner_space, 0, 1373 nentries * sizeof(struct ce_desc)); 1374 1375 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1376 src_ring->sw_index &= src_ring->nentries_mask; 1377 src_ring->hw_index = src_ring->sw_index; 1378 1379 src_ring->write_index = 1380 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 1381 src_ring->write_index &= src_ring->nentries_mask; 1382 1383 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 1384 src_ring->base_addr_ce_space); 1385 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 1386 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 1387 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 1388 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 1389 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 1390 1391 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1392 "boot init ce src ring id %d entries %d base_addr %p\n", 1393 ce_id, nentries, src_ring->base_addr_owner_space); 1394 1395 return 0; 1396 } 1397 1398 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 1399 unsigned int ce_id, 1400 const struct ce_attr *attr) 1401 { 1402 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1403 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1404 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 1405 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1406 1407 nentries = roundup_pow_of_two(attr->dest_nentries); 1408 1409 if (ar->hw_params.target_64bit) 1410 memset(dest_ring->base_addr_owner_space, 0, 1411 nentries * sizeof(struct ce_desc_64)); 1412 else 1413 memset(dest_ring->base_addr_owner_space, 0, 1414 nentries * sizeof(struct ce_desc)); 1415 1416 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 1417 dest_ring->sw_index &= dest_ring->nentries_mask; 1418 dest_ring->write_index = 1419 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 1420 dest_ring->write_index &= dest_ring->nentries_mask; 1421 1422 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 1423 dest_ring->base_addr_ce_space); 1424 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 1425 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 1426 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 1427 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 1428 1429 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1430 "boot ce dest ring id %d entries %d base_addr %p\n", 1431 ce_id, nentries, dest_ring->base_addr_owner_space); 1432 1433 return 0; 1434 } 1435 1436 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar, 1437 struct ath10k_ce_ring *src_ring, 1438 u32 nentries) 1439 { 1440 src_ring->shadow_base_unaligned = kcalloc(nentries, 1441 sizeof(struct ce_desc_64), 1442 GFP_KERNEL); 1443 if (!src_ring->shadow_base_unaligned) 1444 return -ENOMEM; 1445 1446 src_ring->shadow_base = (struct ce_desc_64 *) 1447 PTR_ALIGN(src_ring->shadow_base_unaligned, 1448 CE_DESC_RING_ALIGN); 1449 return 0; 1450 } 1451 1452 static struct ath10k_ce_ring * 1453 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, 1454 const struct ce_attr *attr) 1455 { 1456 struct ath10k_ce_ring *src_ring; 1457 u32 nentries = attr->src_nentries; 1458 dma_addr_t base_addr; 1459 int ret; 1460 1461 nentries = roundup_pow_of_two(nentries); 1462 1463 src_ring = kzalloc_flex(*src_ring, per_transfer_context, nentries); 1464 if (src_ring == NULL) 1465 return ERR_PTR(-ENOMEM); 1466 1467 src_ring->nentries = nentries; 1468 src_ring->nentries_mask = nentries - 1; 1469 1470 /* 1471 * Legacy platforms that do not support cache 1472 * coherent DMA are unsupported 1473 */ 1474 src_ring->base_addr_owner_space_unaligned = 1475 dma_alloc_coherent(ar->dev, 1476 (nentries * sizeof(struct ce_desc) + 1477 CE_DESC_RING_ALIGN), 1478 &base_addr, GFP_KERNEL); 1479 if (!src_ring->base_addr_owner_space_unaligned) { 1480 kfree(src_ring); 1481 return ERR_PTR(-ENOMEM); 1482 } 1483 1484 src_ring->base_addr_ce_space_unaligned = base_addr; 1485 1486 src_ring->base_addr_owner_space = 1487 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1488 CE_DESC_RING_ALIGN); 1489 src_ring->base_addr_ce_space = 1490 ALIGN(src_ring->base_addr_ce_space_unaligned, 1491 CE_DESC_RING_ALIGN); 1492 1493 if (ar->hw_params.shadow_reg_support) { 1494 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); 1495 if (ret) { 1496 dma_free_coherent(ar->dev, 1497 (nentries * sizeof(struct ce_desc) + 1498 CE_DESC_RING_ALIGN), 1499 src_ring->base_addr_owner_space_unaligned, 1500 base_addr); 1501 kfree(src_ring); 1502 return ERR_PTR(ret); 1503 } 1504 } 1505 1506 return src_ring; 1507 } 1508 1509 static struct ath10k_ce_ring * 1510 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id, 1511 const struct ce_attr *attr) 1512 { 1513 struct ath10k_ce_ring *src_ring; 1514 u32 nentries = attr->src_nentries; 1515 dma_addr_t base_addr; 1516 int ret; 1517 1518 nentries = roundup_pow_of_two(nentries); 1519 1520 src_ring = kzalloc_flex(*src_ring, per_transfer_context, nentries); 1521 if (!src_ring) 1522 return ERR_PTR(-ENOMEM); 1523 1524 src_ring->nentries = nentries; 1525 src_ring->nentries_mask = nentries - 1; 1526 1527 /* Legacy platforms that do not support cache 1528 * coherent DMA are unsupported 1529 */ 1530 src_ring->base_addr_owner_space_unaligned = 1531 dma_alloc_coherent(ar->dev, 1532 (nentries * sizeof(struct ce_desc_64) + 1533 CE_DESC_RING_ALIGN), 1534 &base_addr, GFP_KERNEL); 1535 if (!src_ring->base_addr_owner_space_unaligned) { 1536 kfree(src_ring); 1537 return ERR_PTR(-ENOMEM); 1538 } 1539 1540 src_ring->base_addr_ce_space_unaligned = base_addr; 1541 1542 src_ring->base_addr_owner_space = 1543 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1544 CE_DESC_RING_ALIGN); 1545 src_ring->base_addr_ce_space = 1546 ALIGN(src_ring->base_addr_ce_space_unaligned, 1547 CE_DESC_RING_ALIGN); 1548 1549 if (ar->hw_params.shadow_reg_support) { 1550 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); 1551 if (ret) { 1552 dma_free_coherent(ar->dev, 1553 (nentries * sizeof(struct ce_desc_64) + 1554 CE_DESC_RING_ALIGN), 1555 src_ring->base_addr_owner_space_unaligned, 1556 base_addr); 1557 kfree(src_ring); 1558 return ERR_PTR(ret); 1559 } 1560 } 1561 1562 return src_ring; 1563 } 1564 1565 static struct ath10k_ce_ring * 1566 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, 1567 const struct ce_attr *attr) 1568 { 1569 struct ath10k_ce_ring *dest_ring; 1570 u32 nentries; 1571 dma_addr_t base_addr; 1572 1573 nentries = roundup_pow_of_two(attr->dest_nentries); 1574 1575 dest_ring = kzalloc_flex(*dest_ring, per_transfer_context, nentries); 1576 if (dest_ring == NULL) 1577 return ERR_PTR(-ENOMEM); 1578 1579 dest_ring->nentries = nentries; 1580 dest_ring->nentries_mask = nentries - 1; 1581 1582 /* 1583 * Legacy platforms that do not support cache 1584 * coherent DMA are unsupported 1585 */ 1586 dest_ring->base_addr_owner_space_unaligned = 1587 dma_alloc_coherent(ar->dev, 1588 (nentries * sizeof(struct ce_desc) + 1589 CE_DESC_RING_ALIGN), 1590 &base_addr, GFP_KERNEL); 1591 if (!dest_ring->base_addr_owner_space_unaligned) { 1592 kfree(dest_ring); 1593 return ERR_PTR(-ENOMEM); 1594 } 1595 1596 dest_ring->base_addr_ce_space_unaligned = base_addr; 1597 1598 dest_ring->base_addr_owner_space = 1599 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1600 CE_DESC_RING_ALIGN); 1601 dest_ring->base_addr_ce_space = 1602 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1603 CE_DESC_RING_ALIGN); 1604 1605 return dest_ring; 1606 } 1607 1608 static struct ath10k_ce_ring * 1609 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id, 1610 const struct ce_attr *attr) 1611 { 1612 struct ath10k_ce_ring *dest_ring; 1613 u32 nentries; 1614 dma_addr_t base_addr; 1615 1616 nentries = roundup_pow_of_two(attr->dest_nentries); 1617 1618 dest_ring = kzalloc_flex(*dest_ring, per_transfer_context, nentries); 1619 if (!dest_ring) 1620 return ERR_PTR(-ENOMEM); 1621 1622 dest_ring->nentries = nentries; 1623 dest_ring->nentries_mask = nentries - 1; 1624 1625 /* Legacy platforms that do not support cache 1626 * coherent DMA are unsupported 1627 */ 1628 dest_ring->base_addr_owner_space_unaligned = 1629 dma_alloc_coherent(ar->dev, 1630 (nentries * sizeof(struct ce_desc_64) + 1631 CE_DESC_RING_ALIGN), 1632 &base_addr, GFP_KERNEL); 1633 if (!dest_ring->base_addr_owner_space_unaligned) { 1634 kfree(dest_ring); 1635 return ERR_PTR(-ENOMEM); 1636 } 1637 1638 dest_ring->base_addr_ce_space_unaligned = base_addr; 1639 1640 /* Correctly initialize memory to 0 to prevent garbage 1641 * data crashing system when download firmware 1642 */ 1643 dest_ring->base_addr_owner_space = 1644 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1645 CE_DESC_RING_ALIGN); 1646 dest_ring->base_addr_ce_space = 1647 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1648 CE_DESC_RING_ALIGN); 1649 1650 return dest_ring; 1651 } 1652 1653 /* 1654 * Initialize a Copy Engine based on caller-supplied attributes. 1655 * This may be called once to initialize both source and destination 1656 * rings or it may be called twice for separate source and destination 1657 * initialization. It may be that only one side or the other is 1658 * initialized by software/firmware. 1659 */ 1660 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, 1661 const struct ce_attr *attr) 1662 { 1663 int ret; 1664 1665 if (attr->src_nentries) { 1666 ret = ath10k_ce_init_src_ring(ar, ce_id, attr); 1667 if (ret) { 1668 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n", 1669 ce_id, ret); 1670 return ret; 1671 } 1672 } 1673 1674 if (attr->dest_nentries) { 1675 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); 1676 if (ret) { 1677 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n", 1678 ce_id, ret); 1679 return ret; 1680 } 1681 } 1682 1683 return 0; 1684 } 1685 EXPORT_SYMBOL(ath10k_ce_init_pipe); 1686 1687 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) 1688 { 1689 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1690 1691 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0); 1692 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0); 1693 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0); 1694 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0); 1695 } 1696 1697 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) 1698 { 1699 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1700 1701 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0); 1702 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0); 1703 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0); 1704 } 1705 1706 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) 1707 { 1708 ath10k_ce_deinit_src_ring(ar, ce_id); 1709 ath10k_ce_deinit_dest_ring(ar, ce_id); 1710 } 1711 EXPORT_SYMBOL(ath10k_ce_deinit_pipe); 1712 1713 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1714 { 1715 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1716 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1717 1718 if (ce_state->src_ring) { 1719 if (ar->hw_params.shadow_reg_support) 1720 kfree(ce_state->src_ring->shadow_base_unaligned); 1721 dma_free_coherent(ar->dev, 1722 (ce_state->src_ring->nentries * 1723 sizeof(struct ce_desc) + 1724 CE_DESC_RING_ALIGN), 1725 ce_state->src_ring->base_addr_owner_space_unaligned, 1726 ce_state->src_ring->base_addr_ce_space_unaligned); 1727 kfree(ce_state->src_ring); 1728 } 1729 1730 if (ce_state->dest_ring) { 1731 dma_free_coherent(ar->dev, 1732 (ce_state->dest_ring->nentries * 1733 sizeof(struct ce_desc) + 1734 CE_DESC_RING_ALIGN), 1735 ce_state->dest_ring->base_addr_owner_space_unaligned, 1736 ce_state->dest_ring->base_addr_ce_space_unaligned); 1737 kfree(ce_state->dest_ring); 1738 } 1739 1740 ce_state->src_ring = NULL; 1741 ce_state->dest_ring = NULL; 1742 } 1743 1744 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id) 1745 { 1746 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1747 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1748 1749 if (ce_state->src_ring) { 1750 if (ar->hw_params.shadow_reg_support) 1751 kfree(ce_state->src_ring->shadow_base_unaligned); 1752 dma_free_coherent(ar->dev, 1753 (ce_state->src_ring->nentries * 1754 sizeof(struct ce_desc_64) + 1755 CE_DESC_RING_ALIGN), 1756 ce_state->src_ring->base_addr_owner_space_unaligned, 1757 ce_state->src_ring->base_addr_ce_space_unaligned); 1758 kfree(ce_state->src_ring); 1759 } 1760 1761 if (ce_state->dest_ring) { 1762 dma_free_coherent(ar->dev, 1763 (ce_state->dest_ring->nentries * 1764 sizeof(struct ce_desc_64) + 1765 CE_DESC_RING_ALIGN), 1766 ce_state->dest_ring->base_addr_owner_space_unaligned, 1767 ce_state->dest_ring->base_addr_ce_space_unaligned); 1768 kfree(ce_state->dest_ring); 1769 } 1770 1771 ce_state->src_ring = NULL; 1772 ce_state->dest_ring = NULL; 1773 } 1774 1775 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1776 { 1777 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1778 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1779 1780 ce_state->ops->ce_free_pipe(ar, ce_id); 1781 } 1782 EXPORT_SYMBOL(ath10k_ce_free_pipe); 1783 1784 void ath10k_ce_dump_registers(struct ath10k *ar, 1785 struct ath10k_fw_crash_data *crash_data) 1786 { 1787 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1788 struct ath10k_ce_crash_data ce_data; 1789 u32 addr, id; 1790 1791 lockdep_assert_held(&ar->dump_mutex); 1792 1793 ath10k_err(ar, "Copy Engine register dump:\n"); 1794 1795 spin_lock_bh(&ce->ce_lock); 1796 for (id = 0; id < CE_COUNT; id++) { 1797 addr = ath10k_ce_base_address(ar, id); 1798 ce_data.base_addr = cpu_to_le32(addr); 1799 1800 ce_data.src_wr_idx = 1801 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr)); 1802 ce_data.src_r_idx = 1803 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr)); 1804 ce_data.dst_wr_idx = 1805 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr)); 1806 ce_data.dst_r_idx = 1807 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr)); 1808 1809 if (crash_data) 1810 crash_data->ce_crash_data[id] = ce_data; 1811 1812 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id, 1813 le32_to_cpu(ce_data.base_addr), 1814 le32_to_cpu(ce_data.src_wr_idx), 1815 le32_to_cpu(ce_data.src_r_idx), 1816 le32_to_cpu(ce_data.dst_wr_idx), 1817 le32_to_cpu(ce_data.dst_r_idx)); 1818 } 1819 1820 spin_unlock_bh(&ce->ce_lock); 1821 } 1822 EXPORT_SYMBOL(ath10k_ce_dump_registers); 1823 1824 static const struct ath10k_ce_ops ce_ops = { 1825 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring, 1826 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring, 1827 .ce_rx_post_buf = __ath10k_ce_rx_post_buf, 1828 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock, 1829 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next, 1830 .ce_extract_desc_data = ath10k_ce_extract_desc_data, 1831 .ce_free_pipe = _ath10k_ce_free_pipe, 1832 .ce_send_nolock = _ath10k_ce_send_nolock, 1833 .ce_set_src_ring_base_addr_hi = NULL, 1834 .ce_set_dest_ring_base_addr_hi = NULL, 1835 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock, 1836 }; 1837 1838 static const struct ath10k_ce_ops ce_64_ops = { 1839 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64, 1840 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64, 1841 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64, 1842 .ce_completed_recv_next_nolock = 1843 _ath10k_ce_completed_recv_next_nolock_64, 1844 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64, 1845 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64, 1846 .ce_free_pipe = _ath10k_ce_free_pipe_64, 1847 .ce_send_nolock = _ath10k_ce_send_nolock_64, 1848 .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi, 1849 .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi, 1850 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64, 1851 }; 1852 1853 static void ath10k_ce_set_ops(struct ath10k *ar, 1854 struct ath10k_ce_pipe *ce_state) 1855 { 1856 switch (ar->hw_rev) { 1857 case ATH10K_HW_WCN3990: 1858 ce_state->ops = &ce_64_ops; 1859 break; 1860 default: 1861 ce_state->ops = &ce_ops; 1862 break; 1863 } 1864 } 1865 1866 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, 1867 const struct ce_attr *attr) 1868 { 1869 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1870 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1871 int ret; 1872 1873 ath10k_ce_set_ops(ar, ce_state); 1874 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid 1875 * additional TX locking checks. 1876 * 1877 * For the lack of a better place do the check here. 1878 */ 1879 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC > 1880 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1881 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC > 1882 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1883 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC > 1884 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1885 1886 ce_state->ar = ar; 1887 ce_state->id = ce_id; 1888 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1889 ce_state->attr_flags = attr->flags; 1890 ce_state->src_sz_max = attr->src_sz_max; 1891 1892 if (attr->src_nentries) 1893 ce_state->send_cb = attr->send_cb; 1894 1895 if (attr->dest_nentries) 1896 ce_state->recv_cb = attr->recv_cb; 1897 1898 if (attr->src_nentries) { 1899 ce_state->src_ring = 1900 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr); 1901 if (IS_ERR(ce_state->src_ring)) { 1902 ret = PTR_ERR(ce_state->src_ring); 1903 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n", 1904 ce_id, ret); 1905 ce_state->src_ring = NULL; 1906 return ret; 1907 } 1908 } 1909 1910 if (attr->dest_nentries) { 1911 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar, 1912 ce_id, 1913 attr); 1914 if (IS_ERR(ce_state->dest_ring)) { 1915 ret = PTR_ERR(ce_state->dest_ring); 1916 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n", 1917 ce_id, ret); 1918 ce_state->dest_ring = NULL; 1919 return ret; 1920 } 1921 } 1922 1923 return 0; 1924 } 1925 EXPORT_SYMBOL(ath10k_ce_alloc_pipe); 1926 1927 void ath10k_ce_alloc_rri(struct ath10k *ar) 1928 { 1929 int i; 1930 u32 value; 1931 u32 ctrl1_regs; 1932 u32 ce_base_addr; 1933 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1934 1935 ce->vaddr_rri = dma_alloc_coherent(ar->dev, 1936 (CE_COUNT * sizeof(u32)), 1937 &ce->paddr_rri, GFP_KERNEL); 1938 1939 if (!ce->vaddr_rri) 1940 return; 1941 1942 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low, 1943 lower_32_bits(ce->paddr_rri)); 1944 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high, 1945 (upper_32_bits(ce->paddr_rri) & 1946 CE_DESC_ADDR_HI_MASK)); 1947 1948 for (i = 0; i < CE_COUNT; i++) { 1949 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr; 1950 ce_base_addr = ath10k_ce_base_address(ar, i); 1951 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs); 1952 value |= ar->hw_ce_regs->upd->mask; 1953 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value); 1954 } 1955 } 1956 EXPORT_SYMBOL(ath10k_ce_alloc_rri); 1957 1958 void ath10k_ce_free_rri(struct ath10k *ar) 1959 { 1960 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1961 1962 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)), 1963 ce->vaddr_rri, 1964 ce->paddr_rri); 1965 } 1966 EXPORT_SYMBOL(ath10k_ce_free_rri); 1967