1 // SPDX-License-Identifier: ISC 2 /* 3 * Copyright (c) 2005-2011 Atheros Communications Inc. 4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. 5 * Copyright (c) 2018 The Linux Foundation. All rights reserved. 6 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. 7 */ 8 9 #include "hif.h" 10 #include "ce.h" 11 #include "debug.h" 12 13 /* 14 * Support for Copy Engine hardware, which is mainly used for 15 * communication between Host and Target over a PCIe interconnect. 16 */ 17 18 /* 19 * A single CopyEngine (CE) comprises two "rings": 20 * a source ring 21 * a destination ring 22 * 23 * Each ring consists of a number of descriptors which specify 24 * an address, length, and meta-data. 25 * 26 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target) 27 * controls one ring and the other side controls the other ring. 28 * The source side chooses when to initiate a transfer and it 29 * chooses what to send (buffer address, length). The destination 30 * side keeps a supply of "anonymous receive buffers" available and 31 * it handles incoming data as it arrives (when the destination 32 * receives an interrupt). 33 * 34 * The sender may send a simple buffer (address/length) or it may 35 * send a small list of buffers. When a small list is sent, hardware 36 * "gathers" these and they end up in a single destination buffer 37 * with a single interrupt. 38 * 39 * There are several "contexts" managed by this layer -- more, it 40 * may seem -- than should be needed. These are provided mainly for 41 * maximum flexibility and especially to facilitate a simpler HIF 42 * implementation. There are per-CopyEngine recv, send, and watermark 43 * contexts. These are supplied by the caller when a recv, send, 44 * or watermark handler is established and they are echoed back to 45 * the caller when the respective callbacks are invoked. There is 46 * also a per-transfer context supplied by the caller when a buffer 47 * (or sendlist) is sent and when a buffer is enqueued for recv. 48 * These per-transfer contexts are echoed back to the caller when 49 * the buffer is sent/received. 50 */ 51 52 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar, 53 struct ath10k_ce_pipe *ce_state) 54 { 55 u32 ce_id = ce_state->id; 56 u32 addr = 0; 57 58 switch (ce_id) { 59 case 0: 60 addr = 0x00032000; 61 break; 62 case 3: 63 addr = 0x0003200C; 64 break; 65 case 4: 66 addr = 0x00032010; 67 break; 68 case 5: 69 addr = 0x00032014; 70 break; 71 case 7: 72 addr = 0x0003201C; 73 break; 74 default: 75 ath10k_warn(ar, "invalid CE id: %d", ce_id); 76 break; 77 } 78 return addr; 79 } 80 81 static inline unsigned int 82 ath10k_set_ring_byte(unsigned int offset, 83 struct ath10k_hw_ce_regs_addr_map *addr_map) 84 { 85 return ((offset << addr_map->lsb) & addr_map->mask); 86 } 87 88 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset) 89 { 90 struct ath10k_ce *ce = ath10k_ce_priv(ar); 91 92 return ce->bus_ops->read32(ar, offset); 93 } 94 95 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value) 96 { 97 struct ath10k_ce *ce = ath10k_ce_priv(ar); 98 99 ce->bus_ops->write32(ar, offset, value); 100 } 101 102 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar, 103 u32 ce_ctrl_addr, 104 unsigned int n) 105 { 106 ath10k_ce_write32(ar, ce_ctrl_addr + 107 ar->hw_ce_regs->dst_wr_index_addr, n); 108 } 109 110 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar, 111 u32 ce_ctrl_addr) 112 { 113 return ath10k_ce_read32(ar, ce_ctrl_addr + 114 ar->hw_ce_regs->dst_wr_index_addr); 115 } 116 117 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar, 118 u32 ce_ctrl_addr, 119 unsigned int n) 120 { 121 ath10k_ce_write32(ar, ce_ctrl_addr + 122 ar->hw_ce_regs->sr_wr_index_addr, n); 123 } 124 125 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar, 126 u32 ce_ctrl_addr) 127 { 128 return ath10k_ce_read32(ar, ce_ctrl_addr + 129 ar->hw_ce_regs->sr_wr_index_addr); 130 } 131 132 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar, 133 u32 ce_id) 134 { 135 struct ath10k_ce *ce = ath10k_ce_priv(ar); 136 137 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK; 138 } 139 140 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar, 141 u32 ce_ctrl_addr) 142 { 143 struct ath10k_ce *ce = ath10k_ce_priv(ar); 144 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); 145 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 146 u32 index; 147 148 if (ar->hw_params.rri_on_ddr && 149 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) 150 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id); 151 else 152 index = ath10k_ce_read32(ar, ce_ctrl_addr + 153 ar->hw_ce_regs->current_srri_addr); 154 155 return index; 156 } 157 158 static inline void 159 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar, 160 struct ath10k_ce_pipe *ce_state, 161 unsigned int value) 162 { 163 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value); 164 } 165 166 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar, 167 u32 ce_id, 168 u64 addr) 169 { 170 struct ath10k_ce *ce = ath10k_ce_priv(ar); 171 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 172 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id); 173 u32 addr_lo = lower_32_bits(addr); 174 175 ath10k_ce_write32(ar, ce_ctrl_addr + 176 ar->hw_ce_regs->sr_base_addr_lo, addr_lo); 177 178 if (ce_state->ops->ce_set_src_ring_base_addr_hi) { 179 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr, 180 addr); 181 } 182 } 183 184 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar, 185 u32 ce_ctrl_addr, 186 u64 addr) 187 { 188 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK; 189 190 ath10k_ce_write32(ar, ce_ctrl_addr + 191 ar->hw_ce_regs->sr_base_addr_hi, addr_hi); 192 } 193 194 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar, 195 u32 ce_ctrl_addr, 196 unsigned int n) 197 { 198 ath10k_ce_write32(ar, ce_ctrl_addr + 199 ar->hw_ce_regs->sr_size_addr, n); 200 } 201 202 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar, 203 u32 ce_ctrl_addr, 204 unsigned int n) 205 { 206 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 207 208 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 209 ctrl_regs->addr); 210 211 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 212 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) | 213 ath10k_set_ring_byte(n, ctrl_regs->dmax)); 214 } 215 216 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar, 217 u32 ce_ctrl_addr, 218 unsigned int n) 219 { 220 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 221 222 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 223 ctrl_regs->addr); 224 225 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 226 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) | 227 ath10k_set_ring_byte(n, ctrl_regs->src_ring)); 228 } 229 230 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar, 231 u32 ce_ctrl_addr, 232 unsigned int n) 233 { 234 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs; 235 236 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 237 ctrl_regs->addr); 238 239 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr, 240 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) | 241 ath10k_set_ring_byte(n, ctrl_regs->dst_ring)); 242 } 243 244 static inline 245 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id) 246 { 247 struct ath10k_ce *ce = ath10k_ce_priv(ar); 248 249 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) & 250 CE_DDR_RRI_MASK; 251 } 252 253 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar, 254 u32 ce_ctrl_addr) 255 { 256 struct ath10k_ce *ce = ath10k_ce_priv(ar); 257 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr); 258 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 259 u32 index; 260 261 if (ar->hw_params.rri_on_ddr && 262 (ce_state->attr_flags & CE_ATTR_DIS_INTR)) 263 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id); 264 else 265 index = ath10k_ce_read32(ar, ce_ctrl_addr + 266 ar->hw_ce_regs->current_drri_addr); 267 268 return index; 269 } 270 271 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar, 272 u32 ce_id, 273 u64 addr) 274 { 275 struct ath10k_ce *ce = ath10k_ce_priv(ar); 276 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 277 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id); 278 u32 addr_lo = lower_32_bits(addr); 279 280 ath10k_ce_write32(ar, ce_ctrl_addr + 281 ar->hw_ce_regs->dr_base_addr_lo, addr_lo); 282 283 if (ce_state->ops->ce_set_dest_ring_base_addr_hi) { 284 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr, 285 addr); 286 } 287 } 288 289 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar, 290 u32 ce_ctrl_addr, 291 u64 addr) 292 { 293 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK; 294 u32 reg_value; 295 296 reg_value = ath10k_ce_read32(ar, ce_ctrl_addr + 297 ar->hw_ce_regs->dr_base_addr_hi); 298 reg_value &= ~CE_DESC_ADDR_HI_MASK; 299 reg_value |= addr_hi; 300 ath10k_ce_write32(ar, ce_ctrl_addr + 301 ar->hw_ce_regs->dr_base_addr_hi, reg_value); 302 } 303 304 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar, 305 u32 ce_ctrl_addr, 306 unsigned int n) 307 { 308 ath10k_ce_write32(ar, ce_ctrl_addr + 309 ar->hw_ce_regs->dr_size_addr, n); 310 } 311 312 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar, 313 u32 ce_ctrl_addr, 314 unsigned int n) 315 { 316 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 317 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 318 319 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 320 (addr & ~(srcr_wm->wm_high->mask)) | 321 (ath10k_set_ring_byte(n, srcr_wm->wm_high))); 322 } 323 324 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar, 325 u32 ce_ctrl_addr, 326 unsigned int n) 327 { 328 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr; 329 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr); 330 331 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr, 332 (addr & ~(srcr_wm->wm_low->mask)) | 333 (ath10k_set_ring_byte(n, srcr_wm->wm_low))); 334 } 335 336 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar, 337 u32 ce_ctrl_addr, 338 unsigned int n) 339 { 340 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 341 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 342 343 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 344 (addr & ~(dstr_wm->wm_high->mask)) | 345 (ath10k_set_ring_byte(n, dstr_wm->wm_high))); 346 } 347 348 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar, 349 u32 ce_ctrl_addr, 350 unsigned int n) 351 { 352 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr; 353 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr); 354 355 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr, 356 (addr & ~(dstr_wm->wm_low->mask)) | 357 (ath10k_set_ring_byte(n, dstr_wm->wm_low))); 358 } 359 360 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar, 361 u32 ce_ctrl_addr) 362 { 363 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 364 365 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 366 ar->hw_ce_regs->host_ie_addr); 367 368 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 369 host_ie_addr | host_ie->copy_complete->mask); 370 } 371 372 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar, 373 u32 ce_ctrl_addr) 374 { 375 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie; 376 377 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 378 ar->hw_ce_regs->host_ie_addr); 379 380 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 381 host_ie_addr & ~(host_ie->copy_complete->mask)); 382 } 383 384 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar, 385 u32 ce_ctrl_addr) 386 { 387 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 388 389 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr + 390 ar->hw_ce_regs->host_ie_addr); 391 392 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr, 393 host_ie_addr & ~(wm_regs->wm_mask)); 394 } 395 396 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar, 397 u32 ce_ctrl_addr) 398 { 399 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs; 400 401 u32 misc_ie_addr = ath10k_ce_read32(ar, 402 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr); 403 404 ath10k_ce_write32(ar, 405 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr, 406 misc_ie_addr & ~(misc_regs->err_mask)); 407 } 408 409 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar, 410 u32 ce_ctrl_addr, 411 unsigned int mask) 412 { 413 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 414 415 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask); 416 } 417 418 /* 419 * Guts of ath10k_ce_send. 420 * The caller takes responsibility for any needed locking. 421 */ 422 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 423 void *per_transfer_context, 424 dma_addr_t buffer, 425 unsigned int nbytes, 426 unsigned int transfer_id, 427 unsigned int flags) 428 { 429 struct ath10k *ar = ce_state->ar; 430 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 431 struct ce_desc *desc, sdesc; 432 unsigned int nentries_mask = src_ring->nentries_mask; 433 unsigned int sw_index = src_ring->sw_index; 434 unsigned int write_index = src_ring->write_index; 435 u32 ctrl_addr = ce_state->ctrl_addr; 436 u32 desc_flags = 0; 437 int ret = 0; 438 439 if (nbytes > ce_state->src_sz_max) 440 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 441 __func__, nbytes, ce_state->src_sz_max); 442 443 if (unlikely(CE_RING_DELTA(nentries_mask, 444 write_index, sw_index - 1) <= 0)) { 445 ret = -ENOSR; 446 goto exit; 447 } 448 449 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 450 write_index); 451 452 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 453 454 if (flags & CE_SEND_FLAG_GATHER) 455 desc_flags |= CE_DESC_FLAGS_GATHER; 456 if (flags & CE_SEND_FLAG_BYTE_SWAP) 457 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 458 459 sdesc.addr = __cpu_to_le32(buffer); 460 sdesc.nbytes = __cpu_to_le16(nbytes); 461 sdesc.flags = __cpu_to_le16(desc_flags); 462 463 *desc = sdesc; 464 465 src_ring->per_transfer_context[write_index] = per_transfer_context; 466 467 /* Update Source Ring Write Index */ 468 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 469 470 /* WORKAROUND */ 471 if (!(flags & CE_SEND_FLAG_GATHER)) 472 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index); 473 474 src_ring->write_index = write_index; 475 exit: 476 return ret; 477 } 478 479 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state, 480 void *per_transfer_context, 481 dma_addr_t buffer, 482 unsigned int nbytes, 483 unsigned int transfer_id, 484 unsigned int flags) 485 { 486 struct ath10k *ar = ce_state->ar; 487 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 488 struct ce_desc_64 *desc, sdesc; 489 unsigned int nentries_mask = src_ring->nentries_mask; 490 unsigned int sw_index; 491 unsigned int write_index = src_ring->write_index; 492 u32 ctrl_addr = ce_state->ctrl_addr; 493 __le32 *addr; 494 u32 desc_flags = 0; 495 int ret = 0; 496 497 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags)) 498 return -ESHUTDOWN; 499 500 if (nbytes > ce_state->src_sz_max) 501 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n", 502 __func__, nbytes, ce_state->src_sz_max); 503 504 if (ar->hw_params.rri_on_ddr) 505 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id); 506 else 507 sw_index = src_ring->sw_index; 508 509 if (unlikely(CE_RING_DELTA(nentries_mask, 510 write_index, sw_index - 1) <= 0)) { 511 ret = -ENOSR; 512 goto exit; 513 } 514 515 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, 516 write_index); 517 518 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA); 519 520 if (flags & CE_SEND_FLAG_GATHER) 521 desc_flags |= CE_DESC_FLAGS_GATHER; 522 523 if (flags & CE_SEND_FLAG_BYTE_SWAP) 524 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP; 525 526 addr = (__le32 *)&sdesc.addr; 527 528 flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK; 529 addr[0] = __cpu_to_le32(buffer); 530 addr[1] = __cpu_to_le32(flags); 531 if (flags & CE_SEND_FLAG_GATHER) 532 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER); 533 else 534 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER)); 535 536 sdesc.nbytes = __cpu_to_le16(nbytes); 537 sdesc.flags = __cpu_to_le16(desc_flags); 538 539 *desc = sdesc; 540 541 src_ring->per_transfer_context[write_index] = per_transfer_context; 542 543 /* Update Source Ring Write Index */ 544 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 545 546 if (!(flags & CE_SEND_FLAG_GATHER)) { 547 if (ar->hw_params.shadow_reg_support) 548 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state, 549 write_index); 550 else 551 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, 552 write_index); 553 } 554 555 src_ring->write_index = write_index; 556 exit: 557 return ret; 558 } 559 560 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state, 561 void *per_transfer_context, 562 dma_addr_t buffer, 563 unsigned int nbytes, 564 unsigned int transfer_id, 565 unsigned int flags) 566 { 567 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context, 568 buffer, nbytes, transfer_id, flags); 569 } 570 EXPORT_SYMBOL(ath10k_ce_send_nolock); 571 572 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe) 573 { 574 struct ath10k *ar = pipe->ar; 575 struct ath10k_ce *ce = ath10k_ce_priv(ar); 576 struct ath10k_ce_ring *src_ring = pipe->src_ring; 577 u32 ctrl_addr = pipe->ctrl_addr; 578 579 lockdep_assert_held(&ce->ce_lock); 580 581 /* 582 * This function must be called only if there is an incomplete 583 * scatter-gather transfer (before index register is updated) 584 * that needs to be cleaned up. 585 */ 586 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index)) 587 return; 588 589 if (WARN_ON_ONCE(src_ring->write_index == 590 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr))) 591 return; 592 593 src_ring->write_index--; 594 src_ring->write_index &= src_ring->nentries_mask; 595 596 src_ring->per_transfer_context[src_ring->write_index] = NULL; 597 } 598 EXPORT_SYMBOL(__ath10k_ce_send_revert); 599 600 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state, 601 void *per_transfer_context, 602 dma_addr_t buffer, 603 unsigned int nbytes, 604 unsigned int transfer_id, 605 unsigned int flags) 606 { 607 struct ath10k *ar = ce_state->ar; 608 struct ath10k_ce *ce = ath10k_ce_priv(ar); 609 int ret; 610 611 spin_lock_bh(&ce->ce_lock); 612 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context, 613 buffer, nbytes, transfer_id, flags); 614 spin_unlock_bh(&ce->ce_lock); 615 616 return ret; 617 } 618 EXPORT_SYMBOL(ath10k_ce_send); 619 620 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe) 621 { 622 struct ath10k *ar = pipe->ar; 623 struct ath10k_ce *ce = ath10k_ce_priv(ar); 624 int delta; 625 626 spin_lock_bh(&ce->ce_lock); 627 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask, 628 pipe->src_ring->write_index, 629 pipe->src_ring->sw_index - 1); 630 spin_unlock_bh(&ce->ce_lock); 631 632 return delta; 633 } 634 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries); 635 636 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe) 637 { 638 struct ath10k *ar = pipe->ar; 639 struct ath10k_ce *ce = ath10k_ce_priv(ar); 640 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 641 unsigned int nentries_mask = dest_ring->nentries_mask; 642 unsigned int write_index = dest_ring->write_index; 643 unsigned int sw_index = dest_ring->sw_index; 644 645 lockdep_assert_held(&ce->ce_lock); 646 647 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1); 648 } 649 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs); 650 651 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 652 dma_addr_t paddr) 653 { 654 struct ath10k *ar = pipe->ar; 655 struct ath10k_ce *ce = ath10k_ce_priv(ar); 656 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 657 unsigned int nentries_mask = dest_ring->nentries_mask; 658 unsigned int write_index = dest_ring->write_index; 659 unsigned int sw_index = dest_ring->sw_index; 660 struct ce_desc *base = dest_ring->base_addr_owner_space; 661 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index); 662 u32 ctrl_addr = pipe->ctrl_addr; 663 664 lockdep_assert_held(&ce->ce_lock); 665 666 if ((pipe->id != 5) && 667 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 668 return -ENOSPC; 669 670 desc->addr = __cpu_to_le32(paddr); 671 desc->nbytes = 0; 672 673 dest_ring->per_transfer_context[write_index] = ctx; 674 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 675 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 676 dest_ring->write_index = write_index; 677 678 return 0; 679 } 680 681 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe, 682 void *ctx, 683 dma_addr_t paddr) 684 { 685 struct ath10k *ar = pipe->ar; 686 struct ath10k_ce *ce = ath10k_ce_priv(ar); 687 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 688 unsigned int nentries_mask = dest_ring->nentries_mask; 689 unsigned int write_index = dest_ring->write_index; 690 unsigned int sw_index = dest_ring->sw_index; 691 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 692 struct ce_desc_64 *desc = 693 CE_DEST_RING_TO_DESC_64(base, write_index); 694 u32 ctrl_addr = pipe->ctrl_addr; 695 696 lockdep_assert_held(&ce->ce_lock); 697 698 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0) 699 return -ENOSPC; 700 701 desc->addr = __cpu_to_le64(paddr); 702 desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK); 703 704 desc->nbytes = 0; 705 706 dest_ring->per_transfer_context[write_index] = ctx; 707 write_index = CE_RING_IDX_INCR(nentries_mask, write_index); 708 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 709 dest_ring->write_index = write_index; 710 711 return 0; 712 } 713 714 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries) 715 { 716 struct ath10k *ar = pipe->ar; 717 struct ath10k_ce_ring *dest_ring = pipe->dest_ring; 718 unsigned int nentries_mask = dest_ring->nentries_mask; 719 unsigned int write_index = dest_ring->write_index; 720 u32 ctrl_addr = pipe->ctrl_addr; 721 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 722 723 /* Prevent CE ring stuck issue that will occur when ring is full. 724 * Make sure that write index is 1 less than read index. 725 */ 726 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index) 727 nentries -= 1; 728 729 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries); 730 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index); 731 dest_ring->write_index = write_index; 732 } 733 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx); 734 735 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx, 736 dma_addr_t paddr) 737 { 738 struct ath10k *ar = pipe->ar; 739 struct ath10k_ce *ce = ath10k_ce_priv(ar); 740 int ret; 741 742 spin_lock_bh(&ce->ce_lock); 743 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr); 744 spin_unlock_bh(&ce->ce_lock); 745 746 return ret; 747 } 748 EXPORT_SYMBOL(ath10k_ce_rx_post_buf); 749 750 /* 751 * Guts of ath10k_ce_completed_recv_next. 752 * The caller takes responsibility for any necessary locking. 753 */ 754 static int 755 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 756 void **per_transfer_contextp, 757 unsigned int *nbytesp) 758 { 759 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 760 unsigned int nentries_mask = dest_ring->nentries_mask; 761 unsigned int sw_index = dest_ring->sw_index; 762 763 struct ce_desc *base = dest_ring->base_addr_owner_space; 764 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 765 struct ce_desc sdesc; 766 u16 nbytes; 767 768 /* Copy in one go for performance reasons */ 769 sdesc = *desc; 770 771 nbytes = __le16_to_cpu(sdesc.nbytes); 772 if (nbytes == 0) { 773 /* 774 * This closes a relatively unusual race where the Host 775 * sees the updated DRRI before the update to the 776 * corresponding descriptor has completed. We treat this 777 * as a descriptor that is not yet done. 778 */ 779 return -EIO; 780 } 781 782 desc->nbytes = 0; 783 784 /* Return data from completed destination descriptor */ 785 *nbytesp = nbytes; 786 787 if (per_transfer_contextp) 788 *per_transfer_contextp = 789 dest_ring->per_transfer_context[sw_index]; 790 791 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 792 * So update transfer context all CEs except CE5. 793 */ 794 if (ce_state->id != 5) 795 dest_ring->per_transfer_context[sw_index] = NULL; 796 797 /* Update sw_index */ 798 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 799 dest_ring->sw_index = sw_index; 800 801 return 0; 802 } 803 804 static int 805 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state, 806 void **per_transfer_contextp, 807 unsigned int *nbytesp) 808 { 809 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 810 unsigned int nentries_mask = dest_ring->nentries_mask; 811 unsigned int sw_index = dest_ring->sw_index; 812 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 813 struct ce_desc_64 *desc = 814 CE_DEST_RING_TO_DESC_64(base, sw_index); 815 struct ce_desc_64 sdesc; 816 u16 nbytes; 817 818 /* Copy in one go for performance reasons */ 819 sdesc = *desc; 820 821 nbytes = __le16_to_cpu(sdesc.nbytes); 822 if (nbytes == 0) { 823 /* This closes a relatively unusual race where the Host 824 * sees the updated DRRI before the update to the 825 * corresponding descriptor has completed. We treat this 826 * as a descriptor that is not yet done. 827 */ 828 return -EIO; 829 } 830 831 desc->nbytes = 0; 832 833 /* Return data from completed destination descriptor */ 834 *nbytesp = nbytes; 835 836 if (per_transfer_contextp) 837 *per_transfer_contextp = 838 dest_ring->per_transfer_context[sw_index]; 839 840 /* Copy engine 5 (HTT Rx) will reuse the same transfer context. 841 * So update transfer context all CEs except CE5. 842 */ 843 if (ce_state->id != 5) 844 dest_ring->per_transfer_context[sw_index] = NULL; 845 846 /* Update sw_index */ 847 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 848 dest_ring->sw_index = sw_index; 849 850 return 0; 851 } 852 853 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state, 854 void **per_transfer_ctx, 855 unsigned int *nbytesp) 856 { 857 return ce_state->ops->ce_completed_recv_next_nolock(ce_state, 858 per_transfer_ctx, 859 nbytesp); 860 } 861 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock); 862 863 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state, 864 void **per_transfer_contextp, 865 unsigned int *nbytesp) 866 { 867 struct ath10k *ar = ce_state->ar; 868 struct ath10k_ce *ce = ath10k_ce_priv(ar); 869 int ret; 870 871 spin_lock_bh(&ce->ce_lock); 872 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state, 873 per_transfer_contextp, 874 nbytesp); 875 876 spin_unlock_bh(&ce->ce_lock); 877 878 return ret; 879 } 880 EXPORT_SYMBOL(ath10k_ce_completed_recv_next); 881 882 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 883 void **per_transfer_contextp, 884 dma_addr_t *bufferp) 885 { 886 struct ath10k_ce_ring *dest_ring; 887 unsigned int nentries_mask; 888 unsigned int sw_index; 889 unsigned int write_index; 890 int ret; 891 struct ath10k *ar; 892 struct ath10k_ce *ce; 893 894 dest_ring = ce_state->dest_ring; 895 896 if (!dest_ring) 897 return -EIO; 898 899 ar = ce_state->ar; 900 ce = ath10k_ce_priv(ar); 901 902 spin_lock_bh(&ce->ce_lock); 903 904 nentries_mask = dest_ring->nentries_mask; 905 sw_index = dest_ring->sw_index; 906 write_index = dest_ring->write_index; 907 if (write_index != sw_index) { 908 struct ce_desc *base = dest_ring->base_addr_owner_space; 909 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index); 910 911 /* Return data from completed destination descriptor */ 912 *bufferp = __le32_to_cpu(desc->addr); 913 914 if (per_transfer_contextp) 915 *per_transfer_contextp = 916 dest_ring->per_transfer_context[sw_index]; 917 918 /* sanity */ 919 dest_ring->per_transfer_context[sw_index] = NULL; 920 desc->nbytes = 0; 921 922 /* Update sw_index */ 923 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 924 dest_ring->sw_index = sw_index; 925 ret = 0; 926 } else { 927 ret = -EIO; 928 } 929 930 spin_unlock_bh(&ce->ce_lock); 931 932 return ret; 933 } 934 935 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state, 936 void **per_transfer_contextp, 937 dma_addr_t *bufferp) 938 { 939 struct ath10k_ce_ring *dest_ring; 940 unsigned int nentries_mask; 941 unsigned int sw_index; 942 unsigned int write_index; 943 int ret; 944 struct ath10k *ar; 945 struct ath10k_ce *ce; 946 947 dest_ring = ce_state->dest_ring; 948 949 if (!dest_ring) 950 return -EIO; 951 952 ar = ce_state->ar; 953 ce = ath10k_ce_priv(ar); 954 955 spin_lock_bh(&ce->ce_lock); 956 957 nentries_mask = dest_ring->nentries_mask; 958 sw_index = dest_ring->sw_index; 959 write_index = dest_ring->write_index; 960 if (write_index != sw_index) { 961 struct ce_desc_64 *base = dest_ring->base_addr_owner_space; 962 struct ce_desc_64 *desc = 963 CE_DEST_RING_TO_DESC_64(base, sw_index); 964 965 /* Return data from completed destination descriptor */ 966 *bufferp = __le64_to_cpu(desc->addr); 967 968 if (per_transfer_contextp) 969 *per_transfer_contextp = 970 dest_ring->per_transfer_context[sw_index]; 971 972 /* sanity */ 973 dest_ring->per_transfer_context[sw_index] = NULL; 974 desc->nbytes = 0; 975 976 /* Update sw_index */ 977 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 978 dest_ring->sw_index = sw_index; 979 ret = 0; 980 } else { 981 ret = -EIO; 982 } 983 984 spin_unlock_bh(&ce->ce_lock); 985 986 return ret; 987 } 988 989 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state, 990 void **per_transfer_contextp, 991 dma_addr_t *bufferp) 992 { 993 return ce_state->ops->ce_revoke_recv_next(ce_state, 994 per_transfer_contextp, 995 bufferp); 996 } 997 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next); 998 999 /* 1000 * Guts of ath10k_ce_completed_send_next. 1001 * The caller takes responsibility for any necessary locking. 1002 */ 1003 static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 1004 void **per_transfer_contextp) 1005 { 1006 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1007 u32 ctrl_addr = ce_state->ctrl_addr; 1008 struct ath10k *ar = ce_state->ar; 1009 unsigned int nentries_mask = src_ring->nentries_mask; 1010 unsigned int sw_index = src_ring->sw_index; 1011 unsigned int read_index; 1012 struct ce_desc *desc; 1013 1014 if (src_ring->hw_index == sw_index) { 1015 /* 1016 * The SW completion index has caught up with the cached 1017 * version of the HW completion index. 1018 * Update the cached HW completion index to see whether 1019 * the SW has really caught up to the HW, or if the cached 1020 * value of the HW index has become stale. 1021 */ 1022 1023 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1024 if (read_index == 0xffffffff) 1025 return -ENODEV; 1026 1027 read_index &= nentries_mask; 1028 src_ring->hw_index = read_index; 1029 } 1030 1031 if (ar->hw_params.rri_on_ddr) 1032 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1033 else 1034 read_index = src_ring->hw_index; 1035 1036 if (read_index == sw_index) 1037 return -EIO; 1038 1039 if (per_transfer_contextp) 1040 *per_transfer_contextp = 1041 src_ring->per_transfer_context[sw_index]; 1042 1043 /* sanity */ 1044 src_ring->per_transfer_context[sw_index] = NULL; 1045 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space, 1046 sw_index); 1047 desc->nbytes = 0; 1048 1049 /* Update sw_index */ 1050 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1051 src_ring->sw_index = sw_index; 1052 1053 return 0; 1054 } 1055 1056 static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state, 1057 void **per_transfer_contextp) 1058 { 1059 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1060 u32 ctrl_addr = ce_state->ctrl_addr; 1061 struct ath10k *ar = ce_state->ar; 1062 unsigned int nentries_mask = src_ring->nentries_mask; 1063 unsigned int sw_index = src_ring->sw_index; 1064 unsigned int read_index; 1065 struct ce_desc_64 *desc; 1066 1067 if (src_ring->hw_index == sw_index) { 1068 /* 1069 * The SW completion index has caught up with the cached 1070 * version of the HW completion index. 1071 * Update the cached HW completion index to see whether 1072 * the SW has really caught up to the HW, or if the cached 1073 * value of the HW index has become stale. 1074 */ 1075 1076 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1077 if (read_index == 0xffffffff) 1078 return -ENODEV; 1079 1080 read_index &= nentries_mask; 1081 src_ring->hw_index = read_index; 1082 } 1083 1084 if (ar->hw_params.rri_on_ddr) 1085 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1086 else 1087 read_index = src_ring->hw_index; 1088 1089 if (read_index == sw_index) 1090 return -EIO; 1091 1092 if (per_transfer_contextp) 1093 *per_transfer_contextp = 1094 src_ring->per_transfer_context[sw_index]; 1095 1096 /* sanity */ 1097 src_ring->per_transfer_context[sw_index] = NULL; 1098 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space, 1099 sw_index); 1100 desc->nbytes = 0; 1101 1102 /* Update sw_index */ 1103 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1104 src_ring->sw_index = sw_index; 1105 1106 return 0; 1107 } 1108 1109 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state, 1110 void **per_transfer_contextp) 1111 { 1112 return ce_state->ops->ce_completed_send_next_nolock(ce_state, 1113 per_transfer_contextp); 1114 } 1115 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock); 1116 1117 static void ath10k_ce_extract_desc_data(struct ath10k *ar, 1118 struct ath10k_ce_ring *src_ring, 1119 u32 sw_index, 1120 dma_addr_t *bufferp, 1121 u32 *nbytesp, 1122 u32 *transfer_idp) 1123 { 1124 struct ce_desc *base = src_ring->base_addr_owner_space; 1125 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index); 1126 1127 /* Return data from completed source descriptor */ 1128 *bufferp = __le32_to_cpu(desc->addr); 1129 *nbytesp = __le16_to_cpu(desc->nbytes); 1130 *transfer_idp = MS(__le16_to_cpu(desc->flags), 1131 CE_DESC_FLAGS_META_DATA); 1132 } 1133 1134 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar, 1135 struct ath10k_ce_ring *src_ring, 1136 u32 sw_index, 1137 dma_addr_t *bufferp, 1138 u32 *nbytesp, 1139 u32 *transfer_idp) 1140 { 1141 struct ce_desc_64 *base = src_ring->base_addr_owner_space; 1142 struct ce_desc_64 *desc = 1143 CE_SRC_RING_TO_DESC_64(base, sw_index); 1144 1145 /* Return data from completed source descriptor */ 1146 *bufferp = __le64_to_cpu(desc->addr); 1147 *nbytesp = __le16_to_cpu(desc->nbytes); 1148 *transfer_idp = MS(__le16_to_cpu(desc->flags), 1149 CE_DESC_FLAGS_META_DATA); 1150 } 1151 1152 /* NB: Modeled after ath10k_ce_completed_send_next */ 1153 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state, 1154 void **per_transfer_contextp, 1155 dma_addr_t *bufferp, 1156 unsigned int *nbytesp, 1157 unsigned int *transfer_idp) 1158 { 1159 struct ath10k_ce_ring *src_ring; 1160 unsigned int nentries_mask; 1161 unsigned int sw_index; 1162 unsigned int write_index; 1163 int ret; 1164 struct ath10k *ar; 1165 struct ath10k_ce *ce; 1166 1167 src_ring = ce_state->src_ring; 1168 1169 if (!src_ring) 1170 return -EIO; 1171 1172 ar = ce_state->ar; 1173 ce = ath10k_ce_priv(ar); 1174 1175 spin_lock_bh(&ce->ce_lock); 1176 1177 nentries_mask = src_ring->nentries_mask; 1178 sw_index = src_ring->sw_index; 1179 write_index = src_ring->write_index; 1180 1181 if (write_index != sw_index) { 1182 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index, 1183 bufferp, nbytesp, 1184 transfer_idp); 1185 1186 if (per_transfer_contextp) 1187 *per_transfer_contextp = 1188 src_ring->per_transfer_context[sw_index]; 1189 1190 /* sanity */ 1191 src_ring->per_transfer_context[sw_index] = NULL; 1192 1193 /* Update sw_index */ 1194 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index); 1195 src_ring->sw_index = sw_index; 1196 ret = 0; 1197 } else { 1198 ret = -EIO; 1199 } 1200 1201 spin_unlock_bh(&ce->ce_lock); 1202 1203 return ret; 1204 } 1205 EXPORT_SYMBOL(ath10k_ce_cancel_send_next); 1206 1207 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state, 1208 void **per_transfer_contextp) 1209 { 1210 struct ath10k *ar = ce_state->ar; 1211 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1212 int ret; 1213 1214 spin_lock_bh(&ce->ce_lock); 1215 ret = ath10k_ce_completed_send_next_nolock(ce_state, 1216 per_transfer_contextp); 1217 spin_unlock_bh(&ce->ce_lock); 1218 1219 return ret; 1220 } 1221 EXPORT_SYMBOL(ath10k_ce_completed_send_next); 1222 1223 /* 1224 * Guts of interrupt handler for per-engine interrupts on a particular CE. 1225 * 1226 * Invokes registered callbacks for recv_complete, 1227 * send_complete, and watermarks. 1228 */ 1229 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id) 1230 { 1231 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1232 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1233 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs; 1234 u32 ctrl_addr = ce_state->ctrl_addr; 1235 1236 /* 1237 * Clear before handling 1238 * 1239 * Misc CE interrupts are not being handled, but still need 1240 * to be cleared. 1241 * 1242 * NOTE: When the last copy engine interrupt is cleared the 1243 * hardware will go to sleep. Once this happens any access to 1244 * the CE registers can cause a hardware fault. 1245 */ 1246 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, 1247 wm_regs->cc_mask | wm_regs->wm_mask); 1248 1249 if (ce_state->recv_cb) 1250 ce_state->recv_cb(ce_state); 1251 1252 if (ce_state->send_cb) 1253 ce_state->send_cb(ce_state); 1254 } 1255 EXPORT_SYMBOL(ath10k_ce_per_engine_service); 1256 1257 /* 1258 * Handler for per-engine interrupts on ALL active CEs. 1259 * This is used in cases where the system is sharing a 1260 * single interrupt for all CEs 1261 */ 1262 1263 void ath10k_ce_per_engine_service_any(struct ath10k *ar) 1264 { 1265 int ce_id; 1266 u32 intr_summary; 1267 1268 intr_summary = ath10k_ce_interrupt_summary(ar); 1269 1270 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) { 1271 if (intr_summary & (1 << ce_id)) 1272 intr_summary &= ~(1 << ce_id); 1273 else 1274 /* no intr pending on this CE */ 1275 continue; 1276 1277 ath10k_ce_per_engine_service(ar, ce_id); 1278 } 1279 } 1280 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any); 1281 1282 /* 1283 * Adjust interrupts for the copy complete handler. 1284 * If it's needed for either send or recv, then unmask 1285 * this interrupt; otherwise, mask it. 1286 * 1287 * Called with ce_lock held. 1288 */ 1289 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state) 1290 { 1291 u32 ctrl_addr = ce_state->ctrl_addr; 1292 struct ath10k *ar = ce_state->ar; 1293 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR; 1294 1295 if ((!disable_copy_compl_intr) && 1296 (ce_state->send_cb || ce_state->recv_cb)) 1297 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr); 1298 else 1299 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1300 1301 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1302 } 1303 1304 void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id) 1305 { 1306 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1307 struct ath10k_ce_pipe *ce_state; 1308 u32 ctrl_addr; 1309 1310 ce_state = &ce->ce_states[ce_id]; 1311 if (ce_state->attr_flags & CE_ATTR_POLL) 1312 return; 1313 1314 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1315 1316 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr); 1317 ath10k_ce_error_intr_disable(ar, ctrl_addr); 1318 ath10k_ce_watermark_intr_disable(ar, ctrl_addr); 1319 } 1320 EXPORT_SYMBOL(ath10k_ce_disable_interrupt); 1321 1322 void ath10k_ce_disable_interrupts(struct ath10k *ar) 1323 { 1324 int ce_id; 1325 1326 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) 1327 ath10k_ce_disable_interrupt(ar, ce_id); 1328 } 1329 EXPORT_SYMBOL(ath10k_ce_disable_interrupts); 1330 1331 void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id) 1332 { 1333 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1334 struct ath10k_ce_pipe *ce_state; 1335 1336 ce_state = &ce->ce_states[ce_id]; 1337 if (ce_state->attr_flags & CE_ATTR_POLL) 1338 return; 1339 1340 ath10k_ce_per_engine_handler_adjust(ce_state); 1341 } 1342 EXPORT_SYMBOL(ath10k_ce_enable_interrupt); 1343 1344 void ath10k_ce_enable_interrupts(struct ath10k *ar) 1345 { 1346 int ce_id; 1347 1348 /* Enable interrupts for copy engine that 1349 * are not using polling mode. 1350 */ 1351 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) 1352 ath10k_ce_enable_interrupt(ar, ce_id); 1353 } 1354 EXPORT_SYMBOL(ath10k_ce_enable_interrupts); 1355 1356 static int ath10k_ce_init_src_ring(struct ath10k *ar, 1357 unsigned int ce_id, 1358 const struct ce_attr *attr) 1359 { 1360 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1361 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1362 struct ath10k_ce_ring *src_ring = ce_state->src_ring; 1363 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1364 1365 nentries = roundup_pow_of_two(attr->src_nentries); 1366 1367 if (ar->hw_params.target_64bit) 1368 memset(src_ring->base_addr_owner_space, 0, 1369 nentries * sizeof(struct ce_desc_64)); 1370 else 1371 memset(src_ring->base_addr_owner_space, 0, 1372 nentries * sizeof(struct ce_desc)); 1373 1374 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr); 1375 src_ring->sw_index &= src_ring->nentries_mask; 1376 src_ring->hw_index = src_ring->sw_index; 1377 1378 src_ring->write_index = 1379 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr); 1380 src_ring->write_index &= src_ring->nentries_mask; 1381 1382 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 1383 src_ring->base_addr_ce_space); 1384 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries); 1385 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max); 1386 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0); 1387 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0); 1388 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries); 1389 1390 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1391 "boot init ce src ring id %d entries %d base_addr %pK\n", 1392 ce_id, nentries, src_ring->base_addr_owner_space); 1393 1394 return 0; 1395 } 1396 1397 static int ath10k_ce_init_dest_ring(struct ath10k *ar, 1398 unsigned int ce_id, 1399 const struct ce_attr *attr) 1400 { 1401 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1402 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1403 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring; 1404 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1405 1406 nentries = roundup_pow_of_two(attr->dest_nentries); 1407 1408 if (ar->hw_params.target_64bit) 1409 memset(dest_ring->base_addr_owner_space, 0, 1410 nentries * sizeof(struct ce_desc_64)); 1411 else 1412 memset(dest_ring->base_addr_owner_space, 0, 1413 nentries * sizeof(struct ce_desc)); 1414 1415 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr); 1416 dest_ring->sw_index &= dest_ring->nentries_mask; 1417 dest_ring->write_index = 1418 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr); 1419 dest_ring->write_index &= dest_ring->nentries_mask; 1420 1421 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 1422 dest_ring->base_addr_ce_space); 1423 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries); 1424 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0); 1425 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0); 1426 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries); 1427 1428 ath10k_dbg(ar, ATH10K_DBG_BOOT, 1429 "boot ce dest ring id %d entries %d base_addr %pK\n", 1430 ce_id, nentries, dest_ring->base_addr_owner_space); 1431 1432 return 0; 1433 } 1434 1435 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar, 1436 struct ath10k_ce_ring *src_ring, 1437 u32 nentries) 1438 { 1439 src_ring->shadow_base_unaligned = kcalloc(nentries, 1440 sizeof(struct ce_desc_64), 1441 GFP_KERNEL); 1442 if (!src_ring->shadow_base_unaligned) 1443 return -ENOMEM; 1444 1445 src_ring->shadow_base = (struct ce_desc_64 *) 1446 PTR_ALIGN(src_ring->shadow_base_unaligned, 1447 CE_DESC_RING_ALIGN); 1448 return 0; 1449 } 1450 1451 static struct ath10k_ce_ring * 1452 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id, 1453 const struct ce_attr *attr) 1454 { 1455 struct ath10k_ce_ring *src_ring; 1456 u32 nentries = attr->src_nentries; 1457 dma_addr_t base_addr; 1458 int ret; 1459 1460 nentries = roundup_pow_of_two(nentries); 1461 1462 src_ring = kzalloc(struct_size(src_ring, per_transfer_context, 1463 nentries), GFP_KERNEL); 1464 if (src_ring == NULL) 1465 return ERR_PTR(-ENOMEM); 1466 1467 src_ring->nentries = nentries; 1468 src_ring->nentries_mask = nentries - 1; 1469 1470 /* 1471 * Legacy platforms that do not support cache 1472 * coherent DMA are unsupported 1473 */ 1474 src_ring->base_addr_owner_space_unaligned = 1475 dma_alloc_coherent(ar->dev, 1476 (nentries * sizeof(struct ce_desc) + 1477 CE_DESC_RING_ALIGN), 1478 &base_addr, GFP_KERNEL); 1479 if (!src_ring->base_addr_owner_space_unaligned) { 1480 kfree(src_ring); 1481 return ERR_PTR(-ENOMEM); 1482 } 1483 1484 src_ring->base_addr_ce_space_unaligned = base_addr; 1485 1486 src_ring->base_addr_owner_space = 1487 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1488 CE_DESC_RING_ALIGN); 1489 src_ring->base_addr_ce_space = 1490 ALIGN(src_ring->base_addr_ce_space_unaligned, 1491 CE_DESC_RING_ALIGN); 1492 1493 if (ar->hw_params.shadow_reg_support) { 1494 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); 1495 if (ret) { 1496 dma_free_coherent(ar->dev, 1497 (nentries * sizeof(struct ce_desc) + 1498 CE_DESC_RING_ALIGN), 1499 src_ring->base_addr_owner_space_unaligned, 1500 base_addr); 1501 kfree(src_ring); 1502 return ERR_PTR(ret); 1503 } 1504 } 1505 1506 return src_ring; 1507 } 1508 1509 static struct ath10k_ce_ring * 1510 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id, 1511 const struct ce_attr *attr) 1512 { 1513 struct ath10k_ce_ring *src_ring; 1514 u32 nentries = attr->src_nentries; 1515 dma_addr_t base_addr; 1516 int ret; 1517 1518 nentries = roundup_pow_of_two(nentries); 1519 1520 src_ring = kzalloc(struct_size(src_ring, per_transfer_context, 1521 nentries), GFP_KERNEL); 1522 if (!src_ring) 1523 return ERR_PTR(-ENOMEM); 1524 1525 src_ring->nentries = nentries; 1526 src_ring->nentries_mask = nentries - 1; 1527 1528 /* Legacy platforms that do not support cache 1529 * coherent DMA are unsupported 1530 */ 1531 src_ring->base_addr_owner_space_unaligned = 1532 dma_alloc_coherent(ar->dev, 1533 (nentries * sizeof(struct ce_desc_64) + 1534 CE_DESC_RING_ALIGN), 1535 &base_addr, GFP_KERNEL); 1536 if (!src_ring->base_addr_owner_space_unaligned) { 1537 kfree(src_ring); 1538 return ERR_PTR(-ENOMEM); 1539 } 1540 1541 src_ring->base_addr_ce_space_unaligned = base_addr; 1542 1543 src_ring->base_addr_owner_space = 1544 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned, 1545 CE_DESC_RING_ALIGN); 1546 src_ring->base_addr_ce_space = 1547 ALIGN(src_ring->base_addr_ce_space_unaligned, 1548 CE_DESC_RING_ALIGN); 1549 1550 if (ar->hw_params.shadow_reg_support) { 1551 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries); 1552 if (ret) { 1553 dma_free_coherent(ar->dev, 1554 (nentries * sizeof(struct ce_desc_64) + 1555 CE_DESC_RING_ALIGN), 1556 src_ring->base_addr_owner_space_unaligned, 1557 base_addr); 1558 kfree(src_ring); 1559 return ERR_PTR(ret); 1560 } 1561 } 1562 1563 return src_ring; 1564 } 1565 1566 static struct ath10k_ce_ring * 1567 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id, 1568 const struct ce_attr *attr) 1569 { 1570 struct ath10k_ce_ring *dest_ring; 1571 u32 nentries; 1572 dma_addr_t base_addr; 1573 1574 nentries = roundup_pow_of_two(attr->dest_nentries); 1575 1576 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context, 1577 nentries), GFP_KERNEL); 1578 if (dest_ring == NULL) 1579 return ERR_PTR(-ENOMEM); 1580 1581 dest_ring->nentries = nentries; 1582 dest_ring->nentries_mask = nentries - 1; 1583 1584 /* 1585 * Legacy platforms that do not support cache 1586 * coherent DMA are unsupported 1587 */ 1588 dest_ring->base_addr_owner_space_unaligned = 1589 dma_alloc_coherent(ar->dev, 1590 (nentries * sizeof(struct ce_desc) + 1591 CE_DESC_RING_ALIGN), 1592 &base_addr, GFP_KERNEL); 1593 if (!dest_ring->base_addr_owner_space_unaligned) { 1594 kfree(dest_ring); 1595 return ERR_PTR(-ENOMEM); 1596 } 1597 1598 dest_ring->base_addr_ce_space_unaligned = base_addr; 1599 1600 dest_ring->base_addr_owner_space = 1601 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1602 CE_DESC_RING_ALIGN); 1603 dest_ring->base_addr_ce_space = 1604 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1605 CE_DESC_RING_ALIGN); 1606 1607 return dest_ring; 1608 } 1609 1610 static struct ath10k_ce_ring * 1611 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id, 1612 const struct ce_attr *attr) 1613 { 1614 struct ath10k_ce_ring *dest_ring; 1615 u32 nentries; 1616 dma_addr_t base_addr; 1617 1618 nentries = roundup_pow_of_two(attr->dest_nentries); 1619 1620 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context, 1621 nentries), GFP_KERNEL); 1622 if (!dest_ring) 1623 return ERR_PTR(-ENOMEM); 1624 1625 dest_ring->nentries = nentries; 1626 dest_ring->nentries_mask = nentries - 1; 1627 1628 /* Legacy platforms that do not support cache 1629 * coherent DMA are unsupported 1630 */ 1631 dest_ring->base_addr_owner_space_unaligned = 1632 dma_alloc_coherent(ar->dev, 1633 (nentries * sizeof(struct ce_desc_64) + 1634 CE_DESC_RING_ALIGN), 1635 &base_addr, GFP_KERNEL); 1636 if (!dest_ring->base_addr_owner_space_unaligned) { 1637 kfree(dest_ring); 1638 return ERR_PTR(-ENOMEM); 1639 } 1640 1641 dest_ring->base_addr_ce_space_unaligned = base_addr; 1642 1643 /* Correctly initialize memory to 0 to prevent garbage 1644 * data crashing system when download firmware 1645 */ 1646 dest_ring->base_addr_owner_space = 1647 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned, 1648 CE_DESC_RING_ALIGN); 1649 dest_ring->base_addr_ce_space = 1650 ALIGN(dest_ring->base_addr_ce_space_unaligned, 1651 CE_DESC_RING_ALIGN); 1652 1653 return dest_ring; 1654 } 1655 1656 /* 1657 * Initialize a Copy Engine based on caller-supplied attributes. 1658 * This may be called once to initialize both source and destination 1659 * rings or it may be called twice for separate source and destination 1660 * initialization. It may be that only one side or the other is 1661 * initialized by software/firmware. 1662 */ 1663 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id, 1664 const struct ce_attr *attr) 1665 { 1666 int ret; 1667 1668 if (attr->src_nentries) { 1669 ret = ath10k_ce_init_src_ring(ar, ce_id, attr); 1670 if (ret) { 1671 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n", 1672 ce_id, ret); 1673 return ret; 1674 } 1675 } 1676 1677 if (attr->dest_nentries) { 1678 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr); 1679 if (ret) { 1680 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n", 1681 ce_id, ret); 1682 return ret; 1683 } 1684 } 1685 1686 return 0; 1687 } 1688 EXPORT_SYMBOL(ath10k_ce_init_pipe); 1689 1690 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id) 1691 { 1692 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1693 1694 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0); 1695 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0); 1696 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0); 1697 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0); 1698 } 1699 1700 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id) 1701 { 1702 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1703 1704 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0); 1705 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0); 1706 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0); 1707 } 1708 1709 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id) 1710 { 1711 ath10k_ce_deinit_src_ring(ar, ce_id); 1712 ath10k_ce_deinit_dest_ring(ar, ce_id); 1713 } 1714 EXPORT_SYMBOL(ath10k_ce_deinit_pipe); 1715 1716 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1717 { 1718 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1719 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1720 1721 if (ce_state->src_ring) { 1722 if (ar->hw_params.shadow_reg_support) 1723 kfree(ce_state->src_ring->shadow_base_unaligned); 1724 dma_free_coherent(ar->dev, 1725 (ce_state->src_ring->nentries * 1726 sizeof(struct ce_desc) + 1727 CE_DESC_RING_ALIGN), 1728 ce_state->src_ring->base_addr_owner_space, 1729 ce_state->src_ring->base_addr_ce_space); 1730 kfree(ce_state->src_ring); 1731 } 1732 1733 if (ce_state->dest_ring) { 1734 dma_free_coherent(ar->dev, 1735 (ce_state->dest_ring->nentries * 1736 sizeof(struct ce_desc) + 1737 CE_DESC_RING_ALIGN), 1738 ce_state->dest_ring->base_addr_owner_space, 1739 ce_state->dest_ring->base_addr_ce_space); 1740 kfree(ce_state->dest_ring); 1741 } 1742 1743 ce_state->src_ring = NULL; 1744 ce_state->dest_ring = NULL; 1745 } 1746 1747 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id) 1748 { 1749 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1750 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1751 1752 if (ce_state->src_ring) { 1753 if (ar->hw_params.shadow_reg_support) 1754 kfree(ce_state->src_ring->shadow_base_unaligned); 1755 dma_free_coherent(ar->dev, 1756 (ce_state->src_ring->nentries * 1757 sizeof(struct ce_desc_64) + 1758 CE_DESC_RING_ALIGN), 1759 ce_state->src_ring->base_addr_owner_space, 1760 ce_state->src_ring->base_addr_ce_space); 1761 kfree(ce_state->src_ring); 1762 } 1763 1764 if (ce_state->dest_ring) { 1765 dma_free_coherent(ar->dev, 1766 (ce_state->dest_ring->nentries * 1767 sizeof(struct ce_desc_64) + 1768 CE_DESC_RING_ALIGN), 1769 ce_state->dest_ring->base_addr_owner_space, 1770 ce_state->dest_ring->base_addr_ce_space); 1771 kfree(ce_state->dest_ring); 1772 } 1773 1774 ce_state->src_ring = NULL; 1775 ce_state->dest_ring = NULL; 1776 } 1777 1778 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id) 1779 { 1780 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1781 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1782 1783 ce_state->ops->ce_free_pipe(ar, ce_id); 1784 } 1785 EXPORT_SYMBOL(ath10k_ce_free_pipe); 1786 1787 void ath10k_ce_dump_registers(struct ath10k *ar, 1788 struct ath10k_fw_crash_data *crash_data) 1789 { 1790 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1791 struct ath10k_ce_crash_data ce_data; 1792 u32 addr, id; 1793 1794 lockdep_assert_held(&ar->dump_mutex); 1795 1796 ath10k_err(ar, "Copy Engine register dump:\n"); 1797 1798 spin_lock_bh(&ce->ce_lock); 1799 for (id = 0; id < CE_COUNT; id++) { 1800 addr = ath10k_ce_base_address(ar, id); 1801 ce_data.base_addr = cpu_to_le32(addr); 1802 1803 ce_data.src_wr_idx = 1804 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr)); 1805 ce_data.src_r_idx = 1806 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr)); 1807 ce_data.dst_wr_idx = 1808 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr)); 1809 ce_data.dst_r_idx = 1810 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr)); 1811 1812 if (crash_data) 1813 crash_data->ce_crash_data[id] = ce_data; 1814 1815 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id, 1816 le32_to_cpu(ce_data.base_addr), 1817 le32_to_cpu(ce_data.src_wr_idx), 1818 le32_to_cpu(ce_data.src_r_idx), 1819 le32_to_cpu(ce_data.dst_wr_idx), 1820 le32_to_cpu(ce_data.dst_r_idx)); 1821 } 1822 1823 spin_unlock_bh(&ce->ce_lock); 1824 } 1825 EXPORT_SYMBOL(ath10k_ce_dump_registers); 1826 1827 static const struct ath10k_ce_ops ce_ops = { 1828 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring, 1829 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring, 1830 .ce_rx_post_buf = __ath10k_ce_rx_post_buf, 1831 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock, 1832 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next, 1833 .ce_extract_desc_data = ath10k_ce_extract_desc_data, 1834 .ce_free_pipe = _ath10k_ce_free_pipe, 1835 .ce_send_nolock = _ath10k_ce_send_nolock, 1836 .ce_set_src_ring_base_addr_hi = NULL, 1837 .ce_set_dest_ring_base_addr_hi = NULL, 1838 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock, 1839 }; 1840 1841 static const struct ath10k_ce_ops ce_64_ops = { 1842 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64, 1843 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64, 1844 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64, 1845 .ce_completed_recv_next_nolock = 1846 _ath10k_ce_completed_recv_next_nolock_64, 1847 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64, 1848 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64, 1849 .ce_free_pipe = _ath10k_ce_free_pipe_64, 1850 .ce_send_nolock = _ath10k_ce_send_nolock_64, 1851 .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi, 1852 .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi, 1853 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64, 1854 }; 1855 1856 static void ath10k_ce_set_ops(struct ath10k *ar, 1857 struct ath10k_ce_pipe *ce_state) 1858 { 1859 switch (ar->hw_rev) { 1860 case ATH10K_HW_WCN3990: 1861 ce_state->ops = &ce_64_ops; 1862 break; 1863 default: 1864 ce_state->ops = &ce_ops; 1865 break; 1866 } 1867 } 1868 1869 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id, 1870 const struct ce_attr *attr) 1871 { 1872 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1873 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id]; 1874 int ret; 1875 1876 ath10k_ce_set_ops(ar, ce_state); 1877 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid 1878 * additional TX locking checks. 1879 * 1880 * For the lack of a better place do the check here. 1881 */ 1882 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC > 1883 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1884 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC > 1885 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1886 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC > 1887 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1)); 1888 1889 ce_state->ar = ar; 1890 ce_state->id = ce_id; 1891 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id); 1892 ce_state->attr_flags = attr->flags; 1893 ce_state->src_sz_max = attr->src_sz_max; 1894 1895 if (attr->src_nentries) 1896 ce_state->send_cb = attr->send_cb; 1897 1898 if (attr->dest_nentries) 1899 ce_state->recv_cb = attr->recv_cb; 1900 1901 if (attr->src_nentries) { 1902 ce_state->src_ring = 1903 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr); 1904 if (IS_ERR(ce_state->src_ring)) { 1905 ret = PTR_ERR(ce_state->src_ring); 1906 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n", 1907 ce_id, ret); 1908 ce_state->src_ring = NULL; 1909 return ret; 1910 } 1911 } 1912 1913 if (attr->dest_nentries) { 1914 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar, 1915 ce_id, 1916 attr); 1917 if (IS_ERR(ce_state->dest_ring)) { 1918 ret = PTR_ERR(ce_state->dest_ring); 1919 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n", 1920 ce_id, ret); 1921 ce_state->dest_ring = NULL; 1922 return ret; 1923 } 1924 } 1925 1926 return 0; 1927 } 1928 EXPORT_SYMBOL(ath10k_ce_alloc_pipe); 1929 1930 void ath10k_ce_alloc_rri(struct ath10k *ar) 1931 { 1932 int i; 1933 u32 value; 1934 u32 ctrl1_regs; 1935 u32 ce_base_addr; 1936 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1937 1938 ce->vaddr_rri = dma_alloc_coherent(ar->dev, 1939 (CE_COUNT * sizeof(u32)), 1940 &ce->paddr_rri, GFP_KERNEL); 1941 1942 if (!ce->vaddr_rri) 1943 return; 1944 1945 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low, 1946 lower_32_bits(ce->paddr_rri)); 1947 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high, 1948 (upper_32_bits(ce->paddr_rri) & 1949 CE_DESC_ADDR_HI_MASK)); 1950 1951 for (i = 0; i < CE_COUNT; i++) { 1952 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr; 1953 ce_base_addr = ath10k_ce_base_address(ar, i); 1954 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs); 1955 value |= ar->hw_ce_regs->upd->mask; 1956 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value); 1957 } 1958 } 1959 EXPORT_SYMBOL(ath10k_ce_alloc_rri); 1960 1961 void ath10k_ce_free_rri(struct ath10k *ar) 1962 { 1963 struct ath10k_ce *ce = ath10k_ce_priv(ar); 1964 1965 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)), 1966 ce->vaddr_rri, 1967 ce->paddr_rri); 1968 } 1969 EXPORT_SYMBOL(ath10k_ce_free_rri); 1970