1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * TI K3 NAVSS Ring Accelerator subsystem driver 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/io.h> 10 #include <linux/init.h> 11 #include <linux/of.h> 12 #include <linux/platform_device.h> 13 #include <linux/sys_soc.h> 14 #include <linux/dma/ti-cppi5.h> 15 #include <linux/soc/ti/k3-ringacc.h> 16 #include <linux/soc/ti/ti_sci_protocol.h> 17 #include <linux/soc/ti/ti_sci_inta_msi.h> 18 #include <linux/of_irq.h> 19 #include <linux/irqdomain.h> 20 21 static LIST_HEAD(k3_ringacc_list); 22 static DEFINE_MUTEX(k3_ringacc_list_lock); 23 24 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 25 #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0) 26 27 /** 28 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 29 * 30 * @resv_16: Reserved 31 * @db: Ring Doorbell Register 32 * @resv_4: Reserved 33 * @occ: Ring Occupancy Register 34 * @indx: Ring Current Index Register 35 * @hwocc: Ring Hardware Occupancy Register 36 * @hwindx: Ring Hardware Current Index Register 37 */ 38 struct k3_ring_rt_regs { 39 u32 resv_16[4]; 40 u32 db; 41 u32 resv_4[1]; 42 u32 occ; 43 u32 indx; 44 u32 hwocc; 45 u32 hwindx; 46 }; 47 48 #define K3_RINGACC_RT_REGS_STEP 0x1000 49 #define K3_DMARING_RT_REGS_STEP 0x2000 50 #define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000 51 #define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0) 52 #define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31) 53 #define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0) 54 #define K3_DMARING_RT_DB_TDOWN_ACK BIT(31) 55 56 /** 57 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 58 * 59 * @head_data: Ring Head Entry Data Registers 60 * @tail_data: Ring Tail Entry Data Registers 61 * @peek_head_data: Ring Peek Head Entry Data Regs 62 * @peek_tail_data: Ring Peek Tail Entry Data Regs 63 */ 64 struct k3_ring_fifo_regs { 65 u32 head_data[128]; 66 u32 tail_data[128]; 67 u32 peek_head_data[128]; 68 u32 peek_tail_data[128]; 69 }; 70 71 /** 72 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 73 * 74 * @revision: Revision Register 75 * @config: Config Register 76 */ 77 struct k3_ringacc_proxy_gcfg_regs { 78 u32 revision; 79 u32 config; 80 }; 81 82 #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 83 84 /** 85 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 86 * 87 * @control: Proxy Control Register 88 * @status: Proxy Status Register 89 * @resv_512: Reserved 90 * @data: Proxy Data Register 91 */ 92 struct k3_ringacc_proxy_target_regs { 93 u32 control; 94 u32 status; 95 u8 resv_512[504]; 96 u32 data[128]; 97 }; 98 99 #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 100 #define K3_RINGACC_PROXY_NOT_USED (-1) 101 102 enum k3_ringacc_proxy_access_mode { 103 PROXY_ACCESS_MODE_HEAD = 0, 104 PROXY_ACCESS_MODE_TAIL = 1, 105 PROXY_ACCESS_MODE_PEEK_HEAD = 2, 106 PROXY_ACCESS_MODE_PEEK_TAIL = 3, 107 }; 108 109 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 110 #define K3_RINGACC_FIFO_REGS_STEP 0x1000 111 #define K3_RINGACC_MAX_DB_RING_CNT (127U) 112 113 struct k3_ring_ops { 114 int (*push_tail)(struct k3_ring *ring, void *elm); 115 int (*push_head)(struct k3_ring *ring, void *elm); 116 int (*pop_tail)(struct k3_ring *ring, void *elm); 117 int (*pop_head)(struct k3_ring *ring, void *elm); 118 }; 119 120 /** 121 * struct k3_ring_state - Internal state tracking structure 122 * 123 * @free: Number of free entries 124 * @occ: Occupancy 125 * @windex: Write index 126 * @rindex: Read index 127 */ 128 struct k3_ring_state { 129 u32 free; 130 u32 occ; 131 u32 windex; 132 u32 rindex; 133 u32 tdown_complete:1; 134 }; 135 136 /** 137 * struct k3_ring - RA Ring descriptor 138 * 139 * @rt: Ring control/status registers 140 * @fifos: Ring queues registers 141 * @proxy: Ring Proxy Datapath registers 142 * @ring_mem_dma: Ring buffer dma address 143 * @ring_mem_virt: Ring buffer virt address 144 * @ops: Ring operations 145 * @size: Ring size in elements 146 * @elm_size: Size of the ring element 147 * @mode: Ring mode 148 * @flags: flags 149 * @state: Ring state 150 * @ring_id: Ring Id 151 * @parent: Pointer on struct @k3_ringacc 152 * @use_count: Use count for shared rings 153 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 154 * @dma_dev: device to be used for DMA API (allocation, mapping) 155 * @asel: Address Space Select value for physical addresses 156 */ 157 struct k3_ring { 158 struct k3_ring_rt_regs __iomem *rt; 159 struct k3_ring_fifo_regs __iomem *fifos; 160 struct k3_ringacc_proxy_target_regs __iomem *proxy; 161 dma_addr_t ring_mem_dma; 162 void *ring_mem_virt; 163 struct k3_ring_ops *ops; 164 u32 size; 165 enum k3_ring_size elm_size; 166 enum k3_ring_mode mode; 167 u32 flags; 168 #define K3_RING_FLAG_BUSY BIT(1) 169 #define K3_RING_FLAG_SHARED BIT(2) 170 #define K3_RING_FLAG_REVERSE BIT(3) 171 struct k3_ring_state state; 172 u32 ring_id; 173 struct k3_ringacc *parent; 174 u32 use_count; 175 int proxy_id; 176 struct device *dma_dev; 177 u32 asel; 178 #define K3_ADDRESS_ASEL_SHIFT 48 179 }; 180 181 struct k3_ringacc_ops { 182 int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc); 183 }; 184 185 /** 186 * struct k3_ringacc - Rings accelerator descriptor 187 * 188 * @dev: pointer on RA device 189 * @proxy_gcfg: RA proxy global config registers 190 * @proxy_target_base: RA proxy datapath region 191 * @num_rings: number of ring in RA 192 * @rings_inuse: bitfield for ring usage tracking 193 * @rm_gp_range: general purpose rings range from tisci 194 * @dma_ring_reset_quirk: DMA reset w/a enable 195 * @num_proxies: number of RA proxies 196 * @proxy_inuse: bitfield for proxy usage tracking 197 * @rings: array of rings descriptors (struct @k3_ring) 198 * @list: list of RAs in the system 199 * @req_lock: protect rings allocation 200 * @tisci: pointer ti-sci handle 201 * @tisci_ring_ops: ti-sci rings ops 202 * @tisci_dev_id: ti-sci device id 203 * @ops: SoC specific ringacc operation 204 * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA) 205 */ 206 struct k3_ringacc { 207 struct device *dev; 208 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 209 void __iomem *proxy_target_base; 210 u32 num_rings; /* number of rings in Ringacc module */ 211 unsigned long *rings_inuse; 212 struct ti_sci_resource *rm_gp_range; 213 214 bool dma_ring_reset_quirk; 215 u32 num_proxies; 216 unsigned long *proxy_inuse; 217 218 struct k3_ring *rings; 219 struct list_head list; 220 struct mutex req_lock; /* protect rings allocation */ 221 222 const struct ti_sci_handle *tisci; 223 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 224 u32 tisci_dev_id; 225 226 const struct k3_ringacc_ops *ops; 227 bool dma_rings; 228 }; 229 230 /** 231 * struct k3_ringacc - Rings accelerator SoC data 232 * 233 * @dma_ring_reset_quirk: DMA reset w/a enable 234 */ 235 struct k3_ringacc_soc_data { 236 unsigned dma_ring_reset_quirk:1; 237 }; 238 239 static int k3_ringacc_ring_read_occ(struct k3_ring *ring) 240 { 241 return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK; 242 } 243 244 static void k3_ringacc_ring_update_occ(struct k3_ring *ring) 245 { 246 u32 val; 247 248 val = readl(&ring->rt->occ); 249 250 ring->state.occ = val & K3_RINGACC_RT_OCC_MASK; 251 ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE); 252 } 253 254 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 255 { 256 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 257 (4 << ring->elm_size); 258 } 259 260 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 261 { 262 return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 263 } 264 265 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 266 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 267 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem); 268 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem); 269 270 static struct k3_ring_ops k3_ring_mode_ring_ops = { 271 .push_tail = k3_ringacc_ring_push_mem, 272 .pop_head = k3_ringacc_ring_pop_mem, 273 }; 274 275 static struct k3_ring_ops k3_dmaring_fwd_ops = { 276 .push_tail = k3_ringacc_ring_push_mem, 277 .pop_head = k3_dmaring_fwd_pop, 278 }; 279 280 static struct k3_ring_ops k3_dmaring_reverse_ops = { 281 /* Reverse side of the DMA ring can only be popped by SW */ 282 .pop_head = k3_dmaring_reverse_pop, 283 }; 284 285 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 286 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 287 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 288 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 289 290 static struct k3_ring_ops k3_ring_mode_msg_ops = { 291 .push_tail = k3_ringacc_ring_push_io, 292 .push_head = k3_ringacc_ring_push_head_io, 293 .pop_tail = k3_ringacc_ring_pop_tail_io, 294 .pop_head = k3_ringacc_ring_pop_io, 295 }; 296 297 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 298 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 299 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 300 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 301 302 static struct k3_ring_ops k3_ring_mode_proxy_ops = { 303 .push_tail = k3_ringacc_ring_push_tail_proxy, 304 .push_head = k3_ringacc_ring_push_head_proxy, 305 .pop_tail = k3_ringacc_ring_pop_tail_proxy, 306 .pop_head = k3_ringacc_ring_pop_head_proxy, 307 }; 308 309 static void k3_ringacc_ring_dump(struct k3_ring *ring) 310 { 311 struct device *dev = ring->parent->dev; 312 313 dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 314 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 315 &ring->ring_mem_dma); 316 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 317 ring->elm_size, ring->size, ring->mode, ring->proxy_id); 318 dev_dbg(dev, "dump flags %08X\n", ring->flags); 319 320 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 321 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 322 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 323 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 324 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 325 326 if (ring->ring_mem_virt) 327 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 328 16, 1, ring->ring_mem_virt, 16 * 8, false); 329 } 330 331 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 332 int id, u32 flags) 333 { 334 int proxy_id = K3_RINGACC_PROXY_NOT_USED; 335 336 mutex_lock(&ringacc->req_lock); 337 338 if (id == K3_RINGACC_RING_ID_ANY) { 339 /* Request for any general purpose ring */ 340 struct ti_sci_resource_desc *gp_rings = 341 &ringacc->rm_gp_range->desc[0]; 342 unsigned long size; 343 344 size = gp_rings->start + gp_rings->num; 345 id = find_next_zero_bit(ringacc->rings_inuse, size, 346 gp_rings->start); 347 if (id == size) 348 goto error; 349 } else if (id < 0) { 350 goto error; 351 } 352 353 if (test_bit(id, ringacc->rings_inuse) && 354 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 355 goto error; 356 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 357 goto out; 358 359 if (flags & K3_RINGACC_RING_USE_PROXY) { 360 proxy_id = find_next_zero_bit(ringacc->proxy_inuse, 361 ringacc->num_proxies, 0); 362 if (proxy_id == ringacc->num_proxies) 363 goto error; 364 } 365 366 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 367 set_bit(proxy_id, ringacc->proxy_inuse); 368 ringacc->rings[id].proxy_id = proxy_id; 369 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 370 proxy_id); 371 } else { 372 dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 373 } 374 375 set_bit(id, ringacc->rings_inuse); 376 out: 377 ringacc->rings[id].use_count++; 378 mutex_unlock(&ringacc->req_lock); 379 return &ringacc->rings[id]; 380 381 error: 382 mutex_unlock(&ringacc->req_lock); 383 return NULL; 384 } 385 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 386 387 static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id, 388 struct k3_ring **fwd_ring, 389 struct k3_ring **compl_ring) 390 { 391 int ret = 0; 392 393 /* 394 * DMA rings must be requested by ID, completion ring is the reverse 395 * side of the forward ring 396 */ 397 if (fwd_id < 0) 398 return -EINVAL; 399 400 mutex_lock(&ringacc->req_lock); 401 402 if (test_bit(fwd_id, ringacc->rings_inuse)) { 403 ret = -EBUSY; 404 goto error; 405 } 406 407 *fwd_ring = &ringacc->rings[fwd_id]; 408 *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings]; 409 set_bit(fwd_id, ringacc->rings_inuse); 410 ringacc->rings[fwd_id].use_count++; 411 dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id); 412 413 mutex_unlock(&ringacc->req_lock); 414 return 0; 415 416 error: 417 mutex_unlock(&ringacc->req_lock); 418 return ret; 419 } 420 421 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, 422 int fwd_id, int compl_id, 423 struct k3_ring **fwd_ring, 424 struct k3_ring **compl_ring) 425 { 426 int ret = 0; 427 428 if (!fwd_ring || !compl_ring) 429 return -EINVAL; 430 431 if (ringacc->dma_rings) 432 return k3_dmaring_request_dual_ring(ringacc, fwd_id, 433 fwd_ring, compl_ring); 434 435 *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); 436 if (!(*fwd_ring)) 437 return -ENODEV; 438 439 *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); 440 if (!(*compl_ring)) { 441 k3_ringacc_ring_free(*fwd_ring); 442 ret = -ENODEV; 443 } 444 445 return ret; 446 } 447 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); 448 449 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 450 { 451 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 452 struct k3_ringacc *ringacc = ring->parent; 453 int ret; 454 455 ring_cfg.nav_id = ringacc->tisci_dev_id; 456 ring_cfg.index = ring->ring_id; 457 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID; 458 ring_cfg.count = ring->size; 459 460 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 461 if (ret) 462 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 463 ret, ring->ring_id); 464 } 465 466 void k3_ringacc_ring_reset(struct k3_ring *ring) 467 { 468 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 469 return; 470 471 memset(&ring->state, 0, sizeof(ring->state)); 472 473 k3_ringacc_ring_reset_sci(ring); 474 } 475 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 476 477 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 478 enum k3_ring_mode mode) 479 { 480 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 481 struct k3_ringacc *ringacc = ring->parent; 482 int ret; 483 484 ring_cfg.nav_id = ringacc->tisci_dev_id; 485 ring_cfg.index = ring->ring_id; 486 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID; 487 ring_cfg.mode = mode; 488 489 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 490 if (ret) 491 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 492 ret, ring->ring_id); 493 } 494 495 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 496 { 497 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 498 return; 499 500 if (!ring->parent->dma_ring_reset_quirk) 501 goto reset; 502 503 if (!occ) 504 occ = k3_ringacc_ring_read_occ(ring); 505 506 if (occ) { 507 u32 db_ring_cnt, db_ring_cnt_cur; 508 509 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 510 ring->ring_id, occ); 511 /* TI-SCI ring reset */ 512 k3_ringacc_ring_reset_sci(ring); 513 514 /* 515 * Setup the ring in ring/doorbell mode (if not already in this 516 * mode) 517 */ 518 if (ring->mode != K3_RINGACC_RING_MODE_RING) 519 k3_ringacc_ring_reconfig_qmode_sci( 520 ring, K3_RINGACC_RING_MODE_RING); 521 /* 522 * Ring the doorbell 2**22 – ringOcc times. 523 * This will wrap the internal UDMAP ring state occupancy 524 * counter (which is 21-bits wide) to 0. 525 */ 526 db_ring_cnt = (1U << 22) - occ; 527 528 while (db_ring_cnt != 0) { 529 /* 530 * Ring the doorbell with the maximum count each 531 * iteration if possible to minimize the total 532 * of writes 533 */ 534 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 535 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 536 else 537 db_ring_cnt_cur = db_ring_cnt; 538 539 writel(db_ring_cnt_cur, &ring->rt->db); 540 db_ring_cnt -= db_ring_cnt_cur; 541 } 542 543 /* Restore the original ring mode (if not ring mode) */ 544 if (ring->mode != K3_RINGACC_RING_MODE_RING) 545 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 546 } 547 548 reset: 549 /* Reset the ring */ 550 k3_ringacc_ring_reset(ring); 551 } 552 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 553 554 static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 555 { 556 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 557 struct k3_ringacc *ringacc = ring->parent; 558 int ret; 559 560 ring_cfg.nav_id = ringacc->tisci_dev_id; 561 ring_cfg.index = ring->ring_id; 562 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 563 564 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 565 if (ret) 566 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 567 ret, ring->ring_id); 568 } 569 570 int k3_ringacc_ring_free(struct k3_ring *ring) 571 { 572 struct k3_ringacc *ringacc; 573 574 if (!ring) 575 return -EINVAL; 576 577 ringacc = ring->parent; 578 579 /* 580 * DMA rings: rings shared memory and configuration, only forward ring 581 * is configured and reverse ring considered as slave. 582 */ 583 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 584 return 0; 585 586 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 587 588 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 589 return -EINVAL; 590 591 mutex_lock(&ringacc->req_lock); 592 593 if (--ring->use_count) 594 goto out; 595 596 if (!(ring->flags & K3_RING_FLAG_BUSY)) 597 goto no_init; 598 599 k3_ringacc_ring_free_sci(ring); 600 601 dma_free_coherent(ring->dma_dev, 602 ring->size * (4 << ring->elm_size), 603 ring->ring_mem_virt, ring->ring_mem_dma); 604 ring->flags = 0; 605 ring->ops = NULL; 606 ring->dma_dev = NULL; 607 ring->asel = 0; 608 609 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 610 clear_bit(ring->proxy_id, ringacc->proxy_inuse); 611 ring->proxy = NULL; 612 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 613 } 614 615 no_init: 616 clear_bit(ring->ring_id, ringacc->rings_inuse); 617 618 out: 619 mutex_unlock(&ringacc->req_lock); 620 return 0; 621 } 622 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 623 624 u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 625 { 626 if (!ring) 627 return -EINVAL; 628 629 return ring->ring_id; 630 } 631 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 632 633 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 634 { 635 if (!ring) 636 return -EINVAL; 637 638 return ring->parent->tisci_dev_id; 639 } 640 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 641 642 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 643 { 644 int irq_num; 645 646 if (!ring) 647 return -EINVAL; 648 649 irq_num = ti_sci_inta_msi_get_virq(ring->parent->dev, ring->ring_id); 650 if (irq_num <= 0) 651 irq_num = -EINVAL; 652 return irq_num; 653 } 654 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 655 656 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 657 { 658 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 659 struct k3_ringacc *ringacc = ring->parent; 660 int ret; 661 662 if (!ringacc->tisci) 663 return -EINVAL; 664 665 ring_cfg.nav_id = ringacc->tisci_dev_id; 666 ring_cfg.index = ring->ring_id; 667 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 668 ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma); 669 ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma); 670 ring_cfg.count = ring->size; 671 ring_cfg.mode = ring->mode; 672 ring_cfg.size = ring->elm_size; 673 ring_cfg.asel = ring->asel; 674 675 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 676 if (ret) 677 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 678 ret, ring->ring_id); 679 680 return ret; 681 } 682 683 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 684 { 685 struct k3_ringacc *ringacc; 686 struct k3_ring *reverse_ring; 687 int ret = 0; 688 689 if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 || 690 cfg->mode != K3_RINGACC_RING_MODE_RING || 691 cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK) 692 return -EINVAL; 693 694 ringacc = ring->parent; 695 696 /* 697 * DMA rings: rings shared memory and configuration, only forward ring 698 * is configured and reverse ring considered as slave. 699 */ 700 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 701 return 0; 702 703 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 704 return -EINVAL; 705 706 ring->size = cfg->size; 707 ring->elm_size = cfg->elm_size; 708 ring->mode = cfg->mode; 709 ring->asel = cfg->asel; 710 ring->dma_dev = cfg->dma_dev; 711 if (!ring->dma_dev) { 712 dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n", 713 ring->ring_id); 714 ring->dma_dev = ringacc->dev; 715 } 716 717 memset(&ring->state, 0, sizeof(ring->state)); 718 719 ring->ops = &k3_dmaring_fwd_ops; 720 721 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 722 ring->size * (4 << ring->elm_size), 723 &ring->ring_mem_dma, GFP_KERNEL); 724 if (!ring->ring_mem_virt) { 725 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 726 ret = -ENOMEM; 727 goto err_free_ops; 728 } 729 730 ret = k3_ringacc_ring_cfg_sci(ring); 731 if (ret) 732 goto err_free_mem; 733 734 ring->flags |= K3_RING_FLAG_BUSY; 735 736 k3_ringacc_ring_dump(ring); 737 738 /* DMA rings: configure reverse ring */ 739 reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings]; 740 reverse_ring->size = cfg->size; 741 reverse_ring->elm_size = cfg->elm_size; 742 reverse_ring->mode = cfg->mode; 743 reverse_ring->asel = cfg->asel; 744 memset(&reverse_ring->state, 0, sizeof(reverse_ring->state)); 745 reverse_ring->ops = &k3_dmaring_reverse_ops; 746 747 reverse_ring->ring_mem_virt = ring->ring_mem_virt; 748 reverse_ring->ring_mem_dma = ring->ring_mem_dma; 749 reverse_ring->flags |= K3_RING_FLAG_BUSY; 750 k3_ringacc_ring_dump(reverse_ring); 751 752 return 0; 753 754 err_free_mem: 755 dma_free_coherent(ring->dma_dev, 756 ring->size * (4 << ring->elm_size), 757 ring->ring_mem_virt, 758 ring->ring_mem_dma); 759 err_free_ops: 760 ring->ops = NULL; 761 ring->proxy = NULL; 762 ring->dma_dev = NULL; 763 ring->asel = 0; 764 return ret; 765 } 766 767 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 768 { 769 struct k3_ringacc *ringacc; 770 int ret = 0; 771 772 if (!ring || !cfg) 773 return -EINVAL; 774 775 ringacc = ring->parent; 776 777 if (ringacc->dma_rings) 778 return k3_dmaring_cfg(ring, cfg); 779 780 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 781 cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 782 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 783 !test_bit(ring->ring_id, ringacc->rings_inuse)) 784 return -EINVAL; 785 786 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 787 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 788 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 789 dev_err(ringacc->dev, 790 "Message mode must use proxy for %u element size\n", 791 4 << ring->elm_size); 792 return -EINVAL; 793 } 794 795 /* 796 * In case of shared ring only the first user (master user) can 797 * configure the ring. The sequence should be by the client: 798 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 799 * k3_ringacc_ring_cfg(ring, cfg); # master configuration 800 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 801 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 802 */ 803 if (ring->use_count != 1) 804 return 0; 805 806 ring->size = cfg->size; 807 ring->elm_size = cfg->elm_size; 808 ring->mode = cfg->mode; 809 memset(&ring->state, 0, sizeof(ring->state)); 810 811 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 812 ring->proxy = ringacc->proxy_target_base + 813 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 814 815 switch (ring->mode) { 816 case K3_RINGACC_RING_MODE_RING: 817 ring->ops = &k3_ring_mode_ring_ops; 818 ring->dma_dev = cfg->dma_dev; 819 if (!ring->dma_dev) 820 ring->dma_dev = ringacc->dev; 821 break; 822 case K3_RINGACC_RING_MODE_MESSAGE: 823 ring->dma_dev = ringacc->dev; 824 if (ring->proxy) 825 ring->ops = &k3_ring_mode_proxy_ops; 826 else 827 ring->ops = &k3_ring_mode_msg_ops; 828 break; 829 default: 830 ring->ops = NULL; 831 ret = -EINVAL; 832 goto err_free_proxy; 833 } 834 835 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 836 ring->size * (4 << ring->elm_size), 837 &ring->ring_mem_dma, GFP_KERNEL); 838 if (!ring->ring_mem_virt) { 839 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 840 ret = -ENOMEM; 841 goto err_free_ops; 842 } 843 844 ret = k3_ringacc_ring_cfg_sci(ring); 845 846 if (ret) 847 goto err_free_mem; 848 849 ring->flags |= K3_RING_FLAG_BUSY; 850 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 851 K3_RING_FLAG_SHARED : 0; 852 853 k3_ringacc_ring_dump(ring); 854 855 return 0; 856 857 err_free_mem: 858 dma_free_coherent(ring->dma_dev, 859 ring->size * (4 << ring->elm_size), 860 ring->ring_mem_virt, 861 ring->ring_mem_dma); 862 err_free_ops: 863 ring->ops = NULL; 864 ring->dma_dev = NULL; 865 err_free_proxy: 866 ring->proxy = NULL; 867 return ret; 868 } 869 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 870 871 u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 872 { 873 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 874 return -EINVAL; 875 876 return ring->size; 877 } 878 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 879 880 u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 881 { 882 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 883 return -EINVAL; 884 885 if (!ring->state.free) 886 ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring); 887 888 return ring->state.free; 889 } 890 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 891 892 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 893 { 894 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 895 return -EINVAL; 896 897 return k3_ringacc_ring_read_occ(ring); 898 } 899 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 900 901 u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 902 { 903 return !k3_ringacc_ring_get_free(ring); 904 } 905 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 906 907 enum k3_ringacc_access_mode { 908 K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 909 K3_RINGACC_ACCESS_MODE_POP_HEAD, 910 K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 911 K3_RINGACC_ACCESS_MODE_POP_TAIL, 912 K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 913 K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 914 }; 915 916 #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 917 #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 918 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 919 enum k3_ringacc_proxy_access_mode mode) 920 { 921 u32 val; 922 923 val = ring->ring_id; 924 val |= K3_RINGACC_PROXY_MODE(mode); 925 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 926 writel(val, &ring->proxy->control); 927 return 0; 928 } 929 930 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 931 enum k3_ringacc_access_mode access_mode) 932 { 933 void __iomem *ptr; 934 935 ptr = (void __iomem *)&ring->proxy->data; 936 937 switch (access_mode) { 938 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 939 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 940 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 941 break; 942 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 943 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 944 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 945 break; 946 default: 947 return -EINVAL; 948 } 949 950 ptr += k3_ringacc_ring_get_fifo_pos(ring); 951 952 switch (access_mode) { 953 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 954 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 955 dev_dbg(ring->parent->dev, 956 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 957 access_mode); 958 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 959 ring->state.occ--; 960 break; 961 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 962 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 963 dev_dbg(ring->parent->dev, 964 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 965 access_mode); 966 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 967 ring->state.free--; 968 break; 969 default: 970 return -EINVAL; 971 } 972 973 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, 974 ring->state.occ); 975 return 0; 976 } 977 978 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 979 { 980 return k3_ringacc_ring_access_proxy(ring, elem, 981 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 982 } 983 984 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 985 { 986 return k3_ringacc_ring_access_proxy(ring, elem, 987 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 988 } 989 990 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 991 { 992 return k3_ringacc_ring_access_proxy(ring, elem, 993 K3_RINGACC_ACCESS_MODE_POP_HEAD); 994 } 995 996 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 997 { 998 return k3_ringacc_ring_access_proxy(ring, elem, 999 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1000 } 1001 1002 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 1003 enum k3_ringacc_access_mode access_mode) 1004 { 1005 void __iomem *ptr; 1006 1007 switch (access_mode) { 1008 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 1009 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 1010 ptr = (void __iomem *)&ring->fifos->head_data; 1011 break; 1012 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 1013 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 1014 ptr = (void __iomem *)&ring->fifos->tail_data; 1015 break; 1016 default: 1017 return -EINVAL; 1018 } 1019 1020 ptr += k3_ringacc_ring_get_fifo_pos(ring); 1021 1022 switch (access_mode) { 1023 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 1024 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 1025 dev_dbg(ring->parent->dev, 1026 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 1027 access_mode); 1028 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 1029 ring->state.occ--; 1030 break; 1031 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 1032 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 1033 dev_dbg(ring->parent->dev, 1034 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 1035 access_mode); 1036 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 1037 ring->state.free--; 1038 break; 1039 default: 1040 return -EINVAL; 1041 } 1042 1043 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", 1044 ring->state.free, ring->state.windex, ring->state.occ, 1045 ring->state.rindex); 1046 return 0; 1047 } 1048 1049 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 1050 { 1051 return k3_ringacc_ring_access_io(ring, elem, 1052 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 1053 } 1054 1055 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 1056 { 1057 return k3_ringacc_ring_access_io(ring, elem, 1058 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 1059 } 1060 1061 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 1062 { 1063 return k3_ringacc_ring_access_io(ring, elem, 1064 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1065 } 1066 1067 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 1068 { 1069 return k3_ringacc_ring_access_io(ring, elem, 1070 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1071 } 1072 1073 /* 1074 * The element is 48 bits of address + ASEL bits in the ring. 1075 * ASEL is used by the DMAs and should be removed for the kernel as it is not 1076 * part of the physical memory address. 1077 */ 1078 static void k3_dmaring_remove_asel_from_elem(u64 *elem) 1079 { 1080 *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0); 1081 } 1082 1083 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem) 1084 { 1085 void *elem_ptr; 1086 u32 elem_idx; 1087 1088 /* 1089 * DMA rings: forward ring is always tied DMA channel and HW does not 1090 * maintain any state data required for POP operation and its unknown 1091 * how much elements were consumed by HW. So, to actually 1092 * do POP, the read pointer has to be recalculated every time. 1093 */ 1094 ring->state.occ = k3_ringacc_ring_read_occ(ring); 1095 if (ring->state.windex >= ring->state.occ) 1096 elem_idx = ring->state.windex - ring->state.occ; 1097 else 1098 elem_idx = ring->size - (ring->state.occ - ring->state.windex); 1099 1100 elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx); 1101 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1102 k3_dmaring_remove_asel_from_elem(elem); 1103 1104 ring->state.occ--; 1105 writel(-1, &ring->rt->db); 1106 1107 dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n", 1108 __func__, ring->state.occ, ring->state.windex, elem_idx, 1109 elem_ptr); 1110 return 0; 1111 } 1112 1113 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem) 1114 { 1115 void *elem_ptr; 1116 1117 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 1118 1119 if (ring->state.occ) { 1120 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1121 k3_dmaring_remove_asel_from_elem(elem); 1122 1123 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 1124 ring->state.occ--; 1125 writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db); 1126 } else if (ring->state.tdown_complete) { 1127 dma_addr_t *value = elem; 1128 1129 *value = CPPI5_TDCM_MARKER; 1130 writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db); 1131 ring->state.tdown_complete = false; 1132 } 1133 1134 dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n", 1135 __func__, ring->state.occ, ring->state.rindex, elem_ptr); 1136 return 0; 1137 } 1138 1139 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 1140 { 1141 void *elem_ptr; 1142 1143 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); 1144 1145 memcpy(elem_ptr, elem, (4 << ring->elm_size)); 1146 if (ring->parent->dma_rings) { 1147 u64 *addr = elem_ptr; 1148 1149 *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT); 1150 } 1151 1152 ring->state.windex = (ring->state.windex + 1) % ring->size; 1153 ring->state.free--; 1154 writel(1, &ring->rt->db); 1155 1156 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 1157 ring->state.free, ring->state.windex); 1158 1159 return 0; 1160 } 1161 1162 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 1163 { 1164 void *elem_ptr; 1165 1166 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 1167 1168 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1169 1170 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 1171 ring->state.occ--; 1172 writel(-1, &ring->rt->db); 1173 1174 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 1175 ring->state.occ, ring->state.rindex, elem_ptr); 1176 return 0; 1177 } 1178 1179 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 1180 { 1181 int ret = -EOPNOTSUPP; 1182 1183 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1184 return -EINVAL; 1185 1186 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", 1187 ring->state.free, ring->state.windex); 1188 1189 if (k3_ringacc_ring_is_full(ring)) 1190 return -ENOMEM; 1191 1192 if (ring->ops && ring->ops->push_tail) 1193 ret = ring->ops->push_tail(ring, elem); 1194 1195 return ret; 1196 } 1197 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 1198 1199 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 1200 { 1201 int ret = -EOPNOTSUPP; 1202 1203 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1204 return -EINVAL; 1205 1206 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 1207 ring->state.free, ring->state.windex); 1208 1209 if (k3_ringacc_ring_is_full(ring)) 1210 return -ENOMEM; 1211 1212 if (ring->ops && ring->ops->push_head) 1213 ret = ring->ops->push_head(ring, elem); 1214 1215 return ret; 1216 } 1217 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 1218 1219 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 1220 { 1221 int ret = -EOPNOTSUPP; 1222 1223 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1224 return -EINVAL; 1225 1226 if (!ring->state.occ) 1227 k3_ringacc_ring_update_occ(ring); 1228 1229 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, 1230 ring->state.rindex); 1231 1232 if (!ring->state.occ && !ring->state.tdown_complete) 1233 return -ENODATA; 1234 1235 if (ring->ops && ring->ops->pop_head) 1236 ret = ring->ops->pop_head(ring, elem); 1237 1238 return ret; 1239 } 1240 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 1241 1242 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 1243 { 1244 int ret = -EOPNOTSUPP; 1245 1246 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1247 return -EINVAL; 1248 1249 if (!ring->state.occ) 1250 k3_ringacc_ring_update_occ(ring); 1251 1252 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", 1253 ring->state.occ, ring->state.rindex); 1254 1255 if (!ring->state.occ) 1256 return -ENODATA; 1257 1258 if (ring->ops && ring->ops->pop_tail) 1259 ret = ring->ops->pop_tail(ring, elem); 1260 1261 return ret; 1262 } 1263 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 1264 1265 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 1266 const char *property) 1267 { 1268 struct device_node *ringacc_np; 1269 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 1270 struct k3_ringacc *entry; 1271 1272 ringacc_np = of_parse_phandle(np, property, 0); 1273 if (!ringacc_np) 1274 return ERR_PTR(-ENODEV); 1275 1276 mutex_lock(&k3_ringacc_list_lock); 1277 list_for_each_entry(entry, &k3_ringacc_list, list) 1278 if (entry->dev->of_node == ringacc_np) { 1279 ringacc = entry; 1280 break; 1281 } 1282 mutex_unlock(&k3_ringacc_list_lock); 1283 of_node_put(ringacc_np); 1284 1285 return ringacc; 1286 } 1287 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 1288 1289 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 1290 { 1291 struct device_node *node = ringacc->dev->of_node; 1292 struct device *dev = ringacc->dev; 1293 struct platform_device *pdev = to_platform_device(dev); 1294 int ret; 1295 1296 if (!node) { 1297 dev_err(dev, "device tree info unavailable\n"); 1298 return -ENODEV; 1299 } 1300 1301 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 1302 if (ret) { 1303 dev_err(dev, "ti,num-rings read failure %d\n", ret); 1304 return ret; 1305 } 1306 1307 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 1308 if (IS_ERR(ringacc->tisci)) { 1309 ret = PTR_ERR(ringacc->tisci); 1310 if (ret != -EPROBE_DEFER) 1311 dev_err(dev, "ti,sci read fail %d\n", ret); 1312 ringacc->tisci = NULL; 1313 return ret; 1314 } 1315 1316 ret = of_property_read_u32(node, "ti,sci-dev-id", 1317 &ringacc->tisci_dev_id); 1318 if (ret) { 1319 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 1320 return ret; 1321 } 1322 1323 pdev->id = ringacc->tisci_dev_id; 1324 1325 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 1326 ringacc->tisci_dev_id, 1327 "ti,sci-rm-range-gp-rings"); 1328 if (IS_ERR(ringacc->rm_gp_range)) { 1329 dev_err(dev, "Failed to allocate MSI interrupts\n"); 1330 return PTR_ERR(ringacc->rm_gp_range); 1331 } 1332 1333 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 1334 ringacc->rm_gp_range); 1335 } 1336 1337 static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = { 1338 .dma_ring_reset_quirk = 1, 1339 }; 1340 1341 static const struct soc_device_attribute k3_ringacc_socinfo[] = { 1342 { .family = "AM65X", 1343 .revision = "SR1.0", 1344 .data = &k3_ringacc_soc_data_sr1 1345 }, 1346 {/* sentinel */} 1347 }; 1348 1349 static int k3_ringacc_init(struct platform_device *pdev, 1350 struct k3_ringacc *ringacc) 1351 { 1352 const struct soc_device_attribute *soc; 1353 void __iomem *base_fifo, *base_rt; 1354 struct device *dev = &pdev->dev; 1355 struct resource *res; 1356 int ret, i; 1357 1358 dev->msi_domain = of_msi_get_domain(dev, dev->of_node, 1359 DOMAIN_BUS_TI_SCI_INTA_MSI); 1360 if (!dev->msi_domain) { 1361 dev_err(dev, "Failed to get MSI domain\n"); 1362 return -EPROBE_DEFER; 1363 } 1364 1365 ret = k3_ringacc_probe_dt(ringacc); 1366 if (ret) 1367 return ret; 1368 1369 soc = soc_device_match(k3_ringacc_socinfo); 1370 if (soc && soc->data) { 1371 const struct k3_ringacc_soc_data *soc_data = soc->data; 1372 1373 ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk; 1374 } 1375 1376 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "rt"); 1377 base_rt = devm_ioremap_resource(dev, res); 1378 if (IS_ERR(base_rt)) 1379 return PTR_ERR(base_rt); 1380 1381 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fifos"); 1382 base_fifo = devm_ioremap_resource(dev, res); 1383 if (IS_ERR(base_fifo)) 1384 return PTR_ERR(base_fifo); 1385 1386 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "proxy_gcfg"); 1387 ringacc->proxy_gcfg = devm_ioremap_resource(dev, res); 1388 if (IS_ERR(ringacc->proxy_gcfg)) 1389 return PTR_ERR(ringacc->proxy_gcfg); 1390 1391 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, 1392 "proxy_target"); 1393 ringacc->proxy_target_base = devm_ioremap_resource(dev, res); 1394 if (IS_ERR(ringacc->proxy_target_base)) 1395 return PTR_ERR(ringacc->proxy_target_base); 1396 1397 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 1398 K3_RINGACC_PROXY_CFG_THREADS_MASK; 1399 1400 ringacc->rings = devm_kzalloc(dev, 1401 sizeof(*ringacc->rings) * 1402 ringacc->num_rings, 1403 GFP_KERNEL); 1404 ringacc->rings_inuse = devm_kcalloc(dev, 1405 BITS_TO_LONGS(ringacc->num_rings), 1406 sizeof(unsigned long), GFP_KERNEL); 1407 ringacc->proxy_inuse = devm_kcalloc(dev, 1408 BITS_TO_LONGS(ringacc->num_proxies), 1409 sizeof(unsigned long), GFP_KERNEL); 1410 1411 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 1412 return -ENOMEM; 1413 1414 for (i = 0; i < ringacc->num_rings; i++) { 1415 ringacc->rings[i].rt = base_rt + 1416 K3_RINGACC_RT_REGS_STEP * i; 1417 ringacc->rings[i].fifos = base_fifo + 1418 K3_RINGACC_FIFO_REGS_STEP * i; 1419 ringacc->rings[i].parent = ringacc; 1420 ringacc->rings[i].ring_id = i; 1421 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 1422 } 1423 1424 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1425 1426 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 1427 ringacc->num_rings, 1428 ringacc->rm_gp_range->desc[0].start, 1429 ringacc->rm_gp_range->desc[0].num, 1430 ringacc->tisci_dev_id); 1431 dev_info(dev, "dma-ring-reset-quirk: %s\n", 1432 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled"); 1433 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 1434 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 1435 1436 return 0; 1437 } 1438 1439 struct ringacc_match_data { 1440 struct k3_ringacc_ops ops; 1441 }; 1442 1443 static struct ringacc_match_data k3_ringacc_data = { 1444 .ops = { 1445 .init = k3_ringacc_init, 1446 }, 1447 }; 1448 1449 /* Match table for of_platform binding */ 1450 static const struct of_device_id k3_ringacc_of_match[] = { 1451 { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, }, 1452 {}, 1453 }; 1454 1455 struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev, 1456 struct k3_ringacc_init_data *data) 1457 { 1458 struct device *dev = &pdev->dev; 1459 struct k3_ringacc *ringacc; 1460 void __iomem *base_rt; 1461 struct resource *res; 1462 int i; 1463 1464 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1465 if (!ringacc) 1466 return ERR_PTR(-ENOMEM); 1467 1468 ringacc->dev = dev; 1469 ringacc->dma_rings = true; 1470 ringacc->num_rings = data->num_rings; 1471 ringacc->tisci = data->tisci; 1472 ringacc->tisci_dev_id = data->tisci_dev_id; 1473 1474 mutex_init(&ringacc->req_lock); 1475 1476 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "ringrt"); 1477 base_rt = devm_ioremap_resource(dev, res); 1478 if (IS_ERR(base_rt)) 1479 return ERR_CAST(base_rt); 1480 1481 ringacc->rings = devm_kzalloc(dev, 1482 sizeof(*ringacc->rings) * 1483 ringacc->num_rings * 2, 1484 GFP_KERNEL); 1485 ringacc->rings_inuse = devm_kcalloc(dev, 1486 BITS_TO_LONGS(ringacc->num_rings), 1487 sizeof(unsigned long), GFP_KERNEL); 1488 1489 if (!ringacc->rings || !ringacc->rings_inuse) 1490 return ERR_PTR(-ENOMEM); 1491 1492 for (i = 0; i < ringacc->num_rings; i++) { 1493 struct k3_ring *ring = &ringacc->rings[i]; 1494 1495 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i; 1496 ring->parent = ringacc; 1497 ring->ring_id = i; 1498 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1499 1500 ring = &ringacc->rings[ringacc->num_rings + i]; 1501 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i + 1502 K3_DMARING_RT_REGS_REVERSE_OFS; 1503 ring->parent = ringacc; 1504 ring->ring_id = i; 1505 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1506 ring->flags = K3_RING_FLAG_REVERSE; 1507 } 1508 1509 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1510 1511 dev_info(dev, "Number of rings: %u\n", ringacc->num_rings); 1512 1513 return ringacc; 1514 } 1515 EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init); 1516 1517 static int k3_ringacc_probe(struct platform_device *pdev) 1518 { 1519 const struct ringacc_match_data *match_data; 1520 const struct of_device_id *match; 1521 struct device *dev = &pdev->dev; 1522 struct k3_ringacc *ringacc; 1523 int ret; 1524 1525 match = of_match_node(k3_ringacc_of_match, dev->of_node); 1526 if (!match) 1527 return -ENODEV; 1528 match_data = match->data; 1529 1530 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1531 if (!ringacc) 1532 return -ENOMEM; 1533 1534 ringacc->dev = dev; 1535 mutex_init(&ringacc->req_lock); 1536 ringacc->ops = &match_data->ops; 1537 1538 ret = ringacc->ops->init(pdev, ringacc); 1539 if (ret) 1540 return ret; 1541 1542 dev_set_drvdata(dev, ringacc); 1543 1544 mutex_lock(&k3_ringacc_list_lock); 1545 list_add_tail(&ringacc->list, &k3_ringacc_list); 1546 mutex_unlock(&k3_ringacc_list_lock); 1547 1548 return 0; 1549 } 1550 1551 static struct platform_driver k3_ringacc_driver = { 1552 .probe = k3_ringacc_probe, 1553 .driver = { 1554 .name = "k3-ringacc", 1555 .of_match_table = k3_ringacc_of_match, 1556 .suppress_bind_attrs = true, 1557 }, 1558 }; 1559 builtin_platform_driver(k3_ringacc_driver); 1560