1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * TI K3 NAVSS Ring Accelerator subsystem driver 4 * 5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com 6 */ 7 8 #include <linux/dma-mapping.h> 9 #include <linux/io.h> 10 #include <linux/module.h> 11 #include <linux/of.h> 12 #include <linux/platform_device.h> 13 #include <linux/string_choices.h> 14 #include <linux/sys_soc.h> 15 #include <linux/dma/ti-cppi5.h> 16 #include <linux/soc/ti/k3-ringacc.h> 17 #include <linux/soc/ti/ti_sci_protocol.h> 18 #include <linux/soc/ti/ti_sci_inta_msi.h> 19 #include <linux/of_irq.h> 20 #include <linux/irqdomain.h> 21 22 static LIST_HEAD(k3_ringacc_list); 23 static DEFINE_MUTEX(k3_ringacc_list_lock); 24 25 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0) 26 #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0) 27 28 /** 29 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region 30 * 31 * @resv_16: Reserved 32 * @db: Ring Doorbell Register 33 * @resv_4: Reserved 34 * @occ: Ring Occupancy Register 35 * @indx: Ring Current Index Register 36 * @hwocc: Ring Hardware Occupancy Register 37 * @hwindx: Ring Hardware Current Index Register 38 */ 39 struct k3_ring_rt_regs { 40 u32 resv_16[4]; 41 u32 db; 42 u32 resv_4[1]; 43 u32 occ; 44 u32 indx; 45 u32 hwocc; 46 u32 hwindx; 47 }; 48 49 #define K3_RINGACC_RT_REGS_STEP 0x1000 50 #define K3_DMARING_RT_REGS_STEP 0x2000 51 #define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000 52 #define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0) 53 #define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31) 54 #define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0) 55 #define K3_DMARING_RT_DB_TDOWN_ACK BIT(31) 56 57 /** 58 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region 59 * 60 * @head_data: Ring Head Entry Data Registers 61 * @tail_data: Ring Tail Entry Data Registers 62 * @peek_head_data: Ring Peek Head Entry Data Regs 63 * @peek_tail_data: Ring Peek Tail Entry Data Regs 64 */ 65 struct k3_ring_fifo_regs { 66 u32 head_data[128]; 67 u32 tail_data[128]; 68 u32 peek_head_data[128]; 69 u32 peek_tail_data[128]; 70 }; 71 72 /** 73 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region 74 * 75 * @revision: Revision Register 76 * @config: Config Register 77 */ 78 struct k3_ringacc_proxy_gcfg_regs { 79 u32 revision; 80 u32 config; 81 }; 82 83 #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0) 84 85 /** 86 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region 87 * 88 * @control: Proxy Control Register 89 * @status: Proxy Status Register 90 * @resv_512: Reserved 91 * @data: Proxy Data Register 92 */ 93 struct k3_ringacc_proxy_target_regs { 94 u32 control; 95 u32 status; 96 u8 resv_512[504]; 97 u32 data[128]; 98 }; 99 100 #define K3_RINGACC_PROXY_TARGET_STEP 0x1000 101 #define K3_RINGACC_PROXY_NOT_USED (-1) 102 103 enum k3_ringacc_proxy_access_mode { 104 PROXY_ACCESS_MODE_HEAD = 0, 105 PROXY_ACCESS_MODE_TAIL = 1, 106 PROXY_ACCESS_MODE_PEEK_HEAD = 2, 107 PROXY_ACCESS_MODE_PEEK_TAIL = 3, 108 }; 109 110 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U) 111 #define K3_RINGACC_FIFO_REGS_STEP 0x1000 112 #define K3_RINGACC_MAX_DB_RING_CNT (127U) 113 114 struct k3_ring_ops { 115 int (*push_tail)(struct k3_ring *ring, void *elm); 116 int (*push_head)(struct k3_ring *ring, void *elm); 117 int (*pop_tail)(struct k3_ring *ring, void *elm); 118 int (*pop_head)(struct k3_ring *ring, void *elm); 119 }; 120 121 /** 122 * struct k3_ring_state - Internal state tracking structure 123 * 124 * @free: Number of free entries 125 * @occ: Occupancy 126 * @windex: Write index 127 * @rindex: Read index 128 * @tdown_complete: Tear down complete state 129 */ 130 struct k3_ring_state { 131 u32 free; 132 u32 occ; 133 u32 windex; 134 u32 rindex; 135 u32 tdown_complete:1; 136 }; 137 138 /** 139 * struct k3_ring - RA Ring descriptor 140 * 141 * @rt: Ring control/status registers 142 * @fifos: Ring queues registers 143 * @proxy: Ring Proxy Datapath registers 144 * @ring_mem_dma: Ring buffer dma address 145 * @ring_mem_virt: Ring buffer virt address 146 * @ops: Ring operations 147 * @size: Ring size in elements 148 * @elm_size: Size of the ring element 149 * @mode: Ring mode 150 * @flags: flags 151 * @state: Ring state 152 * @ring_id: Ring Id 153 * @parent: Pointer on struct @k3_ringacc 154 * @use_count: Use count for shared rings 155 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY) 156 * @dma_dev: device to be used for DMA API (allocation, mapping) 157 * @asel: Address Space Select value for physical addresses 158 */ 159 struct k3_ring { 160 struct k3_ring_rt_regs __iomem *rt; 161 struct k3_ring_fifo_regs __iomem *fifos; 162 struct k3_ringacc_proxy_target_regs __iomem *proxy; 163 dma_addr_t ring_mem_dma; 164 void *ring_mem_virt; 165 const struct k3_ring_ops *ops; 166 u32 size; 167 enum k3_ring_size elm_size; 168 enum k3_ring_mode mode; 169 u32 flags; 170 #define K3_RING_FLAG_BUSY BIT(1) 171 #define K3_RING_FLAG_SHARED BIT(2) 172 #define K3_RING_FLAG_REVERSE BIT(3) 173 struct k3_ring_state state; 174 u32 ring_id; 175 struct k3_ringacc *parent; 176 u32 use_count; 177 int proxy_id; 178 struct device *dma_dev; 179 u32 asel; 180 #define K3_ADDRESS_ASEL_SHIFT 48 181 }; 182 183 struct k3_ringacc_ops { 184 int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc); 185 }; 186 187 /** 188 * struct k3_ringacc - Rings accelerator descriptor 189 * 190 * @dev: pointer on RA device 191 * @proxy_gcfg: RA proxy global config registers 192 * @proxy_target_base: RA proxy datapath region 193 * @num_rings: number of ring in RA 194 * @rings_inuse: bitfield for ring usage tracking 195 * @rm_gp_range: general purpose rings range from tisci 196 * @dma_ring_reset_quirk: DMA reset workaround enable 197 * @num_proxies: number of RA proxies 198 * @proxy_inuse: bitfield for proxy usage tracking 199 * @rings: array of rings descriptors (struct @k3_ring) 200 * @list: list of RAs in the system 201 * @req_lock: protect rings allocation 202 * @tisci: pointer ti-sci handle 203 * @tisci_ring_ops: ti-sci rings ops 204 * @tisci_dev_id: ti-sci device id 205 * @ops: SoC specific ringacc operation 206 * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA) 207 */ 208 struct k3_ringacc { 209 struct device *dev; 210 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg; 211 void __iomem *proxy_target_base; 212 u32 num_rings; /* number of rings in Ringacc module */ 213 unsigned long *rings_inuse; 214 struct ti_sci_resource *rm_gp_range; 215 216 bool dma_ring_reset_quirk; 217 u32 num_proxies; 218 unsigned long *proxy_inuse; 219 220 struct k3_ring *rings; 221 struct list_head list; 222 struct mutex req_lock; /* protect rings allocation */ 223 224 const struct ti_sci_handle *tisci; 225 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops; 226 u32 tisci_dev_id; 227 228 const struct k3_ringacc_ops *ops; 229 bool dma_rings; 230 }; 231 232 /** 233 * struct k3_ringacc_soc_data - Rings accelerator SoC data 234 * 235 * @dma_ring_reset_quirk: DMA reset workaround enable 236 */ 237 struct k3_ringacc_soc_data { 238 unsigned dma_ring_reset_quirk:1; 239 }; 240 241 static int k3_ringacc_ring_read_occ(struct k3_ring *ring) 242 { 243 return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK; 244 } 245 246 static void k3_ringacc_ring_update_occ(struct k3_ring *ring) 247 { 248 u32 val; 249 250 val = readl(&ring->rt->occ); 251 252 ring->state.occ = val & K3_RINGACC_RT_OCC_MASK; 253 ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE); 254 } 255 256 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring) 257 { 258 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES - 259 (4 << ring->elm_size); 260 } 261 262 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx) 263 { 264 return (ring->ring_mem_virt + idx * (4 << ring->elm_size)); 265 } 266 267 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem); 268 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem); 269 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem); 270 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem); 271 272 static const struct k3_ring_ops k3_ring_mode_ring_ops = { 273 .push_tail = k3_ringacc_ring_push_mem, 274 .pop_head = k3_ringacc_ring_pop_mem, 275 }; 276 277 static const struct k3_ring_ops k3_dmaring_fwd_ops = { 278 .push_tail = k3_ringacc_ring_push_mem, 279 .pop_head = k3_dmaring_fwd_pop, 280 }; 281 282 static const struct k3_ring_ops k3_dmaring_reverse_ops = { 283 /* Reverse side of the DMA ring can only be popped by SW */ 284 .pop_head = k3_dmaring_reverse_pop, 285 }; 286 287 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem); 288 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem); 289 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem); 290 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem); 291 292 static const struct k3_ring_ops k3_ring_mode_msg_ops = { 293 .push_tail = k3_ringacc_ring_push_io, 294 .push_head = k3_ringacc_ring_push_head_io, 295 .pop_tail = k3_ringacc_ring_pop_tail_io, 296 .pop_head = k3_ringacc_ring_pop_io, 297 }; 298 299 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem); 300 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem); 301 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem); 302 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem); 303 304 static const struct k3_ring_ops k3_ring_mode_proxy_ops = { 305 .push_tail = k3_ringacc_ring_push_tail_proxy, 306 .push_head = k3_ringacc_ring_push_head_proxy, 307 .pop_tail = k3_ringacc_ring_pop_tail_proxy, 308 .pop_head = k3_ringacc_ring_pop_head_proxy, 309 }; 310 311 static void k3_ringacc_ring_dump(struct k3_ring *ring) 312 { 313 struct device *dev = ring->parent->dev; 314 315 dev_dbg(dev, "dump ring: %d\n", ring->ring_id); 316 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt, 317 &ring->ring_mem_dma); 318 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n", 319 ring->elm_size, ring->size, ring->mode, ring->proxy_id); 320 dev_dbg(dev, "dump flags %08X\n", ring->flags); 321 322 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db)); 323 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ)); 324 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx)); 325 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc)); 326 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx)); 327 328 if (ring->ring_mem_virt) 329 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE, 330 16, 1, ring->ring_mem_virt, 16 * 8, false); 331 } 332 333 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc, 334 int id, u32 flags) 335 { 336 int proxy_id = K3_RINGACC_PROXY_NOT_USED; 337 338 mutex_lock(&ringacc->req_lock); 339 340 if (!try_module_get(ringacc->dev->driver->owner)) 341 goto err_module_get; 342 343 if (id == K3_RINGACC_RING_ID_ANY) { 344 /* Request for any general purpose ring */ 345 struct ti_sci_resource_desc *gp_rings = 346 &ringacc->rm_gp_range->desc[0]; 347 unsigned long size; 348 349 size = gp_rings->start + gp_rings->num; 350 id = find_next_zero_bit(ringacc->rings_inuse, size, 351 gp_rings->start); 352 if (id == size) 353 goto error; 354 } else if (id < 0) { 355 goto error; 356 } 357 358 if (test_bit(id, ringacc->rings_inuse) && 359 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED)) 360 goto error; 361 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED) 362 goto out; 363 364 if (flags & K3_RINGACC_RING_USE_PROXY) { 365 proxy_id = find_first_zero_bit(ringacc->proxy_inuse, 366 ringacc->num_proxies); 367 if (proxy_id == ringacc->num_proxies) 368 goto error; 369 } 370 371 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) { 372 set_bit(proxy_id, ringacc->proxy_inuse); 373 ringacc->rings[id].proxy_id = proxy_id; 374 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id, 375 proxy_id); 376 } else { 377 dev_dbg(ringacc->dev, "Giving ring#%d\n", id); 378 } 379 380 set_bit(id, ringacc->rings_inuse); 381 out: 382 ringacc->rings[id].use_count++; 383 mutex_unlock(&ringacc->req_lock); 384 return &ringacc->rings[id]; 385 386 error: 387 module_put(ringacc->dev->driver->owner); 388 389 err_module_get: 390 mutex_unlock(&ringacc->req_lock); 391 return NULL; 392 } 393 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring); 394 395 static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id, 396 struct k3_ring **fwd_ring, 397 struct k3_ring **compl_ring) 398 { 399 int ret = 0; 400 401 /* 402 * DMA rings must be requested by ID, completion ring is the reverse 403 * side of the forward ring 404 */ 405 if (fwd_id < 0) 406 return -EINVAL; 407 408 mutex_lock(&ringacc->req_lock); 409 410 if (!try_module_get(ringacc->dev->driver->owner)) { 411 ret = -EINVAL; 412 goto err_module_get; 413 } 414 415 if (test_bit(fwd_id, ringacc->rings_inuse)) { 416 ret = -EBUSY; 417 goto error; 418 } 419 420 *fwd_ring = &ringacc->rings[fwd_id]; 421 *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings]; 422 set_bit(fwd_id, ringacc->rings_inuse); 423 ringacc->rings[fwd_id].use_count++; 424 dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id); 425 426 mutex_unlock(&ringacc->req_lock); 427 return 0; 428 429 error: 430 module_put(ringacc->dev->driver->owner); 431 err_module_get: 432 mutex_unlock(&ringacc->req_lock); 433 return ret; 434 } 435 436 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, 437 int fwd_id, int compl_id, 438 struct k3_ring **fwd_ring, 439 struct k3_ring **compl_ring) 440 { 441 int ret = 0; 442 443 if (!fwd_ring || !compl_ring) 444 return -EINVAL; 445 446 if (ringacc->dma_rings) 447 return k3_dmaring_request_dual_ring(ringacc, fwd_id, 448 fwd_ring, compl_ring); 449 450 *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0); 451 if (!(*fwd_ring)) 452 return -ENODEV; 453 454 *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0); 455 if (!(*compl_ring)) { 456 k3_ringacc_ring_free(*fwd_ring); 457 ret = -ENODEV; 458 } 459 460 return ret; 461 } 462 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair); 463 464 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring) 465 { 466 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 467 struct k3_ringacc *ringacc = ring->parent; 468 int ret; 469 470 ring_cfg.nav_id = ringacc->tisci_dev_id; 471 ring_cfg.index = ring->ring_id; 472 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID; 473 ring_cfg.count = ring->size; 474 475 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 476 if (ret) 477 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n", 478 ret, ring->ring_id); 479 } 480 481 void k3_ringacc_ring_reset(struct k3_ring *ring) 482 { 483 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 484 return; 485 486 memset(&ring->state, 0, sizeof(ring->state)); 487 488 k3_ringacc_ring_reset_sci(ring); 489 } 490 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset); 491 492 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring, 493 enum k3_ring_mode mode) 494 { 495 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 496 struct k3_ringacc *ringacc = ring->parent; 497 int ret; 498 499 ring_cfg.nav_id = ringacc->tisci_dev_id; 500 ring_cfg.index = ring->ring_id; 501 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID; 502 ring_cfg.mode = mode; 503 504 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 505 if (ret) 506 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n", 507 ret, ring->ring_id); 508 } 509 510 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ) 511 { 512 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 513 return; 514 515 if (!ring->parent->dma_ring_reset_quirk) 516 goto reset; 517 518 if (!occ) 519 occ = k3_ringacc_ring_read_occ(ring); 520 521 if (occ) { 522 u32 db_ring_cnt, db_ring_cnt_cur; 523 524 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__, 525 ring->ring_id, occ); 526 /* TI-SCI ring reset */ 527 k3_ringacc_ring_reset_sci(ring); 528 529 /* 530 * Setup the ring in ring/doorbell mode (if not already in this 531 * mode) 532 */ 533 if (ring->mode != K3_RINGACC_RING_MODE_RING) 534 k3_ringacc_ring_reconfig_qmode_sci( 535 ring, K3_RINGACC_RING_MODE_RING); 536 /* 537 * Ring the doorbell 2**22 – ringOcc times. 538 * This will wrap the internal UDMAP ring state occupancy 539 * counter (which is 21-bits wide) to 0. 540 */ 541 db_ring_cnt = (1U << 22) - occ; 542 543 while (db_ring_cnt != 0) { 544 /* 545 * Ring the doorbell with the maximum count each 546 * iteration if possible to minimize the total 547 * of writes 548 */ 549 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT) 550 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT; 551 else 552 db_ring_cnt_cur = db_ring_cnt; 553 554 writel(db_ring_cnt_cur, &ring->rt->db); 555 db_ring_cnt -= db_ring_cnt_cur; 556 } 557 558 /* Restore the original ring mode (if not ring mode) */ 559 if (ring->mode != K3_RINGACC_RING_MODE_RING) 560 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode); 561 } 562 563 reset: 564 /* Reset the ring */ 565 k3_ringacc_ring_reset(ring); 566 } 567 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma); 568 569 static void k3_ringacc_ring_free_sci(struct k3_ring *ring) 570 { 571 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 572 struct k3_ringacc *ringacc = ring->parent; 573 int ret; 574 575 ring_cfg.nav_id = ringacc->tisci_dev_id; 576 ring_cfg.index = ring->ring_id; 577 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 578 579 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 580 if (ret) 581 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n", 582 ret, ring->ring_id); 583 } 584 585 int k3_ringacc_ring_free(struct k3_ring *ring) 586 { 587 struct k3_ringacc *ringacc; 588 589 if (!ring) 590 return -EINVAL; 591 592 ringacc = ring->parent; 593 594 /* 595 * DMA rings: rings shared memory and configuration, only forward ring 596 * is configured and reverse ring considered as slave. 597 */ 598 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 599 return 0; 600 601 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags); 602 603 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 604 return -EINVAL; 605 606 mutex_lock(&ringacc->req_lock); 607 608 if (--ring->use_count) 609 goto out; 610 611 if (!(ring->flags & K3_RING_FLAG_BUSY)) 612 goto no_init; 613 614 k3_ringacc_ring_free_sci(ring); 615 616 dma_free_coherent(ring->dma_dev, 617 ring->size * (4 << ring->elm_size), 618 ring->ring_mem_virt, ring->ring_mem_dma); 619 ring->flags = 0; 620 ring->ops = NULL; 621 ring->dma_dev = NULL; 622 ring->asel = 0; 623 624 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) { 625 clear_bit(ring->proxy_id, ringacc->proxy_inuse); 626 ring->proxy = NULL; 627 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 628 } 629 630 no_init: 631 clear_bit(ring->ring_id, ringacc->rings_inuse); 632 633 module_put(ringacc->dev->driver->owner); 634 635 out: 636 mutex_unlock(&ringacc->req_lock); 637 return 0; 638 } 639 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free); 640 641 u32 k3_ringacc_get_ring_id(struct k3_ring *ring) 642 { 643 if (!ring) 644 return -EINVAL; 645 646 return ring->ring_id; 647 } 648 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id); 649 650 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring) 651 { 652 if (!ring) 653 return -EINVAL; 654 655 return ring->parent->tisci_dev_id; 656 } 657 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id); 658 659 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring) 660 { 661 int irq_num; 662 663 if (!ring) 664 return -EINVAL; 665 666 irq_num = msi_get_virq(ring->parent->dev, ring->ring_id); 667 if (irq_num <= 0) 668 irq_num = -EINVAL; 669 return irq_num; 670 } 671 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num); 672 673 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring) 674 { 675 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 }; 676 struct k3_ringacc *ringacc = ring->parent; 677 int ret; 678 679 if (!ringacc->tisci) 680 return -EINVAL; 681 682 ring_cfg.nav_id = ringacc->tisci_dev_id; 683 ring_cfg.index = ring->ring_id; 684 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER; 685 ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma); 686 ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma); 687 ring_cfg.count = ring->size; 688 ring_cfg.mode = ring->mode; 689 ring_cfg.size = ring->elm_size; 690 ring_cfg.asel = ring->asel; 691 692 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg); 693 if (ret) 694 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n", 695 ret, ring->ring_id); 696 697 return ret; 698 } 699 700 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 701 { 702 struct k3_ringacc *ringacc; 703 struct k3_ring *reverse_ring; 704 int ret = 0; 705 706 if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 || 707 cfg->mode != K3_RINGACC_RING_MODE_RING || 708 cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK) 709 return -EINVAL; 710 711 ringacc = ring->parent; 712 713 /* 714 * DMA rings: rings shared memory and configuration, only forward ring 715 * is configured and reverse ring considered as slave. 716 */ 717 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE)) 718 return 0; 719 720 if (!test_bit(ring->ring_id, ringacc->rings_inuse)) 721 return -EINVAL; 722 723 ring->size = cfg->size; 724 ring->elm_size = cfg->elm_size; 725 ring->mode = cfg->mode; 726 ring->asel = cfg->asel; 727 ring->dma_dev = cfg->dma_dev; 728 if (!ring->dma_dev) { 729 dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n", 730 ring->ring_id); 731 ring->dma_dev = ringacc->dev; 732 } 733 734 memset(&ring->state, 0, sizeof(ring->state)); 735 736 ring->ops = &k3_dmaring_fwd_ops; 737 738 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 739 ring->size * (4 << ring->elm_size), 740 &ring->ring_mem_dma, GFP_KERNEL); 741 if (!ring->ring_mem_virt) { 742 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 743 ret = -ENOMEM; 744 goto err_free_ops; 745 } 746 747 ret = k3_ringacc_ring_cfg_sci(ring); 748 if (ret) 749 goto err_free_mem; 750 751 ring->flags |= K3_RING_FLAG_BUSY; 752 753 k3_ringacc_ring_dump(ring); 754 755 /* DMA rings: configure reverse ring */ 756 reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings]; 757 reverse_ring->size = cfg->size; 758 reverse_ring->elm_size = cfg->elm_size; 759 reverse_ring->mode = cfg->mode; 760 reverse_ring->asel = cfg->asel; 761 memset(&reverse_ring->state, 0, sizeof(reverse_ring->state)); 762 reverse_ring->ops = &k3_dmaring_reverse_ops; 763 764 reverse_ring->ring_mem_virt = ring->ring_mem_virt; 765 reverse_ring->ring_mem_dma = ring->ring_mem_dma; 766 reverse_ring->flags |= K3_RING_FLAG_BUSY; 767 k3_ringacc_ring_dump(reverse_ring); 768 769 return 0; 770 771 err_free_mem: 772 dma_free_coherent(ring->dma_dev, 773 ring->size * (4 << ring->elm_size), 774 ring->ring_mem_virt, 775 ring->ring_mem_dma); 776 err_free_ops: 777 ring->ops = NULL; 778 ring->proxy = NULL; 779 ring->dma_dev = NULL; 780 ring->asel = 0; 781 return ret; 782 } 783 784 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg) 785 { 786 struct k3_ringacc *ringacc; 787 int ret = 0; 788 789 if (!ring || !cfg) 790 return -EINVAL; 791 792 ringacc = ring->parent; 793 794 if (ringacc->dma_rings) 795 return k3_dmaring_cfg(ring, cfg); 796 797 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 || 798 cfg->mode >= K3_RINGACC_RING_MODE_INVALID || 799 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK || 800 !test_bit(ring->ring_id, ringacc->rings_inuse)) 801 return -EINVAL; 802 803 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE && 804 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED && 805 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) { 806 dev_err(ringacc->dev, 807 "Message mode must use proxy for %u element size\n", 808 4 << ring->elm_size); 809 return -EINVAL; 810 } 811 812 /* 813 * In case of shared ring only the first user (master user) can 814 * configure the ring. The sequence should be by the client: 815 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user 816 * k3_ringacc_ring_cfg(ring, cfg); # master configuration 817 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 818 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED); 819 */ 820 if (ring->use_count != 1) 821 return 0; 822 823 ring->size = cfg->size; 824 ring->elm_size = cfg->elm_size; 825 ring->mode = cfg->mode; 826 memset(&ring->state, 0, sizeof(ring->state)); 827 828 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) 829 ring->proxy = ringacc->proxy_target_base + 830 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP; 831 832 switch (ring->mode) { 833 case K3_RINGACC_RING_MODE_RING: 834 ring->ops = &k3_ring_mode_ring_ops; 835 ring->dma_dev = cfg->dma_dev; 836 if (!ring->dma_dev) 837 ring->dma_dev = ringacc->dev; 838 break; 839 case K3_RINGACC_RING_MODE_MESSAGE: 840 ring->dma_dev = ringacc->dev; 841 if (ring->proxy) 842 ring->ops = &k3_ring_mode_proxy_ops; 843 else 844 ring->ops = &k3_ring_mode_msg_ops; 845 break; 846 default: 847 ring->ops = NULL; 848 ret = -EINVAL; 849 goto err_free_proxy; 850 } 851 852 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev, 853 ring->size * (4 << ring->elm_size), 854 &ring->ring_mem_dma, GFP_KERNEL); 855 if (!ring->ring_mem_virt) { 856 dev_err(ringacc->dev, "Failed to alloc ring mem\n"); 857 ret = -ENOMEM; 858 goto err_free_ops; 859 } 860 861 ret = k3_ringacc_ring_cfg_sci(ring); 862 863 if (ret) 864 goto err_free_mem; 865 866 ring->flags |= K3_RING_FLAG_BUSY; 867 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ? 868 K3_RING_FLAG_SHARED : 0; 869 870 k3_ringacc_ring_dump(ring); 871 872 return 0; 873 874 err_free_mem: 875 dma_free_coherent(ring->dma_dev, 876 ring->size * (4 << ring->elm_size), 877 ring->ring_mem_virt, 878 ring->ring_mem_dma); 879 err_free_ops: 880 ring->ops = NULL; 881 ring->dma_dev = NULL; 882 err_free_proxy: 883 ring->proxy = NULL; 884 return ret; 885 } 886 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg); 887 888 u32 k3_ringacc_ring_get_size(struct k3_ring *ring) 889 { 890 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 891 return -EINVAL; 892 893 return ring->size; 894 } 895 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size); 896 897 u32 k3_ringacc_ring_get_free(struct k3_ring *ring) 898 { 899 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 900 return -EINVAL; 901 902 if (!ring->state.free) 903 ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring); 904 905 return ring->state.free; 906 } 907 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free); 908 909 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring) 910 { 911 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 912 return -EINVAL; 913 914 return k3_ringacc_ring_read_occ(ring); 915 } 916 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ); 917 918 u32 k3_ringacc_ring_is_full(struct k3_ring *ring) 919 { 920 return !k3_ringacc_ring_get_free(ring); 921 } 922 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full); 923 924 enum k3_ringacc_access_mode { 925 K3_RINGACC_ACCESS_MODE_PUSH_HEAD, 926 K3_RINGACC_ACCESS_MODE_POP_HEAD, 927 K3_RINGACC_ACCESS_MODE_PUSH_TAIL, 928 K3_RINGACC_ACCESS_MODE_POP_TAIL, 929 K3_RINGACC_ACCESS_MODE_PEEK_HEAD, 930 K3_RINGACC_ACCESS_MODE_PEEK_TAIL, 931 }; 932 933 #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16) 934 #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24) 935 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring, 936 enum k3_ringacc_proxy_access_mode mode) 937 { 938 u32 val; 939 940 val = ring->ring_id; 941 val |= K3_RINGACC_PROXY_MODE(mode); 942 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size); 943 writel(val, &ring->proxy->control); 944 return 0; 945 } 946 947 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem, 948 enum k3_ringacc_access_mode access_mode) 949 { 950 void __iomem *ptr; 951 952 ptr = (void __iomem *)&ring->proxy->data; 953 954 switch (access_mode) { 955 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 956 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 957 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD); 958 break; 959 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 960 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 961 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL); 962 break; 963 default: 964 return -EINVAL; 965 } 966 967 ptr += k3_ringacc_ring_get_fifo_pos(ring); 968 969 switch (access_mode) { 970 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 971 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 972 dev_dbg(ring->parent->dev, 973 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 974 access_mode); 975 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 976 ring->state.occ--; 977 break; 978 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 979 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 980 dev_dbg(ring->parent->dev, 981 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 982 access_mode); 983 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 984 ring->state.free--; 985 break; 986 default: 987 return -EINVAL; 988 } 989 990 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free, 991 ring->state.occ); 992 return 0; 993 } 994 995 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem) 996 { 997 return k3_ringacc_ring_access_proxy(ring, elem, 998 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 999 } 1000 1001 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem) 1002 { 1003 return k3_ringacc_ring_access_proxy(ring, elem, 1004 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 1005 } 1006 1007 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem) 1008 { 1009 return k3_ringacc_ring_access_proxy(ring, elem, 1010 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1011 } 1012 1013 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem) 1014 { 1015 return k3_ringacc_ring_access_proxy(ring, elem, 1016 K3_RINGACC_ACCESS_MODE_POP_TAIL); 1017 } 1018 1019 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem, 1020 enum k3_ringacc_access_mode access_mode) 1021 { 1022 void __iomem *ptr; 1023 1024 switch (access_mode) { 1025 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 1026 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 1027 ptr = (void __iomem *)&ring->fifos->head_data; 1028 break; 1029 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 1030 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 1031 ptr = (void __iomem *)&ring->fifos->tail_data; 1032 break; 1033 default: 1034 return -EINVAL; 1035 } 1036 1037 ptr += k3_ringacc_ring_get_fifo_pos(ring); 1038 1039 switch (access_mode) { 1040 case K3_RINGACC_ACCESS_MODE_POP_HEAD: 1041 case K3_RINGACC_ACCESS_MODE_POP_TAIL: 1042 dev_dbg(ring->parent->dev, 1043 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr, 1044 access_mode); 1045 memcpy_fromio(elem, ptr, (4 << ring->elm_size)); 1046 ring->state.occ--; 1047 break; 1048 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL: 1049 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD: 1050 dev_dbg(ring->parent->dev, 1051 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr, 1052 access_mode); 1053 memcpy_toio(ptr, elem, (4 << ring->elm_size)); 1054 ring->state.free--; 1055 break; 1056 default: 1057 return -EINVAL; 1058 } 1059 1060 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n", 1061 ring->state.free, ring->state.windex, ring->state.occ, 1062 ring->state.rindex); 1063 return 0; 1064 } 1065 1066 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem) 1067 { 1068 return k3_ringacc_ring_access_io(ring, elem, 1069 K3_RINGACC_ACCESS_MODE_PUSH_HEAD); 1070 } 1071 1072 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem) 1073 { 1074 return k3_ringacc_ring_access_io(ring, elem, 1075 K3_RINGACC_ACCESS_MODE_PUSH_TAIL); 1076 } 1077 1078 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem) 1079 { 1080 return k3_ringacc_ring_access_io(ring, elem, 1081 K3_RINGACC_ACCESS_MODE_POP_HEAD); 1082 } 1083 1084 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem) 1085 { 1086 return k3_ringacc_ring_access_io(ring, elem, 1087 K3_RINGACC_ACCESS_MODE_POP_TAIL); 1088 } 1089 1090 /* 1091 * The element is 48 bits of address + ASEL bits in the ring. 1092 * ASEL is used by the DMAs and should be removed for the kernel as it is not 1093 * part of the physical memory address. 1094 */ 1095 static void k3_dmaring_remove_asel_from_elem(u64 *elem) 1096 { 1097 *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0); 1098 } 1099 1100 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem) 1101 { 1102 void *elem_ptr; 1103 u32 elem_idx; 1104 1105 /* 1106 * DMA rings: forward ring is always tied DMA channel and HW does not 1107 * maintain any state data required for POP operation and its unknown 1108 * how much elements were consumed by HW. So, to actually 1109 * do POP, the read pointer has to be recalculated every time. 1110 */ 1111 ring->state.occ = k3_ringacc_ring_read_occ(ring); 1112 if (ring->state.windex >= ring->state.occ) 1113 elem_idx = ring->state.windex - ring->state.occ; 1114 else 1115 elem_idx = ring->size - (ring->state.occ - ring->state.windex); 1116 1117 elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx); 1118 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1119 k3_dmaring_remove_asel_from_elem(elem); 1120 1121 ring->state.occ--; 1122 writel(-1, &ring->rt->db); 1123 1124 dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n", 1125 __func__, ring->state.occ, ring->state.windex, elem_idx, 1126 elem_ptr); 1127 return 0; 1128 } 1129 1130 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem) 1131 { 1132 void *elem_ptr; 1133 1134 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 1135 1136 if (ring->state.occ) { 1137 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1138 k3_dmaring_remove_asel_from_elem(elem); 1139 1140 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 1141 ring->state.occ--; 1142 writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db); 1143 } else if (ring->state.tdown_complete) { 1144 dma_addr_t *value = elem; 1145 1146 *value = CPPI5_TDCM_MARKER; 1147 writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db); 1148 ring->state.tdown_complete = false; 1149 } 1150 1151 dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n", 1152 __func__, ring->state.occ, ring->state.rindex, elem_ptr); 1153 return 0; 1154 } 1155 1156 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem) 1157 { 1158 void *elem_ptr; 1159 1160 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex); 1161 1162 memcpy(elem_ptr, elem, (4 << ring->elm_size)); 1163 if (ring->parent->dma_rings) { 1164 u64 *addr = elem_ptr; 1165 1166 *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT); 1167 } 1168 1169 ring->state.windex = (ring->state.windex + 1) % ring->size; 1170 ring->state.free--; 1171 writel(1, &ring->rt->db); 1172 1173 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n", 1174 ring->state.free, ring->state.windex); 1175 1176 return 0; 1177 } 1178 1179 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem) 1180 { 1181 void *elem_ptr; 1182 1183 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex); 1184 1185 memcpy(elem, elem_ptr, (4 << ring->elm_size)); 1186 1187 ring->state.rindex = (ring->state.rindex + 1) % ring->size; 1188 ring->state.occ--; 1189 writel(-1, &ring->rt->db); 1190 1191 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n", 1192 ring->state.occ, ring->state.rindex, elem_ptr); 1193 return 0; 1194 } 1195 1196 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem) 1197 { 1198 int ret = -EOPNOTSUPP; 1199 1200 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1201 return -EINVAL; 1202 1203 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n", 1204 ring->state.free, ring->state.windex); 1205 1206 if (k3_ringacc_ring_is_full(ring)) 1207 return -ENOMEM; 1208 1209 if (ring->ops && ring->ops->push_tail) 1210 ret = ring->ops->push_tail(ring, elem); 1211 1212 return ret; 1213 } 1214 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push); 1215 1216 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem) 1217 { 1218 int ret = -EOPNOTSUPP; 1219 1220 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1221 return -EINVAL; 1222 1223 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n", 1224 ring->state.free, ring->state.windex); 1225 1226 if (k3_ringacc_ring_is_full(ring)) 1227 return -ENOMEM; 1228 1229 if (ring->ops && ring->ops->push_head) 1230 ret = ring->ops->push_head(ring, elem); 1231 1232 return ret; 1233 } 1234 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head); 1235 1236 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem) 1237 { 1238 int ret = -EOPNOTSUPP; 1239 1240 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1241 return -EINVAL; 1242 1243 if (!ring->state.occ) 1244 k3_ringacc_ring_update_occ(ring); 1245 1246 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ, 1247 ring->state.rindex); 1248 1249 if (!ring->state.occ && !ring->state.tdown_complete) 1250 return -ENODATA; 1251 1252 if (ring->ops && ring->ops->pop_head) 1253 ret = ring->ops->pop_head(ring, elem); 1254 1255 return ret; 1256 } 1257 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop); 1258 1259 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem) 1260 { 1261 int ret = -EOPNOTSUPP; 1262 1263 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY)) 1264 return -EINVAL; 1265 1266 if (!ring->state.occ) 1267 k3_ringacc_ring_update_occ(ring); 1268 1269 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n", 1270 ring->state.occ, ring->state.rindex); 1271 1272 if (!ring->state.occ) 1273 return -ENODATA; 1274 1275 if (ring->ops && ring->ops->pop_tail) 1276 ret = ring->ops->pop_tail(ring, elem); 1277 1278 return ret; 1279 } 1280 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail); 1281 1282 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, 1283 const char *property) 1284 { 1285 struct device_node *ringacc_np; 1286 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER); 1287 struct k3_ringacc *entry; 1288 1289 ringacc_np = of_parse_phandle(np, property, 0); 1290 if (!ringacc_np) 1291 return ERR_PTR(-ENODEV); 1292 1293 mutex_lock(&k3_ringacc_list_lock); 1294 list_for_each_entry(entry, &k3_ringacc_list, list) 1295 if (device_match_of_node(entry->dev, ringacc_np)) { 1296 ringacc = entry; 1297 break; 1298 } 1299 mutex_unlock(&k3_ringacc_list_lock); 1300 of_node_put(ringacc_np); 1301 1302 return ringacc; 1303 } 1304 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle); 1305 1306 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc) 1307 { 1308 struct device_node *node = ringacc->dev->of_node; 1309 struct device *dev = ringacc->dev; 1310 struct platform_device *pdev = to_platform_device(dev); 1311 int ret; 1312 1313 if (!node) { 1314 dev_err(dev, "device tree info unavailable\n"); 1315 return -ENODEV; 1316 } 1317 1318 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings); 1319 if (ret) { 1320 dev_err(dev, "ti,num-rings read failure %d\n", ret); 1321 return ret; 1322 } 1323 1324 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci"); 1325 if (IS_ERR(ringacc->tisci)) { 1326 ret = PTR_ERR(ringacc->tisci); 1327 if (ret != -EPROBE_DEFER) 1328 dev_err(dev, "ti,sci read fail %d\n", ret); 1329 ringacc->tisci = NULL; 1330 return ret; 1331 } 1332 1333 ret = of_property_read_u32(node, "ti,sci-dev-id", 1334 &ringacc->tisci_dev_id); 1335 if (ret) { 1336 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret); 1337 return ret; 1338 } 1339 1340 pdev->id = ringacc->tisci_dev_id; 1341 1342 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev, 1343 ringacc->tisci_dev_id, 1344 "ti,sci-rm-range-gp-rings"); 1345 if (IS_ERR(ringacc->rm_gp_range)) { 1346 dev_err(dev, "Failed to allocate MSI interrupts\n"); 1347 return PTR_ERR(ringacc->rm_gp_range); 1348 } 1349 1350 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev, 1351 ringacc->rm_gp_range); 1352 } 1353 1354 static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = { 1355 .dma_ring_reset_quirk = 1, 1356 }; 1357 1358 static const struct soc_device_attribute k3_ringacc_socinfo[] = { 1359 { .family = "AM65X", 1360 .revision = "SR1.0", 1361 .data = &k3_ringacc_soc_data_sr1 1362 }, 1363 {/* sentinel */} 1364 }; 1365 1366 static int k3_ringacc_init(struct platform_device *pdev, 1367 struct k3_ringacc *ringacc) 1368 { 1369 const struct soc_device_attribute *soc; 1370 void __iomem *base_fifo, *base_rt; 1371 struct device *dev = &pdev->dev; 1372 int ret, i; 1373 1374 dev->msi.domain = of_msi_get_domain(dev, dev->of_node, 1375 DOMAIN_BUS_TI_SCI_INTA_MSI); 1376 if (!dev->msi.domain) 1377 return -EPROBE_DEFER; 1378 1379 ret = k3_ringacc_probe_dt(ringacc); 1380 if (ret) 1381 return ret; 1382 1383 soc = soc_device_match(k3_ringacc_socinfo); 1384 if (soc && soc->data) { 1385 const struct k3_ringacc_soc_data *soc_data = soc->data; 1386 1387 ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk; 1388 } 1389 1390 base_rt = devm_platform_ioremap_resource_byname(pdev, "rt"); 1391 if (IS_ERR(base_rt)) 1392 return PTR_ERR(base_rt); 1393 1394 base_fifo = devm_platform_ioremap_resource_byname(pdev, "fifos"); 1395 if (IS_ERR(base_fifo)) 1396 return PTR_ERR(base_fifo); 1397 1398 ringacc->proxy_gcfg = devm_platform_ioremap_resource_byname(pdev, "proxy_gcfg"); 1399 if (IS_ERR(ringacc->proxy_gcfg)) 1400 return PTR_ERR(ringacc->proxy_gcfg); 1401 1402 ringacc->proxy_target_base = devm_platform_ioremap_resource_byname(pdev, 1403 "proxy_target"); 1404 if (IS_ERR(ringacc->proxy_target_base)) 1405 return PTR_ERR(ringacc->proxy_target_base); 1406 1407 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) & 1408 K3_RINGACC_PROXY_CFG_THREADS_MASK; 1409 1410 ringacc->rings = devm_kzalloc(dev, 1411 sizeof(*ringacc->rings) * 1412 ringacc->num_rings, 1413 GFP_KERNEL); 1414 ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings, 1415 GFP_KERNEL); 1416 ringacc->proxy_inuse = devm_bitmap_zalloc(dev, ringacc->num_proxies, 1417 GFP_KERNEL); 1418 1419 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse) 1420 return -ENOMEM; 1421 1422 for (i = 0; i < ringacc->num_rings; i++) { 1423 ringacc->rings[i].rt = base_rt + 1424 K3_RINGACC_RT_REGS_STEP * i; 1425 ringacc->rings[i].fifos = base_fifo + 1426 K3_RINGACC_FIFO_REGS_STEP * i; 1427 ringacc->rings[i].parent = ringacc; 1428 ringacc->rings[i].ring_id = i; 1429 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED; 1430 } 1431 1432 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1433 1434 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n", 1435 ringacc->num_rings, 1436 ringacc->rm_gp_range->desc[0].start, 1437 ringacc->rm_gp_range->desc[0].num, 1438 ringacc->tisci_dev_id); 1439 dev_info(dev, "dma-ring-reset-quirk: %s\n", 1440 str_enabled_disabled(ringacc->dma_ring_reset_quirk)); 1441 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n", 1442 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies); 1443 1444 return 0; 1445 } 1446 1447 struct ringacc_match_data { 1448 struct k3_ringacc_ops ops; 1449 }; 1450 1451 static struct ringacc_match_data k3_ringacc_data = { 1452 .ops = { 1453 .init = k3_ringacc_init, 1454 }, 1455 }; 1456 1457 /* Match table for of_platform binding */ 1458 static const struct of_device_id k3_ringacc_of_match[] = { 1459 { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, }, 1460 {}, 1461 }; 1462 MODULE_DEVICE_TABLE(of, k3_ringacc_of_match); 1463 1464 struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev, 1465 struct k3_ringacc_init_data *data) 1466 { 1467 struct device *dev = &pdev->dev; 1468 struct k3_ringacc *ringacc; 1469 void __iomem *base_rt; 1470 int i; 1471 1472 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1473 if (!ringacc) 1474 return ERR_PTR(-ENOMEM); 1475 1476 ringacc->dev = dev; 1477 ringacc->dma_rings = true; 1478 ringacc->num_rings = data->num_rings; 1479 ringacc->tisci = data->tisci; 1480 ringacc->tisci_dev_id = data->tisci_dev_id; 1481 1482 mutex_init(&ringacc->req_lock); 1483 1484 base_rt = devm_platform_ioremap_resource_byname(pdev, "ringrt"); 1485 if (IS_ERR(base_rt)) 1486 return ERR_CAST(base_rt); 1487 1488 ringacc->rings = devm_kzalloc(dev, 1489 sizeof(*ringacc->rings) * 1490 ringacc->num_rings * 2, 1491 GFP_KERNEL); 1492 ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings, 1493 GFP_KERNEL); 1494 1495 if (!ringacc->rings || !ringacc->rings_inuse) 1496 return ERR_PTR(-ENOMEM); 1497 1498 for (i = 0; i < ringacc->num_rings; i++) { 1499 struct k3_ring *ring = &ringacc->rings[i]; 1500 1501 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i; 1502 ring->parent = ringacc; 1503 ring->ring_id = i; 1504 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1505 1506 ring = &ringacc->rings[ringacc->num_rings + i]; 1507 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i + 1508 K3_DMARING_RT_REGS_REVERSE_OFS; 1509 ring->parent = ringacc; 1510 ring->ring_id = i; 1511 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED; 1512 ring->flags = K3_RING_FLAG_REVERSE; 1513 } 1514 1515 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops; 1516 1517 dev_info(dev, "Number of rings: %u\n", ringacc->num_rings); 1518 1519 return ringacc; 1520 } 1521 EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init); 1522 1523 static int k3_ringacc_probe(struct platform_device *pdev) 1524 { 1525 const struct ringacc_match_data *match_data; 1526 struct device *dev = &pdev->dev; 1527 struct k3_ringacc *ringacc; 1528 int ret; 1529 1530 match_data = of_device_get_match_data(&pdev->dev); 1531 if (!match_data) 1532 return -ENODEV; 1533 1534 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL); 1535 if (!ringacc) 1536 return -ENOMEM; 1537 1538 ringacc->dev = dev; 1539 mutex_init(&ringacc->req_lock); 1540 ringacc->ops = &match_data->ops; 1541 1542 ret = ringacc->ops->init(pdev, ringacc); 1543 if (ret) 1544 return ret; 1545 1546 dev_set_drvdata(dev, ringacc); 1547 1548 mutex_lock(&k3_ringacc_list_lock); 1549 list_add_tail(&ringacc->list, &k3_ringacc_list); 1550 mutex_unlock(&k3_ringacc_list_lock); 1551 1552 return 0; 1553 } 1554 1555 static void k3_ringacc_remove(struct platform_device *pdev) 1556 { 1557 struct k3_ringacc *ringacc = dev_get_drvdata(&pdev->dev); 1558 1559 mutex_lock(&k3_ringacc_list_lock); 1560 list_del(&ringacc->list); 1561 mutex_unlock(&k3_ringacc_list_lock); 1562 } 1563 1564 static struct platform_driver k3_ringacc_driver = { 1565 .probe = k3_ringacc_probe, 1566 .remove = k3_ringacc_remove, 1567 .driver = { 1568 .name = "k3-ringacc", 1569 .of_match_table = k3_ringacc_of_match, 1570 .suppress_bind_attrs = true, 1571 }, 1572 }; 1573 module_platform_driver(k3_ringacc_driver); 1574 1575 MODULE_LICENSE("GPL"); 1576 MODULE_DESCRIPTION("TI Ringacc driver for K3 SOCs"); 1577 MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>"); 1578