1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * TI K3 NAVSS Ring Accelerator subsystem driver
4 *
5 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
6 */
7
8 #include <linux/dma-mapping.h>
9 #include <linux/io.h>
10 #include <linux/module.h>
11 #include <linux/of.h>
12 #include <linux/platform_device.h>
13 #include <linux/sys_soc.h>
14 #include <linux/dma/ti-cppi5.h>
15 #include <linux/soc/ti/k3-ringacc.h>
16 #include <linux/soc/ti/ti_sci_protocol.h>
17 #include <linux/soc/ti/ti_sci_inta_msi.h>
18 #include <linux/of_irq.h>
19 #include <linux/irqdomain.h>
20
21 static LIST_HEAD(k3_ringacc_list);
22 static DEFINE_MUTEX(k3_ringacc_list_lock);
23
24 #define K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK GENMASK(19, 0)
25 #define K3_DMARING_CFG_RING_SIZE_ELCNT_MASK GENMASK(15, 0)
26
27 /**
28 * struct k3_ring_rt_regs - The RA realtime Control/Status Registers region
29 *
30 * @resv_16: Reserved
31 * @db: Ring Doorbell Register
32 * @resv_4: Reserved
33 * @occ: Ring Occupancy Register
34 * @indx: Ring Current Index Register
35 * @hwocc: Ring Hardware Occupancy Register
36 * @hwindx: Ring Hardware Current Index Register
37 */
38 struct k3_ring_rt_regs {
39 u32 resv_16[4];
40 u32 db;
41 u32 resv_4[1];
42 u32 occ;
43 u32 indx;
44 u32 hwocc;
45 u32 hwindx;
46 };
47
48 #define K3_RINGACC_RT_REGS_STEP 0x1000
49 #define K3_DMARING_RT_REGS_STEP 0x2000
50 #define K3_DMARING_RT_REGS_REVERSE_OFS 0x1000
51 #define K3_RINGACC_RT_OCC_MASK GENMASK(20, 0)
52 #define K3_DMARING_RT_OCC_TDOWN_COMPLETE BIT(31)
53 #define K3_DMARING_RT_DB_ENTRY_MASK GENMASK(7, 0)
54 #define K3_DMARING_RT_DB_TDOWN_ACK BIT(31)
55
56 /**
57 * struct k3_ring_fifo_regs - The Ring Accelerator Queues Registers region
58 *
59 * @head_data: Ring Head Entry Data Registers
60 * @tail_data: Ring Tail Entry Data Registers
61 * @peek_head_data: Ring Peek Head Entry Data Regs
62 * @peek_tail_data: Ring Peek Tail Entry Data Regs
63 */
64 struct k3_ring_fifo_regs {
65 u32 head_data[128];
66 u32 tail_data[128];
67 u32 peek_head_data[128];
68 u32 peek_tail_data[128];
69 };
70
71 /**
72 * struct k3_ringacc_proxy_gcfg_regs - RA Proxy Global Config MMIO Region
73 *
74 * @revision: Revision Register
75 * @config: Config Register
76 */
77 struct k3_ringacc_proxy_gcfg_regs {
78 u32 revision;
79 u32 config;
80 };
81
82 #define K3_RINGACC_PROXY_CFG_THREADS_MASK GENMASK(15, 0)
83
84 /**
85 * struct k3_ringacc_proxy_target_regs - Proxy Datapath MMIO Region
86 *
87 * @control: Proxy Control Register
88 * @status: Proxy Status Register
89 * @resv_512: Reserved
90 * @data: Proxy Data Register
91 */
92 struct k3_ringacc_proxy_target_regs {
93 u32 control;
94 u32 status;
95 u8 resv_512[504];
96 u32 data[128];
97 };
98
99 #define K3_RINGACC_PROXY_TARGET_STEP 0x1000
100 #define K3_RINGACC_PROXY_NOT_USED (-1)
101
102 enum k3_ringacc_proxy_access_mode {
103 PROXY_ACCESS_MODE_HEAD = 0,
104 PROXY_ACCESS_MODE_TAIL = 1,
105 PROXY_ACCESS_MODE_PEEK_HEAD = 2,
106 PROXY_ACCESS_MODE_PEEK_TAIL = 3,
107 };
108
109 #define K3_RINGACC_FIFO_WINDOW_SIZE_BYTES (512U)
110 #define K3_RINGACC_FIFO_REGS_STEP 0x1000
111 #define K3_RINGACC_MAX_DB_RING_CNT (127U)
112
113 struct k3_ring_ops {
114 int (*push_tail)(struct k3_ring *ring, void *elm);
115 int (*push_head)(struct k3_ring *ring, void *elm);
116 int (*pop_tail)(struct k3_ring *ring, void *elm);
117 int (*pop_head)(struct k3_ring *ring, void *elm);
118 };
119
120 /**
121 * struct k3_ring_state - Internal state tracking structure
122 *
123 * @free: Number of free entries
124 * @occ: Occupancy
125 * @windex: Write index
126 * @rindex: Read index
127 * @tdown_complete: Tear down complete state
128 */
129 struct k3_ring_state {
130 u32 free;
131 u32 occ;
132 u32 windex;
133 u32 rindex;
134 u32 tdown_complete:1;
135 };
136
137 /**
138 * struct k3_ring - RA Ring descriptor
139 *
140 * @rt: Ring control/status registers
141 * @fifos: Ring queues registers
142 * @proxy: Ring Proxy Datapath registers
143 * @ring_mem_dma: Ring buffer dma address
144 * @ring_mem_virt: Ring buffer virt address
145 * @ops: Ring operations
146 * @size: Ring size in elements
147 * @elm_size: Size of the ring element
148 * @mode: Ring mode
149 * @flags: flags
150 * @state: Ring state
151 * @ring_id: Ring Id
152 * @parent: Pointer on struct @k3_ringacc
153 * @use_count: Use count for shared rings
154 * @proxy_id: RA Ring Proxy Id (only if @K3_RINGACC_RING_USE_PROXY)
155 * @dma_dev: device to be used for DMA API (allocation, mapping)
156 * @asel: Address Space Select value for physical addresses
157 */
158 struct k3_ring {
159 struct k3_ring_rt_regs __iomem *rt;
160 struct k3_ring_fifo_regs __iomem *fifos;
161 struct k3_ringacc_proxy_target_regs __iomem *proxy;
162 dma_addr_t ring_mem_dma;
163 void *ring_mem_virt;
164 const struct k3_ring_ops *ops;
165 u32 size;
166 enum k3_ring_size elm_size;
167 enum k3_ring_mode mode;
168 u32 flags;
169 #define K3_RING_FLAG_BUSY BIT(1)
170 #define K3_RING_FLAG_SHARED BIT(2)
171 #define K3_RING_FLAG_REVERSE BIT(3)
172 struct k3_ring_state state;
173 u32 ring_id;
174 struct k3_ringacc *parent;
175 u32 use_count;
176 int proxy_id;
177 struct device *dma_dev;
178 u32 asel;
179 #define K3_ADDRESS_ASEL_SHIFT 48
180 };
181
182 struct k3_ringacc_ops {
183 int (*init)(struct platform_device *pdev, struct k3_ringacc *ringacc);
184 };
185
186 /**
187 * struct k3_ringacc - Rings accelerator descriptor
188 *
189 * @dev: pointer on RA device
190 * @proxy_gcfg: RA proxy global config registers
191 * @proxy_target_base: RA proxy datapath region
192 * @num_rings: number of ring in RA
193 * @rings_inuse: bitfield for ring usage tracking
194 * @rm_gp_range: general purpose rings range from tisci
195 * @dma_ring_reset_quirk: DMA reset workaround enable
196 * @num_proxies: number of RA proxies
197 * @proxy_inuse: bitfield for proxy usage tracking
198 * @rings: array of rings descriptors (struct @k3_ring)
199 * @list: list of RAs in the system
200 * @req_lock: protect rings allocation
201 * @tisci: pointer ti-sci handle
202 * @tisci_ring_ops: ti-sci rings ops
203 * @tisci_dev_id: ti-sci device id
204 * @ops: SoC specific ringacc operation
205 * @dma_rings: indicate DMA ring (dual ring within BCDMA/PKTDMA)
206 */
207 struct k3_ringacc {
208 struct device *dev;
209 struct k3_ringacc_proxy_gcfg_regs __iomem *proxy_gcfg;
210 void __iomem *proxy_target_base;
211 u32 num_rings; /* number of rings in Ringacc module */
212 unsigned long *rings_inuse;
213 struct ti_sci_resource *rm_gp_range;
214
215 bool dma_ring_reset_quirk;
216 u32 num_proxies;
217 unsigned long *proxy_inuse;
218
219 struct k3_ring *rings;
220 struct list_head list;
221 struct mutex req_lock; /* protect rings allocation */
222
223 const struct ti_sci_handle *tisci;
224 const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
225 u32 tisci_dev_id;
226
227 const struct k3_ringacc_ops *ops;
228 bool dma_rings;
229 };
230
231 /**
232 * struct k3_ringacc_soc_data - Rings accelerator SoC data
233 *
234 * @dma_ring_reset_quirk: DMA reset workaround enable
235 */
236 struct k3_ringacc_soc_data {
237 unsigned dma_ring_reset_quirk:1;
238 };
239
k3_ringacc_ring_read_occ(struct k3_ring * ring)240 static int k3_ringacc_ring_read_occ(struct k3_ring *ring)
241 {
242 return readl(&ring->rt->occ) & K3_RINGACC_RT_OCC_MASK;
243 }
244
k3_ringacc_ring_update_occ(struct k3_ring * ring)245 static void k3_ringacc_ring_update_occ(struct k3_ring *ring)
246 {
247 u32 val;
248
249 val = readl(&ring->rt->occ);
250
251 ring->state.occ = val & K3_RINGACC_RT_OCC_MASK;
252 ring->state.tdown_complete = !!(val & K3_DMARING_RT_OCC_TDOWN_COMPLETE);
253 }
254
k3_ringacc_ring_get_fifo_pos(struct k3_ring * ring)255 static long k3_ringacc_ring_get_fifo_pos(struct k3_ring *ring)
256 {
257 return K3_RINGACC_FIFO_WINDOW_SIZE_BYTES -
258 (4 << ring->elm_size);
259 }
260
k3_ringacc_get_elm_addr(struct k3_ring * ring,u32 idx)261 static void *k3_ringacc_get_elm_addr(struct k3_ring *ring, u32 idx)
262 {
263 return (ring->ring_mem_virt + idx * (4 << ring->elm_size));
264 }
265
266 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem);
267 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem);
268 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem);
269 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem);
270
271 static const struct k3_ring_ops k3_ring_mode_ring_ops = {
272 .push_tail = k3_ringacc_ring_push_mem,
273 .pop_head = k3_ringacc_ring_pop_mem,
274 };
275
276 static const struct k3_ring_ops k3_dmaring_fwd_ops = {
277 .push_tail = k3_ringacc_ring_push_mem,
278 .pop_head = k3_dmaring_fwd_pop,
279 };
280
281 static const struct k3_ring_ops k3_dmaring_reverse_ops = {
282 /* Reverse side of the DMA ring can only be popped by SW */
283 .pop_head = k3_dmaring_reverse_pop,
284 };
285
286 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem);
287 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem);
288 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem);
289 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem);
290
291 static const struct k3_ring_ops k3_ring_mode_msg_ops = {
292 .push_tail = k3_ringacc_ring_push_io,
293 .push_head = k3_ringacc_ring_push_head_io,
294 .pop_tail = k3_ringacc_ring_pop_tail_io,
295 .pop_head = k3_ringacc_ring_pop_io,
296 };
297
298 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem);
299 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem);
300 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem);
301 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem);
302
303 static const struct k3_ring_ops k3_ring_mode_proxy_ops = {
304 .push_tail = k3_ringacc_ring_push_tail_proxy,
305 .push_head = k3_ringacc_ring_push_head_proxy,
306 .pop_tail = k3_ringacc_ring_pop_tail_proxy,
307 .pop_head = k3_ringacc_ring_pop_head_proxy,
308 };
309
k3_ringacc_ring_dump(struct k3_ring * ring)310 static void k3_ringacc_ring_dump(struct k3_ring *ring)
311 {
312 struct device *dev = ring->parent->dev;
313
314 dev_dbg(dev, "dump ring: %d\n", ring->ring_id);
315 dev_dbg(dev, "dump mem virt %p, dma %pad\n", ring->ring_mem_virt,
316 &ring->ring_mem_dma);
317 dev_dbg(dev, "dump elmsize %d, size %d, mode %d, proxy_id %d\n",
318 ring->elm_size, ring->size, ring->mode, ring->proxy_id);
319 dev_dbg(dev, "dump flags %08X\n", ring->flags);
320
321 dev_dbg(dev, "dump ring_rt_regs: db%08x\n", readl(&ring->rt->db));
322 dev_dbg(dev, "dump occ%08x\n", readl(&ring->rt->occ));
323 dev_dbg(dev, "dump indx%08x\n", readl(&ring->rt->indx));
324 dev_dbg(dev, "dump hwocc%08x\n", readl(&ring->rt->hwocc));
325 dev_dbg(dev, "dump hwindx%08x\n", readl(&ring->rt->hwindx));
326
327 if (ring->ring_mem_virt)
328 print_hex_dump_debug("dump ring_mem_virt ", DUMP_PREFIX_NONE,
329 16, 1, ring->ring_mem_virt, 16 * 8, false);
330 }
331
k3_ringacc_request_ring(struct k3_ringacc * ringacc,int id,u32 flags)332 struct k3_ring *k3_ringacc_request_ring(struct k3_ringacc *ringacc,
333 int id, u32 flags)
334 {
335 int proxy_id = K3_RINGACC_PROXY_NOT_USED;
336
337 mutex_lock(&ringacc->req_lock);
338
339 if (!try_module_get(ringacc->dev->driver->owner))
340 goto err_module_get;
341
342 if (id == K3_RINGACC_RING_ID_ANY) {
343 /* Request for any general purpose ring */
344 struct ti_sci_resource_desc *gp_rings =
345 &ringacc->rm_gp_range->desc[0];
346 unsigned long size;
347
348 size = gp_rings->start + gp_rings->num;
349 id = find_next_zero_bit(ringacc->rings_inuse, size,
350 gp_rings->start);
351 if (id == size)
352 goto error;
353 } else if (id < 0) {
354 goto error;
355 }
356
357 if (test_bit(id, ringacc->rings_inuse) &&
358 !(ringacc->rings[id].flags & K3_RING_FLAG_SHARED))
359 goto error;
360 else if (ringacc->rings[id].flags & K3_RING_FLAG_SHARED)
361 goto out;
362
363 if (flags & K3_RINGACC_RING_USE_PROXY) {
364 proxy_id = find_first_zero_bit(ringacc->proxy_inuse,
365 ringacc->num_proxies);
366 if (proxy_id == ringacc->num_proxies)
367 goto error;
368 }
369
370 if (proxy_id != K3_RINGACC_PROXY_NOT_USED) {
371 set_bit(proxy_id, ringacc->proxy_inuse);
372 ringacc->rings[id].proxy_id = proxy_id;
373 dev_dbg(ringacc->dev, "Giving ring#%d proxy#%d\n", id,
374 proxy_id);
375 } else {
376 dev_dbg(ringacc->dev, "Giving ring#%d\n", id);
377 }
378
379 set_bit(id, ringacc->rings_inuse);
380 out:
381 ringacc->rings[id].use_count++;
382 mutex_unlock(&ringacc->req_lock);
383 return &ringacc->rings[id];
384
385 error:
386 module_put(ringacc->dev->driver->owner);
387
388 err_module_get:
389 mutex_unlock(&ringacc->req_lock);
390 return NULL;
391 }
392 EXPORT_SYMBOL_GPL(k3_ringacc_request_ring);
393
k3_dmaring_request_dual_ring(struct k3_ringacc * ringacc,int fwd_id,struct k3_ring ** fwd_ring,struct k3_ring ** compl_ring)394 static int k3_dmaring_request_dual_ring(struct k3_ringacc *ringacc, int fwd_id,
395 struct k3_ring **fwd_ring,
396 struct k3_ring **compl_ring)
397 {
398 int ret = 0;
399
400 /*
401 * DMA rings must be requested by ID, completion ring is the reverse
402 * side of the forward ring
403 */
404 if (fwd_id < 0)
405 return -EINVAL;
406
407 mutex_lock(&ringacc->req_lock);
408
409 if (!try_module_get(ringacc->dev->driver->owner)) {
410 ret = -EINVAL;
411 goto err_module_get;
412 }
413
414 if (test_bit(fwd_id, ringacc->rings_inuse)) {
415 ret = -EBUSY;
416 goto error;
417 }
418
419 *fwd_ring = &ringacc->rings[fwd_id];
420 *compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
421 set_bit(fwd_id, ringacc->rings_inuse);
422 ringacc->rings[fwd_id].use_count++;
423 dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);
424
425 mutex_unlock(&ringacc->req_lock);
426 return 0;
427
428 error:
429 module_put(ringacc->dev->driver->owner);
430 err_module_get:
431 mutex_unlock(&ringacc->req_lock);
432 return ret;
433 }
434
k3_ringacc_request_rings_pair(struct k3_ringacc * ringacc,int fwd_id,int compl_id,struct k3_ring ** fwd_ring,struct k3_ring ** compl_ring)435 int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc,
436 int fwd_id, int compl_id,
437 struct k3_ring **fwd_ring,
438 struct k3_ring **compl_ring)
439 {
440 int ret = 0;
441
442 if (!fwd_ring || !compl_ring)
443 return -EINVAL;
444
445 if (ringacc->dma_rings)
446 return k3_dmaring_request_dual_ring(ringacc, fwd_id,
447 fwd_ring, compl_ring);
448
449 *fwd_ring = k3_ringacc_request_ring(ringacc, fwd_id, 0);
450 if (!(*fwd_ring))
451 return -ENODEV;
452
453 *compl_ring = k3_ringacc_request_ring(ringacc, compl_id, 0);
454 if (!(*compl_ring)) {
455 k3_ringacc_ring_free(*fwd_ring);
456 ret = -ENODEV;
457 }
458
459 return ret;
460 }
461 EXPORT_SYMBOL_GPL(k3_ringacc_request_rings_pair);
462
k3_ringacc_ring_reset_sci(struct k3_ring * ring)463 static void k3_ringacc_ring_reset_sci(struct k3_ring *ring)
464 {
465 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
466 struct k3_ringacc *ringacc = ring->parent;
467 int ret;
468
469 ring_cfg.nav_id = ringacc->tisci_dev_id;
470 ring_cfg.index = ring->ring_id;
471 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID;
472 ring_cfg.count = ring->size;
473
474 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
475 if (ret)
476 dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
477 ret, ring->ring_id);
478 }
479
k3_ringacc_ring_reset(struct k3_ring * ring)480 void k3_ringacc_ring_reset(struct k3_ring *ring)
481 {
482 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
483 return;
484
485 memset(&ring->state, 0, sizeof(ring->state));
486
487 k3_ringacc_ring_reset_sci(ring);
488 }
489 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset);
490
k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring * ring,enum k3_ring_mode mode)491 static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_ring *ring,
492 enum k3_ring_mode mode)
493 {
494 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
495 struct k3_ringacc *ringacc = ring->parent;
496 int ret;
497
498 ring_cfg.nav_id = ringacc->tisci_dev_id;
499 ring_cfg.index = ring->ring_id;
500 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_RING_MODE_VALID;
501 ring_cfg.mode = mode;
502
503 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
504 if (ret)
505 dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
506 ret, ring->ring_id);
507 }
508
k3_ringacc_ring_reset_dma(struct k3_ring * ring,u32 occ)509 void k3_ringacc_ring_reset_dma(struct k3_ring *ring, u32 occ)
510 {
511 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
512 return;
513
514 if (!ring->parent->dma_ring_reset_quirk)
515 goto reset;
516
517 if (!occ)
518 occ = k3_ringacc_ring_read_occ(ring);
519
520 if (occ) {
521 u32 db_ring_cnt, db_ring_cnt_cur;
522
523 dev_dbg(ring->parent->dev, "%s %u occ: %u\n", __func__,
524 ring->ring_id, occ);
525 /* TI-SCI ring reset */
526 k3_ringacc_ring_reset_sci(ring);
527
528 /*
529 * Setup the ring in ring/doorbell mode (if not already in this
530 * mode)
531 */
532 if (ring->mode != K3_RINGACC_RING_MODE_RING)
533 k3_ringacc_ring_reconfig_qmode_sci(
534 ring, K3_RINGACC_RING_MODE_RING);
535 /*
536 * Ring the doorbell 2**22 – ringOcc times.
537 * This will wrap the internal UDMAP ring state occupancy
538 * counter (which is 21-bits wide) to 0.
539 */
540 db_ring_cnt = (1U << 22) - occ;
541
542 while (db_ring_cnt != 0) {
543 /*
544 * Ring the doorbell with the maximum count each
545 * iteration if possible to minimize the total
546 * of writes
547 */
548 if (db_ring_cnt > K3_RINGACC_MAX_DB_RING_CNT)
549 db_ring_cnt_cur = K3_RINGACC_MAX_DB_RING_CNT;
550 else
551 db_ring_cnt_cur = db_ring_cnt;
552
553 writel(db_ring_cnt_cur, &ring->rt->db);
554 db_ring_cnt -= db_ring_cnt_cur;
555 }
556
557 /* Restore the original ring mode (if not ring mode) */
558 if (ring->mode != K3_RINGACC_RING_MODE_RING)
559 k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
560 }
561
562 reset:
563 /* Reset the ring */
564 k3_ringacc_ring_reset(ring);
565 }
566 EXPORT_SYMBOL_GPL(k3_ringacc_ring_reset_dma);
567
k3_ringacc_ring_free_sci(struct k3_ring * ring)568 static void k3_ringacc_ring_free_sci(struct k3_ring *ring)
569 {
570 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
571 struct k3_ringacc *ringacc = ring->parent;
572 int ret;
573
574 ring_cfg.nav_id = ringacc->tisci_dev_id;
575 ring_cfg.index = ring->ring_id;
576 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
577
578 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
579 if (ret)
580 dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
581 ret, ring->ring_id);
582 }
583
k3_ringacc_ring_free(struct k3_ring * ring)584 int k3_ringacc_ring_free(struct k3_ring *ring)
585 {
586 struct k3_ringacc *ringacc;
587
588 if (!ring)
589 return -EINVAL;
590
591 ringacc = ring->parent;
592
593 /*
594 * DMA rings: rings shared memory and configuration, only forward ring
595 * is configured and reverse ring considered as slave.
596 */
597 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
598 return 0;
599
600 dev_dbg(ring->parent->dev, "flags: 0x%08x\n", ring->flags);
601
602 if (!test_bit(ring->ring_id, ringacc->rings_inuse))
603 return -EINVAL;
604
605 mutex_lock(&ringacc->req_lock);
606
607 if (--ring->use_count)
608 goto out;
609
610 if (!(ring->flags & K3_RING_FLAG_BUSY))
611 goto no_init;
612
613 k3_ringacc_ring_free_sci(ring);
614
615 dma_free_coherent(ring->dma_dev,
616 ring->size * (4 << ring->elm_size),
617 ring->ring_mem_virt, ring->ring_mem_dma);
618 ring->flags = 0;
619 ring->ops = NULL;
620 ring->dma_dev = NULL;
621 ring->asel = 0;
622
623 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED) {
624 clear_bit(ring->proxy_id, ringacc->proxy_inuse);
625 ring->proxy = NULL;
626 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
627 }
628
629 no_init:
630 clear_bit(ring->ring_id, ringacc->rings_inuse);
631
632 module_put(ringacc->dev->driver->owner);
633
634 out:
635 mutex_unlock(&ringacc->req_lock);
636 return 0;
637 }
638 EXPORT_SYMBOL_GPL(k3_ringacc_ring_free);
639
k3_ringacc_get_ring_id(struct k3_ring * ring)640 u32 k3_ringacc_get_ring_id(struct k3_ring *ring)
641 {
642 if (!ring)
643 return -EINVAL;
644
645 return ring->ring_id;
646 }
647 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_id);
648
k3_ringacc_get_tisci_dev_id(struct k3_ring * ring)649 u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring)
650 {
651 if (!ring)
652 return -EINVAL;
653
654 return ring->parent->tisci_dev_id;
655 }
656 EXPORT_SYMBOL_GPL(k3_ringacc_get_tisci_dev_id);
657
k3_ringacc_get_ring_irq_num(struct k3_ring * ring)658 int k3_ringacc_get_ring_irq_num(struct k3_ring *ring)
659 {
660 int irq_num;
661
662 if (!ring)
663 return -EINVAL;
664
665 irq_num = msi_get_virq(ring->parent->dev, ring->ring_id);
666 if (irq_num <= 0)
667 irq_num = -EINVAL;
668 return irq_num;
669 }
670 EXPORT_SYMBOL_GPL(k3_ringacc_get_ring_irq_num);
671
k3_ringacc_ring_cfg_sci(struct k3_ring * ring)672 static int k3_ringacc_ring_cfg_sci(struct k3_ring *ring)
673 {
674 struct ti_sci_msg_rm_ring_cfg ring_cfg = { 0 };
675 struct k3_ringacc *ringacc = ring->parent;
676 int ret;
677
678 if (!ringacc->tisci)
679 return -EINVAL;
680
681 ring_cfg.nav_id = ringacc->tisci_dev_id;
682 ring_cfg.index = ring->ring_id;
683 ring_cfg.valid_params = TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER;
684 ring_cfg.addr_lo = lower_32_bits(ring->ring_mem_dma);
685 ring_cfg.addr_hi = upper_32_bits(ring->ring_mem_dma);
686 ring_cfg.count = ring->size;
687 ring_cfg.mode = ring->mode;
688 ring_cfg.size = ring->elm_size;
689 ring_cfg.asel = ring->asel;
690
691 ret = ringacc->tisci_ring_ops->set_cfg(ringacc->tisci, &ring_cfg);
692 if (ret)
693 dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
694 ret, ring->ring_id);
695
696 return ret;
697 }
698
k3_dmaring_cfg(struct k3_ring * ring,struct k3_ring_cfg * cfg)699 static int k3_dmaring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
700 {
701 struct k3_ringacc *ringacc;
702 struct k3_ring *reverse_ring;
703 int ret = 0;
704
705 if (cfg->elm_size != K3_RINGACC_RING_ELSIZE_8 ||
706 cfg->mode != K3_RINGACC_RING_MODE_RING ||
707 cfg->size & ~K3_DMARING_CFG_RING_SIZE_ELCNT_MASK)
708 return -EINVAL;
709
710 ringacc = ring->parent;
711
712 /*
713 * DMA rings: rings shared memory and configuration, only forward ring
714 * is configured and reverse ring considered as slave.
715 */
716 if (ringacc->dma_rings && (ring->flags & K3_RING_FLAG_REVERSE))
717 return 0;
718
719 if (!test_bit(ring->ring_id, ringacc->rings_inuse))
720 return -EINVAL;
721
722 ring->size = cfg->size;
723 ring->elm_size = cfg->elm_size;
724 ring->mode = cfg->mode;
725 ring->asel = cfg->asel;
726 ring->dma_dev = cfg->dma_dev;
727 if (!ring->dma_dev) {
728 dev_warn(ringacc->dev, "dma_dev is not provided for ring%d\n",
729 ring->ring_id);
730 ring->dma_dev = ringacc->dev;
731 }
732
733 memset(&ring->state, 0, sizeof(ring->state));
734
735 ring->ops = &k3_dmaring_fwd_ops;
736
737 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
738 ring->size * (4 << ring->elm_size),
739 &ring->ring_mem_dma, GFP_KERNEL);
740 if (!ring->ring_mem_virt) {
741 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
742 ret = -ENOMEM;
743 goto err_free_ops;
744 }
745
746 ret = k3_ringacc_ring_cfg_sci(ring);
747 if (ret)
748 goto err_free_mem;
749
750 ring->flags |= K3_RING_FLAG_BUSY;
751
752 k3_ringacc_ring_dump(ring);
753
754 /* DMA rings: configure reverse ring */
755 reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
756 reverse_ring->size = cfg->size;
757 reverse_ring->elm_size = cfg->elm_size;
758 reverse_ring->mode = cfg->mode;
759 reverse_ring->asel = cfg->asel;
760 memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
761 reverse_ring->ops = &k3_dmaring_reverse_ops;
762
763 reverse_ring->ring_mem_virt = ring->ring_mem_virt;
764 reverse_ring->ring_mem_dma = ring->ring_mem_dma;
765 reverse_ring->flags |= K3_RING_FLAG_BUSY;
766 k3_ringacc_ring_dump(reverse_ring);
767
768 return 0;
769
770 err_free_mem:
771 dma_free_coherent(ring->dma_dev,
772 ring->size * (4 << ring->elm_size),
773 ring->ring_mem_virt,
774 ring->ring_mem_dma);
775 err_free_ops:
776 ring->ops = NULL;
777 ring->proxy = NULL;
778 ring->dma_dev = NULL;
779 ring->asel = 0;
780 return ret;
781 }
782
k3_ringacc_ring_cfg(struct k3_ring * ring,struct k3_ring_cfg * cfg)783 int k3_ringacc_ring_cfg(struct k3_ring *ring, struct k3_ring_cfg *cfg)
784 {
785 struct k3_ringacc *ringacc;
786 int ret = 0;
787
788 if (!ring || !cfg)
789 return -EINVAL;
790
791 ringacc = ring->parent;
792
793 if (ringacc->dma_rings)
794 return k3_dmaring_cfg(ring, cfg);
795
796 if (cfg->elm_size > K3_RINGACC_RING_ELSIZE_256 ||
797 cfg->mode >= K3_RINGACC_RING_MODE_INVALID ||
798 cfg->size & ~K3_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
799 !test_bit(ring->ring_id, ringacc->rings_inuse))
800 return -EINVAL;
801
802 if (cfg->mode == K3_RINGACC_RING_MODE_MESSAGE &&
803 ring->proxy_id == K3_RINGACC_PROXY_NOT_USED &&
804 cfg->elm_size > K3_RINGACC_RING_ELSIZE_8) {
805 dev_err(ringacc->dev,
806 "Message mode must use proxy for %u element size\n",
807 4 << ring->elm_size);
808 return -EINVAL;
809 }
810
811 /*
812 * In case of shared ring only the first user (master user) can
813 * configure the ring. The sequence should be by the client:
814 * ring = k3_ringacc_request_ring(ringacc, ring_id, 0); # master user
815 * k3_ringacc_ring_cfg(ring, cfg); # master configuration
816 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
817 * k3_ringacc_request_ring(ringacc, ring_id, K3_RING_FLAG_SHARED);
818 */
819 if (ring->use_count != 1)
820 return 0;
821
822 ring->size = cfg->size;
823 ring->elm_size = cfg->elm_size;
824 ring->mode = cfg->mode;
825 memset(&ring->state, 0, sizeof(ring->state));
826
827 if (ring->proxy_id != K3_RINGACC_PROXY_NOT_USED)
828 ring->proxy = ringacc->proxy_target_base +
829 ring->proxy_id * K3_RINGACC_PROXY_TARGET_STEP;
830
831 switch (ring->mode) {
832 case K3_RINGACC_RING_MODE_RING:
833 ring->ops = &k3_ring_mode_ring_ops;
834 ring->dma_dev = cfg->dma_dev;
835 if (!ring->dma_dev)
836 ring->dma_dev = ringacc->dev;
837 break;
838 case K3_RINGACC_RING_MODE_MESSAGE:
839 ring->dma_dev = ringacc->dev;
840 if (ring->proxy)
841 ring->ops = &k3_ring_mode_proxy_ops;
842 else
843 ring->ops = &k3_ring_mode_msg_ops;
844 break;
845 default:
846 ring->ops = NULL;
847 ret = -EINVAL;
848 goto err_free_proxy;
849 }
850
851 ring->ring_mem_virt = dma_alloc_coherent(ring->dma_dev,
852 ring->size * (4 << ring->elm_size),
853 &ring->ring_mem_dma, GFP_KERNEL);
854 if (!ring->ring_mem_virt) {
855 dev_err(ringacc->dev, "Failed to alloc ring mem\n");
856 ret = -ENOMEM;
857 goto err_free_ops;
858 }
859
860 ret = k3_ringacc_ring_cfg_sci(ring);
861
862 if (ret)
863 goto err_free_mem;
864
865 ring->flags |= K3_RING_FLAG_BUSY;
866 ring->flags |= (cfg->flags & K3_RINGACC_RING_SHARED) ?
867 K3_RING_FLAG_SHARED : 0;
868
869 k3_ringacc_ring_dump(ring);
870
871 return 0;
872
873 err_free_mem:
874 dma_free_coherent(ring->dma_dev,
875 ring->size * (4 << ring->elm_size),
876 ring->ring_mem_virt,
877 ring->ring_mem_dma);
878 err_free_ops:
879 ring->ops = NULL;
880 ring->dma_dev = NULL;
881 err_free_proxy:
882 ring->proxy = NULL;
883 return ret;
884 }
885 EXPORT_SYMBOL_GPL(k3_ringacc_ring_cfg);
886
k3_ringacc_ring_get_size(struct k3_ring * ring)887 u32 k3_ringacc_ring_get_size(struct k3_ring *ring)
888 {
889 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
890 return -EINVAL;
891
892 return ring->size;
893 }
894 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_size);
895
k3_ringacc_ring_get_free(struct k3_ring * ring)896 u32 k3_ringacc_ring_get_free(struct k3_ring *ring)
897 {
898 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
899 return -EINVAL;
900
901 if (!ring->state.free)
902 ring->state.free = ring->size - k3_ringacc_ring_read_occ(ring);
903
904 return ring->state.free;
905 }
906 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_free);
907
k3_ringacc_ring_get_occ(struct k3_ring * ring)908 u32 k3_ringacc_ring_get_occ(struct k3_ring *ring)
909 {
910 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
911 return -EINVAL;
912
913 return k3_ringacc_ring_read_occ(ring);
914 }
915 EXPORT_SYMBOL_GPL(k3_ringacc_ring_get_occ);
916
k3_ringacc_ring_is_full(struct k3_ring * ring)917 u32 k3_ringacc_ring_is_full(struct k3_ring *ring)
918 {
919 return !k3_ringacc_ring_get_free(ring);
920 }
921 EXPORT_SYMBOL_GPL(k3_ringacc_ring_is_full);
922
923 enum k3_ringacc_access_mode {
924 K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
925 K3_RINGACC_ACCESS_MODE_POP_HEAD,
926 K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
927 K3_RINGACC_ACCESS_MODE_POP_TAIL,
928 K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
929 K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
930 };
931
932 #define K3_RINGACC_PROXY_MODE(x) (((x) & 0x3) << 16)
933 #define K3_RINGACC_PROXY_ELSIZE(x) (((x) & 0x7) << 24)
k3_ringacc_ring_cfg_proxy(struct k3_ring * ring,enum k3_ringacc_proxy_access_mode mode)934 static int k3_ringacc_ring_cfg_proxy(struct k3_ring *ring,
935 enum k3_ringacc_proxy_access_mode mode)
936 {
937 u32 val;
938
939 val = ring->ring_id;
940 val |= K3_RINGACC_PROXY_MODE(mode);
941 val |= K3_RINGACC_PROXY_ELSIZE(ring->elm_size);
942 writel(val, &ring->proxy->control);
943 return 0;
944 }
945
k3_ringacc_ring_access_proxy(struct k3_ring * ring,void * elem,enum k3_ringacc_access_mode access_mode)946 static int k3_ringacc_ring_access_proxy(struct k3_ring *ring, void *elem,
947 enum k3_ringacc_access_mode access_mode)
948 {
949 void __iomem *ptr;
950
951 ptr = (void __iomem *)&ring->proxy->data;
952
953 switch (access_mode) {
954 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
955 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
956 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_HEAD);
957 break;
958 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
959 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
960 k3_ringacc_ring_cfg_proxy(ring, PROXY_ACCESS_MODE_TAIL);
961 break;
962 default:
963 return -EINVAL;
964 }
965
966 ptr += k3_ringacc_ring_get_fifo_pos(ring);
967
968 switch (access_mode) {
969 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
970 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
971 dev_dbg(ring->parent->dev,
972 "proxy:memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
973 access_mode);
974 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
975 ring->state.occ--;
976 break;
977 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
978 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
979 dev_dbg(ring->parent->dev,
980 "proxy:memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
981 access_mode);
982 memcpy_toio(ptr, elem, (4 << ring->elm_size));
983 ring->state.free--;
984 break;
985 default:
986 return -EINVAL;
987 }
988
989 dev_dbg(ring->parent->dev, "proxy: free%d occ%d\n", ring->state.free,
990 ring->state.occ);
991 return 0;
992 }
993
k3_ringacc_ring_push_head_proxy(struct k3_ring * ring,void * elem)994 static int k3_ringacc_ring_push_head_proxy(struct k3_ring *ring, void *elem)
995 {
996 return k3_ringacc_ring_access_proxy(ring, elem,
997 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
998 }
999
k3_ringacc_ring_push_tail_proxy(struct k3_ring * ring,void * elem)1000 static int k3_ringacc_ring_push_tail_proxy(struct k3_ring *ring, void *elem)
1001 {
1002 return k3_ringacc_ring_access_proxy(ring, elem,
1003 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
1004 }
1005
k3_ringacc_ring_pop_head_proxy(struct k3_ring * ring,void * elem)1006 static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
1007 {
1008 return k3_ringacc_ring_access_proxy(ring, elem,
1009 K3_RINGACC_ACCESS_MODE_POP_HEAD);
1010 }
1011
k3_ringacc_ring_pop_tail_proxy(struct k3_ring * ring,void * elem)1012 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
1013 {
1014 return k3_ringacc_ring_access_proxy(ring, elem,
1015 K3_RINGACC_ACCESS_MODE_POP_HEAD);
1016 }
1017
k3_ringacc_ring_access_io(struct k3_ring * ring,void * elem,enum k3_ringacc_access_mode access_mode)1018 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
1019 enum k3_ringacc_access_mode access_mode)
1020 {
1021 void __iomem *ptr;
1022
1023 switch (access_mode) {
1024 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1025 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1026 ptr = (void __iomem *)&ring->fifos->head_data;
1027 break;
1028 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1029 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1030 ptr = (void __iomem *)&ring->fifos->tail_data;
1031 break;
1032 default:
1033 return -EINVAL;
1034 }
1035
1036 ptr += k3_ringacc_ring_get_fifo_pos(ring);
1037
1038 switch (access_mode) {
1039 case K3_RINGACC_ACCESS_MODE_POP_HEAD:
1040 case K3_RINGACC_ACCESS_MODE_POP_TAIL:
1041 dev_dbg(ring->parent->dev,
1042 "memcpy_fromio(x): --> ptr(%p), mode:%d\n", ptr,
1043 access_mode);
1044 memcpy_fromio(elem, ptr, (4 << ring->elm_size));
1045 ring->state.occ--;
1046 break;
1047 case K3_RINGACC_ACCESS_MODE_PUSH_TAIL:
1048 case K3_RINGACC_ACCESS_MODE_PUSH_HEAD:
1049 dev_dbg(ring->parent->dev,
1050 "memcpy_toio(x): --> ptr(%p), mode:%d\n", ptr,
1051 access_mode);
1052 memcpy_toio(ptr, elem, (4 << ring->elm_size));
1053 ring->state.free--;
1054 break;
1055 default:
1056 return -EINVAL;
1057 }
1058
1059 dev_dbg(ring->parent->dev, "free%d index%d occ%d index%d\n",
1060 ring->state.free, ring->state.windex, ring->state.occ,
1061 ring->state.rindex);
1062 return 0;
1063 }
1064
k3_ringacc_ring_push_head_io(struct k3_ring * ring,void * elem)1065 static int k3_ringacc_ring_push_head_io(struct k3_ring *ring, void *elem)
1066 {
1067 return k3_ringacc_ring_access_io(ring, elem,
1068 K3_RINGACC_ACCESS_MODE_PUSH_HEAD);
1069 }
1070
k3_ringacc_ring_push_io(struct k3_ring * ring,void * elem)1071 static int k3_ringacc_ring_push_io(struct k3_ring *ring, void *elem)
1072 {
1073 return k3_ringacc_ring_access_io(ring, elem,
1074 K3_RINGACC_ACCESS_MODE_PUSH_TAIL);
1075 }
1076
k3_ringacc_ring_pop_io(struct k3_ring * ring,void * elem)1077 static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
1078 {
1079 return k3_ringacc_ring_access_io(ring, elem,
1080 K3_RINGACC_ACCESS_MODE_POP_HEAD);
1081 }
1082
k3_ringacc_ring_pop_tail_io(struct k3_ring * ring,void * elem)1083 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
1084 {
1085 return k3_ringacc_ring_access_io(ring, elem,
1086 K3_RINGACC_ACCESS_MODE_POP_HEAD);
1087 }
1088
1089 /*
1090 * The element is 48 bits of address + ASEL bits in the ring.
1091 * ASEL is used by the DMAs and should be removed for the kernel as it is not
1092 * part of the physical memory address.
1093 */
k3_dmaring_remove_asel_from_elem(u64 * elem)1094 static void k3_dmaring_remove_asel_from_elem(u64 *elem)
1095 {
1096 *elem &= GENMASK_ULL(K3_ADDRESS_ASEL_SHIFT - 1, 0);
1097 }
1098
k3_dmaring_fwd_pop(struct k3_ring * ring,void * elem)1099 static int k3_dmaring_fwd_pop(struct k3_ring *ring, void *elem)
1100 {
1101 void *elem_ptr;
1102 u32 elem_idx;
1103
1104 /*
1105 * DMA rings: forward ring is always tied DMA channel and HW does not
1106 * maintain any state data required for POP operation and its unknown
1107 * how much elements were consumed by HW. So, to actually
1108 * do POP, the read pointer has to be recalculated every time.
1109 */
1110 ring->state.occ = k3_ringacc_ring_read_occ(ring);
1111 if (ring->state.windex >= ring->state.occ)
1112 elem_idx = ring->state.windex - ring->state.occ;
1113 else
1114 elem_idx = ring->size - (ring->state.occ - ring->state.windex);
1115
1116 elem_ptr = k3_ringacc_get_elm_addr(ring, elem_idx);
1117 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1118 k3_dmaring_remove_asel_from_elem(elem);
1119
1120 ring->state.occ--;
1121 writel(-1, &ring->rt->db);
1122
1123 dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
1124 __func__, ring->state.occ, ring->state.windex, elem_idx,
1125 elem_ptr);
1126 return 0;
1127 }
1128
k3_dmaring_reverse_pop(struct k3_ring * ring,void * elem)1129 static int k3_dmaring_reverse_pop(struct k3_ring *ring, void *elem)
1130 {
1131 void *elem_ptr;
1132
1133 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1134
1135 if (ring->state.occ) {
1136 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1137 k3_dmaring_remove_asel_from_elem(elem);
1138
1139 ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1140 ring->state.occ--;
1141 writel(-1 & K3_DMARING_RT_DB_ENTRY_MASK, &ring->rt->db);
1142 } else if (ring->state.tdown_complete) {
1143 dma_addr_t *value = elem;
1144
1145 *value = CPPI5_TDCM_MARKER;
1146 writel(K3_DMARING_RT_DB_TDOWN_ACK, &ring->rt->db);
1147 ring->state.tdown_complete = false;
1148 }
1149
1150 dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
1151 __func__, ring->state.occ, ring->state.rindex, elem_ptr);
1152 return 0;
1153 }
1154
k3_ringacc_ring_push_mem(struct k3_ring * ring,void * elem)1155 static int k3_ringacc_ring_push_mem(struct k3_ring *ring, void *elem)
1156 {
1157 void *elem_ptr;
1158
1159 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.windex);
1160
1161 memcpy(elem_ptr, elem, (4 << ring->elm_size));
1162 if (ring->parent->dma_rings) {
1163 u64 *addr = elem_ptr;
1164
1165 *addr |= ((u64)ring->asel << K3_ADDRESS_ASEL_SHIFT);
1166 }
1167
1168 ring->state.windex = (ring->state.windex + 1) % ring->size;
1169 ring->state.free--;
1170 writel(1, &ring->rt->db);
1171
1172 dev_dbg(ring->parent->dev, "ring_push_mem: free%d index%d\n",
1173 ring->state.free, ring->state.windex);
1174
1175 return 0;
1176 }
1177
k3_ringacc_ring_pop_mem(struct k3_ring * ring,void * elem)1178 static int k3_ringacc_ring_pop_mem(struct k3_ring *ring, void *elem)
1179 {
1180 void *elem_ptr;
1181
1182 elem_ptr = k3_ringacc_get_elm_addr(ring, ring->state.rindex);
1183
1184 memcpy(elem, elem_ptr, (4 << ring->elm_size));
1185
1186 ring->state.rindex = (ring->state.rindex + 1) % ring->size;
1187 ring->state.occ--;
1188 writel(-1, &ring->rt->db);
1189
1190 dev_dbg(ring->parent->dev, "ring_pop_mem: occ%d index%d pos_ptr%p\n",
1191 ring->state.occ, ring->state.rindex, elem_ptr);
1192 return 0;
1193 }
1194
k3_ringacc_ring_push(struct k3_ring * ring,void * elem)1195 int k3_ringacc_ring_push(struct k3_ring *ring, void *elem)
1196 {
1197 int ret = -EOPNOTSUPP;
1198
1199 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1200 return -EINVAL;
1201
1202 dev_dbg(ring->parent->dev, "ring_push: free%d index%d\n",
1203 ring->state.free, ring->state.windex);
1204
1205 if (k3_ringacc_ring_is_full(ring))
1206 return -ENOMEM;
1207
1208 if (ring->ops && ring->ops->push_tail)
1209 ret = ring->ops->push_tail(ring, elem);
1210
1211 return ret;
1212 }
1213 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push);
1214
k3_ringacc_ring_push_head(struct k3_ring * ring,void * elem)1215 int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem)
1216 {
1217 int ret = -EOPNOTSUPP;
1218
1219 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1220 return -EINVAL;
1221
1222 dev_dbg(ring->parent->dev, "ring_push_head: free%d index%d\n",
1223 ring->state.free, ring->state.windex);
1224
1225 if (k3_ringacc_ring_is_full(ring))
1226 return -ENOMEM;
1227
1228 if (ring->ops && ring->ops->push_head)
1229 ret = ring->ops->push_head(ring, elem);
1230
1231 return ret;
1232 }
1233 EXPORT_SYMBOL_GPL(k3_ringacc_ring_push_head);
1234
k3_ringacc_ring_pop(struct k3_ring * ring,void * elem)1235 int k3_ringacc_ring_pop(struct k3_ring *ring, void *elem)
1236 {
1237 int ret = -EOPNOTSUPP;
1238
1239 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1240 return -EINVAL;
1241
1242 if (!ring->state.occ)
1243 k3_ringacc_ring_update_occ(ring);
1244
1245 dev_dbg(ring->parent->dev, "ring_pop: occ%d index%d\n", ring->state.occ,
1246 ring->state.rindex);
1247
1248 if (!ring->state.occ && !ring->state.tdown_complete)
1249 return -ENODATA;
1250
1251 if (ring->ops && ring->ops->pop_head)
1252 ret = ring->ops->pop_head(ring, elem);
1253
1254 return ret;
1255 }
1256 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop);
1257
k3_ringacc_ring_pop_tail(struct k3_ring * ring,void * elem)1258 int k3_ringacc_ring_pop_tail(struct k3_ring *ring, void *elem)
1259 {
1260 int ret = -EOPNOTSUPP;
1261
1262 if (!ring || !(ring->flags & K3_RING_FLAG_BUSY))
1263 return -EINVAL;
1264
1265 if (!ring->state.occ)
1266 k3_ringacc_ring_update_occ(ring);
1267
1268 dev_dbg(ring->parent->dev, "ring_pop_tail: occ%d index%d\n",
1269 ring->state.occ, ring->state.rindex);
1270
1271 if (!ring->state.occ)
1272 return -ENODATA;
1273
1274 if (ring->ops && ring->ops->pop_tail)
1275 ret = ring->ops->pop_tail(ring, elem);
1276
1277 return ret;
1278 }
1279 EXPORT_SYMBOL_GPL(k3_ringacc_ring_pop_tail);
1280
of_k3_ringacc_get_by_phandle(struct device_node * np,const char * property)1281 struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np,
1282 const char *property)
1283 {
1284 struct device_node *ringacc_np;
1285 struct k3_ringacc *ringacc = ERR_PTR(-EPROBE_DEFER);
1286 struct k3_ringacc *entry;
1287
1288 ringacc_np = of_parse_phandle(np, property, 0);
1289 if (!ringacc_np)
1290 return ERR_PTR(-ENODEV);
1291
1292 mutex_lock(&k3_ringacc_list_lock);
1293 list_for_each_entry(entry, &k3_ringacc_list, list)
1294 if (entry->dev->of_node == ringacc_np) {
1295 ringacc = entry;
1296 break;
1297 }
1298 mutex_unlock(&k3_ringacc_list_lock);
1299 of_node_put(ringacc_np);
1300
1301 return ringacc;
1302 }
1303 EXPORT_SYMBOL_GPL(of_k3_ringacc_get_by_phandle);
1304
k3_ringacc_probe_dt(struct k3_ringacc * ringacc)1305 static int k3_ringacc_probe_dt(struct k3_ringacc *ringacc)
1306 {
1307 struct device_node *node = ringacc->dev->of_node;
1308 struct device *dev = ringacc->dev;
1309 struct platform_device *pdev = to_platform_device(dev);
1310 int ret;
1311
1312 if (!node) {
1313 dev_err(dev, "device tree info unavailable\n");
1314 return -ENODEV;
1315 }
1316
1317 ret = of_property_read_u32(node, "ti,num-rings", &ringacc->num_rings);
1318 if (ret) {
1319 dev_err(dev, "ti,num-rings read failure %d\n", ret);
1320 return ret;
1321 }
1322
1323 ringacc->tisci = ti_sci_get_by_phandle(node, "ti,sci");
1324 if (IS_ERR(ringacc->tisci)) {
1325 ret = PTR_ERR(ringacc->tisci);
1326 if (ret != -EPROBE_DEFER)
1327 dev_err(dev, "ti,sci read fail %d\n", ret);
1328 ringacc->tisci = NULL;
1329 return ret;
1330 }
1331
1332 ret = of_property_read_u32(node, "ti,sci-dev-id",
1333 &ringacc->tisci_dev_id);
1334 if (ret) {
1335 dev_err(dev, "ti,sci-dev-id read fail %d\n", ret);
1336 return ret;
1337 }
1338
1339 pdev->id = ringacc->tisci_dev_id;
1340
1341 ringacc->rm_gp_range = devm_ti_sci_get_of_resource(ringacc->tisci, dev,
1342 ringacc->tisci_dev_id,
1343 "ti,sci-rm-range-gp-rings");
1344 if (IS_ERR(ringacc->rm_gp_range)) {
1345 dev_err(dev, "Failed to allocate MSI interrupts\n");
1346 return PTR_ERR(ringacc->rm_gp_range);
1347 }
1348
1349 return ti_sci_inta_msi_domain_alloc_irqs(ringacc->dev,
1350 ringacc->rm_gp_range);
1351 }
1352
1353 static const struct k3_ringacc_soc_data k3_ringacc_soc_data_sr1 = {
1354 .dma_ring_reset_quirk = 1,
1355 };
1356
1357 static const struct soc_device_attribute k3_ringacc_socinfo[] = {
1358 { .family = "AM65X",
1359 .revision = "SR1.0",
1360 .data = &k3_ringacc_soc_data_sr1
1361 },
1362 {/* sentinel */}
1363 };
1364
k3_ringacc_init(struct platform_device * pdev,struct k3_ringacc * ringacc)1365 static int k3_ringacc_init(struct platform_device *pdev,
1366 struct k3_ringacc *ringacc)
1367 {
1368 const struct soc_device_attribute *soc;
1369 void __iomem *base_fifo, *base_rt;
1370 struct device *dev = &pdev->dev;
1371 int ret, i;
1372
1373 dev->msi.domain = of_msi_get_domain(dev, dev->of_node,
1374 DOMAIN_BUS_TI_SCI_INTA_MSI);
1375 if (!dev->msi.domain)
1376 return -EPROBE_DEFER;
1377
1378 ret = k3_ringacc_probe_dt(ringacc);
1379 if (ret)
1380 return ret;
1381
1382 soc = soc_device_match(k3_ringacc_socinfo);
1383 if (soc && soc->data) {
1384 const struct k3_ringacc_soc_data *soc_data = soc->data;
1385
1386 ringacc->dma_ring_reset_quirk = soc_data->dma_ring_reset_quirk;
1387 }
1388
1389 base_rt = devm_platform_ioremap_resource_byname(pdev, "rt");
1390 if (IS_ERR(base_rt))
1391 return PTR_ERR(base_rt);
1392
1393 base_fifo = devm_platform_ioremap_resource_byname(pdev, "fifos");
1394 if (IS_ERR(base_fifo))
1395 return PTR_ERR(base_fifo);
1396
1397 ringacc->proxy_gcfg = devm_platform_ioremap_resource_byname(pdev, "proxy_gcfg");
1398 if (IS_ERR(ringacc->proxy_gcfg))
1399 return PTR_ERR(ringacc->proxy_gcfg);
1400
1401 ringacc->proxy_target_base = devm_platform_ioremap_resource_byname(pdev,
1402 "proxy_target");
1403 if (IS_ERR(ringacc->proxy_target_base))
1404 return PTR_ERR(ringacc->proxy_target_base);
1405
1406 ringacc->num_proxies = readl(&ringacc->proxy_gcfg->config) &
1407 K3_RINGACC_PROXY_CFG_THREADS_MASK;
1408
1409 ringacc->rings = devm_kzalloc(dev,
1410 sizeof(*ringacc->rings) *
1411 ringacc->num_rings,
1412 GFP_KERNEL);
1413 ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings,
1414 GFP_KERNEL);
1415 ringacc->proxy_inuse = devm_bitmap_zalloc(dev, ringacc->num_proxies,
1416 GFP_KERNEL);
1417
1418 if (!ringacc->rings || !ringacc->rings_inuse || !ringacc->proxy_inuse)
1419 return -ENOMEM;
1420
1421 for (i = 0; i < ringacc->num_rings; i++) {
1422 ringacc->rings[i].rt = base_rt +
1423 K3_RINGACC_RT_REGS_STEP * i;
1424 ringacc->rings[i].fifos = base_fifo +
1425 K3_RINGACC_FIFO_REGS_STEP * i;
1426 ringacc->rings[i].parent = ringacc;
1427 ringacc->rings[i].ring_id = i;
1428 ringacc->rings[i].proxy_id = K3_RINGACC_PROXY_NOT_USED;
1429 }
1430
1431 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1432
1433 dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
1434 ringacc->num_rings,
1435 ringacc->rm_gp_range->desc[0].start,
1436 ringacc->rm_gp_range->desc[0].num,
1437 ringacc->tisci_dev_id);
1438 dev_info(dev, "dma-ring-reset-quirk: %s\n",
1439 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
1440 dev_info(dev, "RA Proxy rev. %08x, num_proxies:%u\n",
1441 readl(&ringacc->proxy_gcfg->revision), ringacc->num_proxies);
1442
1443 return 0;
1444 }
1445
1446 struct ringacc_match_data {
1447 struct k3_ringacc_ops ops;
1448 };
1449
1450 static struct ringacc_match_data k3_ringacc_data = {
1451 .ops = {
1452 .init = k3_ringacc_init,
1453 },
1454 };
1455
1456 /* Match table for of_platform binding */
1457 static const struct of_device_id k3_ringacc_of_match[] = {
1458 { .compatible = "ti,am654-navss-ringacc", .data = &k3_ringacc_data, },
1459 {},
1460 };
1461 MODULE_DEVICE_TABLE(of, k3_ringacc_of_match);
1462
k3_ringacc_dmarings_init(struct platform_device * pdev,struct k3_ringacc_init_data * data)1463 struct k3_ringacc *k3_ringacc_dmarings_init(struct platform_device *pdev,
1464 struct k3_ringacc_init_data *data)
1465 {
1466 struct device *dev = &pdev->dev;
1467 struct k3_ringacc *ringacc;
1468 void __iomem *base_rt;
1469 int i;
1470
1471 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1472 if (!ringacc)
1473 return ERR_PTR(-ENOMEM);
1474
1475 ringacc->dev = dev;
1476 ringacc->dma_rings = true;
1477 ringacc->num_rings = data->num_rings;
1478 ringacc->tisci = data->tisci;
1479 ringacc->tisci_dev_id = data->tisci_dev_id;
1480
1481 mutex_init(&ringacc->req_lock);
1482
1483 base_rt = devm_platform_ioremap_resource_byname(pdev, "ringrt");
1484 if (IS_ERR(base_rt))
1485 return ERR_CAST(base_rt);
1486
1487 ringacc->rings = devm_kzalloc(dev,
1488 sizeof(*ringacc->rings) *
1489 ringacc->num_rings * 2,
1490 GFP_KERNEL);
1491 ringacc->rings_inuse = devm_bitmap_zalloc(dev, ringacc->num_rings,
1492 GFP_KERNEL);
1493
1494 if (!ringacc->rings || !ringacc->rings_inuse)
1495 return ERR_PTR(-ENOMEM);
1496
1497 for (i = 0; i < ringacc->num_rings; i++) {
1498 struct k3_ring *ring = &ringacc->rings[i];
1499
1500 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i;
1501 ring->parent = ringacc;
1502 ring->ring_id = i;
1503 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1504
1505 ring = &ringacc->rings[ringacc->num_rings + i];
1506 ring->rt = base_rt + K3_DMARING_RT_REGS_STEP * i +
1507 K3_DMARING_RT_REGS_REVERSE_OFS;
1508 ring->parent = ringacc;
1509 ring->ring_id = i;
1510 ring->proxy_id = K3_RINGACC_PROXY_NOT_USED;
1511 ring->flags = K3_RING_FLAG_REVERSE;
1512 }
1513
1514 ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;
1515
1516 dev_info(dev, "Number of rings: %u\n", ringacc->num_rings);
1517
1518 return ringacc;
1519 }
1520 EXPORT_SYMBOL_GPL(k3_ringacc_dmarings_init);
1521
k3_ringacc_probe(struct platform_device * pdev)1522 static int k3_ringacc_probe(struct platform_device *pdev)
1523 {
1524 const struct ringacc_match_data *match_data;
1525 struct device *dev = &pdev->dev;
1526 struct k3_ringacc *ringacc;
1527 int ret;
1528
1529 match_data = of_device_get_match_data(&pdev->dev);
1530 if (!match_data)
1531 return -ENODEV;
1532
1533 ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
1534 if (!ringacc)
1535 return -ENOMEM;
1536
1537 ringacc->dev = dev;
1538 mutex_init(&ringacc->req_lock);
1539 ringacc->ops = &match_data->ops;
1540
1541 ret = ringacc->ops->init(pdev, ringacc);
1542 if (ret)
1543 return ret;
1544
1545 dev_set_drvdata(dev, ringacc);
1546
1547 mutex_lock(&k3_ringacc_list_lock);
1548 list_add_tail(&ringacc->list, &k3_ringacc_list);
1549 mutex_unlock(&k3_ringacc_list_lock);
1550
1551 return 0;
1552 }
1553
k3_ringacc_remove(struct platform_device * pdev)1554 static void k3_ringacc_remove(struct platform_device *pdev)
1555 {
1556 struct k3_ringacc *ringacc = dev_get_drvdata(&pdev->dev);
1557
1558 mutex_lock(&k3_ringacc_list_lock);
1559 list_del(&ringacc->list);
1560 mutex_unlock(&k3_ringacc_list_lock);
1561 }
1562
1563 static struct platform_driver k3_ringacc_driver = {
1564 .probe = k3_ringacc_probe,
1565 .remove = k3_ringacc_remove,
1566 .driver = {
1567 .name = "k3-ringacc",
1568 .of_match_table = k3_ringacc_of_match,
1569 .suppress_bind_attrs = true,
1570 },
1571 };
1572 module_platform_driver(k3_ringacc_driver);
1573
1574 MODULE_LICENSE("GPL");
1575 MODULE_DESCRIPTION("TI Ringacc driver for K3 SOCs");
1576 MODULE_AUTHOR("Grygorii Strashko <grygorii.strashko@ti.com>");
1577