xref: /linux/drivers/soc/ti/knav_qmss_acc.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Keystone accumulator queue manager
4  *
5  * Copyright (C) 2014 Texas Instruments Incorporated - http://www.ti.com
6  * Author:	Sandeep Nair <sandeep_n@ti.com>
7  *		Cyril Chemparathy <cyril@ti.com>
8  *		Santosh Shilimkar <santosh.shilimkar@ti.com>
9  */
10 
11 #include <linux/dma-mapping.h>
12 #include <linux/io.h>
13 #include <linux/interrupt.h>
14 #include <linux/module.h>
15 #include <linux/of_address.h>
16 #include <linux/soc/ti/knav_qmss.h>
17 
18 #include "knav_qmss.h"
19 
20 #define knav_range_offset_to_inst(kdev, range, q)	\
21 	(range->queue_base_inst + (q << kdev->inst_shift))
22 
__knav_acc_notify(struct knav_range_info * range,struct knav_acc_channel * acc)23 static void __knav_acc_notify(struct knav_range_info *range,
24 				struct knav_acc_channel *acc)
25 {
26 	struct knav_device *kdev = range->kdev;
27 	struct knav_queue_inst *inst;
28 	int range_base, queue;
29 
30 	range_base = kdev->base_id + range->queue_base;
31 
32 	if (range->flags & RANGE_MULTI_QUEUE) {
33 		for (queue = 0; queue < range->num_queues; queue++) {
34 			inst = knav_range_offset_to_inst(kdev, range,
35 								queue);
36 			if (inst->notify_needed) {
37 				inst->notify_needed = 0;
38 				dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
39 					range_base + queue);
40 				knav_queue_notify(inst);
41 			}
42 		}
43 	} else {
44 		queue = acc->channel - range->acc_info.start_channel;
45 		inst = knav_range_offset_to_inst(kdev, range, queue);
46 		dev_dbg(kdev->dev, "acc-irq: notifying %d\n",
47 			range_base + queue);
48 		knav_queue_notify(inst);
49 	}
50 }
51 
knav_acc_set_notify(struct knav_range_info * range,struct knav_queue_inst * kq,bool enabled)52 static int knav_acc_set_notify(struct knav_range_info *range,
53 				struct knav_queue_inst *kq,
54 				bool enabled)
55 {
56 	struct knav_pdsp_info *pdsp = range->acc_info.pdsp;
57 	struct knav_device *kdev = range->kdev;
58 	u32 mask, offset;
59 
60 	/*
61 	 * when enabling, we need to re-trigger an interrupt if we
62 	 * have descriptors pending
63 	 */
64 	if (!enabled || atomic_read(&kq->desc_count) <= 0)
65 		return 0;
66 
67 	kq->notify_needed = 1;
68 	atomic_inc(&kq->acc->retrigger_count);
69 	mask = BIT(kq->acc->channel % 32);
70 	offset = ACC_INTD_OFFSET_STATUS(kq->acc->channel);
71 	dev_dbg(kdev->dev, "setup-notify: re-triggering irq for %s\n",
72 		kq->acc->name);
73 	writel_relaxed(mask, pdsp->intd + offset);
74 	return 0;
75 }
76 
knav_acc_int_handler(int irq,void * _instdata)77 static irqreturn_t knav_acc_int_handler(int irq, void *_instdata)
78 {
79 	struct knav_acc_channel *acc;
80 	struct knav_queue_inst *kq = NULL;
81 	struct knav_range_info *range;
82 	struct knav_pdsp_info *pdsp;
83 	struct knav_acc_info *info;
84 	struct knav_device *kdev;
85 
86 	u32 *list, *list_cpu, val, idx, notifies;
87 	int range_base, channel, queue = 0;
88 	dma_addr_t list_dma;
89 
90 	range = _instdata;
91 	info  = &range->acc_info;
92 	kdev  = range->kdev;
93 	pdsp  = range->acc_info.pdsp;
94 	acc   = range->acc;
95 
96 	range_base = kdev->base_id + range->queue_base;
97 	if ((range->flags & RANGE_MULTI_QUEUE) == 0) {
98 		for (queue = 0; queue < range->num_irqs; queue++)
99 			if (range->irqs[queue].irq == irq)
100 				break;
101 		kq = knav_range_offset_to_inst(kdev, range, queue);
102 		acc += queue;
103 	}
104 
105 	channel = acc->channel;
106 	list_dma = acc->list_dma[acc->list_index];
107 	list_cpu = acc->list_cpu[acc->list_index];
108 	dev_dbg(kdev->dev, "acc-irq: channel %d, list %d, virt %p, dma %pad\n",
109 		channel, acc->list_index, list_cpu, &list_dma);
110 	if (atomic_read(&acc->retrigger_count)) {
111 		atomic_dec(&acc->retrigger_count);
112 		__knav_acc_notify(range, acc);
113 		writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
114 		/* ack the interrupt */
115 		writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
116 			       pdsp->intd + ACC_INTD_OFFSET_EOI);
117 
118 		return IRQ_HANDLED;
119 	}
120 
121 	notifies = readl_relaxed(pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
122 	WARN_ON(!notifies);
123 	dma_sync_single_for_cpu(kdev->dev, list_dma, info->list_size,
124 				DMA_FROM_DEVICE);
125 
126 	for (list = list_cpu; list < list_cpu + (info->list_size / sizeof(u32));
127 	     list += ACC_LIST_ENTRY_WORDS) {
128 		if (ACC_LIST_ENTRY_WORDS == 1) {
129 			dev_dbg(kdev->dev,
130 				"acc-irq: list %d, entry @%p, %08x\n",
131 				acc->list_index, list, list[0]);
132 		} else if (ACC_LIST_ENTRY_WORDS == 2) {
133 			dev_dbg(kdev->dev,
134 				"acc-irq: list %d, entry @%p, %08x %08x\n",
135 				acc->list_index, list, list[0], list[1]);
136 		} else if (ACC_LIST_ENTRY_WORDS == 4) {
137 			dev_dbg(kdev->dev,
138 				"acc-irq: list %d, entry @%p, %08x %08x %08x %08x\n",
139 				acc->list_index, list, list[0], list[1],
140 				list[2], list[3]);
141 		}
142 
143 		val = list[ACC_LIST_ENTRY_DESC_IDX];
144 		if (!val)
145 			break;
146 
147 		if (range->flags & RANGE_MULTI_QUEUE) {
148 			queue = list[ACC_LIST_ENTRY_QUEUE_IDX] >> 16;
149 			if (queue < range_base ||
150 			    queue >= range_base + range->num_queues) {
151 				dev_err(kdev->dev,
152 					"bad queue %d, expecting %d-%d\n",
153 					queue, range_base,
154 					range_base + range->num_queues);
155 				break;
156 			}
157 			queue -= range_base;
158 			kq = knav_range_offset_to_inst(kdev, range,
159 								queue);
160 		}
161 
162 		if (atomic_inc_return(&kq->desc_count) >= ACC_DESCS_MAX) {
163 			atomic_dec(&kq->desc_count);
164 			dev_err(kdev->dev,
165 				"acc-irq: queue %d full, entry dropped\n",
166 				queue + range_base);
167 			continue;
168 		}
169 
170 		idx = atomic_inc_return(&kq->desc_tail) & ACC_DESCS_MASK;
171 		kq->descs[idx] = val;
172 		kq->notify_needed = 1;
173 		dev_dbg(kdev->dev, "acc-irq: enqueue %08x at %d, queue %d\n",
174 			val, idx, queue + range_base);
175 	}
176 
177 	__knav_acc_notify(range, acc);
178 	memset(list_cpu, 0, info->list_size);
179 	dma_sync_single_for_device(kdev->dev, list_dma, info->list_size,
180 				   DMA_TO_DEVICE);
181 
182 	/* flip to the other list */
183 	acc->list_index ^= 1;
184 
185 	/* reset the interrupt counter */
186 	writel_relaxed(1, pdsp->intd + ACC_INTD_OFFSET_COUNT(channel));
187 
188 	/* ack the interrupt */
189 	writel_relaxed(ACC_CHANNEL_INT_BASE + channel,
190 		       pdsp->intd + ACC_INTD_OFFSET_EOI);
191 
192 	return IRQ_HANDLED;
193 }
194 
knav_range_setup_acc_irq(struct knav_range_info * range,int queue,bool enabled)195 static int knav_range_setup_acc_irq(struct knav_range_info *range,
196 				int queue, bool enabled)
197 {
198 	struct knav_device *kdev = range->kdev;
199 	struct knav_acc_channel *acc;
200 	struct cpumask *cpu_mask;
201 	int ret = 0, irq;
202 	u32 old, new;
203 
204 	if (range->flags & RANGE_MULTI_QUEUE) {
205 		acc = range->acc;
206 		irq = range->irqs[0].irq;
207 		cpu_mask = range->irqs[0].cpu_mask;
208 	} else {
209 		acc = range->acc + queue;
210 		irq = range->irqs[queue].irq;
211 		cpu_mask = range->irqs[queue].cpu_mask;
212 	}
213 
214 	old = acc->open_mask;
215 	if (enabled)
216 		new = old | BIT(queue);
217 	else
218 		new = old & ~BIT(queue);
219 	acc->open_mask = new;
220 
221 	dev_dbg(kdev->dev,
222 		"setup-acc-irq: open mask old %08x, new %08x, channel %s\n",
223 		old, new, acc->name);
224 
225 	if (likely(new == old))
226 		return 0;
227 
228 	if (new && !old) {
229 		dev_dbg(kdev->dev,
230 			"setup-acc-irq: requesting %s for channel %s\n",
231 			acc->name, acc->name);
232 		ret = request_irq(irq, knav_acc_int_handler, 0, acc->name,
233 				  range);
234 		if (!ret && cpu_mask) {
235 			ret = irq_set_affinity_hint(irq, cpu_mask);
236 			if (ret) {
237 				dev_warn(range->kdev->dev,
238 					 "Failed to set IRQ affinity\n");
239 				return ret;
240 			}
241 		}
242 	}
243 
244 	if (old && !new) {
245 		dev_dbg(kdev->dev, "setup-acc-irq: freeing %s for channel %s\n",
246 			acc->name, acc->name);
247 		ret = irq_set_affinity_hint(irq, NULL);
248 		if (ret)
249 			dev_warn(range->kdev->dev,
250 				 "Failed to set IRQ affinity\n");
251 		free_irq(irq, range);
252 	}
253 
254 	return ret;
255 }
256 
knav_acc_result_str(enum knav_acc_result result)257 static const char *knav_acc_result_str(enum knav_acc_result result)
258 {
259 	static const char * const result_str[] = {
260 		[ACC_RET_IDLE]			= "idle",
261 		[ACC_RET_SUCCESS]		= "success",
262 		[ACC_RET_INVALID_COMMAND]	= "invalid command",
263 		[ACC_RET_INVALID_CHANNEL]	= "invalid channel",
264 		[ACC_RET_INACTIVE_CHANNEL]	= "inactive channel",
265 		[ACC_RET_ACTIVE_CHANNEL]	= "active channel",
266 		[ACC_RET_INVALID_QUEUE]		= "invalid queue",
267 		[ACC_RET_INVALID_RET]		= "invalid return code",
268 	};
269 
270 	if (result >= ARRAY_SIZE(result_str))
271 		return result_str[ACC_RET_INVALID_RET];
272 	else
273 		return result_str[result];
274 }
275 
276 static enum knav_acc_result
knav_acc_write(struct knav_device * kdev,struct knav_pdsp_info * pdsp,struct knav_reg_acc_command * cmd)277 knav_acc_write(struct knav_device *kdev, struct knav_pdsp_info *pdsp,
278 		struct knav_reg_acc_command *cmd)
279 {
280 	u32 result;
281 
282 	dev_dbg(kdev->dev, "acc command %08x %08x %08x %08x %08x\n",
283 		cmd->command, cmd->queue_mask, cmd->list_dma,
284 		cmd->queue_num, cmd->timer_config);
285 
286 	writel_relaxed(cmd->timer_config, &pdsp->acc_command->timer_config);
287 	writel_relaxed(cmd->queue_num, &pdsp->acc_command->queue_num);
288 	writel_relaxed(cmd->list_dma, &pdsp->acc_command->list_dma);
289 	writel_relaxed(cmd->queue_mask, &pdsp->acc_command->queue_mask);
290 	writel_relaxed(cmd->command, &pdsp->acc_command->command);
291 
292 	/* wait for the command to clear */
293 	do {
294 		result = readl_relaxed(&pdsp->acc_command->command);
295 	} while ((result >> 8) & 0xff);
296 
297 	return (result >> 24) & 0xff;
298 }
299 
knav_acc_setup_cmd(struct knav_device * kdev,struct knav_range_info * range,struct knav_reg_acc_command * cmd,int queue)300 static void knav_acc_setup_cmd(struct knav_device *kdev,
301 				struct knav_range_info *range,
302 				struct knav_reg_acc_command *cmd,
303 				int queue)
304 {
305 	struct knav_acc_info *info = &range->acc_info;
306 	struct knav_acc_channel *acc;
307 	int queue_base;
308 	u32 queue_mask;
309 
310 	if (range->flags & RANGE_MULTI_QUEUE) {
311 		acc = range->acc;
312 		queue_base = range->queue_base;
313 		queue_mask = BIT(range->num_queues) - 1;
314 	} else {
315 		acc = range->acc + queue;
316 		queue_base = range->queue_base + queue;
317 		queue_mask = 0;
318 	}
319 
320 	memset(cmd, 0, sizeof(*cmd));
321 	cmd->command    = acc->channel;
322 	cmd->queue_mask = queue_mask;
323 	cmd->list_dma   = (u32)acc->list_dma[0];
324 	cmd->queue_num  = info->list_entries << 16;
325 	cmd->queue_num |= queue_base;
326 
327 	cmd->timer_config = ACC_LIST_ENTRY_TYPE << 18;
328 	if (range->flags & RANGE_MULTI_QUEUE)
329 		cmd->timer_config |= ACC_CFG_MULTI_QUEUE;
330 	cmd->timer_config |= info->pacing_mode << 16;
331 	cmd->timer_config |= info->timer_count;
332 }
333 
knav_acc_stop(struct knav_device * kdev,struct knav_range_info * range,int queue)334 static void knav_acc_stop(struct knav_device *kdev,
335 				struct knav_range_info *range,
336 				int queue)
337 {
338 	struct knav_reg_acc_command cmd;
339 	struct knav_acc_channel *acc;
340 	enum knav_acc_result result;
341 
342 	acc = range->acc + queue;
343 
344 	knav_acc_setup_cmd(kdev, range, &cmd, queue);
345 	cmd.command |= ACC_CMD_DISABLE_CHANNEL << 8;
346 	result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
347 
348 	dev_dbg(kdev->dev, "stopped acc channel %s, result %s\n",
349 		acc->name, knav_acc_result_str(result));
350 }
351 
knav_acc_start(struct knav_device * kdev,struct knav_range_info * range,int queue)352 static enum knav_acc_result knav_acc_start(struct knav_device *kdev,
353 						struct knav_range_info *range,
354 						int queue)
355 {
356 	struct knav_reg_acc_command cmd;
357 	struct knav_acc_channel *acc;
358 	enum knav_acc_result result;
359 
360 	acc = range->acc + queue;
361 
362 	knav_acc_setup_cmd(kdev, range, &cmd, queue);
363 	cmd.command |= ACC_CMD_ENABLE_CHANNEL << 8;
364 	result = knav_acc_write(kdev, range->acc_info.pdsp, &cmd);
365 
366 	dev_dbg(kdev->dev, "started acc channel %s, result %s\n",
367 		acc->name, knav_acc_result_str(result));
368 
369 	return result;
370 }
371 
knav_acc_init_range(struct knav_range_info * range)372 static int knav_acc_init_range(struct knav_range_info *range)
373 {
374 	struct knav_device *kdev = range->kdev;
375 	struct knav_acc_channel *acc;
376 	enum knav_acc_result result;
377 	int queue;
378 
379 	for (queue = 0; queue < range->num_queues; queue++) {
380 		acc = range->acc + queue;
381 
382 		knav_acc_stop(kdev, range, queue);
383 		acc->list_index = 0;
384 		result = knav_acc_start(kdev, range, queue);
385 
386 		if (result != ACC_RET_SUCCESS)
387 			return -EIO;
388 
389 		if (range->flags & RANGE_MULTI_QUEUE)
390 			return 0;
391 	}
392 	return 0;
393 }
394 
knav_acc_init_queue(struct knav_range_info * range,struct knav_queue_inst * kq)395 static int knav_acc_init_queue(struct knav_range_info *range,
396 				struct knav_queue_inst *kq)
397 {
398 	unsigned id = kq->id - range->queue_base;
399 
400 	kq->descs = devm_kcalloc(range->kdev->dev,
401 				 ACC_DESCS_MAX, sizeof(u32), GFP_KERNEL);
402 	if (!kq->descs)
403 		return -ENOMEM;
404 
405 	kq->acc = range->acc;
406 	if ((range->flags & RANGE_MULTI_QUEUE) == 0)
407 		kq->acc += id;
408 	return 0;
409 }
410 
knav_acc_open_queue(struct knav_range_info * range,struct knav_queue_inst * inst,unsigned flags)411 static int knav_acc_open_queue(struct knav_range_info *range,
412 				struct knav_queue_inst *inst, unsigned flags)
413 {
414 	unsigned id = inst->id - range->queue_base;
415 
416 	return knav_range_setup_acc_irq(range, id, true);
417 }
418 
knav_acc_close_queue(struct knav_range_info * range,struct knav_queue_inst * inst)419 static int knav_acc_close_queue(struct knav_range_info *range,
420 					struct knav_queue_inst *inst)
421 {
422 	unsigned id = inst->id - range->queue_base;
423 
424 	return knav_range_setup_acc_irq(range, id, false);
425 }
426 
knav_acc_free_range(struct knav_range_info * range)427 static int knav_acc_free_range(struct knav_range_info *range)
428 {
429 	struct knav_device *kdev = range->kdev;
430 	struct knav_acc_channel *acc;
431 	struct knav_acc_info *info;
432 	int channel, channels;
433 
434 	info = &range->acc_info;
435 
436 	if (range->flags & RANGE_MULTI_QUEUE)
437 		channels = 1;
438 	else
439 		channels = range->num_queues;
440 
441 	for (channel = 0; channel < channels; channel++) {
442 		acc = range->acc + channel;
443 		if (!acc->list_cpu[0])
444 			continue;
445 		dma_unmap_single(kdev->dev, acc->list_dma[0],
446 				 info->mem_size, DMA_BIDIRECTIONAL);
447 		free_pages_exact(acc->list_cpu[0], info->mem_size);
448 	}
449 	devm_kfree(range->kdev->dev, range->acc);
450 	return 0;
451 }
452 
453 static const struct knav_range_ops knav_acc_range_ops = {
454 	.set_notify	= knav_acc_set_notify,
455 	.init_queue	= knav_acc_init_queue,
456 	.open_queue	= knav_acc_open_queue,
457 	.close_queue	= knav_acc_close_queue,
458 	.init_range	= knav_acc_init_range,
459 	.free_range	= knav_acc_free_range,
460 };
461 
462 /**
463  * knav_init_acc_range: Initialise accumulator ranges
464  *
465  * @kdev:		qmss device
466  * @node:		device node
467  * @range:		qmms range information
468  *
469  * Return 0 on success or error
470  */
knav_init_acc_range(struct knav_device * kdev,struct device_node * node,struct knav_range_info * range)471 int knav_init_acc_range(struct knav_device *kdev,
472 			struct device_node *node,
473 			struct knav_range_info *range)
474 {
475 	struct knav_acc_channel *acc;
476 	struct knav_pdsp_info *pdsp;
477 	struct knav_acc_info *info;
478 	int ret, channel, channels;
479 	int list_size, mem_size;
480 	dma_addr_t list_dma;
481 	void *list_mem;
482 	u32 config[5];
483 
484 	range->flags |= RANGE_HAS_ACCUMULATOR;
485 	info = &range->acc_info;
486 
487 	ret = of_property_read_u32_array(node, "accumulator", config, 5);
488 	if (ret)
489 		return ret;
490 
491 	info->pdsp_id		= config[0];
492 	info->start_channel	= config[1];
493 	info->list_entries	= config[2];
494 	info->pacing_mode	= config[3];
495 	info->timer_count	= config[4] / ACC_DEFAULT_PERIOD;
496 
497 	if (info->start_channel > ACC_MAX_CHANNEL) {
498 		dev_err(kdev->dev, "channel %d invalid for range %s\n",
499 			info->start_channel, range->name);
500 		return -EINVAL;
501 	}
502 
503 	if (info->pacing_mode > 3) {
504 		dev_err(kdev->dev, "pacing mode %d invalid for range %s\n",
505 			info->pacing_mode, range->name);
506 		return -EINVAL;
507 	}
508 
509 	pdsp = knav_find_pdsp(kdev, info->pdsp_id);
510 	if (!pdsp) {
511 		dev_err(kdev->dev, "pdsp id %d not found for range %s\n",
512 			info->pdsp_id, range->name);
513 		return -EINVAL;
514 	}
515 
516 	if (!pdsp->started) {
517 		dev_err(kdev->dev, "pdsp id %d not started for range %s\n",
518 			info->pdsp_id, range->name);
519 		return -ENODEV;
520 	}
521 
522 	info->pdsp = pdsp;
523 	channels = range->num_queues;
524 	if (of_property_read_bool(node, "multi-queue")) {
525 		range->flags |= RANGE_MULTI_QUEUE;
526 		channels = 1;
527 		if (range->queue_base & (32 - 1)) {
528 			dev_err(kdev->dev,
529 				"misaligned multi-queue accumulator range %s\n",
530 				range->name);
531 			return -EINVAL;
532 		}
533 		if (range->num_queues > 32) {
534 			dev_err(kdev->dev,
535 				"too many queues in accumulator range %s\n",
536 				range->name);
537 			return -EINVAL;
538 		}
539 	}
540 
541 	/* figure out list size */
542 	list_size  = info->list_entries;
543 	list_size *= ACC_LIST_ENTRY_WORDS * sizeof(u32);
544 	info->list_size = list_size;
545 	mem_size   = PAGE_ALIGN(list_size * 2);
546 	info->mem_size  = mem_size;
547 	range->acc = devm_kcalloc(kdev->dev, channels, sizeof(*range->acc),
548 				  GFP_KERNEL);
549 	if (!range->acc)
550 		return -ENOMEM;
551 
552 	for (channel = 0; channel < channels; channel++) {
553 		acc = range->acc + channel;
554 		acc->channel = info->start_channel + channel;
555 
556 		/* allocate memory for the two lists */
557 		list_mem = alloc_pages_exact(mem_size, GFP_KERNEL | GFP_DMA);
558 		if (!list_mem)
559 			return -ENOMEM;
560 
561 		list_dma = dma_map_single(kdev->dev, list_mem, mem_size,
562 					  DMA_BIDIRECTIONAL);
563 		if (dma_mapping_error(kdev->dev, list_dma)) {
564 			free_pages_exact(list_mem, mem_size);
565 			return -ENOMEM;
566 		}
567 
568 		memset(list_mem, 0, mem_size);
569 		dma_sync_single_for_device(kdev->dev, list_dma, mem_size,
570 					   DMA_TO_DEVICE);
571 		scnprintf(acc->name, sizeof(acc->name), "hwqueue-acc-%d",
572 			  acc->channel);
573 		acc->list_cpu[0] = list_mem;
574 		acc->list_cpu[1] = list_mem + list_size;
575 		acc->list_dma[0] = list_dma;
576 		acc->list_dma[1] = list_dma + list_size;
577 		dev_dbg(kdev->dev, "%s: channel %d, dma %pad, virt %8p\n",
578 			acc->name, acc->channel, &list_dma, list_mem);
579 	}
580 
581 	range->ops = &knav_acc_range_ops;
582 	return 0;
583 }
584 EXPORT_SYMBOL_GPL(knav_init_acc_range);
585