59 #define for_each_handle_rcu(qh, inst)				\  argument
60 	list_for_each_entry_rcu(qh, &inst->handles, list,	\
88 	struct knav_queue *qh;  in knav_queue_notify()  local
94 	for_each_handle_rcu(qh, inst) {  in knav_queue_notify()
95 		if (atomic_read(&qh->notifier_enabled) <= 0)  in knav_queue_notify()
97 		if (WARN_ON(!qh->notifier_fn))  in knav_queue_notify()
99 		this_cpu_inc(qh->stats->notifies);  in knav_queue_notify()
100 		qh->notifier_fn(qh->notifier_fn_arg);  in knav_queue_notify()
219 	struct knav_queue *qh;  in __knav_queue_open()  local
223 	qh = devm_kzalloc(inst->kdev->dev, sizeof(*qh), GFP_KERNEL);  in __knav_queue_open()
224 	if (!qh)  in __knav_queue_open()
227 	qh->stats = alloc_percpu(struct knav_queue_stats);  in __knav_queue_open()
228 	if (!qh->stats) {  in __knav_queue_open()
233 	qh->flags = flags;  in __knav_queue_open()
234 	qh->inst = inst;  in __knav_queue_open()
236 	qh->reg_push = &inst->qmgr->reg_push[id];  in __knav_queue_open()
237 	qh->reg_pop = &inst->qmgr->reg_pop[id];  in __knav_queue_open()
238 	qh->reg_peek = &inst->qmgr->reg_peek[id];  in __knav_queue_open()
251 	list_add_tail_rcu(&qh->list, &inst->handles);  in __knav_queue_open()
252 	return qh;  in __knav_queue_open()
255 	if (qh->stats)  in __knav_queue_open()
256 		free_percpu(qh->stats);  in __knav_queue_open()
257 	devm_kfree(inst->kdev->dev, qh);  in __knav_queue_open()
265 	struct knav_queue *qh;  in knav_queue_open_by_id()  local
269 	qh = ERR_PTR(-ENODEV);  in knav_queue_open_by_id()
274 	qh = ERR_PTR(-EEXIST);  in knav_queue_open_by_id()
278 	qh = ERR_PTR(-EBUSY);  in knav_queue_open_by_id()
283 	qh = __knav_queue_open(inst, name, flags);  in knav_queue_open_by_id()
288 	return qh;  in knav_queue_open_by_id()
295 	struct knav_queue *qh = ERR_PTR(-EINVAL);  in knav_queue_open_by_type()  local
307 		qh = __knav_queue_open(inst, name, flags);  in knav_queue_open_by_type()
313 	return qh;  in knav_queue_open_by_type()
324 static int knav_queue_enable_notifier(struct knav_queue *qh)  in knav_queue_enable_notifier()  argument
326 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_enable_notifier()
329 	if (WARN_ON(!qh->notifier_fn))  in knav_queue_enable_notifier()
333 	first = (atomic_inc_return(&qh->notifier_enabled) == 1);  in knav_queue_enable_notifier()
345 static int knav_queue_disable_notifier(struct knav_queue *qh)  in knav_queue_disable_notifier()  argument
347 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_disable_notifier()
350 	last = (atomic_dec_return(&qh->notifier_enabled) == 0);  in knav_queue_disable_notifier()
361 static int knav_queue_set_notifier(struct knav_queue *qh,  in knav_queue_set_notifier()  argument
364 	knav_queue_notify_fn old_fn = qh->notifier_fn;  in knav_queue_set_notifier()
369 	if (!(qh->inst->range->flags & (RANGE_HAS_ACCUMULATOR | RANGE_HAS_IRQ)))  in knav_queue_set_notifier()
373 		knav_queue_disable_notifier(qh);  in knav_queue_set_notifier()
375 	qh->notifier_fn = cfg->fn;  in knav_queue_set_notifier()
376 	qh->notifier_fn_arg = cfg->fn_arg;  in knav_queue_set_notifier()
379 		knav_queue_enable_notifier(qh);  in knav_queue_set_notifier()
422 	struct knav_queue *qh = qhandle;  in knav_queue_get_count()  local
423 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_get_count()
425 	return readl_relaxed(&qh->reg_peek[0].entry_count) +  in knav_queue_get_count()
433 	struct knav_queue *qh;  in knav_queue_debug_show_instance()  local
446 	for_each_handle_rcu(qh, inst) {  in knav_queue_debug_show_instance()
448 			pushes += per_cpu_ptr(qh->stats, cpu)->pushes;  in knav_queue_debug_show_instance()
449 			pops += per_cpu_ptr(qh->stats, cpu)->pops;  in knav_queue_debug_show_instance()
450 			push_errors += per_cpu_ptr(qh->stats, cpu)->push_errors;  in knav_queue_debug_show_instance()
451 			pop_errors += per_cpu_ptr(qh->stats, cpu)->pop_errors;  in knav_queue_debug_show_instance()
452 			notifies += per_cpu_ptr(qh->stats, cpu)->notifies;  in knav_queue_debug_show_instance()
456 				qh,  in knav_queue_debug_show_instance()
459 				knav_queue_get_count(qh),  in knav_queue_debug_show_instance()
503 static int knav_queue_flush(struct knav_queue *qh)  in knav_queue_flush()  argument
505 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_flush()
530 	struct knav_queue *qh = ERR_PTR(-EINVAL);  in knav_queue_open()  local
536 		qh = knav_queue_open_by_type(name, id, flags);  in knav_queue_open()
540 		qh = knav_queue_open_by_id(name, id, flags);  in knav_queue_open()
543 	return qh;  in knav_queue_open()
553 	struct knav_queue *qh = qhandle;  in knav_queue_close()  local
554 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_close()
556 	while (atomic_read(&qh->notifier_enabled) > 0)  in knav_queue_close()
557 		knav_queue_disable_notifier(qh);  in knav_queue_close()
560 	list_del_rcu(&qh->list);  in knav_queue_close()
569 	free_percpu(qh->stats);  in knav_queue_close()
570 	devm_kfree(inst->kdev->dev, qh);  in knav_queue_close()
585 	struct knav_queue *qh = qhandle;  in knav_queue_device_control()  local
591 		ret = qh->inst->kdev->base_id + qh->inst->id;  in knav_queue_device_control()
595 		ret = knav_queue_flush(qh);  in knav_queue_device_control()
600 		ret = knav_queue_set_notifier(qh, cfg);  in knav_queue_device_control()
604 		ret = knav_queue_enable_notifier(qh);  in knav_queue_device_control()
608 		ret = knav_queue_disable_notifier(qh);  in knav_queue_device_control()
612 		ret = knav_queue_get_count(qh);  in knav_queue_device_control()
637 	struct knav_queue *qh = qhandle;  in knav_queue_push()  local
641 	writel_relaxed(val, &qh->reg_push[0].ptr_size_thresh);  in knav_queue_push()
643 	this_cpu_inc(qh->stats->pushes);  in knav_queue_push()
657 	struct knav_queue *qh = qhandle;  in knav_queue_pop()  local
658 	struct knav_queue_inst *inst = qh->inst;  in knav_queue_pop()
672 		val = readl_relaxed(&qh->reg_pop[0].ptr_size_thresh);  in knav_queue_pop()
681 	this_cpu_inc(qh->stats->pops);  in knav_queue_pop()