33 	if (q->uacce->ops->start_queue) {  in uacce_start_queue()
34 		ret = q->uacce->ops->start_queue(q);  in uacce_start_queue()
45 	struct uacce_device *uacce = q->uacce;  in uacce_put_queue()  local
47 	if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue)  in uacce_put_queue()
48 		uacce->ops->stop_queue(q);  in uacce_put_queue()
51 	     uacce->ops->put_queue)  in uacce_put_queue()
52 		uacce->ops->put_queue(q);  in uacce_put_queue()
63 	struct uacce_device *uacce = q->uacce;  in uacce_fops_unl_ioctl()  local
74 	mutex_lock(&uacce->mutex);  in uacce_fops_unl_ioctl()
86 		if (uacce->ops->ioctl)  in uacce_fops_unl_ioctl()
87 			ret = uacce->ops->ioctl(q, cmd, arg);  in uacce_fops_unl_ioctl()
92 	mutex_unlock(&uacce->mutex);  in uacce_fops_unl_ioctl()
106 static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)  in uacce_bind_queue()  argument
111 	if (!(uacce->flags & UACCE_DEV_SVA))  in uacce_bind_queue()
114 	handle = iommu_sva_bind_device(uacce->parent, current->mm);  in uacce_bind_queue()
139 	struct uacce_device *uacce;  in uacce_fops_open()  local
143 	uacce = xa_load(&uacce_xa, iminor(inode));  in uacce_fops_open()
144 	if (!uacce)  in uacce_fops_open()
151 	mutex_lock(&uacce->mutex);  in uacce_fops_open()
153 	if (!uacce->parent) {  in uacce_fops_open()
158 	ret = uacce_bind_queue(uacce, q);  in uacce_fops_open()
162 	q->uacce = uacce;  in uacce_fops_open()
164 	if (uacce->ops->get_queue) {  in uacce_fops_open()
165 		ret = uacce->ops->get_queue(uacce, q->pasid, q);  in uacce_fops_open()
175 	list_add(&q->list, &uacce->queues);  in uacce_fops_open()
176 	mutex_unlock(&uacce->mutex);  in uacce_fops_open()
184 	mutex_unlock(&uacce->mutex);  in uacce_fops_open()
191 	struct uacce_device *uacce = q->uacce;  in uacce_fops_release()  local
193 	mutex_lock(&uacce->mutex);  in uacce_fops_release()
197 	mutex_unlock(&uacce->mutex);  in uacce_fops_release()
224 	struct uacce_device *uacce = q->uacce;  in uacce_fops_mmap()  local
257 		if (!uacce->ops->mmap) {  in uacce_fops_mmap()
262 		ret = uacce->ops->mmap(q, vma, qfr);  in uacce_fops_mmap()
286 	struct uacce_device *uacce = q->uacce;  in uacce_fops_poll()  local
295 	if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q))  in uacce_fops_poll()
320 	struct uacce_device *uacce = to_uacce_device(dev);  in api_show()  local
322 	return sysfs_emit(buf, "%s\n", uacce->api_ver);  in api_show()
328 	struct uacce_device *uacce = to_uacce_device(dev);  in flags_show()  local
330 	return sysfs_emit(buf, "%u\n", uacce->flags);  in flags_show()
337 	struct uacce_device *uacce = to_uacce_device(dev);  in available_instances_show()  local
339 	if (!uacce->ops->get_available_instances)  in available_instances_show()
343 		       uacce->ops->get_available_instances(uacce));  in available_instances_show()
349 	struct uacce_device *uacce = to_uacce_device(dev);  in algorithms_show()  local
351 	return sysfs_emit(buf, "%s\n", uacce->algs);  in algorithms_show()
357 	struct uacce_device *uacce = to_uacce_device(dev);  in region_mmio_size_show()  local
360 		       uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT);  in region_mmio_size_show()
366 	struct uacce_device *uacce = to_uacce_device(dev);  in region_dus_size_show()  local
369 		       uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT);  in region_dus_size_show()
375 	struct uacce_device *uacce = to_uacce_device(dev);  in isolate_show()  local
377 	return sysfs_emit(buf, "%d\n", uacce->ops->get_isolate_state(uacce));  in isolate_show()
382 	struct uacce_device *uacce = to_uacce_device(dev);  in isolate_strategy_show()  local
385 	val = uacce->ops->isolate_err_threshold_read(uacce);  in isolate_strategy_show()
393 	struct uacce_device *uacce = to_uacce_device(dev);  in isolate_strategy_store()  local
403 	ret = uacce->ops->isolate_err_threshold_write(uacce, val);  in isolate_strategy_store()
435 	struct uacce_device *uacce = to_uacce_device(dev);  in uacce_dev_is_visible()  local
438 	    (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) ||  in uacce_dev_is_visible()
440 	    (!uacce->qf_pg_num[UACCE_QFRT_DUS])))  in uacce_dev_is_visible()
444 	    (!uacce->ops->isolate_err_threshold_read &&  in uacce_dev_is_visible()
445 	     !uacce->ops->isolate_err_threshold_write))  in uacce_dev_is_visible()
448 	if (attr == &dev_attr_isolate.attr && !uacce->ops->get_isolate_state)  in uacce_dev_is_visible()
463 	struct uacce_device *uacce = to_uacce_device(dev);  in uacce_release()  local
465 	kfree(uacce);  in uacce_release()
493 static void uacce_disable_sva(struct uacce_device *uacce)  in uacce_disable_sva()  argument
495 	if (!(uacce->flags & UACCE_DEV_SVA))  in uacce_disable_sva()
498 	iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA);  in uacce_disable_sva()
499 	iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_IOPF);  in uacce_disable_sva()
514 	struct uacce_device *uacce;  in uacce_alloc()  local
517 	uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL);  in uacce_alloc()
518 	if (!uacce)  in uacce_alloc()
523 	uacce->parent = parent;  in uacce_alloc()
524 	uacce->flags = flags;  in uacce_alloc()
525 	uacce->ops = interface->ops;  in uacce_alloc()
527 	ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b,  in uacce_alloc()
532 	INIT_LIST_HEAD(&uacce->queues);  in uacce_alloc()
533 	mutex_init(&uacce->mutex);  in uacce_alloc()
534 	device_initialize(&uacce->dev);  in uacce_alloc()
535 	uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id);  in uacce_alloc()
536 	uacce->dev.class = &uacce_class;  in uacce_alloc()
537 	uacce->dev.groups = uacce_dev_groups;  in uacce_alloc()
538 	uacce->dev.parent = uacce->parent;  in uacce_alloc()
539 	uacce->dev.release = uacce_release;  in uacce_alloc()
540 	dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id);  in uacce_alloc()
542 	return uacce;  in uacce_alloc()
545 	uacce_disable_sva(uacce);  in uacce_alloc()
546 	kfree(uacce);  in uacce_alloc()
557 int uacce_register(struct uacce_device *uacce)  in uacce_register()  argument
559 	if (!uacce)  in uacce_register()
562 	uacce->cdev = cdev_alloc();  in uacce_register()
563 	if (!uacce->cdev)  in uacce_register()
566 	uacce->cdev->ops = &uacce_fops;  in uacce_register()
567 	uacce->cdev->owner = THIS_MODULE;  in uacce_register()
569 	return cdev_device_add(uacce->cdev, &uacce->dev);  in uacce_register()
577 void uacce_remove(struct uacce_device *uacce)  in uacce_remove()  argument
581 	if (!uacce)  in uacce_remove()
589 	mutex_lock(&uacce->mutex);  in uacce_remove()
591 	list_for_each_entry_safe(q, next_q, &uacce->queues, list) {  in uacce_remove()
609 	uacce_disable_sva(uacce);  in uacce_remove()
611 	if (uacce->cdev)  in uacce_remove()
612 		cdev_device_del(uacce->cdev, &uacce->dev);  in uacce_remove()
613 	xa_erase(&uacce_xa, uacce->dev_id);  in uacce_remove()
618 	uacce->ops = NULL;  in uacce_remove()
619 	uacce->parent = NULL;  in uacce_remove()
620 	mutex_unlock(&uacce->mutex);  in uacce_remove()
621 	put_device(&uacce->dev);  in uacce_remove()