xref: /freebsd/sys/dev/qat/qat_common/adf_freebsd_uio.c (revision 866dc4bd81398b478daefe4b7447b92422b4a06c)

/* SPDX-License-Identifier: BSD-3-Clause */
/* Copyright(c) 2007-2022 Intel Corporation */
#include "qat_freebsd.h"
#include "adf_cfg.h"
#include "adf_common_drv.h"
#include "adf_accel_devices.h"
#include "icp_qat_uclo.h"
#include "icp_qat_fw.h"
#include "icp_qat_fw_init_admin.h"
#include "adf_cfg_strings.h"
#include "adf_uio_control.h"
#include "adf_uio_cleanup.h"
#include "adf_uio.h"
#include "adf_transport_access_macros.h"
#include "adf_transport_internal.h"
#include <sys/conf.h>
#include <sys/capsicum.h>
#include <sys/kdb.h>
#include <sys/condvar.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/proc.h>
#include <sys/file.h>
#include <sys/lock.h>
#include <sys/rwlock.h>
#include <sys/sglist.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pager.h>

#define ADF_UIO_GET_NAME(accel_dev) (GET_HW_DATA(accel_dev)->dev_class->name)
#define ADF_UIO_GET_TYPE(accel_dev) (GET_HW_DATA(accel_dev)->dev_class->type)
#define ADF_UIO_GET_BAR(accel_dev)                                             \
	(GET_HW_DATA(accel_dev)->get_etr_bar_id(GET_HW_DATA(accel_dev)))

static d_ioctl_t adf_uio_ioctl;
static d_mmap_single_t adf_uio_mmap_single;

static struct cdevsw adf_uio_cdevsw = { .d_ioctl = adf_uio_ioctl,
					.d_mmap_single = adf_uio_mmap_single,
					.d_version = D_VERSION,
					.d_name = "qat" };

struct adf_uio_open_bundle {
	struct adf_uio_control_accel *accel;
	int bundle;
	struct file **mem_files;
	int num_mem_files;
};

static void
adf_release_bundle(void *arg)
{
	struct adf_uio_control_accel *accel = NULL;
	struct adf_uio_open_bundle *handle = NULL;
	struct adf_uio_control_bundle *bundle = NULL;
	struct adf_uio_instance_rings *instance_rings, *tmp;
	int i = 0;

	handle = arg;
	accel = handle->accel;
	bundle = &accel->bundle[handle->bundle];

	mutex_lock(&bundle->lock);
	adf_uio_do_cleanup_orphan(bundle->hardware_bundle_number, accel);
	mutex_unlock(&bundle->lock);

	for (i = 0; i < handle->num_mem_files; i++) {
		/*
		 * Similar to the garbage collection of orphaned file
		 * descriptor references in UNIX domain socket control
		 * messages, the current thread isn't relevant to the
		 * the file descriptor reference being released.  In
		 * particular, the current thread does not hold any
		 * advisory file locks on these file descriptors.
		 */
		fdrop(handle->mem_files[i], NULL);
	}
	free(handle->mem_files, M_QAT);

	mtx_lock(&accel->lock);

	mutex_lock(&bundle->list_lock);
	list_for_each_entry_safe(instance_rings, tmp, &bundle->list, list)
	{
		if (instance_rings->user_pid == curproc->p_pid) {
			list_del(&instance_rings->list);
			free(instance_rings, M_QAT);
			break;
		}
	}
	mutex_unlock(&bundle->list_lock);

	adf_dev_put(accel->accel_dev);
	accel->num_handles--;
	free(handle, M_QAT);
	if (!accel->num_handles) {
		cv_broadcast(&accel->cleanup_ok);
		/* the broadcasting effect happens after releasing accel->lock
		 */
	}
	mtx_unlock(&accel->lock);
}

static int
adf_add_mem_fd(struct adf_accel_dev *accel_dev, int mem_fd)
{
	struct adf_uio_control_accel *accel = NULL;
	struct adf_uio_open_bundle *handle = NULL;
	struct file *fp, **new_files;
	cap_rights_t rights;
	int error = -1, old_count = 0;

	error = devfs_get_cdevpriv((void **)&handle);
	if (error)
		return (error);

	error = fget(curthread, mem_fd, cap_rights_init(&rights), &fp);
	if (error) {
		printf(
		    "Failed to fetch file pointer from current process %d \n",
		    __LINE__);
		return (error);
	}

	accel = accel_dev->accel;
	mtx_lock(&accel->lock);
	for (;;) {
		old_count = handle->num_mem_files;
		mtx_unlock(&accel->lock);
		new_files = malloc((old_count + 1) * sizeof(*new_files),
				   M_QAT,
				   M_WAITOK);
		mtx_lock(&accel->lock);
		if (old_count == handle->num_mem_files) {
			if (old_count != 0) {
				memcpy(new_files,
				       handle->mem_files,
				       old_count * sizeof(*new_files));
				free(handle->mem_files, M_QAT);
			}
			handle->mem_files = new_files;
			new_files[old_count] = fp;
			handle->num_mem_files++;
			break;
		} else
			free(new_files, M_QAT);
	}
	mtx_unlock(&accel->lock);
	return (0);
}

static vm_object_t
adf_uio_map_bar(struct adf_accel_dev *accel_dev, uint8_t bank_offset)
{
	unsigned int ring_bundle_size, offset;
	struct sglist *sg = NULL;
	struct adf_uio_control_accel *accel = accel_dev->accel;
	struct adf_hw_csr_info *csr_info = &accel_dev->hw_device->csr_info;
	vm_object_t obj;

	ring_bundle_size = csr_info->ring_bundle_size;
	offset = bank_offset * ring_bundle_size;

	sg = sglist_alloc(1, M_WAITOK);

	/* Starting from new HW there is an additional offset
	 * for bundle CSRs
	 */
	sglist_append_phys(sg,
			   accel->bar->base_addr + offset +
			       csr_info->csr_addr_offset,
			   ring_bundle_size);

	obj = vm_pager_allocate(
	    OBJT_SG, sg, ring_bundle_size, VM_PROT_RW, 0, NULL);
	if (obj != NULL) {
		VM_OBJECT_WLOCK(obj);
		vm_object_set_memattr(obj, VM_MEMATTR_UNCACHEABLE);
		VM_OBJECT_WUNLOCK(obj);
	}
	sglist_free(sg);

	return obj;
}

static int
adf_alloc_bundle(struct adf_accel_dev *accel_dev, int bundle_nr)
{
	struct adf_uio_control_accel *accel = NULL;
	struct adf_uio_open_bundle *handle = NULL;
	int error;

	if (bundle_nr < 0 || bundle_nr >= GET_MAX_BANKS(accel_dev)) {
		printf("ERROR in %s (%d) %d\n", __func__, bundle_nr, __LINE__);
		return EINVAL;
	}

	accel = accel_dev->accel;
	handle = malloc(sizeof(*handle), M_QAT, M_WAITOK | M_ZERO);
	handle->accel = accel;
	handle->bundle = bundle_nr;

	mtx_lock(&accel->lock);
	adf_dev_get(accel_dev);
	accel->num_handles++;
	mtx_unlock(&accel->lock);

	error = devfs_set_cdevpriv(handle, adf_release_bundle);
	if (error) {
		adf_release_bundle(handle);
		device_printf(GET_DEV(accel_dev),
			      "ERROR in adf_alloc_bundle %d\n",
			      __LINE__);
		return (error);
	}

	return (0);
}

static int
adf_uio_ioctl(struct cdev *dev,
	      u_long cmd,
	      caddr_t data,
	      int fflag,
	      struct thread *td)
{
	struct adf_accel_dev *accel_dev = dev->si_drv1;
	struct adf_hw_csr_info *csr_info = NULL;

	if (!accel_dev) {
		printf("%s - accel_dev is NULL\n", __func__);
		return EFAULT;
	}

	csr_info = &accel_dev->hw_device->csr_info;

	switch (cmd) {
	case IOCTL_GET_BUNDLE_SIZE:
		*(uint32_t *)data = csr_info->ring_bundle_size;
		break;
	case IOCTL_ALLOC_BUNDLE:
		return (adf_alloc_bundle(accel_dev, *(int *)data));
	case IOCTL_GET_ACCEL_TYPE:
		*(uint32_t *)data = ADF_UIO_GET_TYPE(accel_dev);
		break;
	case IOCTL_ADD_MEM_FD:
		return (adf_add_mem_fd(accel_dev, *(int *)data));
	default:
		return (ENOTTY);
	}
	return (0);
}

static int
adf_uio_mmap_single(struct cdev *dev,
		    vm_ooffset_t *offset,
		    vm_size_t size,
		    struct vm_object **object,
		    int nprot)
{
	struct adf_uio_open_bundle *handle = NULL;
	struct adf_uio_control_accel *accel = NULL;
	struct adf_uio_control_bundle *bundle = NULL;
	struct adf_uio_instance_rings *instance_rings;
	int error;

	error = devfs_get_cdevpriv((void **)&handle);
	if (error)
		return (error);

	if (!handle->accel) {
		printf("QAT: Error - no accel in handle\n");
		return EINVAL;
	}
	accel = handle->accel;

	if (!accel->accel_dev) {
		printf("QAT: Error - no accel_dev in accel\n");
		return EINVAL;
	}

	bundle = &accel->bundle[handle->bundle];
	if (!bundle->obj) {
		printf("QAT: Error no vm_object in bundle\n");
		return EINVAL;
	}

	/* Adding pid to bundle list */
	instance_rings =
	    malloc(sizeof(*instance_rings), M_QAT, M_WAITOK | M_ZERO);
	instance_rings->user_pid = curproc->p_pid;
	instance_rings->ring_mask = 0;
	mutex_lock(&bundle->list_lock);
	list_add_tail(&instance_rings->list, &bundle->list);
	mutex_unlock(&bundle->list_lock);

	vm_object_reference(bundle->obj);
	*object = bundle->obj;
	return (0);
}

static inline void
adf_uio_init_accel_ctrl(struct adf_uio_control_accel *accel,
			struct adf_accel_dev *accel_dev,
			unsigned int nb_bundles)
{
	struct adf_uio_control_bundle *bundle;
	struct qat_uio_bundle_dev *priv;
	unsigned int i;

	accel->nb_bundles = nb_bundles;
	accel->total_used_bundles = 0;

	for (i = 0; i < nb_bundles; i++) {
		/*initialize the bundle */
		bundle = &accel->bundle[i];
		priv = &bundle->uio_priv;
		bundle->hardware_bundle_number =
		    GET_MAX_BANKS(accel_dev) - nb_bundles + i;

		INIT_LIST_HEAD(&bundle->list);
		priv->bundle = bundle;
		priv->accel = accel;

		mutex_init(&bundle->lock);
		mutex_init(&bundle->list_lock);
		if (!accel->bar)
			printf("ERROR: bar not defined in accel\n");
		else
			bundle->csr_addr = (void *)accel->bar->virt_addr;
	}
}

/**
 * Initialization bars on dev start.
 */
static inline void
adf_uio_init_bundle_dev(struct adf_uio_control_accel *accel,
			struct adf_accel_dev *accel_dev,
			unsigned int nb_bundles)
{
	struct adf_uio_control_bundle *bundle;
	unsigned int i;

	for (i = 0; i < nb_bundles; i++) {
		bundle = &accel->bundle[i];
		bundle->obj =
		    adf_uio_map_bar(accel_dev, bundle->hardware_bundle_number);
		if (!bundle->obj) {
			device_printf(GET_DEV(accel_dev),
				      "ERROR in adf_alloc_bundle %d\n",
				      __LINE__);
		}
	}
}

int
adf_uio_register(struct adf_accel_dev *accel_dev)
{
	struct adf_uio_control_accel *accel = NULL;
	char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { 0 };
	int nb_bundles;

	if (!accel_dev) {
		printf("%s - accel_dev is NULL\n", __func__);
		return EFAULT;
	}

	if (adf_cfg_get_param_value(
		accel_dev, ADF_GENERAL_SEC, ADF_FIRST_USER_BUNDLE, val)) {
		nb_bundles = 0;
	} else {
		nb_bundles = GET_MAX_BANKS(accel_dev);
	}

	if (nb_bundles) {
		accel = malloc(sizeof(*accel) +
				   nb_bundles *
				       sizeof(struct adf_uio_control_bundle),
			       M_QAT,
			       M_WAITOK | M_ZERO);
		mtx_init(&accel->lock, "qat uio", NULL, MTX_DEF);
		accel->accel_dev = accel_dev;
		accel->bar = accel_dev->accel_pci_dev.pci_bars +
		    ADF_UIO_GET_BAR(accel_dev);

		adf_uio_init_accel_ctrl(accel, accel_dev, nb_bundles);
		accel->cdev = make_dev(&adf_uio_cdevsw,
				       0,
				       UID_ROOT,
				       GID_WHEEL,
				       0600,
				       "%s",
				       device_get_nameunit(GET_DEV(accel_dev)));
		if (accel->cdev == NULL) {
			mtx_destroy(&accel->lock);
			goto fail_clean;
		}
		accel->cdev->si_drv1 = accel_dev;
		accel_dev->accel = accel;
		cv_init(&accel->cleanup_ok, "uio_accel_cv");

		adf_uio_init_bundle_dev(accel, accel_dev, nb_bundles);
	}
	return 0;
fail_clean:
	free(accel, M_QAT);
	device_printf(GET_DEV(accel_dev), "Failed to register UIO devices\n");
	return ENODEV;
}

void
adf_uio_remove(struct adf_accel_dev *accel_dev)
{
	struct adf_uio_control_accel *accel = accel_dev->accel;
	struct adf_uio_control_bundle *bundle;
	unsigned int i;

	if (accel) {
		/* Un-mapping all bars */
		for (i = 0; i < accel->nb_bundles; i++) {
			bundle = &accel->bundle[i];
			vm_object_deallocate(bundle->obj);
		}

		destroy_dev(accel->cdev);
		mtx_lock(&accel->lock);
		while (accel->num_handles) {
			cv_timedwait_sig(&accel->cleanup_ok,
					 &accel->lock,
					 3 * hz);
		}
		mtx_unlock(&accel->lock);
		mtx_destroy(&accel->lock);
		cv_destroy(&accel->cleanup_ok);
		free(accel, M_QAT);
		accel_dev->accel = NULL;
	}
}