cxl/core/memdev.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2020 Intel Corporation. */

#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/firmware.h>
#include <linux/device.h>
#include <linux/slab.h>
#include <linux/idr.h>
#include <linux/pci.h>
#include <cxlmem.h>
#include "trace.h"
#include "core.h"

static DECLARE_RWSEM(cxl_memdev_rwsem);

/*
 * An entire PCI topology full of devices should be enough for any
 * config
 */
#define CXL_MEM_MAX_DEVS 65536

static int cxl_mem_major;
static DEFINE_IDA(cxl_memdev_ida);

static void cxl_memdev_release(struct device *dev)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct device *parent = dev->parent;

	ida_free(&cxl_memdev_ida, cxlmd->id);
	kfree(cxlmd);
	put_device(parent);
}

static char *cxl_memdev_devnode(const struct device *dev, umode_t *mode, kuid_t *uid,
				kgid_t *gid)
{
	return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
}

static ssize_t firmware_version_show(struct device *dev,
				     struct device_attribute *attr, char *buf)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);

	if (!mds)
		return sysfs_emit(buf, "\n");
	return sysfs_emit(buf, "%.16s\n", mds->firmware_version);
}
static DEVICE_ATTR_RO(firmware_version);

static ssize_t payload_max_show(struct device *dev,
				struct device_attribute *attr, char *buf)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);

	if (!mds)
		return sysfs_emit(buf, "\n");
	return sysfs_emit(buf, "%zu\n", cxlds->cxl_mbox.payload_size);
}
static DEVICE_ATTR_RO(payload_max);

static ssize_t label_storage_size_show(struct device *dev,
				       struct device_attribute *attr, char *buf)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);

	if (!mds)
		return sysfs_emit(buf, "\n");
	return sysfs_emit(buf, "%zu\n", mds->lsa_size);
}
static DEVICE_ATTR_RO(label_storage_size);

static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds)
{
	/* Static RAM is only expected at partition 0. */
	if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
		return 0;
	return resource_size(&cxlds->part[0].res);
}

static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
			     char *buf)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	unsigned long long len = cxl_ram_size(cxlds);

	return sysfs_emit(buf, "%#llx\n", len);
}

static struct device_attribute dev_attr_ram_size =
	__ATTR(size, 0444, ram_size_show, NULL);

static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
			      char *buf)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	unsigned long long len = cxl_pmem_size(cxlds);

	return sysfs_emit(buf, "%#llx\n", len);
}

static struct device_attribute dev_attr_pmem_size =
	__ATTR(size, 0444, pmem_size_show, NULL);

static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
			   char *buf)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;

	return sysfs_emit(buf, "%#llx\n", cxlds->serial);
}
static DEVICE_ATTR_RO(serial);

static ssize_t numa_node_show(struct device *dev, struct device_attribute *attr,
			      char *buf)
{
	return sysfs_emit(buf, "%d\n", dev_to_node(dev));
}
static DEVICE_ATTR_RO(numa_node);

static ssize_t security_state_show(struct device *dev,
				   struct device_attribute *attr,
				   char *buf)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
	unsigned long state = mds->security.state;
	int rc = 0;

	/* sync with latest submission state */
	mutex_lock(&cxl_mbox->mbox_mutex);
	if (mds->security.sanitize_active)
		rc = sysfs_emit(buf, "sanitize\n");
	mutex_unlock(&cxl_mbox->mbox_mutex);
	if (rc)
		return rc;

	if (!(state & CXL_PMEM_SEC_STATE_USER_PASS_SET))
		return sysfs_emit(buf, "disabled\n");
	if (state & CXL_PMEM_SEC_STATE_FROZEN ||
	    state & CXL_PMEM_SEC_STATE_MASTER_PLIMIT ||
	    state & CXL_PMEM_SEC_STATE_USER_PLIMIT)
		return sysfs_emit(buf, "frozen\n");
	if (state & CXL_PMEM_SEC_STATE_LOCKED)
		return sysfs_emit(buf, "locked\n");

	return sysfs_emit(buf, "unlocked\n");
}
static struct device_attribute dev_attr_security_state =
	__ATTR(state, 0444, security_state_show, NULL);

static ssize_t security_sanitize_store(struct device *dev,
				       struct device_attribute *attr,
				       const char *buf, size_t len)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	bool sanitize;
	ssize_t rc;

	if (kstrtobool(buf, &sanitize) || !sanitize)
		return -EINVAL;

	rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SANITIZE);
	if (rc)
		return rc;

	return len;
}
static struct device_attribute dev_attr_security_sanitize =
	__ATTR(sanitize, 0200, NULL, security_sanitize_store);

static ssize_t security_erase_store(struct device *dev,
				    struct device_attribute *attr,
				    const char *buf, size_t len)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	ssize_t rc;
	bool erase;

	if (kstrtobool(buf, &erase) || !erase)
		return -EINVAL;

	rc = cxl_mem_sanitize(cxlmd, CXL_MBOX_OP_SECURE_ERASE);
	if (rc)
		return rc;

	return len;
}
static struct device_attribute dev_attr_security_erase =
	__ATTR(erase, 0200, NULL, security_erase_store);

bool cxl_memdev_has_poison_cmd(struct cxl_memdev *cxlmd,
			       enum poison_cmd_enabled_bits cmd)
{
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);

	if (!mds)
		return 0;

	return test_bit(cmd, mds->poison.enabled_cmds);
}

static int cxl_get_poison_by_memdev(struct cxl_memdev *cxlmd)
{
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	u64 offset, length;
	int rc = 0;

	/* CXL 3.0 Spec 8.2.9.8.4.1 Separate pmem and ram poison requests */
	for (int i = 0; i < cxlds->nr_partitions; i++) {
		const struct resource *res = &cxlds->part[i].res;

		offset = res->start;
		length = resource_size(res);
		rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
		/*
		 * Invalid Physical Address is not an error for
		 * volatile addresses. Device support is optional.
		 */
		if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM)
			rc = 0;
	}
	return rc;
}

int cxl_trigger_poison_list(struct cxl_memdev *cxlmd)
{
	struct cxl_port *port;
	int rc;

	port = cxlmd->endpoint;
	if (!port || !is_cxl_endpoint(port))
		return -EINVAL;

	ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
		return rc;

	ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
		return rc;

	if (cxl_num_decoders_committed(port) == 0) {
		/* No regions mapped to this memdev */
		rc = cxl_get_poison_by_memdev(cxlmd);
	} else {
		/* Regions mapped, collect poison by endpoint */
		rc =  cxl_get_poison_by_endpoint(port);
	}

	return rc;
}
EXPORT_SYMBOL_NS_GPL(cxl_trigger_poison_list, "CXL");

static int cxl_validate_poison_dpa(struct cxl_memdev *cxlmd, u64 dpa)
{
	struct cxl_dev_state *cxlds = cxlmd->cxlds;

	if (!IS_ENABLED(CONFIG_DEBUG_FS))
		return 0;

	if (!resource_size(&cxlds->dpa_res)) {
		dev_dbg(cxlds->dev, "device has no dpa resource\n");
		return -EINVAL;
	}
	if (!cxl_resource_contains_addr(&cxlds->dpa_res, dpa)) {
		dev_dbg(cxlds->dev, "dpa:0x%llx not in resource:%pR\n",
			dpa, &cxlds->dpa_res);
		return -EINVAL;
	}
	if (!IS_ALIGNED(dpa, 64)) {
		dev_dbg(cxlds->dev, "dpa:0x%llx is not 64-byte aligned\n", dpa);
		return -EINVAL;
	}

	return 0;
}

int cxl_inject_poison_locked(struct cxl_memdev *cxlmd, u64 dpa)
{
	struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
	struct cxl_mbox_inject_poison inject;
	struct cxl_poison_record record;
	struct cxl_mbox_cmd mbox_cmd;
	struct cxl_region *cxlr;
	int rc;

	if (!IS_ENABLED(CONFIG_DEBUG_FS))
		return 0;

	lockdep_assert_held(&cxl_rwsem.dpa);
	lockdep_assert_held(&cxl_rwsem.region);

	rc = cxl_validate_poison_dpa(cxlmd, dpa);
	if (rc)
		return rc;

	inject.address = cpu_to_le64(dpa);
	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_INJECT_POISON,
		.size_in = sizeof(inject),
		.payload_in = &inject,
	};
	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
	if (rc)
		return rc;

	cxlr = cxl_dpa_to_region(cxlmd, dpa);
	if (cxlr)
		dev_warn_once(cxl_mbox->host,
			      "poison inject dpa:%#llx region: %s\n", dpa,
			      dev_name(&cxlr->dev));

	record = (struct cxl_poison_record) {
		.address = cpu_to_le64(dpa),
		.length = cpu_to_le32(1),
	};
	trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_INJECT);

	return 0;
}

int cxl_inject_poison(struct cxl_memdev *cxlmd, u64 dpa)
{
	int rc;

	ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
		return rc;

	ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
		return rc;

	return cxl_inject_poison_locked(cxlmd, dpa);
}
EXPORT_SYMBOL_NS_GPL(cxl_inject_poison, "CXL");

int cxl_clear_poison_locked(struct cxl_memdev *cxlmd, u64 dpa)
{
	struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
	struct cxl_mbox_clear_poison clear;
	struct cxl_poison_record record;
	struct cxl_mbox_cmd mbox_cmd;
	struct cxl_region *cxlr;
	int rc;

	if (!IS_ENABLED(CONFIG_DEBUG_FS))
		return 0;

	lockdep_assert_held(&cxl_rwsem.dpa);
	lockdep_assert_held(&cxl_rwsem.region);

	rc = cxl_validate_poison_dpa(cxlmd, dpa);
	if (rc)
		return rc;

	/*
	 * In CXL 3.0 Spec 8.2.9.8.4.3, the Clear Poison mailbox command
	 * is defined to accept 64 bytes of write-data, along with the
	 * address to clear. This driver uses zeroes as write-data.
	 */
	clear = (struct cxl_mbox_clear_poison) {
		.address = cpu_to_le64(dpa)
	};

	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_CLEAR_POISON,
		.size_in = sizeof(clear),
		.payload_in = &clear,
	};

	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
	if (rc)
		return rc;

	cxlr = cxl_dpa_to_region(cxlmd, dpa);
	if (cxlr)
		dev_warn_once(cxl_mbox->host,
			      "poison clear dpa:%#llx region: %s\n", dpa,
			      dev_name(&cxlr->dev));

	record = (struct cxl_poison_record) {
		.address = cpu_to_le64(dpa),
		.length = cpu_to_le32(1),
	};
	trace_cxl_poison(cxlmd, cxlr, &record, 0, 0, CXL_POISON_TRACE_CLEAR);

	return 0;
}

int cxl_clear_poison(struct cxl_memdev *cxlmd, u64 dpa)
{
	int rc;

	ACQUIRE(rwsem_read_intr, region_rwsem)(&cxl_rwsem.region);
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &region_rwsem)))
		return rc;

	ACQUIRE(rwsem_read_intr, dpa_rwsem)(&cxl_rwsem.dpa);
	if ((rc = ACQUIRE_ERR(rwsem_read_intr, &dpa_rwsem)))
		return rc;

	return cxl_clear_poison_locked(cxlmd, dpa);
}
EXPORT_SYMBOL_NS_GPL(cxl_clear_poison, "CXL");

static struct attribute *cxl_memdev_attributes[] = {
	&dev_attr_serial.attr,
	&dev_attr_firmware_version.attr,
	&dev_attr_payload_max.attr,
	&dev_attr_label_storage_size.attr,
	&dev_attr_numa_node.attr,
	NULL,
};

static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds)
{
	for (int i = 0; i < cxlds->nr_partitions; i++)
		if (cxlds->part[i].mode == CXL_PARTMODE_PMEM)
			return &cxlds->part[i].perf;
	return NULL;
}

static ssize_t pmem_qos_class_show(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;

	return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class);
}

static struct device_attribute dev_attr_pmem_qos_class =
	__ATTR(qos_class, 0444, pmem_qos_class_show, NULL);

static struct attribute *cxl_memdev_pmem_attributes[] = {
	&dev_attr_pmem_size.attr,
	&dev_attr_pmem_qos_class.attr,
	NULL,
};

static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds)
{
	if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
		return NULL;
	return &cxlds->part[0].perf;
}

static ssize_t ram_qos_class_show(struct device *dev,
				  struct device_attribute *attr, char *buf)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dev_state *cxlds = cxlmd->cxlds;

	return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class);
}

static struct device_attribute dev_attr_ram_qos_class =
	__ATTR(qos_class, 0444, ram_qos_class_show, NULL);

static struct attribute *cxl_memdev_ram_attributes[] = {
	&dev_attr_ram_size.attr,
	&dev_attr_ram_qos_class.attr,
	NULL,
};

static struct attribute *cxl_memdev_security_attributes[] = {
	&dev_attr_security_state.attr,
	&dev_attr_security_sanitize.attr,
	&dev_attr_security_erase.attr,
	NULL,
};

static umode_t cxl_memdev_visible(struct kobject *kobj, struct attribute *a,
				  int n)
{
	if (!IS_ENABLED(CONFIG_NUMA) && a == &dev_attr_numa_node.attr)
		return 0;
	return a->mode;
}

static struct attribute_group cxl_memdev_attribute_group = {
	.attrs = cxl_memdev_attributes,
	.is_visible = cxl_memdev_visible,
};

static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
{
	struct device *dev = kobj_to_dev(kobj);
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds);

	if (a == &dev_attr_ram_qos_class.attr &&
	    (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
		return 0;

	return a->mode;
}

static struct attribute_group cxl_memdev_ram_attribute_group = {
	.name = "ram",
	.attrs = cxl_memdev_ram_attributes,
	.is_visible = cxl_ram_visible,
};

static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n)
{
	struct device *dev = kobj_to_dev(kobj);
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds);

	if (a == &dev_attr_pmem_qos_class.attr &&
	    (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
		return 0;

	return a->mode;
}

static struct attribute_group cxl_memdev_pmem_attribute_group = {
	.name = "pmem",
	.attrs = cxl_memdev_pmem_attributes,
	.is_visible = cxl_pmem_visible,
};

static umode_t cxl_memdev_security_visible(struct kobject *kobj,
					   struct attribute *a, int n)
{
	struct device *dev = kobj_to_dev(kobj);
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);

	if (a == &dev_attr_security_sanitize.attr &&
	    !test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
		return 0;

	if (a == &dev_attr_security_erase.attr &&
	    !test_bit(CXL_SEC_ENABLED_SECURE_ERASE, mds->security.enabled_cmds))
		return 0;

	return a->mode;
}

static struct attribute_group cxl_memdev_security_attribute_group = {
	.name = "security",
	.attrs = cxl_memdev_security_attributes,
	.is_visible = cxl_memdev_security_visible,
};

static const struct attribute_group *cxl_memdev_attribute_groups[] = {
	&cxl_memdev_attribute_group,
	&cxl_memdev_ram_attribute_group,
	&cxl_memdev_pmem_attribute_group,
	&cxl_memdev_security_attribute_group,
	NULL,
};

void cxl_memdev_update_perf(struct cxl_memdev *cxlmd)
{
	sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_ram_attribute_group);
	sysfs_update_group(&cxlmd->dev.kobj, &cxl_memdev_pmem_attribute_group);
}
EXPORT_SYMBOL_NS_GPL(cxl_memdev_update_perf, "CXL");

static const struct device_type cxl_class_memdev_type = {
	.name = "cxl_memdev",
	.release = cxl_memdev_release,
	.devnode = cxl_memdev_devnode,
	.groups = cxl_memdev_attribute_groups,
};

static const struct device_type cxl_memdev_type = {
	.name = "cxl_memdev",
	.release = cxl_memdev_release,
	.devnode = cxl_memdev_devnode,
};

bool is_cxl_memdev(const struct device *dev)
{
	return (dev->type == &cxl_class_memdev_type ||
		dev->type == &cxl_memdev_type);
}
EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL");

static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode)
{
	int i = info->nr_partitions;

	if (size == 0)
		return;

	info->part[i].range = (struct range) {
		.start = start,
		.end = start + size - 1,
	};
	info->part[i].mode = mode;
	info->nr_partitions++;
}

int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info)
{
	struct cxl_dev_state *cxlds = &mds->cxlds;
	struct device *dev = cxlds->dev;
	int rc;

	if (!cxlds->media_ready) {
		info->size = 0;
		return 0;
	}

	info->size = mds->total_bytes;

	if (mds->partition_align_bytes == 0) {
		add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM);
		add_part(info, mds->volatile_only_bytes,
			 mds->persistent_only_bytes, CXL_PARTMODE_PMEM);
		return 0;
	}

	rc = cxl_mem_get_partition_info(mds);
	if (rc) {
		dev_err(dev, "Failed to query partition information\n");
		return rc;
	}

	add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM);
	add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes,
		 CXL_PARTMODE_PMEM);

	return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL");


/**
 * cxl_set_capacity: initialize dpa by a driver without a mailbox.
 *
 * @cxlds: pointer to cxl_dev_state
 * @capacity: device volatile memory size
 */
int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity)
{
	struct cxl_dpa_info range_info = {
		.size = capacity,
	};

	add_part(&range_info, 0, capacity, CXL_PARTMODE_RAM);
	return cxl_dpa_setup(cxlds, &range_info);
}
EXPORT_SYMBOL_NS_GPL(cxl_set_capacity, "CXL");

/**
 * set_exclusive_cxl_commands() - atomically disable user cxl commands
 * @mds: The device state to operate on
 * @cmds: bitmap of commands to mark exclusive
 *
 * Grab the cxl_memdev_rwsem in write mode to flush in-flight
 * invocations of the ioctl path and then disable future execution of
 * commands with the command ids set in @cmds.
 */
void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
				unsigned long *cmds)
{
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;

	guard(rwsem_write)(&cxl_memdev_rwsem);
	bitmap_or(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
		  cmds, CXL_MEM_COMMAND_ID_MAX);
}
EXPORT_SYMBOL_NS_GPL(set_exclusive_cxl_commands, "CXL");

/**
 * clear_exclusive_cxl_commands() - atomically enable user cxl commands
 * @mds: The device state to modify
 * @cmds: bitmap of commands to mark available for userspace
 */
void clear_exclusive_cxl_commands(struct cxl_memdev_state *mds,
				  unsigned long *cmds)
{
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;

	guard(rwsem_write)(&cxl_memdev_rwsem);
	bitmap_andnot(cxl_mbox->exclusive_cmds, cxl_mbox->exclusive_cmds,
		      cmds, CXL_MEM_COMMAND_ID_MAX);
}
EXPORT_SYMBOL_NS_GPL(clear_exclusive_cxl_commands, "CXL");

static void cxl_memdev_shutdown(struct device *dev)
{
	struct cxl_memdev *cxlmd = to_cxl_memdev(dev);

	guard(rwsem_write)(&cxl_memdev_rwsem);
	cxlmd->cxlds = NULL;
}

static void cxl_memdev_unregister(void *_cxlmd)
{
	struct cxl_memdev *cxlmd = _cxlmd;
	struct device *dev = &cxlmd->dev;

	cdev_device_del(&cxlmd->cdev, dev);
	cxl_memdev_shutdown(dev);
	put_device(dev);
}

static void detach_memdev(struct work_struct *work)
{
	struct cxl_memdev *cxlmd;

	cxlmd = container_of(work, typeof(*cxlmd), detach_work);

	/*
	 * When the creator of @cxlmd sets ->attach it indicates CXL operation
	 * is required. In that case, @cxlmd detach escalates to parent device
	 * detach.
	 */
	if (cxlmd->attach)
		device_release_driver(cxlmd->dev.parent);
	else
		device_release_driver(&cxlmd->dev);
	put_device(&cxlmd->dev);
}

static struct lock_class_key cxl_memdev_key;

struct cxl_dev_state *_devm_cxl_dev_state_create(struct device *dev,
						 enum cxl_devtype type,
						 u64 serial, u16 dvsec,
						 size_t size, bool has_mbox)
{
	struct cxl_dev_state *cxlds = devm_kzalloc(dev, size, GFP_KERNEL);

	if (!cxlds)
		return NULL;

	cxlds->dev = dev;
	cxlds->type = type;
	cxlds->serial = serial;
	cxlds->cxl_dvsec = dvsec;
	cxlds->reg_map.host = dev;
	cxlds->reg_map.resource = CXL_RESOURCE_NONE;

	if (has_mbox)
		cxlds->cxl_mbox.host = dev;

	return cxlds;
}
EXPORT_SYMBOL_NS_GPL(_devm_cxl_dev_state_create, "CXL");

static struct cxl_memdev *cxl_memdev_alloc(struct cxl_dev_state *cxlds,
					   const struct file_operations *fops,
					   const struct cxl_memdev_attach *attach)
{
	struct cxl_memdev *cxlmd;
	struct device *dev;
	struct cdev *cdev;
	int rc;

	cxlmd = kzalloc_obj(*cxlmd);
	if (!cxlmd)
		return ERR_PTR(-ENOMEM);

	rc = ida_alloc_max(&cxl_memdev_ida, CXL_MEM_MAX_DEVS - 1, GFP_KERNEL);
	if (rc < 0)
		goto err;
	cxlmd->id = rc;
	cxlmd->depth = -1;
	cxlmd->attach = attach;
	cxlmd->endpoint = ERR_PTR(-ENXIO);

	dev = &cxlmd->dev;
	device_initialize(dev);
	lockdep_set_class(&dev->mutex, &cxl_memdev_key);
	dev->parent = get_device(cxlds->dev);
	dev->bus = &cxl_bus_type;
	dev->devt = MKDEV(cxl_mem_major, cxlmd->id);
	if (cxlds->type == CXL_DEVTYPE_DEVMEM)
		dev->type = &cxl_memdev_type;
	else
		dev->type = &cxl_class_memdev_type;
	device_set_pm_not_required(dev);
	INIT_WORK(&cxlmd->detach_work, detach_memdev);

	cdev = &cxlmd->cdev;
	cdev_init(cdev, fops);
	return cxlmd;

err:
	kfree(cxlmd);
	return ERR_PTR(rc);
}

static long __cxl_memdev_ioctl(struct cxl_memdev *cxlmd, unsigned int cmd,
			       unsigned long arg)
{
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlmd->cxlds);
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;

	switch (cmd) {
	case CXL_MEM_QUERY_COMMANDS:
		return cxl_query_cmd(cxl_mbox, (void __user *)arg);
	case CXL_MEM_SEND_COMMAND:
		return cxl_send_cmd(cxl_mbox, (void __user *)arg);
	default:
		return -ENOTTY;
	}
}

static long cxl_memdev_ioctl(struct file *file, unsigned int cmd,
			     unsigned long arg)
{
	struct cxl_memdev *cxlmd = file->private_data;
	struct cxl_dev_state *cxlds;

	guard(rwsem_read)(&cxl_memdev_rwsem);
	cxlds = cxlmd->cxlds;
	if (cxlds && cxlds->type == CXL_DEVTYPE_CLASSMEM)
		return __cxl_memdev_ioctl(cxlmd, cmd, arg);

	return -ENXIO;
}

static int cxl_memdev_open(struct inode *inode, struct file *file)
{
	struct cxl_memdev *cxlmd =
		container_of(inode->i_cdev, typeof(*cxlmd), cdev);

	get_device(&cxlmd->dev);
	file->private_data = cxlmd;

	return 0;
}

static int cxl_memdev_release_file(struct inode *inode, struct file *file)
{
	struct cxl_memdev *cxlmd =
		container_of(inode->i_cdev, typeof(*cxlmd), cdev);

	put_device(&cxlmd->dev);

	return 0;
}

/**
 * cxl_mem_get_fw_info - Get Firmware info
 * @mds: The device data for the operation
 *
 * Retrieve firmware info for the device specified.
 *
 * Return: 0 if no error: or the result of the mailbox command.
 *
 * See CXL-3.0 8.2.9.3.1 Get FW Info
 */
static int cxl_mem_get_fw_info(struct cxl_memdev_state *mds)
{
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
	struct cxl_mbox_get_fw_info info;
	struct cxl_mbox_cmd mbox_cmd;
	int rc;

	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_GET_FW_INFO,
		.size_out = sizeof(info),
		.payload_out = &info,
	};

	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
	if (rc < 0)
		return rc;

	mds->fw.num_slots = info.num_slots;
	mds->fw.cur_slot = FIELD_GET(CXL_FW_INFO_SLOT_INFO_CUR_MASK,
				       info.slot_info);

	return 0;
}

/**
 * cxl_mem_activate_fw - Activate Firmware
 * @mds: The device data for the operation
 * @slot: slot number to activate
 *
 * Activate firmware in a given slot for the device specified.
 *
 * Return: 0 if no error: or the result of the mailbox command.
 *
 * See CXL-3.0 8.2.9.3.3 Activate FW
 */
static int cxl_mem_activate_fw(struct cxl_memdev_state *mds, int slot)
{
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
	struct cxl_mbox_activate_fw activate;
	struct cxl_mbox_cmd mbox_cmd;

	if (slot == 0 || slot > mds->fw.num_slots)
		return -EINVAL;

	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_ACTIVATE_FW,
		.size_in = sizeof(activate),
		.payload_in = &activate,
	};

	/* Only offline activation supported for now */
	activate.action = CXL_FW_ACTIVATE_OFFLINE;
	activate.slot = slot;

	return cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
}

/**
 * cxl_mem_abort_fw_xfer - Abort an in-progress FW transfer
 * @mds: The device data for the operation
 *
 * Abort an in-progress firmware transfer for the device specified.
 *
 * Return: 0 if no error: or the result of the mailbox command.
 *
 * See CXL-3.0 8.2.9.3.2 Transfer FW
 */
static int cxl_mem_abort_fw_xfer(struct cxl_memdev_state *mds)
{
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
	struct cxl_mbox_transfer_fw *transfer;
	struct cxl_mbox_cmd mbox_cmd;
	int rc;

	transfer = kzalloc_flex(*transfer, data, 0);
	if (!transfer)
		return -ENOMEM;

	/* Set a 1s poll interval and a total wait time of 30s */
	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_TRANSFER_FW,
		.size_in = sizeof(*transfer),
		.payload_in = transfer,
		.poll_interval_ms = 1000,
		.poll_count = 30,
	};

	transfer->action = CXL_FW_TRANSFER_ACTION_ABORT;

	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
	kfree(transfer);
	return rc;
}

static void cxl_fw_cleanup(struct fw_upload *fwl)
{
	struct cxl_memdev_state *mds = fwl->dd_handle;

	mds->fw.next_slot = 0;
}

static int cxl_fw_do_cancel(struct fw_upload *fwl)
{
	struct cxl_memdev_state *mds = fwl->dd_handle;
	struct cxl_dev_state *cxlds = &mds->cxlds;
	struct cxl_memdev *cxlmd = cxlds->cxlmd;
	int rc;

	rc = cxl_mem_abort_fw_xfer(mds);
	if (rc < 0)
		dev_err(&cxlmd->dev, "Error aborting FW transfer: %d\n", rc);

	return FW_UPLOAD_ERR_CANCELED;
}

static enum fw_upload_err cxl_fw_prepare(struct fw_upload *fwl, const u8 *data,
					 u32 size)
{
	struct cxl_memdev_state *mds = fwl->dd_handle;
	struct cxl_mbox_transfer_fw *transfer;
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;

	if (!size)
		return FW_UPLOAD_ERR_INVALID_SIZE;

	mds->fw.oneshot = struct_size(transfer, data, size) <
			    cxl_mbox->payload_size;

	if (cxl_mem_get_fw_info(mds))
		return FW_UPLOAD_ERR_HW_ERROR;

	/*
	 * So far no state has been changed, hence no other cleanup is
	 * necessary. Simply return the cancelled status.
	 */
	if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
		return FW_UPLOAD_ERR_CANCELED;

	return FW_UPLOAD_ERR_NONE;
}

static enum fw_upload_err cxl_fw_write(struct fw_upload *fwl, const u8 *data,
				       u32 offset, u32 size, u32 *written)
{
	struct cxl_memdev_state *mds = fwl->dd_handle;
	struct cxl_dev_state *cxlds = &mds->cxlds;
	struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
	struct cxl_memdev *cxlmd = cxlds->cxlmd;
	struct cxl_mbox_transfer_fw *transfer;
	struct cxl_mbox_cmd mbox_cmd;
	u32 cur_size, remaining;
	size_t size_in;
	int rc;

	*written = 0;

	/* Offset has to be aligned to 128B (CXL-3.0 8.2.9.3.2 Table 8-57) */
	if (!IS_ALIGNED(offset, CXL_FW_TRANSFER_ALIGNMENT)) {
		dev_err(&cxlmd->dev,
			"misaligned offset for FW transfer slice (%u)\n",
			offset);
		return FW_UPLOAD_ERR_RW_ERROR;
	}

	/*
	 * Pick transfer size based on mds->payload_size @size must bw 128-byte
	 * aligned, ->payload_size is a power of 2 starting at 256 bytes, and
	 * sizeof(*transfer) is 128.  These constraints imply that @cur_size
	 * will always be 128b aligned.
	 */
	cur_size = min_t(size_t, size, cxl_mbox->payload_size - sizeof(*transfer));

	remaining = size - cur_size;
	size_in = struct_size(transfer, data, cur_size);

	if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
		return cxl_fw_do_cancel(fwl);

	/*
	 * Slot numbers are 1-indexed
	 * cur_slot is the 0-indexed next_slot (i.e. 'cur_slot - 1 + 1')
	 * Check for rollover using modulo, and 1-index it by adding 1
	 */
	mds->fw.next_slot = (mds->fw.cur_slot % mds->fw.num_slots) + 1;

	/* Do the transfer via mailbox cmd */
	transfer = kzalloc(size_in, GFP_KERNEL);
	if (!transfer)
		return FW_UPLOAD_ERR_RW_ERROR;

	transfer->offset = cpu_to_le32(offset / CXL_FW_TRANSFER_ALIGNMENT);
	memcpy(transfer->data, data + offset, cur_size);
	if (mds->fw.oneshot) {
		transfer->action = CXL_FW_TRANSFER_ACTION_FULL;
		transfer->slot = mds->fw.next_slot;
	} else {
		if (offset == 0) {
			transfer->action = CXL_FW_TRANSFER_ACTION_INITIATE;
		} else if (remaining == 0) {
			transfer->action = CXL_FW_TRANSFER_ACTION_END;
			transfer->slot = mds->fw.next_slot;
		} else {
			transfer->action = CXL_FW_TRANSFER_ACTION_CONTINUE;
		}
	}

	mbox_cmd = (struct cxl_mbox_cmd) {
		.opcode = CXL_MBOX_OP_TRANSFER_FW,
		.size_in = size_in,
		.payload_in = transfer,
		.poll_interval_ms = 1000,
		.poll_count = 30,
	};

	rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
	if (rc < 0) {
		rc = FW_UPLOAD_ERR_RW_ERROR;
		goto out_free;
	}

	*written = cur_size;

	/* Activate FW if oneshot or if the last slice was written */
	if (mds->fw.oneshot || remaining == 0) {
		dev_dbg(&cxlmd->dev, "Activating firmware slot: %d\n",
			mds->fw.next_slot);
		rc = cxl_mem_activate_fw(mds, mds->fw.next_slot);
		if (rc < 0) {
			dev_err(&cxlmd->dev, "Error activating firmware: %d\n",
				rc);
			rc = FW_UPLOAD_ERR_HW_ERROR;
			goto out_free;
		}
	}

	rc = FW_UPLOAD_ERR_NONE;

out_free:
	kfree(transfer);
	return rc;
}

static enum fw_upload_err cxl_fw_poll_complete(struct fw_upload *fwl)
{
	struct cxl_memdev_state *mds = fwl->dd_handle;

	/*
	 * cxl_internal_send_cmd() handles background operations synchronously.
	 * No need to wait for completions here - any errors would've been
	 * reported and handled during the ->write() call(s).
	 * Just check if a cancel request was received, and return success.
	 */
	if (test_and_clear_bit(CXL_FW_CANCEL, mds->fw.state))
		return cxl_fw_do_cancel(fwl);

	return FW_UPLOAD_ERR_NONE;
}

static void cxl_fw_cancel(struct fw_upload *fwl)
{
	struct cxl_memdev_state *mds = fwl->dd_handle;

	set_bit(CXL_FW_CANCEL, mds->fw.state);
}

static const struct fw_upload_ops cxl_memdev_fw_ops = {
        .prepare = cxl_fw_prepare,
        .write = cxl_fw_write,
        .poll_complete = cxl_fw_poll_complete,
        .cancel = cxl_fw_cancel,
        .cleanup = cxl_fw_cleanup,
};

static void cxl_remove_fw_upload(void *fwl)
{
	firmware_upload_unregister(fwl);
}

int devm_cxl_setup_fw_upload(struct device *host, struct cxl_memdev_state *mds)
{
	struct cxl_dev_state *cxlds = &mds->cxlds;
	struct cxl_mailbox *cxl_mbox = &cxlds->cxl_mbox;
	struct device *dev = &cxlds->cxlmd->dev;
	struct fw_upload *fwl;

	if (!test_bit(CXL_MEM_COMMAND_ID_GET_FW_INFO, cxl_mbox->enabled_cmds))
		return 0;

	fwl = firmware_upload_register(THIS_MODULE, dev, dev_name(dev),
				       &cxl_memdev_fw_ops, mds);
	if (IS_ERR(fwl))
		return PTR_ERR(fwl);
	return devm_add_action_or_reset(host, cxl_remove_fw_upload, fwl);
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_fw_upload, "CXL");

static const struct file_operations cxl_memdev_fops = {
	.owner = THIS_MODULE,
	.unlocked_ioctl = cxl_memdev_ioctl,
	.open = cxl_memdev_open,
	.release = cxl_memdev_release_file,
	.compat_ioctl = compat_ptr_ioctl,
	.llseek = noop_llseek,
};

/*
 * Activate ioctl operations, no cxl_memdev_rwsem manipulation needed as this is
 * ordered with cdev_add() publishing the device.
 */
static int cxlmd_add(struct cxl_memdev *cxlmd, struct cxl_dev_state *cxlds)
{
	int rc;

	cxlmd->cxlds = cxlds;
	cxlds->cxlmd = cxlmd;

	rc = cdev_device_add(&cxlmd->cdev, &cxlmd->dev);
	if (rc) {
		/*
		 * The cdev was briefly live, shutdown any ioctl operations that
		 * saw that state.
		 */
		cxl_memdev_shutdown(&cxlmd->dev);
		return rc;
	}

	return 0;
}

DEFINE_FREE(put_cxlmd, struct cxl_memdev *,
	    if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev))

static bool cxl_memdev_attach_failed(struct cxl_memdev *cxlmd)
{
	/*
	 * If @attach is provided fail if the driver is not attached upon
	 * return. Note that failure here could be the result of a race to
	 * teardown the CXL port topology. I.e. cxl_mem_probe() could have
	 * succeeded and then cxl_mem unbound before the lock is acquired.
	 */
	guard(device)(&cxlmd->dev);
	return (cxlmd->attach && !cxlmd->dev.driver);
}

static struct cxl_memdev *cxl_memdev_autoremove(struct cxl_memdev *cxlmd)
{
	int rc;

	if (cxl_memdev_attach_failed(cxlmd)) {
		cxl_memdev_unregister(cxlmd);
		return ERR_PTR(-ENXIO);
	}

	rc = devm_add_action_or_reset(cxlmd->cxlds->dev, cxl_memdev_unregister,
				      cxlmd);
	if (rc)
		return ERR_PTR(rc);

	return cxlmd;
}

/*
 * Core helper for devm_cxl_add_memdev() that wants to both create a device and
 * assert to the caller that upon return cxl_mem::probe() has been invoked.
 */
struct cxl_memdev *__devm_cxl_add_memdev(struct cxl_dev_state *cxlds,
					 const struct cxl_memdev_attach *attach)
{
	struct device *dev;
	int rc;

	struct cxl_memdev *cxlmd __free(put_cxlmd) =
		cxl_memdev_alloc(cxlds, &cxl_memdev_fops, attach);
	if (IS_ERR(cxlmd))
		return cxlmd;

	dev = &cxlmd->dev;
	rc = dev_set_name(dev, "mem%d", cxlmd->id);
	if (rc)
		return ERR_PTR(rc);

	rc = cxlmd_add(cxlmd, cxlds);
	if (rc)
		return ERR_PTR(rc);

	return cxl_memdev_autoremove(no_free_ptr(cxlmd));
}
EXPORT_SYMBOL_FOR_MODULES(__devm_cxl_add_memdev, "cxl_mem");

static void sanitize_teardown_notifier(void *data)
{
	struct cxl_memdev_state *mds = data;
	struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
	struct kernfs_node *state;

	/*
	 * Prevent new irq triggered invocations of the workqueue and
	 * flush inflight invocations.
	 */
	mutex_lock(&cxl_mbox->mbox_mutex);
	state = mds->security.sanitize_node;
	mds->security.sanitize_node = NULL;
	mutex_unlock(&cxl_mbox->mbox_mutex);

	cancel_delayed_work_sync(&mds->security.poll_dwork);
	sysfs_put(state);
}

int devm_cxl_sanitize_setup_notifier(struct device *host,
				     struct cxl_memdev *cxlmd)
{
	struct cxl_dev_state *cxlds = cxlmd->cxlds;
	struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
	struct kernfs_node *sec;

	if (!test_bit(CXL_SEC_ENABLED_SANITIZE, mds->security.enabled_cmds))
		return 0;

	/*
	 * Note, the expectation is that @cxlmd would have failed to be
	 * created if these sysfs_get_dirent calls fail.
	 */
	sec = sysfs_get_dirent(cxlmd->dev.kobj.sd, "security");
	if (!sec)
		return -ENOENT;
	mds->security.sanitize_node = sysfs_get_dirent(sec, "state");
	sysfs_put(sec);
	if (!mds->security.sanitize_node)
		return -ENOENT;

	return devm_add_action_or_reset(host, sanitize_teardown_notifier, mds);
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_sanitize_setup_notifier, "CXL");

__init int cxl_memdev_init(void)
{
	dev_t devt;
	int rc;

	rc = alloc_chrdev_region(&devt, 0, CXL_MEM_MAX_DEVS, "cxl");
	if (rc)
		return rc;

	cxl_mem_major = MAJOR(devt);

	return 0;
}

void cxl_memdev_exit(void)
{
	unregister_chrdev_region(MKDEV(cxl_mem_major, 0), CXL_MEM_MAX_DEVS);
}