1779dd20cSBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 2779dd20cSBen Widawsky /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3779dd20cSBen Widawsky #include <linux/memregion.h> 4779dd20cSBen Widawsky #include <linux/genalloc.h> 5779dd20cSBen Widawsky #include <linux/device.h> 6779dd20cSBen Widawsky #include <linux/module.h> 7779dd20cSBen Widawsky #include <linux/slab.h> 8dd5ba0ebSBen Widawsky #include <linux/uuid.h> 9779dd20cSBen Widawsky #include <linux/idr.h> 1080d10a6cSBen Widawsky #include <cxlmem.h> 11779dd20cSBen Widawsky #include <cxl.h> 12779dd20cSBen Widawsky #include "core.h" 13779dd20cSBen Widawsky 14779dd20cSBen Widawsky /** 15779dd20cSBen Widawsky * DOC: cxl core region 16779dd20cSBen Widawsky * 17779dd20cSBen Widawsky * CXL Regions represent mapped memory capacity in system physical address 18779dd20cSBen Widawsky * space. Whereas the CXL Root Decoders identify the bounds of potential CXL 19779dd20cSBen Widawsky * Memory ranges, Regions represent the active mapped capacity by the HDM 20779dd20cSBen Widawsky * Decoder Capability structures throughout the Host Bridges, Switches, and 21779dd20cSBen Widawsky * Endpoints in the topology. 22dd5ba0ebSBen Widawsky * 23dd5ba0ebSBen Widawsky * Region configuration has ordering constraints. UUID may be set at any time 24dd5ba0ebSBen Widawsky * but is only visible for persistent regions. 2580d10a6cSBen Widawsky * 1. Interleave granularity 2680d10a6cSBen Widawsky * 2. Interleave size 27b9686e8cSDan Williams * 3. Decoder targets 28779dd20cSBen Widawsky */ 29779dd20cSBen Widawsky 30dd5ba0ebSBen Widawsky /* 31dd5ba0ebSBen Widawsky * All changes to the interleave configuration occur with this lock held 32dd5ba0ebSBen Widawsky * for write. 33dd5ba0ebSBen Widawsky */ 34dd5ba0ebSBen Widawsky static DECLARE_RWSEM(cxl_region_rwsem); 35dd5ba0ebSBen Widawsky 36779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev); 37779dd20cSBen Widawsky 38dd5ba0ebSBen Widawsky static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, 39dd5ba0ebSBen Widawsky char *buf) 40dd5ba0ebSBen Widawsky { 41dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 42dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params; 43dd5ba0ebSBen Widawsky ssize_t rc; 44dd5ba0ebSBen Widawsky 45dd5ba0ebSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 46dd5ba0ebSBen Widawsky if (rc) 47dd5ba0ebSBen Widawsky return rc; 48dd5ba0ebSBen Widawsky rc = sysfs_emit(buf, "%pUb\n", &p->uuid); 49dd5ba0ebSBen Widawsky up_read(&cxl_region_rwsem); 50dd5ba0ebSBen Widawsky 51dd5ba0ebSBen Widawsky return rc; 52dd5ba0ebSBen Widawsky } 53dd5ba0ebSBen Widawsky 54dd5ba0ebSBen Widawsky static int is_dup(struct device *match, void *data) 55dd5ba0ebSBen Widawsky { 56dd5ba0ebSBen Widawsky struct cxl_region_params *p; 57dd5ba0ebSBen Widawsky struct cxl_region *cxlr; 58dd5ba0ebSBen Widawsky uuid_t *uuid = data; 59dd5ba0ebSBen Widawsky 60dd5ba0ebSBen Widawsky if (!is_cxl_region(match)) 61dd5ba0ebSBen Widawsky return 0; 62dd5ba0ebSBen Widawsky 63dd5ba0ebSBen Widawsky lockdep_assert_held(&cxl_region_rwsem); 64dd5ba0ebSBen Widawsky cxlr = to_cxl_region(match); 65dd5ba0ebSBen Widawsky p = &cxlr->params; 66dd5ba0ebSBen Widawsky 67dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, uuid)) { 68dd5ba0ebSBen Widawsky dev_dbg(match, "already has uuid: %pUb\n", uuid); 69dd5ba0ebSBen Widawsky return -EBUSY; 70dd5ba0ebSBen Widawsky } 71dd5ba0ebSBen Widawsky 72dd5ba0ebSBen Widawsky return 0; 73dd5ba0ebSBen Widawsky } 74dd5ba0ebSBen Widawsky 75dd5ba0ebSBen Widawsky static ssize_t uuid_store(struct device *dev, struct device_attribute *attr, 76dd5ba0ebSBen Widawsky const char *buf, size_t len) 77dd5ba0ebSBen Widawsky { 78dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 79dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params; 80dd5ba0ebSBen Widawsky uuid_t temp; 81dd5ba0ebSBen Widawsky ssize_t rc; 82dd5ba0ebSBen Widawsky 83dd5ba0ebSBen Widawsky if (len != UUID_STRING_LEN + 1) 84dd5ba0ebSBen Widawsky return -EINVAL; 85dd5ba0ebSBen Widawsky 86dd5ba0ebSBen Widawsky rc = uuid_parse(buf, &temp); 87dd5ba0ebSBen Widawsky if (rc) 88dd5ba0ebSBen Widawsky return rc; 89dd5ba0ebSBen Widawsky 90dd5ba0ebSBen Widawsky if (uuid_is_null(&temp)) 91dd5ba0ebSBen Widawsky return -EINVAL; 92dd5ba0ebSBen Widawsky 93dd5ba0ebSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 94dd5ba0ebSBen Widawsky if (rc) 95dd5ba0ebSBen Widawsky return rc; 96dd5ba0ebSBen Widawsky 97dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, &temp)) 98dd5ba0ebSBen Widawsky goto out; 99dd5ba0ebSBen Widawsky 100dd5ba0ebSBen Widawsky rc = -EBUSY; 101dd5ba0ebSBen Widawsky if (p->state >= CXL_CONFIG_ACTIVE) 102dd5ba0ebSBen Widawsky goto out; 103dd5ba0ebSBen Widawsky 104dd5ba0ebSBen Widawsky rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup); 105dd5ba0ebSBen Widawsky if (rc < 0) 106dd5ba0ebSBen Widawsky goto out; 107dd5ba0ebSBen Widawsky 108dd5ba0ebSBen Widawsky uuid_copy(&p->uuid, &temp); 109dd5ba0ebSBen Widawsky out: 110dd5ba0ebSBen Widawsky up_write(&cxl_region_rwsem); 111dd5ba0ebSBen Widawsky 112dd5ba0ebSBen Widawsky if (rc) 113dd5ba0ebSBen Widawsky return rc; 114dd5ba0ebSBen Widawsky return len; 115dd5ba0ebSBen Widawsky } 116dd5ba0ebSBen Widawsky static DEVICE_ATTR_RW(uuid); 117dd5ba0ebSBen Widawsky 118176baefbSDan Williams static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port, 119176baefbSDan Williams struct cxl_region *cxlr) 120176baefbSDan Williams { 121176baefbSDan Williams return xa_load(&port->regions, (unsigned long)cxlr); 122176baefbSDan Williams } 123176baefbSDan Williams 124176baefbSDan Williams static int cxl_region_decode_reset(struct cxl_region *cxlr, int count) 125176baefbSDan Williams { 126176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 127176baefbSDan Williams int i; 128176baefbSDan Williams 129176baefbSDan Williams for (i = count - 1; i >= 0; i--) { 130176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 131176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 132176baefbSDan Williams struct cxl_port *iter = cxled_to_port(cxled); 133176baefbSDan Williams struct cxl_ep *ep; 134176baefbSDan Williams int rc; 135176baefbSDan Williams 136176baefbSDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 137176baefbSDan Williams iter = to_cxl_port(iter->dev.parent); 138176baefbSDan Williams 139176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 140176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 141176baefbSDan Williams struct cxl_region_ref *cxl_rr; 142176baefbSDan Williams struct cxl_decoder *cxld; 143176baefbSDan Williams 144176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 145176baefbSDan Williams cxld = cxl_rr->decoder; 146176baefbSDan Williams rc = cxld->reset(cxld); 147176baefbSDan Williams if (rc) 148176baefbSDan Williams return rc; 149176baefbSDan Williams } 150176baefbSDan Williams 151176baefbSDan Williams rc = cxled->cxld.reset(&cxled->cxld); 152176baefbSDan Williams if (rc) 153176baefbSDan Williams return rc; 154176baefbSDan Williams } 155176baefbSDan Williams 156176baefbSDan Williams return 0; 157176baefbSDan Williams } 158176baefbSDan Williams 159176baefbSDan Williams static int cxl_region_decode_commit(struct cxl_region *cxlr) 160176baefbSDan Williams { 161176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 16269c99613SDan Williams int i, rc = 0; 163176baefbSDan Williams 164176baefbSDan Williams for (i = 0; i < p->nr_targets; i++) { 165176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 166176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 167176baefbSDan Williams struct cxl_region_ref *cxl_rr; 168176baefbSDan Williams struct cxl_decoder *cxld; 169176baefbSDan Williams struct cxl_port *iter; 170176baefbSDan Williams struct cxl_ep *ep; 171176baefbSDan Williams 172176baefbSDan Williams /* commit bottom up */ 173176baefbSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 174176baefbSDan Williams iter = to_cxl_port(iter->dev.parent)) { 175176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 176176baefbSDan Williams cxld = cxl_rr->decoder; 1772816e24bSJonathan Cameron if (cxld->commit) 178176baefbSDan Williams rc = cxld->commit(cxld); 179176baefbSDan Williams if (rc) 180176baefbSDan Williams break; 181176baefbSDan Williams } 182176baefbSDan Williams 18369c99613SDan Williams if (rc) { 184176baefbSDan Williams /* programming @iter failed, teardown */ 185176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); ep && iter; 186176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 187176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 188176baefbSDan Williams cxld = cxl_rr->decoder; 189176baefbSDan Williams cxld->reset(cxld); 190176baefbSDan Williams } 191176baefbSDan Williams 192176baefbSDan Williams cxled->cxld.reset(&cxled->cxld); 19369c99613SDan Williams goto err; 19469c99613SDan Williams } 195176baefbSDan Williams } 196176baefbSDan Williams 197176baefbSDan Williams return 0; 198176baefbSDan Williams 19969c99613SDan Williams err: 200176baefbSDan Williams /* undo the targets that were successfully committed */ 201176baefbSDan Williams cxl_region_decode_reset(cxlr, i); 202176baefbSDan Williams return rc; 203176baefbSDan Williams } 204176baefbSDan Williams 205176baefbSDan Williams static ssize_t commit_store(struct device *dev, struct device_attribute *attr, 206176baefbSDan Williams const char *buf, size_t len) 207176baefbSDan Williams { 208176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 209176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 210176baefbSDan Williams bool commit; 211176baefbSDan Williams ssize_t rc; 212176baefbSDan Williams 213176baefbSDan Williams rc = kstrtobool(buf, &commit); 214176baefbSDan Williams if (rc) 215176baefbSDan Williams return rc; 216176baefbSDan Williams 217176baefbSDan Williams rc = down_write_killable(&cxl_region_rwsem); 218176baefbSDan Williams if (rc) 219176baefbSDan Williams return rc; 220176baefbSDan Williams 221176baefbSDan Williams /* Already in the requested state? */ 222176baefbSDan Williams if (commit && p->state >= CXL_CONFIG_COMMIT) 223176baefbSDan Williams goto out; 224176baefbSDan Williams if (!commit && p->state < CXL_CONFIG_COMMIT) 225176baefbSDan Williams goto out; 226176baefbSDan Williams 227176baefbSDan Williams /* Not ready to commit? */ 228176baefbSDan Williams if (commit && p->state < CXL_CONFIG_ACTIVE) { 229176baefbSDan Williams rc = -ENXIO; 230176baefbSDan Williams goto out; 231176baefbSDan Williams } 232176baefbSDan Williams 233176baefbSDan Williams if (commit) 234176baefbSDan Williams rc = cxl_region_decode_commit(cxlr); 235176baefbSDan Williams else { 236176baefbSDan Williams p->state = CXL_CONFIG_RESET_PENDING; 237176baefbSDan Williams up_write(&cxl_region_rwsem); 238176baefbSDan Williams device_release_driver(&cxlr->dev); 239176baefbSDan Williams down_write(&cxl_region_rwsem); 240176baefbSDan Williams 241176baefbSDan Williams /* 242176baefbSDan Williams * The lock was dropped, so need to revalidate that the reset is 243176baefbSDan Williams * still pending. 244176baefbSDan Williams */ 245176baefbSDan Williams if (p->state == CXL_CONFIG_RESET_PENDING) 246176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 247176baefbSDan Williams } 248176baefbSDan Williams 249176baefbSDan Williams if (rc) 250176baefbSDan Williams goto out; 251176baefbSDan Williams 252176baefbSDan Williams if (commit) 253176baefbSDan Williams p->state = CXL_CONFIG_COMMIT; 254176baefbSDan Williams else if (p->state == CXL_CONFIG_RESET_PENDING) 255176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE; 256176baefbSDan Williams 257176baefbSDan Williams out: 258176baefbSDan Williams up_write(&cxl_region_rwsem); 259176baefbSDan Williams 260176baefbSDan Williams if (rc) 261176baefbSDan Williams return rc; 262176baefbSDan Williams return len; 263176baefbSDan Williams } 264176baefbSDan Williams 265176baefbSDan Williams static ssize_t commit_show(struct device *dev, struct device_attribute *attr, 266176baefbSDan Williams char *buf) 267176baefbSDan Williams { 268176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 269176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 270176baefbSDan Williams ssize_t rc; 271176baefbSDan Williams 272176baefbSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 273176baefbSDan Williams if (rc) 274176baefbSDan Williams return rc; 275176baefbSDan Williams rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT); 276176baefbSDan Williams up_read(&cxl_region_rwsem); 277176baefbSDan Williams 278176baefbSDan Williams return rc; 279176baefbSDan Williams } 280176baefbSDan Williams static DEVICE_ATTR_RW(commit); 281176baefbSDan Williams 282dd5ba0ebSBen Widawsky static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a, 283dd5ba0ebSBen Widawsky int n) 284dd5ba0ebSBen Widawsky { 285dd5ba0ebSBen Widawsky struct device *dev = kobj_to_dev(kobj); 286dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 287dd5ba0ebSBen Widawsky 288dd5ba0ebSBen Widawsky if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM) 289dd5ba0ebSBen Widawsky return 0; 290dd5ba0ebSBen Widawsky return a->mode; 291dd5ba0ebSBen Widawsky } 292dd5ba0ebSBen Widawsky 29380d10a6cSBen Widawsky static ssize_t interleave_ways_show(struct device *dev, 29480d10a6cSBen Widawsky struct device_attribute *attr, char *buf) 29580d10a6cSBen Widawsky { 29680d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 29780d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 29880d10a6cSBen Widawsky ssize_t rc; 29980d10a6cSBen Widawsky 30080d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 30180d10a6cSBen Widawsky if (rc) 30280d10a6cSBen Widawsky return rc; 30380d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_ways); 30480d10a6cSBen Widawsky up_read(&cxl_region_rwsem); 30580d10a6cSBen Widawsky 30680d10a6cSBen Widawsky return rc; 30780d10a6cSBen Widawsky } 30880d10a6cSBen Widawsky 309b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void); 310b9686e8cSDan Williams 31180d10a6cSBen Widawsky static ssize_t interleave_ways_store(struct device *dev, 31280d10a6cSBen Widawsky struct device_attribute *attr, 31380d10a6cSBen Widawsky const char *buf, size_t len) 31480d10a6cSBen Widawsky { 31580d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 31680d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 31780d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 31880d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 319c7e3548cSDan Carpenter unsigned int val, save; 320c7e3548cSDan Carpenter int rc; 32180d10a6cSBen Widawsky u8 iw; 32280d10a6cSBen Widawsky 323c7e3548cSDan Carpenter rc = kstrtouint(buf, 0, &val); 32480d10a6cSBen Widawsky if (rc) 32580d10a6cSBen Widawsky return rc; 32680d10a6cSBen Widawsky 32780d10a6cSBen Widawsky rc = ways_to_cxl(val, &iw); 32880d10a6cSBen Widawsky if (rc) 32980d10a6cSBen Widawsky return rc; 33080d10a6cSBen Widawsky 33180d10a6cSBen Widawsky /* 33280d10a6cSBen Widawsky * Even for x3, x9, and x12 interleaves the region interleave must be a 33380d10a6cSBen Widawsky * power of 2 multiple of the host bridge interleave. 33480d10a6cSBen Widawsky */ 33580d10a6cSBen Widawsky if (!is_power_of_2(val / cxld->interleave_ways) || 33680d10a6cSBen Widawsky (val % cxld->interleave_ways)) { 33780d10a6cSBen Widawsky dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val); 33880d10a6cSBen Widawsky return -EINVAL; 33980d10a6cSBen Widawsky } 34080d10a6cSBen Widawsky 34180d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 34280d10a6cSBen Widawsky if (rc) 34380d10a6cSBen Widawsky return rc; 34480d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 34580d10a6cSBen Widawsky rc = -EBUSY; 34680d10a6cSBen Widawsky goto out; 34780d10a6cSBen Widawsky } 34880d10a6cSBen Widawsky 349b9686e8cSDan Williams save = p->interleave_ways; 35080d10a6cSBen Widawsky p->interleave_ways = val; 351b9686e8cSDan Williams rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 352b9686e8cSDan Williams if (rc) 353b9686e8cSDan Williams p->interleave_ways = save; 35480d10a6cSBen Widawsky out: 35580d10a6cSBen Widawsky up_write(&cxl_region_rwsem); 35680d10a6cSBen Widawsky if (rc) 35780d10a6cSBen Widawsky return rc; 35880d10a6cSBen Widawsky return len; 35980d10a6cSBen Widawsky } 36080d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_ways); 36180d10a6cSBen Widawsky 36280d10a6cSBen Widawsky static ssize_t interleave_granularity_show(struct device *dev, 36380d10a6cSBen Widawsky struct device_attribute *attr, 36480d10a6cSBen Widawsky char *buf) 36580d10a6cSBen Widawsky { 36680d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 36780d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 36880d10a6cSBen Widawsky ssize_t rc; 36980d10a6cSBen Widawsky 37080d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 37180d10a6cSBen Widawsky if (rc) 37280d10a6cSBen Widawsky return rc; 37380d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_granularity); 37480d10a6cSBen Widawsky up_read(&cxl_region_rwsem); 37580d10a6cSBen Widawsky 37680d10a6cSBen Widawsky return rc; 37780d10a6cSBen Widawsky } 37880d10a6cSBen Widawsky 37980d10a6cSBen Widawsky static ssize_t interleave_granularity_store(struct device *dev, 38080d10a6cSBen Widawsky struct device_attribute *attr, 38180d10a6cSBen Widawsky const char *buf, size_t len) 38280d10a6cSBen Widawsky { 38380d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 38480d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 38580d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 38680d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 38780d10a6cSBen Widawsky int rc, val; 38880d10a6cSBen Widawsky u16 ig; 38980d10a6cSBen Widawsky 39080d10a6cSBen Widawsky rc = kstrtoint(buf, 0, &val); 39180d10a6cSBen Widawsky if (rc) 39280d10a6cSBen Widawsky return rc; 39380d10a6cSBen Widawsky 39480d10a6cSBen Widawsky rc = granularity_to_cxl(val, &ig); 39580d10a6cSBen Widawsky if (rc) 39680d10a6cSBen Widawsky return rc; 39780d10a6cSBen Widawsky 39880d10a6cSBen Widawsky /* 3994d8e4ea5SDan Williams * When the host-bridge is interleaved, disallow region granularity != 4004d8e4ea5SDan Williams * root granularity. Regions with a granularity less than the root 4014d8e4ea5SDan Williams * interleave result in needing multiple endpoints to support a single 4024d8e4ea5SDan Williams * slot in the interleave (possible to suport in the future). Regions 4034d8e4ea5SDan Williams * with a granularity greater than the root interleave result in invalid 4044d8e4ea5SDan Williams * DPA translations (invalid to support). 40580d10a6cSBen Widawsky */ 4064d8e4ea5SDan Williams if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity) 40780d10a6cSBen Widawsky return -EINVAL; 40880d10a6cSBen Widawsky 40980d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 41080d10a6cSBen Widawsky if (rc) 41180d10a6cSBen Widawsky return rc; 41280d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 41380d10a6cSBen Widawsky rc = -EBUSY; 41480d10a6cSBen Widawsky goto out; 41580d10a6cSBen Widawsky } 41680d10a6cSBen Widawsky 41780d10a6cSBen Widawsky p->interleave_granularity = val; 41880d10a6cSBen Widawsky out: 41980d10a6cSBen Widawsky up_write(&cxl_region_rwsem); 42080d10a6cSBen Widawsky if (rc) 42180d10a6cSBen Widawsky return rc; 42280d10a6cSBen Widawsky return len; 42380d10a6cSBen Widawsky } 42480d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_granularity); 42580d10a6cSBen Widawsky 42623a22cd1SDan Williams static ssize_t resource_show(struct device *dev, struct device_attribute *attr, 42723a22cd1SDan Williams char *buf) 42823a22cd1SDan Williams { 42923a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 43023a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 43123a22cd1SDan Williams u64 resource = -1ULL; 43223a22cd1SDan Williams ssize_t rc; 43323a22cd1SDan Williams 43423a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 43523a22cd1SDan Williams if (rc) 43623a22cd1SDan Williams return rc; 43723a22cd1SDan Williams if (p->res) 43823a22cd1SDan Williams resource = p->res->start; 43923a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", resource); 44023a22cd1SDan Williams up_read(&cxl_region_rwsem); 44123a22cd1SDan Williams 44223a22cd1SDan Williams return rc; 44323a22cd1SDan Williams } 44423a22cd1SDan Williams static DEVICE_ATTR_RO(resource); 44523a22cd1SDan Williams 44623a22cd1SDan Williams static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) 44723a22cd1SDan Williams { 44823a22cd1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 44923a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 45023a22cd1SDan Williams struct resource *res; 45123a22cd1SDan Williams u32 remainder = 0; 45223a22cd1SDan Williams 45323a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 45423a22cd1SDan Williams 45523a22cd1SDan Williams /* Nothing to do... */ 45688ab1ddeSDan Carpenter if (p->res && resource_size(p->res) == size) 45723a22cd1SDan Williams return 0; 45823a22cd1SDan Williams 45923a22cd1SDan Williams /* To change size the old size must be freed first */ 46023a22cd1SDan Williams if (p->res) 46123a22cd1SDan Williams return -EBUSY; 46223a22cd1SDan Williams 46323a22cd1SDan Williams if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) 46423a22cd1SDan Williams return -EBUSY; 46523a22cd1SDan Williams 46623a22cd1SDan Williams /* ways, granularity and uuid (if PMEM) need to be set before HPA */ 46723a22cd1SDan Williams if (!p->interleave_ways || !p->interleave_granularity || 46823a22cd1SDan Williams (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid))) 46923a22cd1SDan Williams return -ENXIO; 47023a22cd1SDan Williams 47123a22cd1SDan Williams div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder); 47223a22cd1SDan Williams if (remainder) 47323a22cd1SDan Williams return -EINVAL; 47423a22cd1SDan Williams 47523a22cd1SDan Williams res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, 47623a22cd1SDan Williams dev_name(&cxlr->dev)); 47723a22cd1SDan Williams if (IS_ERR(res)) { 47823a22cd1SDan Williams dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n", 47923a22cd1SDan Williams PTR_ERR(res)); 48023a22cd1SDan Williams return PTR_ERR(res); 48123a22cd1SDan Williams } 48223a22cd1SDan Williams 48323a22cd1SDan Williams p->res = res; 48423a22cd1SDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 48523a22cd1SDan Williams 48623a22cd1SDan Williams return 0; 48723a22cd1SDan Williams } 48823a22cd1SDan Williams 48923a22cd1SDan Williams static void cxl_region_iomem_release(struct cxl_region *cxlr) 49023a22cd1SDan Williams { 49123a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 49223a22cd1SDan Williams 49323a22cd1SDan Williams if (device_is_registered(&cxlr->dev)) 49423a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 49523a22cd1SDan Williams if (p->res) { 49623a22cd1SDan Williams remove_resource(p->res); 49723a22cd1SDan Williams kfree(p->res); 49823a22cd1SDan Williams p->res = NULL; 49923a22cd1SDan Williams } 50023a22cd1SDan Williams } 50123a22cd1SDan Williams 50223a22cd1SDan Williams static int free_hpa(struct cxl_region *cxlr) 50323a22cd1SDan Williams { 50423a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 50523a22cd1SDan Williams 50623a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 50723a22cd1SDan Williams 50823a22cd1SDan Williams if (!p->res) 50923a22cd1SDan Williams return 0; 51023a22cd1SDan Williams 51123a22cd1SDan Williams if (p->state >= CXL_CONFIG_ACTIVE) 51223a22cd1SDan Williams return -EBUSY; 51323a22cd1SDan Williams 51423a22cd1SDan Williams cxl_region_iomem_release(cxlr); 51523a22cd1SDan Williams p->state = CXL_CONFIG_IDLE; 51623a22cd1SDan Williams return 0; 51723a22cd1SDan Williams } 51823a22cd1SDan Williams 51923a22cd1SDan Williams static ssize_t size_store(struct device *dev, struct device_attribute *attr, 52023a22cd1SDan Williams const char *buf, size_t len) 52123a22cd1SDan Williams { 52223a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 52323a22cd1SDan Williams u64 val; 52423a22cd1SDan Williams int rc; 52523a22cd1SDan Williams 52623a22cd1SDan Williams rc = kstrtou64(buf, 0, &val); 52723a22cd1SDan Williams if (rc) 52823a22cd1SDan Williams return rc; 52923a22cd1SDan Williams 53023a22cd1SDan Williams rc = down_write_killable(&cxl_region_rwsem); 53123a22cd1SDan Williams if (rc) 53223a22cd1SDan Williams return rc; 53323a22cd1SDan Williams 53423a22cd1SDan Williams if (val) 53523a22cd1SDan Williams rc = alloc_hpa(cxlr, val); 53623a22cd1SDan Williams else 53723a22cd1SDan Williams rc = free_hpa(cxlr); 53823a22cd1SDan Williams up_write(&cxl_region_rwsem); 53923a22cd1SDan Williams 54023a22cd1SDan Williams if (rc) 54123a22cd1SDan Williams return rc; 54223a22cd1SDan Williams 54323a22cd1SDan Williams return len; 54423a22cd1SDan Williams } 54523a22cd1SDan Williams 54623a22cd1SDan Williams static ssize_t size_show(struct device *dev, struct device_attribute *attr, 54723a22cd1SDan Williams char *buf) 54823a22cd1SDan Williams { 54923a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 55023a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 55123a22cd1SDan Williams u64 size = 0; 55223a22cd1SDan Williams ssize_t rc; 55323a22cd1SDan Williams 55423a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 55523a22cd1SDan Williams if (rc) 55623a22cd1SDan Williams return rc; 55723a22cd1SDan Williams if (p->res) 55823a22cd1SDan Williams size = resource_size(p->res); 55923a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", size); 56023a22cd1SDan Williams up_read(&cxl_region_rwsem); 56123a22cd1SDan Williams 56223a22cd1SDan Williams return rc; 56323a22cd1SDan Williams } 56423a22cd1SDan Williams static DEVICE_ATTR_RW(size); 56523a22cd1SDan Williams 566dd5ba0ebSBen Widawsky static struct attribute *cxl_region_attrs[] = { 567dd5ba0ebSBen Widawsky &dev_attr_uuid.attr, 568176baefbSDan Williams &dev_attr_commit.attr, 56980d10a6cSBen Widawsky &dev_attr_interleave_ways.attr, 57080d10a6cSBen Widawsky &dev_attr_interleave_granularity.attr, 57123a22cd1SDan Williams &dev_attr_resource.attr, 57223a22cd1SDan Williams &dev_attr_size.attr, 573dd5ba0ebSBen Widawsky NULL, 574dd5ba0ebSBen Widawsky }; 575dd5ba0ebSBen Widawsky 576dd5ba0ebSBen Widawsky static const struct attribute_group cxl_region_group = { 577dd5ba0ebSBen Widawsky .attrs = cxl_region_attrs, 578dd5ba0ebSBen Widawsky .is_visible = cxl_region_visible, 579dd5ba0ebSBen Widawsky }; 580dd5ba0ebSBen Widawsky 581b9686e8cSDan Williams static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos) 582b9686e8cSDan Williams { 583b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 584b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled; 585b9686e8cSDan Williams int rc; 586b9686e8cSDan Williams 587b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 588b9686e8cSDan Williams if (rc) 589b9686e8cSDan Williams return rc; 590b9686e8cSDan Williams 591b9686e8cSDan Williams if (pos >= p->interleave_ways) { 592b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 593b9686e8cSDan Williams p->interleave_ways); 594b9686e8cSDan Williams rc = -ENXIO; 595b9686e8cSDan Williams goto out; 596b9686e8cSDan Williams } 597b9686e8cSDan Williams 598b9686e8cSDan Williams cxled = p->targets[pos]; 599b9686e8cSDan Williams if (!cxled) 600b9686e8cSDan Williams rc = sysfs_emit(buf, "\n"); 601b9686e8cSDan Williams else 602b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev)); 603b9686e8cSDan Williams out: 604b9686e8cSDan Williams up_read(&cxl_region_rwsem); 605b9686e8cSDan Williams 606b9686e8cSDan Williams return rc; 607b9686e8cSDan Williams } 608b9686e8cSDan Williams 609384e624bSDan Williams static int match_free_decoder(struct device *dev, void *data) 610384e624bSDan Williams { 611384e624bSDan Williams struct cxl_decoder *cxld; 612384e624bSDan Williams int *id = data; 613384e624bSDan Williams 614384e624bSDan Williams if (!is_switch_decoder(dev)) 615384e624bSDan Williams return 0; 616384e624bSDan Williams 617384e624bSDan Williams cxld = to_cxl_decoder(dev); 618384e624bSDan Williams 619384e624bSDan Williams /* enforce ordered allocation */ 620384e624bSDan Williams if (cxld->id != *id) 621384e624bSDan Williams return 0; 622384e624bSDan Williams 623384e624bSDan Williams if (!cxld->region) 624384e624bSDan Williams return 1; 625384e624bSDan Williams 626384e624bSDan Williams (*id)++; 627384e624bSDan Williams 628384e624bSDan Williams return 0; 629384e624bSDan Williams } 630384e624bSDan Williams 631384e624bSDan Williams static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port, 632384e624bSDan Williams struct cxl_region *cxlr) 633384e624bSDan Williams { 634384e624bSDan Williams struct device *dev; 635384e624bSDan Williams int id = 0; 636384e624bSDan Williams 637384e624bSDan Williams dev = device_find_child(&port->dev, &id, match_free_decoder); 638384e624bSDan Williams if (!dev) 639384e624bSDan Williams return NULL; 640b9686e8cSDan Williams /* 641384e624bSDan Williams * This decoder is pinned registered as long as the endpoint decoder is 642384e624bSDan Williams * registered, and endpoint decoder unregistration holds the 643384e624bSDan Williams * cxl_region_rwsem over unregister events, so no need to hold on to 644384e624bSDan Williams * this extra reference. 645b9686e8cSDan Williams */ 646384e624bSDan Williams put_device(dev); 647384e624bSDan Williams return to_cxl_decoder(dev); 648384e624bSDan Williams } 649384e624bSDan Williams 650384e624bSDan Williams static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port, 651384e624bSDan Williams struct cxl_region *cxlr) 652384e624bSDan Williams { 653e29a8995SDan Williams struct cxl_region_params *p = &cxlr->params; 654e29a8995SDan Williams struct cxl_region_ref *cxl_rr, *iter; 655e29a8995SDan Williams unsigned long index; 656384e624bSDan Williams int rc; 657384e624bSDan Williams 658e29a8995SDan Williams xa_for_each(&port->regions, index, iter) { 659e29a8995SDan Williams struct cxl_region_params *ip = &iter->region->params; 660e29a8995SDan Williams 661a90accb3SDan Williams if (!ip->res) 662a90accb3SDan Williams continue; 663a90accb3SDan Williams 664e29a8995SDan Williams if (ip->res->start > p->res->start) { 665e29a8995SDan Williams dev_dbg(&cxlr->dev, 666e29a8995SDan Williams "%s: HPA order violation %s:%pr vs %pr\n", 667e29a8995SDan Williams dev_name(&port->dev), 668e29a8995SDan Williams dev_name(&iter->region->dev), ip->res, p->res); 669e29a8995SDan Williams return ERR_PTR(-EBUSY); 670e29a8995SDan Williams } 671e29a8995SDan Williams } 672e29a8995SDan Williams 673384e624bSDan Williams cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL); 674384e624bSDan Williams if (!cxl_rr) 675e29a8995SDan Williams return ERR_PTR(-ENOMEM); 676384e624bSDan Williams cxl_rr->port = port; 677384e624bSDan Williams cxl_rr->region = cxlr; 67827b3f8d1SDan Williams cxl_rr->nr_targets = 1; 679384e624bSDan Williams xa_init(&cxl_rr->endpoints); 680384e624bSDan Williams 681384e624bSDan Williams rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL); 682384e624bSDan Williams if (rc) { 683384e624bSDan Williams dev_dbg(&cxlr->dev, 684384e624bSDan Williams "%s: failed to track region reference: %d\n", 685384e624bSDan Williams dev_name(&port->dev), rc); 686384e624bSDan Williams kfree(cxl_rr); 687e29a8995SDan Williams return ERR_PTR(rc); 688384e624bSDan Williams } 689384e624bSDan Williams 690384e624bSDan Williams return cxl_rr; 691384e624bSDan Williams } 692384e624bSDan Williams 69371ee71d7SVishal Verma static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr) 694384e624bSDan Williams { 695384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region; 696384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 697384e624bSDan Williams 69871ee71d7SVishal Verma if (!cxld) 69971ee71d7SVishal Verma return; 70071ee71d7SVishal Verma 701384e624bSDan Williams dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n"); 702384e624bSDan Williams if (cxld->region == cxlr) { 703384e624bSDan Williams cxld->region = NULL; 704384e624bSDan Williams put_device(&cxlr->dev); 705384e624bSDan Williams } 70671ee71d7SVishal Verma } 707384e624bSDan Williams 70871ee71d7SVishal Verma static void free_region_ref(struct cxl_region_ref *cxl_rr) 70971ee71d7SVishal Verma { 71071ee71d7SVishal Verma struct cxl_port *port = cxl_rr->port; 71171ee71d7SVishal Verma struct cxl_region *cxlr = cxl_rr->region; 71271ee71d7SVishal Verma 71371ee71d7SVishal Verma cxl_rr_free_decoder(cxl_rr); 714384e624bSDan Williams xa_erase(&port->regions, (unsigned long)cxlr); 715384e624bSDan Williams xa_destroy(&cxl_rr->endpoints); 716384e624bSDan Williams kfree(cxl_rr); 717384e624bSDan Williams } 718384e624bSDan Williams 719384e624bSDan Williams static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr, 720384e624bSDan Williams struct cxl_endpoint_decoder *cxled) 721384e624bSDan Williams { 722384e624bSDan Williams int rc; 723384e624bSDan Williams struct cxl_port *port = cxl_rr->port; 724384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region; 725384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 726384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled)); 727384e624bSDan Williams 72827b3f8d1SDan Williams if (ep) { 729384e624bSDan Williams rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep, 730384e624bSDan Williams GFP_KERNEL); 731384e624bSDan Williams if (rc) 732384e624bSDan Williams return rc; 73327b3f8d1SDan Williams } 734384e624bSDan Williams cxl_rr->nr_eps++; 735384e624bSDan Williams 736384e624bSDan Williams if (!cxld->region) { 737384e624bSDan Williams cxld->region = cxlr; 738384e624bSDan Williams get_device(&cxlr->dev); 739384e624bSDan Williams } 740384e624bSDan Williams 741384e624bSDan Williams return 0; 742384e624bSDan Williams } 743384e624bSDan Williams 74471ee71d7SVishal Verma static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr, 74571ee71d7SVishal Verma struct cxl_endpoint_decoder *cxled, 74671ee71d7SVishal Verma struct cxl_region_ref *cxl_rr) 74771ee71d7SVishal Verma { 74871ee71d7SVishal Verma struct cxl_decoder *cxld; 74971ee71d7SVishal Verma 75071ee71d7SVishal Verma if (port == cxled_to_port(cxled)) 75171ee71d7SVishal Verma cxld = &cxled->cxld; 75271ee71d7SVishal Verma else 75371ee71d7SVishal Verma cxld = cxl_region_find_decoder(port, cxlr); 75471ee71d7SVishal Verma if (!cxld) { 75571ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: no decoder available\n", 75671ee71d7SVishal Verma dev_name(&port->dev)); 75771ee71d7SVishal Verma return -EBUSY; 75871ee71d7SVishal Verma } 75971ee71d7SVishal Verma 76071ee71d7SVishal Verma if (cxld->region) { 76171ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", 76271ee71d7SVishal Verma dev_name(&port->dev), dev_name(&cxld->dev), 76371ee71d7SVishal Verma dev_name(&cxld->region->dev)); 76471ee71d7SVishal Verma return -EBUSY; 76571ee71d7SVishal Verma } 76671ee71d7SVishal Verma 76771ee71d7SVishal Verma cxl_rr->decoder = cxld; 76871ee71d7SVishal Verma return 0; 76971ee71d7SVishal Verma } 77071ee71d7SVishal Verma 771384e624bSDan Williams /** 772384e624bSDan Williams * cxl_port_attach_region() - track a region's interest in a port by endpoint 773384e624bSDan Williams * @port: port to add a new region reference 'struct cxl_region_ref' 774384e624bSDan Williams * @cxlr: region to attach to @port 775384e624bSDan Williams * @cxled: endpoint decoder used to create or further pin a region reference 776384e624bSDan Williams * @pos: interleave position of @cxled in @cxlr 777384e624bSDan Williams * 778384e624bSDan Williams * The attach event is an opportunity to validate CXL decode setup 779384e624bSDan Williams * constraints and record metadata needed for programming HDM decoders, 780384e624bSDan Williams * in particular decoder target lists. 781384e624bSDan Williams * 782384e624bSDan Williams * The steps are: 783f13da0d9SBagas Sanjaya * 784384e624bSDan Williams * - validate that there are no other regions with a higher HPA already 785384e624bSDan Williams * associated with @port 786384e624bSDan Williams * - establish a region reference if one is not already present 787f13da0d9SBagas Sanjaya * 788384e624bSDan Williams * - additionally allocate a decoder instance that will host @cxlr on 789384e624bSDan Williams * @port 790f13da0d9SBagas Sanjaya * 791384e624bSDan Williams * - pin the region reference by the endpoint 792384e624bSDan Williams * - account for how many entries in @port's target list are needed to 793384e624bSDan Williams * cover all of the added endpoints. 794384e624bSDan Williams */ 795384e624bSDan Williams static int cxl_port_attach_region(struct cxl_port *port, 796384e624bSDan Williams struct cxl_region *cxlr, 797384e624bSDan Williams struct cxl_endpoint_decoder *cxled, int pos) 798384e624bSDan Williams { 799384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 800384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 801e29a8995SDan Williams struct cxl_region_ref *cxl_rr; 802e29a8995SDan Williams bool nr_targets_inc = false; 803e29a8995SDan Williams struct cxl_decoder *cxld; 804384e624bSDan Williams unsigned long index; 805384e624bSDan Williams int rc = -EBUSY; 806384e624bSDan Williams 807384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 808384e624bSDan Williams 809e29a8995SDan Williams cxl_rr = cxl_rr_load(port, cxlr); 810384e624bSDan Williams if (cxl_rr) { 811384e624bSDan Williams struct cxl_ep *ep_iter; 812384e624bSDan Williams int found = 0; 813384e624bSDan Williams 814e29a8995SDan Williams /* 815e29a8995SDan Williams * Walk the existing endpoints that have been attached to 816e29a8995SDan Williams * @cxlr at @port and see if they share the same 'next' port 817e29a8995SDan Williams * in the downstream direction. I.e. endpoints that share common 818e29a8995SDan Williams * upstream switch. 819e29a8995SDan Williams */ 820384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 821384e624bSDan Williams if (ep_iter == ep) 822384e624bSDan Williams continue; 823384e624bSDan Williams if (ep_iter->next == ep->next) { 824384e624bSDan Williams found++; 825384e624bSDan Williams break; 826384e624bSDan Williams } 827384e624bSDan Williams } 828384e624bSDan Williams 829384e624bSDan Williams /* 830e29a8995SDan Williams * New target port, or @port is an endpoint port that always 831e29a8995SDan Williams * accounts its own local decode as a target. 832384e624bSDan Williams */ 833e29a8995SDan Williams if (!found || !ep->next) { 834384e624bSDan Williams cxl_rr->nr_targets++; 835e29a8995SDan Williams nr_targets_inc = true; 836e29a8995SDan Williams } 837384e624bSDan Williams } else { 838384e624bSDan Williams cxl_rr = alloc_region_ref(port, cxlr); 839e29a8995SDan Williams if (IS_ERR(cxl_rr)) { 840384e624bSDan Williams dev_dbg(&cxlr->dev, 841384e624bSDan Williams "%s: failed to allocate region reference\n", 842384e624bSDan Williams dev_name(&port->dev)); 843e29a8995SDan Williams return PTR_ERR(cxl_rr); 844384e624bSDan Williams } 845e29a8995SDan Williams nr_targets_inc = true; 846384e624bSDan Williams 84771ee71d7SVishal Verma rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr); 84871ee71d7SVishal Verma if (rc) 849384e624bSDan Williams goto out_erase; 850384e624bSDan Williams } 85171ee71d7SVishal Verma cxld = cxl_rr->decoder; 852384e624bSDan Williams 853384e624bSDan Williams rc = cxl_rr_ep_add(cxl_rr, cxled); 854384e624bSDan Williams if (rc) { 855384e624bSDan Williams dev_dbg(&cxlr->dev, 856384e624bSDan Williams "%s: failed to track endpoint %s:%s reference\n", 857384e624bSDan Williams dev_name(&port->dev), dev_name(&cxlmd->dev), 858384e624bSDan Williams dev_name(&cxld->dev)); 859384e624bSDan Williams goto out_erase; 860384e624bSDan Williams } 861384e624bSDan Williams 86227b3f8d1SDan Williams dev_dbg(&cxlr->dev, 86327b3f8d1SDan Williams "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n", 86427b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 86527b3f8d1SDan Williams dev_name(&cxld->dev), dev_name(&cxlmd->dev), 86627b3f8d1SDan Williams dev_name(&cxled->cxld.dev), pos, 86727b3f8d1SDan Williams ep ? ep->next ? dev_name(ep->next->uport) : 86827b3f8d1SDan Williams dev_name(&cxlmd->dev) : 86927b3f8d1SDan Williams "none", 87027b3f8d1SDan Williams cxl_rr->nr_eps, cxl_rr->nr_targets); 87127b3f8d1SDan Williams 872384e624bSDan Williams return 0; 873384e624bSDan Williams out_erase: 874e29a8995SDan Williams if (nr_targets_inc) 875e29a8995SDan Williams cxl_rr->nr_targets--; 876384e624bSDan Williams if (cxl_rr->nr_eps == 0) 877384e624bSDan Williams free_region_ref(cxl_rr); 878384e624bSDan Williams return rc; 879384e624bSDan Williams } 880384e624bSDan Williams 881384e624bSDan Williams static void cxl_port_detach_region(struct cxl_port *port, 882384e624bSDan Williams struct cxl_region *cxlr, 883384e624bSDan Williams struct cxl_endpoint_decoder *cxled) 884384e624bSDan Williams { 885384e624bSDan Williams struct cxl_region_ref *cxl_rr; 88627b3f8d1SDan Williams struct cxl_ep *ep = NULL; 887384e624bSDan Williams 888384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 889384e624bSDan Williams 890384e624bSDan Williams cxl_rr = cxl_rr_load(port, cxlr); 891384e624bSDan Williams if (!cxl_rr) 892384e624bSDan Williams return; 893384e624bSDan Williams 89427b3f8d1SDan Williams /* 89527b3f8d1SDan Williams * Endpoint ports do not carry cxl_ep references, and they 89627b3f8d1SDan Williams * never target more than one endpoint by definition 89727b3f8d1SDan Williams */ 89827b3f8d1SDan Williams if (cxl_rr->decoder == &cxled->cxld) 89927b3f8d1SDan Williams cxl_rr->nr_eps--; 90027b3f8d1SDan Williams else 901384e624bSDan Williams ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled); 902384e624bSDan Williams if (ep) { 903384e624bSDan Williams struct cxl_ep *ep_iter; 904384e624bSDan Williams unsigned long index; 905384e624bSDan Williams int found = 0; 906384e624bSDan Williams 907384e624bSDan Williams cxl_rr->nr_eps--; 908384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 909384e624bSDan Williams if (ep_iter->next == ep->next) { 910384e624bSDan Williams found++; 911384e624bSDan Williams break; 912384e624bSDan Williams } 913384e624bSDan Williams } 914384e624bSDan Williams if (!found) 915384e624bSDan Williams cxl_rr->nr_targets--; 916384e624bSDan Williams } 917384e624bSDan Williams 918384e624bSDan Williams if (cxl_rr->nr_eps == 0) 919384e624bSDan Williams free_region_ref(cxl_rr); 920384e624bSDan Williams } 921384e624bSDan Williams 92227b3f8d1SDan Williams static int check_last_peer(struct cxl_endpoint_decoder *cxled, 92327b3f8d1SDan Williams struct cxl_ep *ep, struct cxl_region_ref *cxl_rr, 92427b3f8d1SDan Williams int distance) 92527b3f8d1SDan Williams { 92627b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 92727b3f8d1SDan Williams struct cxl_region *cxlr = cxl_rr->region; 92827b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 92927b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled_peer; 93027b3f8d1SDan Williams struct cxl_port *port = cxl_rr->port; 93127b3f8d1SDan Williams struct cxl_memdev *cxlmd_peer; 93227b3f8d1SDan Williams struct cxl_ep *ep_peer; 93327b3f8d1SDan Williams int pos = cxled->pos; 93427b3f8d1SDan Williams 93527b3f8d1SDan Williams /* 93627b3f8d1SDan Williams * If this position wants to share a dport with the last endpoint mapped 93727b3f8d1SDan Williams * then that endpoint, at index 'position - distance', must also be 93827b3f8d1SDan Williams * mapped by this dport. 93927b3f8d1SDan Williams */ 94027b3f8d1SDan Williams if (pos < distance) { 94127b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", 94227b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 94327b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 94427b3f8d1SDan Williams return -ENXIO; 94527b3f8d1SDan Williams } 94627b3f8d1SDan Williams cxled_peer = p->targets[pos - distance]; 94727b3f8d1SDan Williams cxlmd_peer = cxled_to_memdev(cxled_peer); 94827b3f8d1SDan Williams ep_peer = cxl_ep_load(port, cxlmd_peer); 94927b3f8d1SDan Williams if (ep->dport != ep_peer->dport) { 95027b3f8d1SDan Williams dev_dbg(&cxlr->dev, 95127b3f8d1SDan Williams "%s:%s: %s:%s pos %d mismatched peer %s:%s\n", 95227b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 95327b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, 95427b3f8d1SDan Williams dev_name(&cxlmd_peer->dev), 95527b3f8d1SDan Williams dev_name(&cxled_peer->cxld.dev)); 95627b3f8d1SDan Williams return -ENXIO; 95727b3f8d1SDan Williams } 95827b3f8d1SDan Williams 95927b3f8d1SDan Williams return 0; 96027b3f8d1SDan Williams } 96127b3f8d1SDan Williams 96227b3f8d1SDan Williams static int cxl_port_setup_targets(struct cxl_port *port, 96327b3f8d1SDan Williams struct cxl_region *cxlr, 96427b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled) 96527b3f8d1SDan Williams { 96627b3f8d1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 96727b3f8d1SDan Williams int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos; 96827b3f8d1SDan Williams struct cxl_port *parent_port = to_cxl_port(port->dev.parent); 96927b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 97027b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 97127b3f8d1SDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 97227b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 97327b3f8d1SDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 97427b3f8d1SDan Williams struct cxl_switch_decoder *cxlsd; 97527b3f8d1SDan Williams u16 eig, peig; 97627b3f8d1SDan Williams u8 eiw, peiw; 97727b3f8d1SDan Williams 97827b3f8d1SDan Williams /* 97927b3f8d1SDan Williams * While root level decoders support x3, x6, x12, switch level 98027b3f8d1SDan Williams * decoders only support powers of 2 up to x16. 98127b3f8d1SDan Williams */ 98227b3f8d1SDan Williams if (!is_power_of_2(cxl_rr->nr_targets)) { 98327b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n", 98427b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 98527b3f8d1SDan Williams cxl_rr->nr_targets); 98627b3f8d1SDan Williams return -EINVAL; 98727b3f8d1SDan Williams } 98827b3f8d1SDan Williams 98927b3f8d1SDan Williams cxlsd = to_cxl_switch_decoder(&cxld->dev); 99027b3f8d1SDan Williams if (cxl_rr->nr_targets_set) { 99127b3f8d1SDan Williams int i, distance; 99227b3f8d1SDan Williams 993e4f6dfa9SDan Williams /* 994e4f6dfa9SDan Williams * Passthrough ports impose no distance requirements between 995e4f6dfa9SDan Williams * peers 996e4f6dfa9SDan Williams */ 997e4f6dfa9SDan Williams if (port->nr_dports == 1) 998e4f6dfa9SDan Williams distance = 0; 999e4f6dfa9SDan Williams else 100027b3f8d1SDan Williams distance = p->nr_targets / cxl_rr->nr_targets; 100127b3f8d1SDan Williams for (i = 0; i < cxl_rr->nr_targets_set; i++) 100227b3f8d1SDan Williams if (ep->dport == cxlsd->target[i]) { 100327b3f8d1SDan Williams rc = check_last_peer(cxled, ep, cxl_rr, 100427b3f8d1SDan Williams distance); 100527b3f8d1SDan Williams if (rc) 100627b3f8d1SDan Williams return rc; 100727b3f8d1SDan Williams goto out_target_set; 100827b3f8d1SDan Williams } 100927b3f8d1SDan Williams goto add_target; 101027b3f8d1SDan Williams } 101127b3f8d1SDan Williams 101227b3f8d1SDan Williams if (is_cxl_root(parent_port)) { 101327b3f8d1SDan Williams parent_ig = cxlrd->cxlsd.cxld.interleave_granularity; 101427b3f8d1SDan Williams parent_iw = cxlrd->cxlsd.cxld.interleave_ways; 101527b3f8d1SDan Williams /* 101627b3f8d1SDan Williams * For purposes of address bit routing, use power-of-2 math for 101727b3f8d1SDan Williams * switch ports. 101827b3f8d1SDan Williams */ 101927b3f8d1SDan Williams if (!is_power_of_2(parent_iw)) 102027b3f8d1SDan Williams parent_iw /= 3; 102127b3f8d1SDan Williams } else { 102227b3f8d1SDan Williams struct cxl_region_ref *parent_rr; 102327b3f8d1SDan Williams struct cxl_decoder *parent_cxld; 102427b3f8d1SDan Williams 102527b3f8d1SDan Williams parent_rr = cxl_rr_load(parent_port, cxlr); 102627b3f8d1SDan Williams parent_cxld = parent_rr->decoder; 102727b3f8d1SDan Williams parent_ig = parent_cxld->interleave_granularity; 102827b3f8d1SDan Williams parent_iw = parent_cxld->interleave_ways; 102927b3f8d1SDan Williams } 103027b3f8d1SDan Williams 10318d428542SDan Williams rc = granularity_to_cxl(parent_ig, &peig); 10328d428542SDan Williams if (rc) { 10338d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n", 10348d428542SDan Williams dev_name(parent_port->uport), 10358d428542SDan Williams dev_name(&parent_port->dev), parent_ig); 10368d428542SDan Williams return rc; 10378d428542SDan Williams } 10388d428542SDan Williams 10398d428542SDan Williams rc = ways_to_cxl(parent_iw, &peiw); 10408d428542SDan Williams if (rc) { 10418d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n", 10428d428542SDan Williams dev_name(parent_port->uport), 10438d428542SDan Williams dev_name(&parent_port->dev), parent_iw); 10448d428542SDan Williams return rc; 10458d428542SDan Williams } 104627b3f8d1SDan Williams 104727b3f8d1SDan Williams iw = cxl_rr->nr_targets; 10488d428542SDan Williams rc = ways_to_cxl(iw, &eiw); 10498d428542SDan Williams if (rc) { 10508d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n", 10518d428542SDan Williams dev_name(port->uport), dev_name(&port->dev), iw); 10528d428542SDan Williams return rc; 10538d428542SDan Williams } 10548d428542SDan Williams 1055298d44d0SDan Williams /* 1056298d44d0SDan Williams * If @parent_port is masking address bits, pick the next unused address 1057298d44d0SDan Williams * bit to route @port's targets. 1058298d44d0SDan Williams */ 1059298d44d0SDan Williams if (parent_iw > 1 && cxl_rr->nr_targets > 1) { 106027b3f8d1SDan Williams u32 address_bit = max(peig + peiw, eiw + peig); 106127b3f8d1SDan Williams 106227b3f8d1SDan Williams eig = address_bit - eiw + 1; 106327b3f8d1SDan Williams } else { 106427b3f8d1SDan Williams eiw = peiw; 106527b3f8d1SDan Williams eig = peig; 106627b3f8d1SDan Williams } 106727b3f8d1SDan Williams 106827b3f8d1SDan Williams rc = cxl_to_granularity(eig, &ig); 106927b3f8d1SDan Williams if (rc) { 107027b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n", 107127b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 107227b3f8d1SDan Williams 256 << eig); 107327b3f8d1SDan Williams return rc; 107427b3f8d1SDan Williams } 107527b3f8d1SDan Williams 107627b3f8d1SDan Williams cxld->interleave_ways = iw; 107727b3f8d1SDan Williams cxld->interleave_granularity = ig; 1078910bc55dSDan Williams cxld->hpa_range = (struct range) { 1079910bc55dSDan Williams .start = p->res->start, 1080910bc55dSDan Williams .end = p->res->end, 1081910bc55dSDan Williams }; 108227b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport), 108327b3f8d1SDan Williams dev_name(&port->dev), iw, ig); 108427b3f8d1SDan Williams add_target: 108527b3f8d1SDan Williams if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) { 108627b3f8d1SDan Williams dev_dbg(&cxlr->dev, 108727b3f8d1SDan Williams "%s:%s: targets full trying to add %s:%s at %d\n", 108827b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 108927b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 109027b3f8d1SDan Williams return -ENXIO; 109127b3f8d1SDan Williams } 109227b3f8d1SDan Williams cxlsd->target[cxl_rr->nr_targets_set] = ep->dport; 109327b3f8d1SDan Williams inc = 1; 109427b3f8d1SDan Williams out_target_set: 109527b3f8d1SDan Williams cxl_rr->nr_targets_set += inc; 109627b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", 109727b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 109827b3f8d1SDan Williams cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport), 109927b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 110027b3f8d1SDan Williams 110127b3f8d1SDan Williams return 0; 110227b3f8d1SDan Williams } 110327b3f8d1SDan Williams 110427b3f8d1SDan Williams static void cxl_port_reset_targets(struct cxl_port *port, 110527b3f8d1SDan Williams struct cxl_region *cxlr) 110627b3f8d1SDan Williams { 110727b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 1108910bc55dSDan Williams struct cxl_decoder *cxld; 110927b3f8d1SDan Williams 111027b3f8d1SDan Williams /* 111127b3f8d1SDan Williams * After the last endpoint has been detached the entire cxl_rr may now 111227b3f8d1SDan Williams * be gone. 111327b3f8d1SDan Williams */ 1114910bc55dSDan Williams if (!cxl_rr) 1115910bc55dSDan Williams return; 111627b3f8d1SDan Williams cxl_rr->nr_targets_set = 0; 1117910bc55dSDan Williams 1118910bc55dSDan Williams cxld = cxl_rr->decoder; 1119910bc55dSDan Williams cxld->hpa_range = (struct range) { 1120910bc55dSDan Williams .start = 0, 1121910bc55dSDan Williams .end = -1, 1122910bc55dSDan Williams }; 112327b3f8d1SDan Williams } 112427b3f8d1SDan Williams 112527b3f8d1SDan Williams static void cxl_region_teardown_targets(struct cxl_region *cxlr) 112627b3f8d1SDan Williams { 112727b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 112827b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled; 112927b3f8d1SDan Williams struct cxl_memdev *cxlmd; 113027b3f8d1SDan Williams struct cxl_port *iter; 113127b3f8d1SDan Williams struct cxl_ep *ep; 113227b3f8d1SDan Williams int i; 113327b3f8d1SDan Williams 113427b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) { 113527b3f8d1SDan Williams cxled = p->targets[i]; 113627b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled); 113727b3f8d1SDan Williams 113827b3f8d1SDan Williams iter = cxled_to_port(cxled); 113927b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 114027b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent); 114127b3f8d1SDan Williams 114227b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 114327b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) 114427b3f8d1SDan Williams cxl_port_reset_targets(iter, cxlr); 114527b3f8d1SDan Williams } 114627b3f8d1SDan Williams } 114727b3f8d1SDan Williams 114827b3f8d1SDan Williams static int cxl_region_setup_targets(struct cxl_region *cxlr) 114927b3f8d1SDan Williams { 115027b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 115127b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled; 115227b3f8d1SDan Williams struct cxl_memdev *cxlmd; 115327b3f8d1SDan Williams struct cxl_port *iter; 115427b3f8d1SDan Williams struct cxl_ep *ep; 115527b3f8d1SDan Williams int i, rc; 115627b3f8d1SDan Williams 115727b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) { 115827b3f8d1SDan Williams cxled = p->targets[i]; 115927b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled); 116027b3f8d1SDan Williams 116127b3f8d1SDan Williams iter = cxled_to_port(cxled); 116227b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 116327b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent); 116427b3f8d1SDan Williams 116527b3f8d1SDan Williams /* 116627b3f8d1SDan Williams * Descend the topology tree programming targets while 116727b3f8d1SDan Williams * looking for conflicts. 116827b3f8d1SDan Williams */ 116927b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 117027b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 117127b3f8d1SDan Williams rc = cxl_port_setup_targets(iter, cxlr, cxled); 117227b3f8d1SDan Williams if (rc) { 117327b3f8d1SDan Williams cxl_region_teardown_targets(cxlr); 117427b3f8d1SDan Williams return rc; 117527b3f8d1SDan Williams } 117627b3f8d1SDan Williams } 117727b3f8d1SDan Williams } 117827b3f8d1SDan Williams 117927b3f8d1SDan Williams return 0; 118027b3f8d1SDan Williams } 118127b3f8d1SDan Williams 1182b9686e8cSDan Williams static int cxl_region_attach(struct cxl_region *cxlr, 1183b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled, int pos) 1184b9686e8cSDan Williams { 1185384e624bSDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 1186384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1187384e624bSDan Williams struct cxl_port *ep_port, *root_port, *iter; 1188b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 1189384e624bSDan Williams struct cxl_dport *dport; 1190384e624bSDan Williams int i, rc = -ENXIO; 1191b9686e8cSDan Williams 1192b9686e8cSDan Williams if (cxled->mode == CXL_DECODER_DEAD) { 1193b9686e8cSDan Williams dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev)); 1194b9686e8cSDan Williams return -ENODEV; 1195b9686e8cSDan Williams } 1196b9686e8cSDan Williams 1197384e624bSDan Williams /* all full of members, or interleave config not established? */ 1198384e624bSDan Williams if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) { 1199384e624bSDan Williams dev_dbg(&cxlr->dev, "region already active\n"); 1200384e624bSDan Williams return -EBUSY; 1201384e624bSDan Williams } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) { 1202384e624bSDan Williams dev_dbg(&cxlr->dev, "interleave config missing\n"); 1203384e624bSDan Williams return -ENXIO; 1204384e624bSDan Williams } 1205384e624bSDan Williams 1206384e624bSDan Williams if (pos < 0 || pos >= p->interleave_ways) { 1207b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1208b9686e8cSDan Williams p->interleave_ways); 1209b9686e8cSDan Williams return -ENXIO; 1210b9686e8cSDan Williams } 1211b9686e8cSDan Williams 1212b9686e8cSDan Williams if (p->targets[pos] == cxled) 1213b9686e8cSDan Williams return 0; 1214b9686e8cSDan Williams 1215b9686e8cSDan Williams if (p->targets[pos]) { 1216b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled_target = p->targets[pos]; 1217b9686e8cSDan Williams struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target); 1218b9686e8cSDan Williams 1219b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n", 1220b9686e8cSDan Williams pos, dev_name(&cxlmd_target->dev), 1221b9686e8cSDan Williams dev_name(&cxled_target->cxld.dev)); 1222b9686e8cSDan Williams return -EBUSY; 1223b9686e8cSDan Williams } 1224b9686e8cSDan Williams 1225384e624bSDan Williams for (i = 0; i < p->interleave_ways; i++) { 1226384e624bSDan Williams struct cxl_endpoint_decoder *cxled_target; 1227384e624bSDan Williams struct cxl_memdev *cxlmd_target; 1228384e624bSDan Williams 1229384e624bSDan Williams cxled_target = p->targets[pos]; 1230384e624bSDan Williams if (!cxled_target) 1231384e624bSDan Williams continue; 1232384e624bSDan Williams 1233384e624bSDan Williams cxlmd_target = cxled_to_memdev(cxled_target); 1234384e624bSDan Williams if (cxlmd_target == cxlmd) { 1235384e624bSDan Williams dev_dbg(&cxlr->dev, 1236384e624bSDan Williams "%s already specified at position %d via: %s\n", 1237384e624bSDan Williams dev_name(&cxlmd->dev), pos, 1238384e624bSDan Williams dev_name(&cxled_target->cxld.dev)); 1239384e624bSDan Williams return -EBUSY; 1240384e624bSDan Williams } 1241384e624bSDan Williams } 1242384e624bSDan Williams 1243384e624bSDan Williams ep_port = cxled_to_port(cxled); 1244384e624bSDan Williams root_port = cxlrd_to_port(cxlrd); 1245384e624bSDan Williams dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge); 1246384e624bSDan Williams if (!dport) { 1247384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n", 1248384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1249384e624bSDan Williams dev_name(cxlr->dev.parent)); 1250384e624bSDan Williams return -ENXIO; 1251384e624bSDan Williams } 1252384e624bSDan Williams 1253384e624bSDan Williams if (cxlrd->calc_hb(cxlrd, pos) != dport) { 1254384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n", 1255384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1256384e624bSDan Williams dev_name(&cxlrd->cxlsd.cxld.dev)); 1257384e624bSDan Williams return -ENXIO; 1258384e624bSDan Williams } 1259384e624bSDan Williams 1260384e624bSDan Williams if (cxled->cxld.target_type != cxlr->type) { 1261384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n", 1262384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1263384e624bSDan Williams cxled->cxld.target_type, cxlr->type); 1264384e624bSDan Williams return -ENXIO; 1265384e624bSDan Williams } 1266384e624bSDan Williams 1267384e624bSDan Williams if (!cxled->dpa_res) { 1268384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n", 1269384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev)); 1270384e624bSDan Williams return -ENXIO; 1271384e624bSDan Williams } 1272384e624bSDan Williams 1273384e624bSDan Williams if (resource_size(cxled->dpa_res) * p->interleave_ways != 1274384e624bSDan Williams resource_size(p->res)) { 1275384e624bSDan Williams dev_dbg(&cxlr->dev, 1276384e624bSDan Williams "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n", 1277384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1278384e624bSDan Williams (u64)resource_size(cxled->dpa_res), p->interleave_ways, 1279384e624bSDan Williams (u64)resource_size(p->res)); 1280384e624bSDan Williams return -EINVAL; 1281384e624bSDan Williams } 1282384e624bSDan Williams 1283384e624bSDan Williams for (iter = ep_port; !is_cxl_root(iter); 1284384e624bSDan Williams iter = to_cxl_port(iter->dev.parent)) { 1285384e624bSDan Williams rc = cxl_port_attach_region(iter, cxlr, cxled, pos); 1286384e624bSDan Williams if (rc) 1287384e624bSDan Williams goto err; 1288384e624bSDan Williams } 1289384e624bSDan Williams 1290b9686e8cSDan Williams p->targets[pos] = cxled; 1291b9686e8cSDan Williams cxled->pos = pos; 1292b9686e8cSDan Williams p->nr_targets++; 1293b9686e8cSDan Williams 129427b3f8d1SDan Williams if (p->nr_targets == p->interleave_ways) { 129527b3f8d1SDan Williams rc = cxl_region_setup_targets(cxlr); 129627b3f8d1SDan Williams if (rc) 12975e42bcbcSDan Carpenter goto err_decrement; 1298384e624bSDan Williams p->state = CXL_CONFIG_ACTIVE; 129927b3f8d1SDan Williams } 1300384e624bSDan Williams 13012901c8bdSDan Williams cxled->cxld.interleave_ways = p->interleave_ways; 13022901c8bdSDan Williams cxled->cxld.interleave_granularity = p->interleave_granularity; 1303910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) { 1304910bc55dSDan Williams .start = p->res->start, 1305910bc55dSDan Williams .end = p->res->end, 1306910bc55dSDan Williams }; 13072901c8bdSDan Williams 1308b9686e8cSDan Williams return 0; 1309384e624bSDan Williams 13105e42bcbcSDan Carpenter err_decrement: 13115e42bcbcSDan Carpenter p->nr_targets--; 1312384e624bSDan Williams err: 1313384e624bSDan Williams for (iter = ep_port; !is_cxl_root(iter); 1314384e624bSDan Williams iter = to_cxl_port(iter->dev.parent)) 1315384e624bSDan Williams cxl_port_detach_region(iter, cxlr, cxled); 1316384e624bSDan Williams return rc; 1317b9686e8cSDan Williams } 1318b9686e8cSDan Williams 1319176baefbSDan Williams static int cxl_region_detach(struct cxl_endpoint_decoder *cxled) 1320b9686e8cSDan Williams { 1321384e624bSDan Williams struct cxl_port *iter, *ep_port = cxled_to_port(cxled); 1322b9686e8cSDan Williams struct cxl_region *cxlr = cxled->cxld.region; 1323b9686e8cSDan Williams struct cxl_region_params *p; 1324176baefbSDan Williams int rc = 0; 1325b9686e8cSDan Williams 1326b9686e8cSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 1327b9686e8cSDan Williams 1328b9686e8cSDan Williams if (!cxlr) 1329176baefbSDan Williams return 0; 1330b9686e8cSDan Williams 1331b9686e8cSDan Williams p = &cxlr->params; 1332b9686e8cSDan Williams get_device(&cxlr->dev); 1333b9686e8cSDan Williams 1334176baefbSDan Williams if (p->state > CXL_CONFIG_ACTIVE) { 1335176baefbSDan Williams /* 1336176baefbSDan Williams * TODO: tear down all impacted regions if a device is 1337176baefbSDan Williams * removed out of order 1338176baefbSDan Williams */ 1339176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 1340176baefbSDan Williams if (rc) 1341176baefbSDan Williams goto out; 1342176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE; 1343176baefbSDan Williams } 1344176baefbSDan Williams 1345384e624bSDan Williams for (iter = ep_port; !is_cxl_root(iter); 1346384e624bSDan Williams iter = to_cxl_port(iter->dev.parent)) 1347384e624bSDan Williams cxl_port_detach_region(iter, cxlr, cxled); 1348384e624bSDan Williams 1349b9686e8cSDan Williams if (cxled->pos < 0 || cxled->pos >= p->interleave_ways || 1350b9686e8cSDan Williams p->targets[cxled->pos] != cxled) { 1351b9686e8cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1352b9686e8cSDan Williams 1353b9686e8cSDan Williams dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n", 1354b9686e8cSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1355b9686e8cSDan Williams cxled->pos); 1356b9686e8cSDan Williams goto out; 1357b9686e8cSDan Williams } 1358b9686e8cSDan Williams 135927b3f8d1SDan Williams if (p->state == CXL_CONFIG_ACTIVE) { 1360384e624bSDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 136127b3f8d1SDan Williams cxl_region_teardown_targets(cxlr); 136227b3f8d1SDan Williams } 1363b9686e8cSDan Williams p->targets[cxled->pos] = NULL; 1364b9686e8cSDan Williams p->nr_targets--; 1365910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) { 1366910bc55dSDan Williams .start = 0, 1367910bc55dSDan Williams .end = -1, 1368910bc55dSDan Williams }; 1369b9686e8cSDan Williams 1370384e624bSDan Williams /* notify the region driver that one of its targets has departed */ 1371b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1372b9686e8cSDan Williams device_release_driver(&cxlr->dev); 1373b9686e8cSDan Williams down_write(&cxl_region_rwsem); 1374b9686e8cSDan Williams out: 1375b9686e8cSDan Williams put_device(&cxlr->dev); 1376176baefbSDan Williams return rc; 1377b9686e8cSDan Williams } 1378b9686e8cSDan Williams 1379b9686e8cSDan Williams void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled) 1380b9686e8cSDan Williams { 1381b9686e8cSDan Williams down_write(&cxl_region_rwsem); 1382b9686e8cSDan Williams cxled->mode = CXL_DECODER_DEAD; 1383b9686e8cSDan Williams cxl_region_detach(cxled); 1384b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1385b9686e8cSDan Williams } 1386b9686e8cSDan Williams 1387b9686e8cSDan Williams static int attach_target(struct cxl_region *cxlr, const char *decoder, int pos) 1388b9686e8cSDan Williams { 1389b9686e8cSDan Williams struct device *dev; 1390b9686e8cSDan Williams int rc; 1391b9686e8cSDan Williams 1392b9686e8cSDan Williams dev = bus_find_device_by_name(&cxl_bus_type, NULL, decoder); 1393b9686e8cSDan Williams if (!dev) 1394b9686e8cSDan Williams return -ENODEV; 1395b9686e8cSDan Williams 1396b9686e8cSDan Williams if (!is_endpoint_decoder(dev)) { 1397b9686e8cSDan Williams put_device(dev); 1398b9686e8cSDan Williams return -EINVAL; 1399b9686e8cSDan Williams } 1400b9686e8cSDan Williams 1401b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem); 1402b9686e8cSDan Williams if (rc) 1403b9686e8cSDan Williams goto out; 1404b9686e8cSDan Williams down_read(&cxl_dpa_rwsem); 1405b9686e8cSDan Williams rc = cxl_region_attach(cxlr, to_cxl_endpoint_decoder(dev), pos); 1406d18bc74aSDan Williams if (rc == 0) 1407d18bc74aSDan Williams set_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags); 1408b9686e8cSDan Williams up_read(&cxl_dpa_rwsem); 1409b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1410b9686e8cSDan Williams out: 1411b9686e8cSDan Williams put_device(dev); 1412b9686e8cSDan Williams return rc; 1413b9686e8cSDan Williams } 1414b9686e8cSDan Williams 1415b9686e8cSDan Williams static int detach_target(struct cxl_region *cxlr, int pos) 1416b9686e8cSDan Williams { 1417b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 1418b9686e8cSDan Williams int rc; 1419b9686e8cSDan Williams 1420b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem); 1421b9686e8cSDan Williams if (rc) 1422b9686e8cSDan Williams return rc; 1423b9686e8cSDan Williams 1424b9686e8cSDan Williams if (pos >= p->interleave_ways) { 1425b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1426b9686e8cSDan Williams p->interleave_ways); 1427b9686e8cSDan Williams rc = -ENXIO; 1428b9686e8cSDan Williams goto out; 1429b9686e8cSDan Williams } 1430b9686e8cSDan Williams 1431b9686e8cSDan Williams if (!p->targets[pos]) { 1432b9686e8cSDan Williams rc = 0; 1433b9686e8cSDan Williams goto out; 1434b9686e8cSDan Williams } 1435b9686e8cSDan Williams 1436176baefbSDan Williams rc = cxl_region_detach(p->targets[pos]); 1437b9686e8cSDan Williams out: 1438b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1439b9686e8cSDan Williams return rc; 1440b9686e8cSDan Williams } 1441b9686e8cSDan Williams 1442b9686e8cSDan Williams static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos, 1443b9686e8cSDan Williams size_t len) 1444b9686e8cSDan Williams { 1445b9686e8cSDan Williams int rc; 1446b9686e8cSDan Williams 1447b9686e8cSDan Williams if (sysfs_streq(buf, "\n")) 1448b9686e8cSDan Williams rc = detach_target(cxlr, pos); 1449b9686e8cSDan Williams else 1450b9686e8cSDan Williams rc = attach_target(cxlr, buf, pos); 1451b9686e8cSDan Williams 1452b9686e8cSDan Williams if (rc < 0) 1453b9686e8cSDan Williams return rc; 1454b9686e8cSDan Williams return len; 1455b9686e8cSDan Williams } 1456b9686e8cSDan Williams 1457b9686e8cSDan Williams #define TARGET_ATTR_RW(n) \ 1458b9686e8cSDan Williams static ssize_t target##n##_show( \ 1459b9686e8cSDan Williams struct device *dev, struct device_attribute *attr, char *buf) \ 1460b9686e8cSDan Williams { \ 1461b9686e8cSDan Williams return show_targetN(to_cxl_region(dev), buf, (n)); \ 1462b9686e8cSDan Williams } \ 1463b9686e8cSDan Williams static ssize_t target##n##_store(struct device *dev, \ 1464b9686e8cSDan Williams struct device_attribute *attr, \ 1465b9686e8cSDan Williams const char *buf, size_t len) \ 1466b9686e8cSDan Williams { \ 1467b9686e8cSDan Williams return store_targetN(to_cxl_region(dev), buf, (n), len); \ 1468b9686e8cSDan Williams } \ 1469b9686e8cSDan Williams static DEVICE_ATTR_RW(target##n) 1470b9686e8cSDan Williams 1471b9686e8cSDan Williams TARGET_ATTR_RW(0); 1472b9686e8cSDan Williams TARGET_ATTR_RW(1); 1473b9686e8cSDan Williams TARGET_ATTR_RW(2); 1474b9686e8cSDan Williams TARGET_ATTR_RW(3); 1475b9686e8cSDan Williams TARGET_ATTR_RW(4); 1476b9686e8cSDan Williams TARGET_ATTR_RW(5); 1477b9686e8cSDan Williams TARGET_ATTR_RW(6); 1478b9686e8cSDan Williams TARGET_ATTR_RW(7); 1479b9686e8cSDan Williams TARGET_ATTR_RW(8); 1480b9686e8cSDan Williams TARGET_ATTR_RW(9); 1481b9686e8cSDan Williams TARGET_ATTR_RW(10); 1482b9686e8cSDan Williams TARGET_ATTR_RW(11); 1483b9686e8cSDan Williams TARGET_ATTR_RW(12); 1484b9686e8cSDan Williams TARGET_ATTR_RW(13); 1485b9686e8cSDan Williams TARGET_ATTR_RW(14); 1486b9686e8cSDan Williams TARGET_ATTR_RW(15); 1487b9686e8cSDan Williams 1488b9686e8cSDan Williams static struct attribute *target_attrs[] = { 1489b9686e8cSDan Williams &dev_attr_target0.attr, 1490b9686e8cSDan Williams &dev_attr_target1.attr, 1491b9686e8cSDan Williams &dev_attr_target2.attr, 1492b9686e8cSDan Williams &dev_attr_target3.attr, 1493b9686e8cSDan Williams &dev_attr_target4.attr, 1494b9686e8cSDan Williams &dev_attr_target5.attr, 1495b9686e8cSDan Williams &dev_attr_target6.attr, 1496b9686e8cSDan Williams &dev_attr_target7.attr, 1497b9686e8cSDan Williams &dev_attr_target8.attr, 1498b9686e8cSDan Williams &dev_attr_target9.attr, 1499b9686e8cSDan Williams &dev_attr_target10.attr, 1500b9686e8cSDan Williams &dev_attr_target11.attr, 1501b9686e8cSDan Williams &dev_attr_target12.attr, 1502b9686e8cSDan Williams &dev_attr_target13.attr, 1503b9686e8cSDan Williams &dev_attr_target14.attr, 1504b9686e8cSDan Williams &dev_attr_target15.attr, 1505b9686e8cSDan Williams NULL, 1506b9686e8cSDan Williams }; 1507b9686e8cSDan Williams 1508b9686e8cSDan Williams static umode_t cxl_region_target_visible(struct kobject *kobj, 1509b9686e8cSDan Williams struct attribute *a, int n) 1510b9686e8cSDan Williams { 1511b9686e8cSDan Williams struct device *dev = kobj_to_dev(kobj); 1512b9686e8cSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 1513b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 1514b9686e8cSDan Williams 1515b9686e8cSDan Williams if (n < p->interleave_ways) 1516b9686e8cSDan Williams return a->mode; 1517b9686e8cSDan Williams return 0; 1518b9686e8cSDan Williams } 1519b9686e8cSDan Williams 1520b9686e8cSDan Williams static const struct attribute_group cxl_region_target_group = { 1521b9686e8cSDan Williams .attrs = target_attrs, 1522b9686e8cSDan Williams .is_visible = cxl_region_target_visible, 1523b9686e8cSDan Williams }; 1524b9686e8cSDan Williams 1525b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void) 1526b9686e8cSDan Williams { 1527b9686e8cSDan Williams return &cxl_region_target_group; 1528b9686e8cSDan Williams } 1529b9686e8cSDan Williams 1530dd5ba0ebSBen Widawsky static const struct attribute_group *region_groups[] = { 1531dd5ba0ebSBen Widawsky &cxl_base_attribute_group, 1532dd5ba0ebSBen Widawsky &cxl_region_group, 1533b9686e8cSDan Williams &cxl_region_target_group, 1534dd5ba0ebSBen Widawsky NULL, 1535dd5ba0ebSBen Widawsky }; 1536dd5ba0ebSBen Widawsky 1537779dd20cSBen Widawsky static void cxl_region_release(struct device *dev) 1538779dd20cSBen Widawsky { 15398f401ec1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 1540779dd20cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 15418f401ec1SDan Williams int id = atomic_read(&cxlrd->region_id); 15428f401ec1SDan Williams 15438f401ec1SDan Williams /* 15448f401ec1SDan Williams * Try to reuse the recently idled id rather than the cached 15458f401ec1SDan Williams * next id to prevent the region id space from increasing 15468f401ec1SDan Williams * unnecessarily. 15478f401ec1SDan Williams */ 15488f401ec1SDan Williams if (cxlr->id < id) 15498f401ec1SDan Williams if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) { 15508f401ec1SDan Williams memregion_free(id); 15518f401ec1SDan Williams goto out; 15528f401ec1SDan Williams } 1553779dd20cSBen Widawsky 1554779dd20cSBen Widawsky memregion_free(cxlr->id); 15558f401ec1SDan Williams out: 15568f401ec1SDan Williams put_device(dev->parent); 1557779dd20cSBen Widawsky kfree(cxlr); 1558779dd20cSBen Widawsky } 1559779dd20cSBen Widawsky 15608d48817dSDan Williams const struct device_type cxl_region_type = { 1561779dd20cSBen Widawsky .name = "cxl_region", 1562779dd20cSBen Widawsky .release = cxl_region_release, 1563dd5ba0ebSBen Widawsky .groups = region_groups 1564779dd20cSBen Widawsky }; 1565779dd20cSBen Widawsky 1566779dd20cSBen Widawsky bool is_cxl_region(struct device *dev) 1567779dd20cSBen Widawsky { 1568779dd20cSBen Widawsky return dev->type == &cxl_region_type; 1569779dd20cSBen Widawsky } 1570779dd20cSBen Widawsky EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL); 1571779dd20cSBen Widawsky 1572779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev) 1573779dd20cSBen Widawsky { 1574779dd20cSBen Widawsky if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type, 1575779dd20cSBen Widawsky "not a cxl_region device\n")) 1576779dd20cSBen Widawsky return NULL; 1577779dd20cSBen Widawsky 1578779dd20cSBen Widawsky return container_of(dev, struct cxl_region, dev); 1579779dd20cSBen Widawsky } 1580779dd20cSBen Widawsky 1581779dd20cSBen Widawsky static void unregister_region(void *dev) 1582779dd20cSBen Widawsky { 158323a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 15840d9e7340SDan Williams struct cxl_region_params *p = &cxlr->params; 15850d9e7340SDan Williams int i; 158623a22cd1SDan Williams 158723a22cd1SDan Williams device_del(dev); 15880d9e7340SDan Williams 15890d9e7340SDan Williams /* 15900d9e7340SDan Williams * Now that region sysfs is shutdown, the parameter block is now 15910d9e7340SDan Williams * read-only, so no need to hold the region rwsem to access the 15920d9e7340SDan Williams * region parameters. 15930d9e7340SDan Williams */ 15940d9e7340SDan Williams for (i = 0; i < p->interleave_ways; i++) 15950d9e7340SDan Williams detach_target(cxlr, i); 15960d9e7340SDan Williams 159723a22cd1SDan Williams cxl_region_iomem_release(cxlr); 159823a22cd1SDan Williams put_device(dev); 1599779dd20cSBen Widawsky } 1600779dd20cSBen Widawsky 1601779dd20cSBen Widawsky static struct lock_class_key cxl_region_key; 1602779dd20cSBen Widawsky 1603779dd20cSBen Widawsky static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id) 1604779dd20cSBen Widawsky { 1605779dd20cSBen Widawsky struct cxl_region *cxlr; 1606779dd20cSBen Widawsky struct device *dev; 1607779dd20cSBen Widawsky 1608779dd20cSBen Widawsky cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL); 1609779dd20cSBen Widawsky if (!cxlr) { 1610779dd20cSBen Widawsky memregion_free(id); 1611779dd20cSBen Widawsky return ERR_PTR(-ENOMEM); 1612779dd20cSBen Widawsky } 1613779dd20cSBen Widawsky 1614779dd20cSBen Widawsky dev = &cxlr->dev; 1615779dd20cSBen Widawsky device_initialize(dev); 1616779dd20cSBen Widawsky lockdep_set_class(&dev->mutex, &cxl_region_key); 1617779dd20cSBen Widawsky dev->parent = &cxlrd->cxlsd.cxld.dev; 16188f401ec1SDan Williams /* 16198f401ec1SDan Williams * Keep root decoder pinned through cxl_region_release to fixup 16208f401ec1SDan Williams * region id allocations 16218f401ec1SDan Williams */ 16228f401ec1SDan Williams get_device(dev->parent); 1623779dd20cSBen Widawsky device_set_pm_not_required(dev); 1624779dd20cSBen Widawsky dev->bus = &cxl_bus_type; 1625779dd20cSBen Widawsky dev->type = &cxl_region_type; 1626779dd20cSBen Widawsky cxlr->id = id; 1627779dd20cSBen Widawsky 1628779dd20cSBen Widawsky return cxlr; 1629779dd20cSBen Widawsky } 1630779dd20cSBen Widawsky 1631779dd20cSBen Widawsky /** 1632779dd20cSBen Widawsky * devm_cxl_add_region - Adds a region to a decoder 1633779dd20cSBen Widawsky * @cxlrd: root decoder 1634779dd20cSBen Widawsky * @id: memregion id to create, or memregion_free() on failure 1635779dd20cSBen Widawsky * @mode: mode for the endpoint decoders of this region 1636779dd20cSBen Widawsky * @type: select whether this is an expander or accelerator (type-2 or type-3) 1637779dd20cSBen Widawsky * 1638779dd20cSBen Widawsky * This is the second step of region initialization. Regions exist within an 1639779dd20cSBen Widawsky * address space which is mapped by a @cxlrd. 1640779dd20cSBen Widawsky * 1641779dd20cSBen Widawsky * Return: 0 if the region was added to the @cxlrd, else returns negative error 1642779dd20cSBen Widawsky * code. The region will be named "regionZ" where Z is the unique region number. 1643779dd20cSBen Widawsky */ 1644779dd20cSBen Widawsky static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, 1645779dd20cSBen Widawsky int id, 1646779dd20cSBen Widawsky enum cxl_decoder_mode mode, 1647779dd20cSBen Widawsky enum cxl_decoder_type type) 1648779dd20cSBen Widawsky { 1649779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent); 1650779dd20cSBen Widawsky struct cxl_region *cxlr; 1651779dd20cSBen Widawsky struct device *dev; 1652779dd20cSBen Widawsky int rc; 1653779dd20cSBen Widawsky 1654779dd20cSBen Widawsky cxlr = cxl_region_alloc(cxlrd, id); 1655779dd20cSBen Widawsky if (IS_ERR(cxlr)) 1656779dd20cSBen Widawsky return cxlr; 1657779dd20cSBen Widawsky cxlr->mode = mode; 1658779dd20cSBen Widawsky cxlr->type = type; 1659779dd20cSBen Widawsky 1660779dd20cSBen Widawsky dev = &cxlr->dev; 1661779dd20cSBen Widawsky rc = dev_set_name(dev, "region%d", id); 1662779dd20cSBen Widawsky if (rc) 1663779dd20cSBen Widawsky goto err; 1664779dd20cSBen Widawsky 1665779dd20cSBen Widawsky rc = device_add(dev); 1666779dd20cSBen Widawsky if (rc) 1667779dd20cSBen Widawsky goto err; 1668779dd20cSBen Widawsky 1669779dd20cSBen Widawsky rc = devm_add_action_or_reset(port->uport, unregister_region, cxlr); 1670779dd20cSBen Widawsky if (rc) 1671779dd20cSBen Widawsky return ERR_PTR(rc); 1672779dd20cSBen Widawsky 1673779dd20cSBen Widawsky dev_dbg(port->uport, "%s: created %s\n", 1674779dd20cSBen Widawsky dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); 1675779dd20cSBen Widawsky return cxlr; 1676779dd20cSBen Widawsky 1677779dd20cSBen Widawsky err: 1678779dd20cSBen Widawsky put_device(dev); 1679779dd20cSBen Widawsky return ERR_PTR(rc); 1680779dd20cSBen Widawsky } 1681779dd20cSBen Widawsky 1682779dd20cSBen Widawsky static ssize_t create_pmem_region_show(struct device *dev, 1683779dd20cSBen Widawsky struct device_attribute *attr, char *buf) 1684779dd20cSBen Widawsky { 1685779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 1686779dd20cSBen Widawsky 1687779dd20cSBen Widawsky return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id)); 1688779dd20cSBen Widawsky } 1689779dd20cSBen Widawsky 1690779dd20cSBen Widawsky static ssize_t create_pmem_region_store(struct device *dev, 1691779dd20cSBen Widawsky struct device_attribute *attr, 1692779dd20cSBen Widawsky const char *buf, size_t len) 1693779dd20cSBen Widawsky { 1694779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 1695779dd20cSBen Widawsky struct cxl_region *cxlr; 1696779dd20cSBen Widawsky int id, rc; 1697779dd20cSBen Widawsky 1698779dd20cSBen Widawsky rc = sscanf(buf, "region%d\n", &id); 1699779dd20cSBen Widawsky if (rc != 1) 1700779dd20cSBen Widawsky return -EINVAL; 1701779dd20cSBen Widawsky 1702779dd20cSBen Widawsky rc = memregion_alloc(GFP_KERNEL); 1703779dd20cSBen Widawsky if (rc < 0) 1704779dd20cSBen Widawsky return rc; 1705779dd20cSBen Widawsky 1706779dd20cSBen Widawsky if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) { 1707779dd20cSBen Widawsky memregion_free(rc); 1708779dd20cSBen Widawsky return -EBUSY; 1709779dd20cSBen Widawsky } 1710779dd20cSBen Widawsky 1711779dd20cSBen Widawsky cxlr = devm_cxl_add_region(cxlrd, id, CXL_DECODER_PMEM, 1712779dd20cSBen Widawsky CXL_DECODER_EXPANDER); 1713779dd20cSBen Widawsky if (IS_ERR(cxlr)) 1714779dd20cSBen Widawsky return PTR_ERR(cxlr); 1715779dd20cSBen Widawsky 1716779dd20cSBen Widawsky return len; 1717779dd20cSBen Widawsky } 1718779dd20cSBen Widawsky DEVICE_ATTR_RW(create_pmem_region); 1719779dd20cSBen Widawsky 1720b9686e8cSDan Williams static ssize_t region_show(struct device *dev, struct device_attribute *attr, 1721b9686e8cSDan Williams char *buf) 1722b9686e8cSDan Williams { 1723b9686e8cSDan Williams struct cxl_decoder *cxld = to_cxl_decoder(dev); 1724b9686e8cSDan Williams ssize_t rc; 1725b9686e8cSDan Williams 1726b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 1727b9686e8cSDan Williams if (rc) 1728b9686e8cSDan Williams return rc; 1729b9686e8cSDan Williams 1730b9686e8cSDan Williams if (cxld->region) 1731b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev)); 1732b9686e8cSDan Williams else 1733b9686e8cSDan Williams rc = sysfs_emit(buf, "\n"); 1734b9686e8cSDan Williams up_read(&cxl_region_rwsem); 1735b9686e8cSDan Williams 1736b9686e8cSDan Williams return rc; 1737b9686e8cSDan Williams } 1738b9686e8cSDan Williams DEVICE_ATTR_RO(region); 1739b9686e8cSDan Williams 1740779dd20cSBen Widawsky static struct cxl_region * 1741779dd20cSBen Widawsky cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name) 1742779dd20cSBen Widawsky { 1743779dd20cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 1744779dd20cSBen Widawsky struct device *region_dev; 1745779dd20cSBen Widawsky 1746779dd20cSBen Widawsky region_dev = device_find_child_by_name(&cxld->dev, name); 1747779dd20cSBen Widawsky if (!region_dev) 1748779dd20cSBen Widawsky return ERR_PTR(-ENODEV); 1749779dd20cSBen Widawsky 1750779dd20cSBen Widawsky return to_cxl_region(region_dev); 1751779dd20cSBen Widawsky } 1752779dd20cSBen Widawsky 1753779dd20cSBen Widawsky static ssize_t delete_region_store(struct device *dev, 1754779dd20cSBen Widawsky struct device_attribute *attr, 1755779dd20cSBen Widawsky const char *buf, size_t len) 1756779dd20cSBen Widawsky { 1757779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 1758779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(dev->parent); 1759779dd20cSBen Widawsky struct cxl_region *cxlr; 1760779dd20cSBen Widawsky 1761779dd20cSBen Widawsky cxlr = cxl_find_region_by_name(cxlrd, buf); 1762779dd20cSBen Widawsky if (IS_ERR(cxlr)) 1763779dd20cSBen Widawsky return PTR_ERR(cxlr); 1764779dd20cSBen Widawsky 1765779dd20cSBen Widawsky devm_release_action(port->uport, unregister_region, cxlr); 1766779dd20cSBen Widawsky put_device(&cxlr->dev); 1767779dd20cSBen Widawsky 1768779dd20cSBen Widawsky return len; 1769779dd20cSBen Widawsky } 1770779dd20cSBen Widawsky DEVICE_ATTR_WO(delete_region); 177123a22cd1SDan Williams 177204ad63f0SDan Williams static void cxl_pmem_region_release(struct device *dev) 177304ad63f0SDan Williams { 177404ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); 177504ad63f0SDan Williams int i; 177604ad63f0SDan Williams 177704ad63f0SDan Williams for (i = 0; i < cxlr_pmem->nr_mappings; i++) { 177804ad63f0SDan Williams struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd; 177904ad63f0SDan Williams 178004ad63f0SDan Williams put_device(&cxlmd->dev); 178104ad63f0SDan Williams } 178204ad63f0SDan Williams 178304ad63f0SDan Williams kfree(cxlr_pmem); 178404ad63f0SDan Williams } 178504ad63f0SDan Williams 178604ad63f0SDan Williams static const struct attribute_group *cxl_pmem_region_attribute_groups[] = { 178704ad63f0SDan Williams &cxl_base_attribute_group, 178804ad63f0SDan Williams NULL, 178904ad63f0SDan Williams }; 179004ad63f0SDan Williams 179104ad63f0SDan Williams const struct device_type cxl_pmem_region_type = { 179204ad63f0SDan Williams .name = "cxl_pmem_region", 179304ad63f0SDan Williams .release = cxl_pmem_region_release, 179404ad63f0SDan Williams .groups = cxl_pmem_region_attribute_groups, 179504ad63f0SDan Williams }; 179604ad63f0SDan Williams 179704ad63f0SDan Williams bool is_cxl_pmem_region(struct device *dev) 179804ad63f0SDan Williams { 179904ad63f0SDan Williams return dev->type == &cxl_pmem_region_type; 180004ad63f0SDan Williams } 180104ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL); 180204ad63f0SDan Williams 180304ad63f0SDan Williams struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev) 180404ad63f0SDan Williams { 180504ad63f0SDan Williams if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev), 180604ad63f0SDan Williams "not a cxl_pmem_region device\n")) 180704ad63f0SDan Williams return NULL; 180804ad63f0SDan Williams return container_of(dev, struct cxl_pmem_region, dev); 180904ad63f0SDan Williams } 181004ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL); 181104ad63f0SDan Williams 181204ad63f0SDan Williams static struct lock_class_key cxl_pmem_region_key; 181304ad63f0SDan Williams 181404ad63f0SDan Williams static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr) 181504ad63f0SDan Williams { 181604ad63f0SDan Williams struct cxl_region_params *p = &cxlr->params; 1817f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb; 181804ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem; 181904ad63f0SDan Williams struct device *dev; 182004ad63f0SDan Williams int i; 182104ad63f0SDan Williams 182204ad63f0SDan Williams down_read(&cxl_region_rwsem); 182304ad63f0SDan Williams if (p->state != CXL_CONFIG_COMMIT) { 182404ad63f0SDan Williams cxlr_pmem = ERR_PTR(-ENXIO); 182504ad63f0SDan Williams goto out; 182604ad63f0SDan Williams } 182704ad63f0SDan Williams 182804ad63f0SDan Williams cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), 182904ad63f0SDan Williams GFP_KERNEL); 183004ad63f0SDan Williams if (!cxlr_pmem) { 183104ad63f0SDan Williams cxlr_pmem = ERR_PTR(-ENOMEM); 183204ad63f0SDan Williams goto out; 183304ad63f0SDan Williams } 183404ad63f0SDan Williams 183504ad63f0SDan Williams cxlr_pmem->hpa_range.start = p->res->start; 183604ad63f0SDan Williams cxlr_pmem->hpa_range.end = p->res->end; 183704ad63f0SDan Williams 183804ad63f0SDan Williams /* Snapshot the region configuration underneath the cxl_region_rwsem */ 183904ad63f0SDan Williams cxlr_pmem->nr_mappings = p->nr_targets; 184004ad63f0SDan Williams for (i = 0; i < p->nr_targets; i++) { 184104ad63f0SDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 184204ad63f0SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 184304ad63f0SDan Williams struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; 184404ad63f0SDan Williams 1845f17b558dSDan Williams /* 1846f17b558dSDan Williams * Regions never span CXL root devices, so by definition the 1847f17b558dSDan Williams * bridge for one device is the same for all. 1848f17b558dSDan Williams */ 1849f17b558dSDan Williams if (i == 0) { 1850f17b558dSDan Williams cxl_nvb = cxl_find_nvdimm_bridge(&cxlmd->dev); 1851f17b558dSDan Williams if (!cxl_nvb) { 1852f17b558dSDan Williams cxlr_pmem = ERR_PTR(-ENODEV); 1853f17b558dSDan Williams goto out; 1854f17b558dSDan Williams } 1855f17b558dSDan Williams cxlr->cxl_nvb = cxl_nvb; 1856f17b558dSDan Williams } 185704ad63f0SDan Williams m->cxlmd = cxlmd; 185804ad63f0SDan Williams get_device(&cxlmd->dev); 185904ad63f0SDan Williams m->start = cxled->dpa_res->start; 186004ad63f0SDan Williams m->size = resource_size(cxled->dpa_res); 186104ad63f0SDan Williams m->position = i; 186204ad63f0SDan Williams } 186304ad63f0SDan Williams 186404ad63f0SDan Williams dev = &cxlr_pmem->dev; 186504ad63f0SDan Williams cxlr_pmem->cxlr = cxlr; 1866f17b558dSDan Williams cxlr->cxlr_pmem = cxlr_pmem; 186704ad63f0SDan Williams device_initialize(dev); 186804ad63f0SDan Williams lockdep_set_class(&dev->mutex, &cxl_pmem_region_key); 186904ad63f0SDan Williams device_set_pm_not_required(dev); 187004ad63f0SDan Williams dev->parent = &cxlr->dev; 187104ad63f0SDan Williams dev->bus = &cxl_bus_type; 187204ad63f0SDan Williams dev->type = &cxl_pmem_region_type; 187304ad63f0SDan Williams out: 187404ad63f0SDan Williams up_read(&cxl_region_rwsem); 187504ad63f0SDan Williams 187604ad63f0SDan Williams return cxlr_pmem; 187704ad63f0SDan Williams } 187804ad63f0SDan Williams 1879f17b558dSDan Williams static void cxlr_pmem_unregister(void *_cxlr_pmem) 188004ad63f0SDan Williams { 1881f17b558dSDan Williams struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem; 1882f17b558dSDan Williams struct cxl_region *cxlr = cxlr_pmem->cxlr; 1883f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 1884f17b558dSDan Williams 1885f17b558dSDan Williams /* 1886f17b558dSDan Williams * Either the bridge is in ->remove() context under the device_lock(), 1887f17b558dSDan Williams * or cxlr_release_nvdimm() is cancelling the bridge's release action 1888f17b558dSDan Williams * for @cxlr_pmem and doing it itself (while manually holding the bridge 1889f17b558dSDan Williams * lock). 1890f17b558dSDan Williams */ 1891f17b558dSDan Williams device_lock_assert(&cxl_nvb->dev); 1892f17b558dSDan Williams cxlr->cxlr_pmem = NULL; 1893f17b558dSDan Williams cxlr_pmem->cxlr = NULL; 1894f17b558dSDan Williams device_unregister(&cxlr_pmem->dev); 1895f17b558dSDan Williams } 1896f17b558dSDan Williams 1897f17b558dSDan Williams static void cxlr_release_nvdimm(void *_cxlr) 1898f17b558dSDan Williams { 1899f17b558dSDan Williams struct cxl_region *cxlr = _cxlr; 1900f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 1901f17b558dSDan Williams 1902f17b558dSDan Williams device_lock(&cxl_nvb->dev); 1903f17b558dSDan Williams if (cxlr->cxlr_pmem) 1904f17b558dSDan Williams devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister, 1905f17b558dSDan Williams cxlr->cxlr_pmem); 1906f17b558dSDan Williams device_unlock(&cxl_nvb->dev); 1907f17b558dSDan Williams cxlr->cxl_nvb = NULL; 1908f17b558dSDan Williams put_device(&cxl_nvb->dev); 190904ad63f0SDan Williams } 191004ad63f0SDan Williams 191104ad63f0SDan Williams /** 191204ad63f0SDan Williams * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge 191304ad63f0SDan Williams * @cxlr: parent CXL region for this pmem region bridge device 191404ad63f0SDan Williams * 191504ad63f0SDan Williams * Return: 0 on success negative error code on failure. 191604ad63f0SDan Williams */ 191704ad63f0SDan Williams static int devm_cxl_add_pmem_region(struct cxl_region *cxlr) 191804ad63f0SDan Williams { 191904ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem; 1920f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb; 192104ad63f0SDan Williams struct device *dev; 192204ad63f0SDan Williams int rc; 192304ad63f0SDan Williams 192404ad63f0SDan Williams cxlr_pmem = cxl_pmem_region_alloc(cxlr); 192504ad63f0SDan Williams if (IS_ERR(cxlr_pmem)) 192604ad63f0SDan Williams return PTR_ERR(cxlr_pmem); 1927f17b558dSDan Williams cxl_nvb = cxlr->cxl_nvb; 192804ad63f0SDan Williams 192904ad63f0SDan Williams dev = &cxlr_pmem->dev; 193004ad63f0SDan Williams rc = dev_set_name(dev, "pmem_region%d", cxlr->id); 193104ad63f0SDan Williams if (rc) 193204ad63f0SDan Williams goto err; 193304ad63f0SDan Williams 193404ad63f0SDan Williams rc = device_add(dev); 193504ad63f0SDan Williams if (rc) 193604ad63f0SDan Williams goto err; 193704ad63f0SDan Williams 193804ad63f0SDan Williams dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 193904ad63f0SDan Williams dev_name(dev)); 194004ad63f0SDan Williams 1941f17b558dSDan Williams device_lock(&cxl_nvb->dev); 1942f17b558dSDan Williams if (cxl_nvb->dev.driver) 1943f17b558dSDan Williams rc = devm_add_action_or_reset(&cxl_nvb->dev, 1944f17b558dSDan Williams cxlr_pmem_unregister, cxlr_pmem); 1945f17b558dSDan Williams else 1946f17b558dSDan Williams rc = -ENXIO; 1947f17b558dSDan Williams device_unlock(&cxl_nvb->dev); 1948f17b558dSDan Williams 1949f17b558dSDan Williams if (rc) 1950f17b558dSDan Williams goto err_bridge; 1951f17b558dSDan Williams 1952f17b558dSDan Williams /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */ 1953f17b558dSDan Williams return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr); 195404ad63f0SDan Williams 195504ad63f0SDan Williams err: 195604ad63f0SDan Williams put_device(dev); 1957f17b558dSDan Williams err_bridge: 1958f17b558dSDan Williams put_device(&cxl_nvb->dev); 1959f17b558dSDan Williams cxlr->cxl_nvb = NULL; 196004ad63f0SDan Williams return rc; 196104ad63f0SDan Williams } 196204ad63f0SDan Williams 1963d18bc74aSDan Williams static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) 1964d18bc74aSDan Williams { 1965d18bc74aSDan Williams if (!test_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags)) 1966d18bc74aSDan Williams return 0; 1967d18bc74aSDan Williams 1968d18bc74aSDan Williams if (!cpu_cache_has_invalidate_memregion()) { 1969d18bc74aSDan Williams if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { 1970d18bc74aSDan Williams dev_warn( 1971d18bc74aSDan Williams &cxlr->dev, 1972*cb4cdf74SColin Ian King "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); 1973d18bc74aSDan Williams clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags); 1974d18bc74aSDan Williams return 0; 1975d18bc74aSDan Williams } else { 1976d18bc74aSDan Williams dev_err(&cxlr->dev, 1977d18bc74aSDan Williams "Failed to synchronize CPU cache state\n"); 1978d18bc74aSDan Williams return -ENXIO; 1979d18bc74aSDan Williams } 1980d18bc74aSDan Williams } 1981d18bc74aSDan Williams 1982d18bc74aSDan Williams cpu_cache_invalidate_memregion(IORES_DESC_CXL); 1983d18bc74aSDan Williams clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags); 1984d18bc74aSDan Williams return 0; 1985d18bc74aSDan Williams } 1986d18bc74aSDan Williams 19878d48817dSDan Williams static int cxl_region_probe(struct device *dev) 19888d48817dSDan Williams { 19898d48817dSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 19908d48817dSDan Williams struct cxl_region_params *p = &cxlr->params; 19918d48817dSDan Williams int rc; 19928d48817dSDan Williams 19938d48817dSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 19948d48817dSDan Williams if (rc) { 19958d48817dSDan Williams dev_dbg(&cxlr->dev, "probe interrupted\n"); 19968d48817dSDan Williams return rc; 19978d48817dSDan Williams } 19988d48817dSDan Williams 19998d48817dSDan Williams if (p->state < CXL_CONFIG_COMMIT) { 20008d48817dSDan Williams dev_dbg(&cxlr->dev, "config state: %d\n", p->state); 20018d48817dSDan Williams rc = -ENXIO; 2002d18bc74aSDan Williams goto out; 20038d48817dSDan Williams } 20048d48817dSDan Williams 2005d18bc74aSDan Williams rc = cxl_region_invalidate_memregion(cxlr); 2006d18bc74aSDan Williams 20078d48817dSDan Williams /* 20088d48817dSDan Williams * From this point on any path that changes the region's state away from 20098d48817dSDan Williams * CXL_CONFIG_COMMIT is also responsible for releasing the driver. 20108d48817dSDan Williams */ 2011d18bc74aSDan Williams out: 20128d48817dSDan Williams up_read(&cxl_region_rwsem); 20138d48817dSDan Williams 2014bf3e5da8SDan Williams if (rc) 2015bf3e5da8SDan Williams return rc; 2016bf3e5da8SDan Williams 201704ad63f0SDan Williams switch (cxlr->mode) { 201804ad63f0SDan Williams case CXL_DECODER_PMEM: 201904ad63f0SDan Williams return devm_cxl_add_pmem_region(cxlr); 202004ad63f0SDan Williams default: 202104ad63f0SDan Williams dev_dbg(&cxlr->dev, "unsupported region mode: %d\n", 202204ad63f0SDan Williams cxlr->mode); 202304ad63f0SDan Williams return -ENXIO; 202404ad63f0SDan Williams } 20258d48817dSDan Williams } 20268d48817dSDan Williams 20278d48817dSDan Williams static struct cxl_driver cxl_region_driver = { 20288d48817dSDan Williams .name = "cxl_region", 20298d48817dSDan Williams .probe = cxl_region_probe, 20308d48817dSDan Williams .id = CXL_DEVICE_REGION, 20318d48817dSDan Williams }; 20328d48817dSDan Williams 20338d48817dSDan Williams int cxl_region_init(void) 20348d48817dSDan Williams { 20358d48817dSDan Williams return cxl_driver_register(&cxl_region_driver); 20368d48817dSDan Williams } 20378d48817dSDan Williams 20388d48817dSDan Williams void cxl_region_exit(void) 20398d48817dSDan Williams { 20408d48817dSDan Williams cxl_driver_unregister(&cxl_region_driver); 20418d48817dSDan Williams } 20428d48817dSDan Williams 204323a22cd1SDan Williams MODULE_IMPORT_NS(CXL); 2044d18bc74aSDan Williams MODULE_IMPORT_NS(DEVMEM); 20458d48817dSDan Williams MODULE_ALIAS_CXL(CXL_DEVICE_REGION); 2046