1779dd20cSBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 2779dd20cSBen Widawsky /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3779dd20cSBen Widawsky #include <linux/memregion.h> 4779dd20cSBen Widawsky #include <linux/genalloc.h> 5779dd20cSBen Widawsky #include <linux/device.h> 6779dd20cSBen Widawsky #include <linux/module.h> 7779dd20cSBen Widawsky #include <linux/slab.h> 8dd5ba0ebSBen Widawsky #include <linux/uuid.h> 9a32320b7SDan Williams #include <linux/sort.h> 10779dd20cSBen Widawsky #include <linux/idr.h> 1180d10a6cSBen Widawsky #include <cxlmem.h> 12779dd20cSBen Widawsky #include <cxl.h> 13779dd20cSBen Widawsky #include "core.h" 14779dd20cSBen Widawsky 15779dd20cSBen Widawsky /** 16779dd20cSBen Widawsky * DOC: cxl core region 17779dd20cSBen Widawsky * 18779dd20cSBen Widawsky * CXL Regions represent mapped memory capacity in system physical address 19779dd20cSBen Widawsky * space. Whereas the CXL Root Decoders identify the bounds of potential CXL 20779dd20cSBen Widawsky * Memory ranges, Regions represent the active mapped capacity by the HDM 21779dd20cSBen Widawsky * Decoder Capability structures throughout the Host Bridges, Switches, and 22779dd20cSBen Widawsky * Endpoints in the topology. 23dd5ba0ebSBen Widawsky * 24dd5ba0ebSBen Widawsky * Region configuration has ordering constraints. UUID may be set at any time 25dd5ba0ebSBen Widawsky * but is only visible for persistent regions. 2680d10a6cSBen Widawsky * 1. Interleave granularity 2780d10a6cSBen Widawsky * 2. Interleave size 28b9686e8cSDan Williams * 3. Decoder targets 29779dd20cSBen Widawsky */ 30779dd20cSBen Widawsky 31dd5ba0ebSBen Widawsky /* 32dd5ba0ebSBen Widawsky * All changes to the interleave configuration occur with this lock held 33dd5ba0ebSBen Widawsky * for write. 34dd5ba0ebSBen Widawsky */ 35dd5ba0ebSBen Widawsky static DECLARE_RWSEM(cxl_region_rwsem); 36dd5ba0ebSBen Widawsky 37779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev); 38779dd20cSBen Widawsky 39dd5ba0ebSBen Widawsky static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, 40dd5ba0ebSBen Widawsky char *buf) 41dd5ba0ebSBen Widawsky { 42dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 43dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params; 44dd5ba0ebSBen Widawsky ssize_t rc; 45dd5ba0ebSBen Widawsky 46dd5ba0ebSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 47dd5ba0ebSBen Widawsky if (rc) 48dd5ba0ebSBen Widawsky return rc; 49a8e7d558SDan Williams if (cxlr->mode != CXL_DECODER_PMEM) 50a8e7d558SDan Williams rc = sysfs_emit(buf, "\n"); 51a8e7d558SDan Williams else 52dd5ba0ebSBen Widawsky rc = sysfs_emit(buf, "%pUb\n", &p->uuid); 53dd5ba0ebSBen Widawsky up_read(&cxl_region_rwsem); 54dd5ba0ebSBen Widawsky 55dd5ba0ebSBen Widawsky return rc; 56dd5ba0ebSBen Widawsky } 57dd5ba0ebSBen Widawsky 58dd5ba0ebSBen Widawsky static int is_dup(struct device *match, void *data) 59dd5ba0ebSBen Widawsky { 60dd5ba0ebSBen Widawsky struct cxl_region_params *p; 61dd5ba0ebSBen Widawsky struct cxl_region *cxlr; 62dd5ba0ebSBen Widawsky uuid_t *uuid = data; 63dd5ba0ebSBen Widawsky 64dd5ba0ebSBen Widawsky if (!is_cxl_region(match)) 65dd5ba0ebSBen Widawsky return 0; 66dd5ba0ebSBen Widawsky 67dd5ba0ebSBen Widawsky lockdep_assert_held(&cxl_region_rwsem); 68dd5ba0ebSBen Widawsky cxlr = to_cxl_region(match); 69dd5ba0ebSBen Widawsky p = &cxlr->params; 70dd5ba0ebSBen Widawsky 71dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, uuid)) { 72dd5ba0ebSBen Widawsky dev_dbg(match, "already has uuid: %pUb\n", uuid); 73dd5ba0ebSBen Widawsky return -EBUSY; 74dd5ba0ebSBen Widawsky } 75dd5ba0ebSBen Widawsky 76dd5ba0ebSBen Widawsky return 0; 77dd5ba0ebSBen Widawsky } 78dd5ba0ebSBen Widawsky 79dd5ba0ebSBen Widawsky static ssize_t uuid_store(struct device *dev, struct device_attribute *attr, 80dd5ba0ebSBen Widawsky const char *buf, size_t len) 81dd5ba0ebSBen Widawsky { 82dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 83dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params; 84dd5ba0ebSBen Widawsky uuid_t temp; 85dd5ba0ebSBen Widawsky ssize_t rc; 86dd5ba0ebSBen Widawsky 87dd5ba0ebSBen Widawsky if (len != UUID_STRING_LEN + 1) 88dd5ba0ebSBen Widawsky return -EINVAL; 89dd5ba0ebSBen Widawsky 90dd5ba0ebSBen Widawsky rc = uuid_parse(buf, &temp); 91dd5ba0ebSBen Widawsky if (rc) 92dd5ba0ebSBen Widawsky return rc; 93dd5ba0ebSBen Widawsky 94dd5ba0ebSBen Widawsky if (uuid_is_null(&temp)) 95dd5ba0ebSBen Widawsky return -EINVAL; 96dd5ba0ebSBen Widawsky 97dd5ba0ebSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 98dd5ba0ebSBen Widawsky if (rc) 99dd5ba0ebSBen Widawsky return rc; 100dd5ba0ebSBen Widawsky 101dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, &temp)) 102dd5ba0ebSBen Widawsky goto out; 103dd5ba0ebSBen Widawsky 104dd5ba0ebSBen Widawsky rc = -EBUSY; 105dd5ba0ebSBen Widawsky if (p->state >= CXL_CONFIG_ACTIVE) 106dd5ba0ebSBen Widawsky goto out; 107dd5ba0ebSBen Widawsky 108dd5ba0ebSBen Widawsky rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup); 109dd5ba0ebSBen Widawsky if (rc < 0) 110dd5ba0ebSBen Widawsky goto out; 111dd5ba0ebSBen Widawsky 112dd5ba0ebSBen Widawsky uuid_copy(&p->uuid, &temp); 113dd5ba0ebSBen Widawsky out: 114dd5ba0ebSBen Widawsky up_write(&cxl_region_rwsem); 115dd5ba0ebSBen Widawsky 116dd5ba0ebSBen Widawsky if (rc) 117dd5ba0ebSBen Widawsky return rc; 118dd5ba0ebSBen Widawsky return len; 119dd5ba0ebSBen Widawsky } 120dd5ba0ebSBen Widawsky static DEVICE_ATTR_RW(uuid); 121dd5ba0ebSBen Widawsky 122176baefbSDan Williams static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port, 123176baefbSDan Williams struct cxl_region *cxlr) 124176baefbSDan Williams { 125176baefbSDan Williams return xa_load(&port->regions, (unsigned long)cxlr); 126176baefbSDan Williams } 127176baefbSDan Williams 128176baefbSDan Williams static int cxl_region_decode_reset(struct cxl_region *cxlr, int count) 129176baefbSDan Williams { 130176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 131176baefbSDan Williams int i; 132176baefbSDan Williams 133176baefbSDan Williams for (i = count - 1; i >= 0; i--) { 134176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 135176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 136176baefbSDan Williams struct cxl_port *iter = cxled_to_port(cxled); 137*030f8803SDan Williams struct cxl_dev_state *cxlds = cxlmd->cxlds; 138176baefbSDan Williams struct cxl_ep *ep; 1394fa4302dSFan Ni int rc = 0; 140176baefbSDan Williams 141*030f8803SDan Williams if (cxlds->rcd) 142*030f8803SDan Williams goto endpoint_reset; 143*030f8803SDan Williams 144176baefbSDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 145176baefbSDan Williams iter = to_cxl_port(iter->dev.parent); 146176baefbSDan Williams 147176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 148176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 149176baefbSDan Williams struct cxl_region_ref *cxl_rr; 150176baefbSDan Williams struct cxl_decoder *cxld; 151176baefbSDan Williams 152176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 153176baefbSDan Williams cxld = cxl_rr->decoder; 1544fa4302dSFan Ni if (cxld->reset) 155176baefbSDan Williams rc = cxld->reset(cxld); 156176baefbSDan Williams if (rc) 157176baefbSDan Williams return rc; 158176baefbSDan Williams } 159176baefbSDan Williams 160*030f8803SDan Williams endpoint_reset: 161176baefbSDan Williams rc = cxled->cxld.reset(&cxled->cxld); 162176baefbSDan Williams if (rc) 163176baefbSDan Williams return rc; 164176baefbSDan Williams } 165176baefbSDan Williams 166176baefbSDan Williams return 0; 167176baefbSDan Williams } 168176baefbSDan Williams 169af3ea9abSDan Williams static int commit_decoder(struct cxl_decoder *cxld) 170af3ea9abSDan Williams { 171af3ea9abSDan Williams struct cxl_switch_decoder *cxlsd = NULL; 172af3ea9abSDan Williams 173af3ea9abSDan Williams if (cxld->commit) 174af3ea9abSDan Williams return cxld->commit(cxld); 175af3ea9abSDan Williams 176af3ea9abSDan Williams if (is_switch_decoder(&cxld->dev)) 177af3ea9abSDan Williams cxlsd = to_cxl_switch_decoder(&cxld->dev); 178af3ea9abSDan Williams 179af3ea9abSDan Williams if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1, 180af3ea9abSDan Williams "->commit() is required\n")) 181af3ea9abSDan Williams return -ENXIO; 182af3ea9abSDan Williams return 0; 183af3ea9abSDan Williams } 184af3ea9abSDan Williams 185176baefbSDan Williams static int cxl_region_decode_commit(struct cxl_region *cxlr) 186176baefbSDan Williams { 187176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 18869c99613SDan Williams int i, rc = 0; 189176baefbSDan Williams 190176baefbSDan Williams for (i = 0; i < p->nr_targets; i++) { 191176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 192176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 193176baefbSDan Williams struct cxl_region_ref *cxl_rr; 194176baefbSDan Williams struct cxl_decoder *cxld; 195176baefbSDan Williams struct cxl_port *iter; 196176baefbSDan Williams struct cxl_ep *ep; 197176baefbSDan Williams 198176baefbSDan Williams /* commit bottom up */ 199176baefbSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 200176baefbSDan Williams iter = to_cxl_port(iter->dev.parent)) { 201176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 202176baefbSDan Williams cxld = cxl_rr->decoder; 203af3ea9abSDan Williams rc = commit_decoder(cxld); 204176baefbSDan Williams if (rc) 205176baefbSDan Williams break; 206176baefbSDan Williams } 207176baefbSDan Williams 20869c99613SDan Williams if (rc) { 209176baefbSDan Williams /* programming @iter failed, teardown */ 210176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); ep && iter; 211176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 212176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 213176baefbSDan Williams cxld = cxl_rr->decoder; 2144fa4302dSFan Ni if (cxld->reset) 215176baefbSDan Williams cxld->reset(cxld); 216176baefbSDan Williams } 217176baefbSDan Williams 218176baefbSDan Williams cxled->cxld.reset(&cxled->cxld); 21969c99613SDan Williams goto err; 22069c99613SDan Williams } 221176baefbSDan Williams } 222176baefbSDan Williams 223176baefbSDan Williams return 0; 224176baefbSDan Williams 22569c99613SDan Williams err: 226176baefbSDan Williams /* undo the targets that were successfully committed */ 227176baefbSDan Williams cxl_region_decode_reset(cxlr, i); 228176baefbSDan Williams return rc; 229176baefbSDan Williams } 230176baefbSDan Williams 231176baefbSDan Williams static ssize_t commit_store(struct device *dev, struct device_attribute *attr, 232176baefbSDan Williams const char *buf, size_t len) 233176baefbSDan Williams { 234176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 235176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 236176baefbSDan Williams bool commit; 237176baefbSDan Williams ssize_t rc; 238176baefbSDan Williams 239176baefbSDan Williams rc = kstrtobool(buf, &commit); 240176baefbSDan Williams if (rc) 241176baefbSDan Williams return rc; 242176baefbSDan Williams 243176baefbSDan Williams rc = down_write_killable(&cxl_region_rwsem); 244176baefbSDan Williams if (rc) 245176baefbSDan Williams return rc; 246176baefbSDan Williams 247176baefbSDan Williams /* Already in the requested state? */ 248176baefbSDan Williams if (commit && p->state >= CXL_CONFIG_COMMIT) 249176baefbSDan Williams goto out; 250176baefbSDan Williams if (!commit && p->state < CXL_CONFIG_COMMIT) 251176baefbSDan Williams goto out; 252176baefbSDan Williams 253176baefbSDan Williams /* Not ready to commit? */ 254176baefbSDan Williams if (commit && p->state < CXL_CONFIG_ACTIVE) { 255176baefbSDan Williams rc = -ENXIO; 256176baefbSDan Williams goto out; 257176baefbSDan Williams } 258176baefbSDan Williams 259176baefbSDan Williams if (commit) 260176baefbSDan Williams rc = cxl_region_decode_commit(cxlr); 261176baefbSDan Williams else { 262176baefbSDan Williams p->state = CXL_CONFIG_RESET_PENDING; 263176baefbSDan Williams up_write(&cxl_region_rwsem); 264176baefbSDan Williams device_release_driver(&cxlr->dev); 265176baefbSDan Williams down_write(&cxl_region_rwsem); 266176baefbSDan Williams 267176baefbSDan Williams /* 268176baefbSDan Williams * The lock was dropped, so need to revalidate that the reset is 269176baefbSDan Williams * still pending. 270176baefbSDan Williams */ 271176baefbSDan Williams if (p->state == CXL_CONFIG_RESET_PENDING) 272176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 273176baefbSDan Williams } 274176baefbSDan Williams 275176baefbSDan Williams if (rc) 276176baefbSDan Williams goto out; 277176baefbSDan Williams 278176baefbSDan Williams if (commit) 279176baefbSDan Williams p->state = CXL_CONFIG_COMMIT; 280176baefbSDan Williams else if (p->state == CXL_CONFIG_RESET_PENDING) 281176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE; 282176baefbSDan Williams 283176baefbSDan Williams out: 284176baefbSDan Williams up_write(&cxl_region_rwsem); 285176baefbSDan Williams 286176baefbSDan Williams if (rc) 287176baefbSDan Williams return rc; 288176baefbSDan Williams return len; 289176baefbSDan Williams } 290176baefbSDan Williams 291176baefbSDan Williams static ssize_t commit_show(struct device *dev, struct device_attribute *attr, 292176baefbSDan Williams char *buf) 293176baefbSDan Williams { 294176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 295176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 296176baefbSDan Williams ssize_t rc; 297176baefbSDan Williams 298176baefbSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 299176baefbSDan Williams if (rc) 300176baefbSDan Williams return rc; 301176baefbSDan Williams rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT); 302176baefbSDan Williams up_read(&cxl_region_rwsem); 303176baefbSDan Williams 304176baefbSDan Williams return rc; 305176baefbSDan Williams } 306176baefbSDan Williams static DEVICE_ATTR_RW(commit); 307176baefbSDan Williams 308dd5ba0ebSBen Widawsky static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a, 309dd5ba0ebSBen Widawsky int n) 310dd5ba0ebSBen Widawsky { 311dd5ba0ebSBen Widawsky struct device *dev = kobj_to_dev(kobj); 312dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 313dd5ba0ebSBen Widawsky 314a8e7d558SDan Williams /* 315a8e7d558SDan Williams * Support tooling that expects to find a 'uuid' attribute for all 316a8e7d558SDan Williams * regions regardless of mode. 317a8e7d558SDan Williams */ 318dd5ba0ebSBen Widawsky if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM) 319a8e7d558SDan Williams return 0444; 320dd5ba0ebSBen Widawsky return a->mode; 321dd5ba0ebSBen Widawsky } 322dd5ba0ebSBen Widawsky 32380d10a6cSBen Widawsky static ssize_t interleave_ways_show(struct device *dev, 32480d10a6cSBen Widawsky struct device_attribute *attr, char *buf) 32580d10a6cSBen Widawsky { 32680d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 32780d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 32880d10a6cSBen Widawsky ssize_t rc; 32980d10a6cSBen Widawsky 33080d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 33180d10a6cSBen Widawsky if (rc) 33280d10a6cSBen Widawsky return rc; 33380d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_ways); 33480d10a6cSBen Widawsky up_read(&cxl_region_rwsem); 33580d10a6cSBen Widawsky 33680d10a6cSBen Widawsky return rc; 33780d10a6cSBen Widawsky } 33880d10a6cSBen Widawsky 339b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void); 340b9686e8cSDan Williams 34180d10a6cSBen Widawsky static ssize_t interleave_ways_store(struct device *dev, 34280d10a6cSBen Widawsky struct device_attribute *attr, 34380d10a6cSBen Widawsky const char *buf, size_t len) 34480d10a6cSBen Widawsky { 34580d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 34680d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 34780d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 34880d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 349c7e3548cSDan Carpenter unsigned int val, save; 350c7e3548cSDan Carpenter int rc; 35180d10a6cSBen Widawsky u8 iw; 35280d10a6cSBen Widawsky 353c7e3548cSDan Carpenter rc = kstrtouint(buf, 0, &val); 35480d10a6cSBen Widawsky if (rc) 35580d10a6cSBen Widawsky return rc; 35680d10a6cSBen Widawsky 357c99b2e8cSDave Jiang rc = ways_to_eiw(val, &iw); 35880d10a6cSBen Widawsky if (rc) 35980d10a6cSBen Widawsky return rc; 36080d10a6cSBen Widawsky 36180d10a6cSBen Widawsky /* 36280d10a6cSBen Widawsky * Even for x3, x9, and x12 interleaves the region interleave must be a 36380d10a6cSBen Widawsky * power of 2 multiple of the host bridge interleave. 36480d10a6cSBen Widawsky */ 36580d10a6cSBen Widawsky if (!is_power_of_2(val / cxld->interleave_ways) || 36680d10a6cSBen Widawsky (val % cxld->interleave_ways)) { 36780d10a6cSBen Widawsky dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val); 36880d10a6cSBen Widawsky return -EINVAL; 36980d10a6cSBen Widawsky } 37080d10a6cSBen Widawsky 37180d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 37280d10a6cSBen Widawsky if (rc) 37380d10a6cSBen Widawsky return rc; 37480d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 37580d10a6cSBen Widawsky rc = -EBUSY; 37680d10a6cSBen Widawsky goto out; 37780d10a6cSBen Widawsky } 37880d10a6cSBen Widawsky 379b9686e8cSDan Williams save = p->interleave_ways; 38080d10a6cSBen Widawsky p->interleave_ways = val; 381b9686e8cSDan Williams rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 382b9686e8cSDan Williams if (rc) 383b9686e8cSDan Williams p->interleave_ways = save; 38480d10a6cSBen Widawsky out: 38580d10a6cSBen Widawsky up_write(&cxl_region_rwsem); 38680d10a6cSBen Widawsky if (rc) 38780d10a6cSBen Widawsky return rc; 38880d10a6cSBen Widawsky return len; 38980d10a6cSBen Widawsky } 39080d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_ways); 39180d10a6cSBen Widawsky 39280d10a6cSBen Widawsky static ssize_t interleave_granularity_show(struct device *dev, 39380d10a6cSBen Widawsky struct device_attribute *attr, 39480d10a6cSBen Widawsky char *buf) 39580d10a6cSBen Widawsky { 39680d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 39780d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 39880d10a6cSBen Widawsky ssize_t rc; 39980d10a6cSBen Widawsky 40080d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 40180d10a6cSBen Widawsky if (rc) 40280d10a6cSBen Widawsky return rc; 40380d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_granularity); 40480d10a6cSBen Widawsky up_read(&cxl_region_rwsem); 40580d10a6cSBen Widawsky 40680d10a6cSBen Widawsky return rc; 40780d10a6cSBen Widawsky } 40880d10a6cSBen Widawsky 40980d10a6cSBen Widawsky static ssize_t interleave_granularity_store(struct device *dev, 41080d10a6cSBen Widawsky struct device_attribute *attr, 41180d10a6cSBen Widawsky const char *buf, size_t len) 41280d10a6cSBen Widawsky { 41380d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 41480d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 41580d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 41680d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 41780d10a6cSBen Widawsky int rc, val; 41880d10a6cSBen Widawsky u16 ig; 41980d10a6cSBen Widawsky 42080d10a6cSBen Widawsky rc = kstrtoint(buf, 0, &val); 42180d10a6cSBen Widawsky if (rc) 42280d10a6cSBen Widawsky return rc; 42380d10a6cSBen Widawsky 42483351ddbSDave Jiang rc = granularity_to_eig(val, &ig); 42580d10a6cSBen Widawsky if (rc) 42680d10a6cSBen Widawsky return rc; 42780d10a6cSBen Widawsky 42880d10a6cSBen Widawsky /* 4294d8e4ea5SDan Williams * When the host-bridge is interleaved, disallow region granularity != 4304d8e4ea5SDan Williams * root granularity. Regions with a granularity less than the root 4314d8e4ea5SDan Williams * interleave result in needing multiple endpoints to support a single 432cbbd05d0SRandy Dunlap * slot in the interleave (possible to support in the future). Regions 4334d8e4ea5SDan Williams * with a granularity greater than the root interleave result in invalid 4344d8e4ea5SDan Williams * DPA translations (invalid to support). 43580d10a6cSBen Widawsky */ 4364d8e4ea5SDan Williams if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity) 43780d10a6cSBen Widawsky return -EINVAL; 43880d10a6cSBen Widawsky 43980d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 44080d10a6cSBen Widawsky if (rc) 44180d10a6cSBen Widawsky return rc; 44280d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 44380d10a6cSBen Widawsky rc = -EBUSY; 44480d10a6cSBen Widawsky goto out; 44580d10a6cSBen Widawsky } 44680d10a6cSBen Widawsky 44780d10a6cSBen Widawsky p->interleave_granularity = val; 44880d10a6cSBen Widawsky out: 44980d10a6cSBen Widawsky up_write(&cxl_region_rwsem); 45080d10a6cSBen Widawsky if (rc) 45180d10a6cSBen Widawsky return rc; 45280d10a6cSBen Widawsky return len; 45380d10a6cSBen Widawsky } 45480d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_granularity); 45580d10a6cSBen Widawsky 45623a22cd1SDan Williams static ssize_t resource_show(struct device *dev, struct device_attribute *attr, 45723a22cd1SDan Williams char *buf) 45823a22cd1SDan Williams { 45923a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 46023a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 46123a22cd1SDan Williams u64 resource = -1ULL; 46223a22cd1SDan Williams ssize_t rc; 46323a22cd1SDan Williams 46423a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 46523a22cd1SDan Williams if (rc) 46623a22cd1SDan Williams return rc; 46723a22cd1SDan Williams if (p->res) 46823a22cd1SDan Williams resource = p->res->start; 46923a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", resource); 47023a22cd1SDan Williams up_read(&cxl_region_rwsem); 47123a22cd1SDan Williams 47223a22cd1SDan Williams return rc; 47323a22cd1SDan Williams } 47423a22cd1SDan Williams static DEVICE_ATTR_RO(resource); 47523a22cd1SDan Williams 4767d505f98SDan Williams static ssize_t mode_show(struct device *dev, struct device_attribute *attr, 4777d505f98SDan Williams char *buf) 4787d505f98SDan Williams { 4797d505f98SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 4807d505f98SDan Williams 4817d505f98SDan Williams return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode)); 4827d505f98SDan Williams } 4837d505f98SDan Williams static DEVICE_ATTR_RO(mode); 4847d505f98SDan Williams 48523a22cd1SDan Williams static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) 48623a22cd1SDan Williams { 48723a22cd1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 48823a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 48923a22cd1SDan Williams struct resource *res; 49023a22cd1SDan Williams u32 remainder = 0; 49123a22cd1SDan Williams 49223a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 49323a22cd1SDan Williams 49423a22cd1SDan Williams /* Nothing to do... */ 49588ab1ddeSDan Carpenter if (p->res && resource_size(p->res) == size) 49623a22cd1SDan Williams return 0; 49723a22cd1SDan Williams 49823a22cd1SDan Williams /* To change size the old size must be freed first */ 49923a22cd1SDan Williams if (p->res) 50023a22cd1SDan Williams return -EBUSY; 50123a22cd1SDan Williams 50223a22cd1SDan Williams if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) 50323a22cd1SDan Williams return -EBUSY; 50423a22cd1SDan Williams 50523a22cd1SDan Williams /* ways, granularity and uuid (if PMEM) need to be set before HPA */ 50623a22cd1SDan Williams if (!p->interleave_ways || !p->interleave_granularity || 50723a22cd1SDan Williams (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid))) 50823a22cd1SDan Williams return -ENXIO; 50923a22cd1SDan Williams 51023a22cd1SDan Williams div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder); 51123a22cd1SDan Williams if (remainder) 51223a22cd1SDan Williams return -EINVAL; 51323a22cd1SDan Williams 51423a22cd1SDan Williams res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, 51523a22cd1SDan Williams dev_name(&cxlr->dev)); 51623a22cd1SDan Williams if (IS_ERR(res)) { 51723a22cd1SDan Williams dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n", 51823a22cd1SDan Williams PTR_ERR(res)); 51923a22cd1SDan Williams return PTR_ERR(res); 52023a22cd1SDan Williams } 52123a22cd1SDan Williams 52223a22cd1SDan Williams p->res = res; 52323a22cd1SDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 52423a22cd1SDan Williams 52523a22cd1SDan Williams return 0; 52623a22cd1SDan Williams } 52723a22cd1SDan Williams 52823a22cd1SDan Williams static void cxl_region_iomem_release(struct cxl_region *cxlr) 52923a22cd1SDan Williams { 53023a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 53123a22cd1SDan Williams 53223a22cd1SDan Williams if (device_is_registered(&cxlr->dev)) 53323a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 53423a22cd1SDan Williams if (p->res) { 535a32320b7SDan Williams /* 536a32320b7SDan Williams * Autodiscovered regions may not have been able to insert their 537a32320b7SDan Williams * resource. 538a32320b7SDan Williams */ 539a32320b7SDan Williams if (p->res->parent) 54023a22cd1SDan Williams remove_resource(p->res); 54123a22cd1SDan Williams kfree(p->res); 54223a22cd1SDan Williams p->res = NULL; 54323a22cd1SDan Williams } 54423a22cd1SDan Williams } 54523a22cd1SDan Williams 54623a22cd1SDan Williams static int free_hpa(struct cxl_region *cxlr) 54723a22cd1SDan Williams { 54823a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 54923a22cd1SDan Williams 55023a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 55123a22cd1SDan Williams 55223a22cd1SDan Williams if (!p->res) 55323a22cd1SDan Williams return 0; 55423a22cd1SDan Williams 55523a22cd1SDan Williams if (p->state >= CXL_CONFIG_ACTIVE) 55623a22cd1SDan Williams return -EBUSY; 55723a22cd1SDan Williams 55823a22cd1SDan Williams cxl_region_iomem_release(cxlr); 55923a22cd1SDan Williams p->state = CXL_CONFIG_IDLE; 56023a22cd1SDan Williams return 0; 56123a22cd1SDan Williams } 56223a22cd1SDan Williams 56323a22cd1SDan Williams static ssize_t size_store(struct device *dev, struct device_attribute *attr, 56423a22cd1SDan Williams const char *buf, size_t len) 56523a22cd1SDan Williams { 56623a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 56723a22cd1SDan Williams u64 val; 56823a22cd1SDan Williams int rc; 56923a22cd1SDan Williams 57023a22cd1SDan Williams rc = kstrtou64(buf, 0, &val); 57123a22cd1SDan Williams if (rc) 57223a22cd1SDan Williams return rc; 57323a22cd1SDan Williams 57423a22cd1SDan Williams rc = down_write_killable(&cxl_region_rwsem); 57523a22cd1SDan Williams if (rc) 57623a22cd1SDan Williams return rc; 57723a22cd1SDan Williams 57823a22cd1SDan Williams if (val) 57923a22cd1SDan Williams rc = alloc_hpa(cxlr, val); 58023a22cd1SDan Williams else 58123a22cd1SDan Williams rc = free_hpa(cxlr); 58223a22cd1SDan Williams up_write(&cxl_region_rwsem); 58323a22cd1SDan Williams 58423a22cd1SDan Williams if (rc) 58523a22cd1SDan Williams return rc; 58623a22cd1SDan Williams 58723a22cd1SDan Williams return len; 58823a22cd1SDan Williams } 58923a22cd1SDan Williams 59023a22cd1SDan Williams static ssize_t size_show(struct device *dev, struct device_attribute *attr, 59123a22cd1SDan Williams char *buf) 59223a22cd1SDan Williams { 59323a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 59423a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 59523a22cd1SDan Williams u64 size = 0; 59623a22cd1SDan Williams ssize_t rc; 59723a22cd1SDan Williams 59823a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 59923a22cd1SDan Williams if (rc) 60023a22cd1SDan Williams return rc; 60123a22cd1SDan Williams if (p->res) 60223a22cd1SDan Williams size = resource_size(p->res); 60323a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", size); 60423a22cd1SDan Williams up_read(&cxl_region_rwsem); 60523a22cd1SDan Williams 60623a22cd1SDan Williams return rc; 60723a22cd1SDan Williams } 60823a22cd1SDan Williams static DEVICE_ATTR_RW(size); 60923a22cd1SDan Williams 610dd5ba0ebSBen Widawsky static struct attribute *cxl_region_attrs[] = { 611dd5ba0ebSBen Widawsky &dev_attr_uuid.attr, 612176baefbSDan Williams &dev_attr_commit.attr, 61380d10a6cSBen Widawsky &dev_attr_interleave_ways.attr, 61480d10a6cSBen Widawsky &dev_attr_interleave_granularity.attr, 61523a22cd1SDan Williams &dev_attr_resource.attr, 61623a22cd1SDan Williams &dev_attr_size.attr, 6177d505f98SDan Williams &dev_attr_mode.attr, 618dd5ba0ebSBen Widawsky NULL, 619dd5ba0ebSBen Widawsky }; 620dd5ba0ebSBen Widawsky 621dd5ba0ebSBen Widawsky static const struct attribute_group cxl_region_group = { 622dd5ba0ebSBen Widawsky .attrs = cxl_region_attrs, 623dd5ba0ebSBen Widawsky .is_visible = cxl_region_visible, 624dd5ba0ebSBen Widawsky }; 625dd5ba0ebSBen Widawsky 626b9686e8cSDan Williams static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos) 627b9686e8cSDan Williams { 628b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 629b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled; 630b9686e8cSDan Williams int rc; 631b9686e8cSDan Williams 632b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 633b9686e8cSDan Williams if (rc) 634b9686e8cSDan Williams return rc; 635b9686e8cSDan Williams 636b9686e8cSDan Williams if (pos >= p->interleave_ways) { 637b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 638b9686e8cSDan Williams p->interleave_ways); 639b9686e8cSDan Williams rc = -ENXIO; 640b9686e8cSDan Williams goto out; 641b9686e8cSDan Williams } 642b9686e8cSDan Williams 643b9686e8cSDan Williams cxled = p->targets[pos]; 644b9686e8cSDan Williams if (!cxled) 645b9686e8cSDan Williams rc = sysfs_emit(buf, "\n"); 646b9686e8cSDan Williams else 647b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev)); 648b9686e8cSDan Williams out: 649b9686e8cSDan Williams up_read(&cxl_region_rwsem); 650b9686e8cSDan Williams 651b9686e8cSDan Williams return rc; 652b9686e8cSDan Williams } 653b9686e8cSDan Williams 654384e624bSDan Williams static int match_free_decoder(struct device *dev, void *data) 655384e624bSDan Williams { 656384e624bSDan Williams struct cxl_decoder *cxld; 657384e624bSDan Williams int *id = data; 658384e624bSDan Williams 659384e624bSDan Williams if (!is_switch_decoder(dev)) 660384e624bSDan Williams return 0; 661384e624bSDan Williams 662384e624bSDan Williams cxld = to_cxl_decoder(dev); 663384e624bSDan Williams 664384e624bSDan Williams /* enforce ordered allocation */ 665384e624bSDan Williams if (cxld->id != *id) 666384e624bSDan Williams return 0; 667384e624bSDan Williams 668384e624bSDan Williams if (!cxld->region) 669384e624bSDan Williams return 1; 670384e624bSDan Williams 671384e624bSDan Williams (*id)++; 672384e624bSDan Williams 673384e624bSDan Williams return 0; 674384e624bSDan Williams } 675384e624bSDan Williams 676384e624bSDan Williams static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port, 677384e624bSDan Williams struct cxl_region *cxlr) 678384e624bSDan Williams { 679384e624bSDan Williams struct device *dev; 680384e624bSDan Williams int id = 0; 681384e624bSDan Williams 682384e624bSDan Williams dev = device_find_child(&port->dev, &id, match_free_decoder); 683384e624bSDan Williams if (!dev) 684384e624bSDan Williams return NULL; 685b9686e8cSDan Williams /* 686384e624bSDan Williams * This decoder is pinned registered as long as the endpoint decoder is 687384e624bSDan Williams * registered, and endpoint decoder unregistration holds the 688384e624bSDan Williams * cxl_region_rwsem over unregister events, so no need to hold on to 689384e624bSDan Williams * this extra reference. 690b9686e8cSDan Williams */ 691384e624bSDan Williams put_device(dev); 692384e624bSDan Williams return to_cxl_decoder(dev); 693384e624bSDan Williams } 694384e624bSDan Williams 695384e624bSDan Williams static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port, 696384e624bSDan Williams struct cxl_region *cxlr) 697384e624bSDan Williams { 698e29a8995SDan Williams struct cxl_region_params *p = &cxlr->params; 699e29a8995SDan Williams struct cxl_region_ref *cxl_rr, *iter; 700e29a8995SDan Williams unsigned long index; 701384e624bSDan Williams int rc; 702384e624bSDan Williams 703e29a8995SDan Williams xa_for_each(&port->regions, index, iter) { 704e29a8995SDan Williams struct cxl_region_params *ip = &iter->region->params; 705e29a8995SDan Williams 706a90accb3SDan Williams if (!ip->res) 707a90accb3SDan Williams continue; 708a90accb3SDan Williams 709e29a8995SDan Williams if (ip->res->start > p->res->start) { 710e29a8995SDan Williams dev_dbg(&cxlr->dev, 711e29a8995SDan Williams "%s: HPA order violation %s:%pr vs %pr\n", 712e29a8995SDan Williams dev_name(&port->dev), 713e29a8995SDan Williams dev_name(&iter->region->dev), ip->res, p->res); 714e29a8995SDan Williams return ERR_PTR(-EBUSY); 715e29a8995SDan Williams } 716e29a8995SDan Williams } 717e29a8995SDan Williams 718384e624bSDan Williams cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL); 719384e624bSDan Williams if (!cxl_rr) 720e29a8995SDan Williams return ERR_PTR(-ENOMEM); 721384e624bSDan Williams cxl_rr->port = port; 722384e624bSDan Williams cxl_rr->region = cxlr; 72327b3f8d1SDan Williams cxl_rr->nr_targets = 1; 724384e624bSDan Williams xa_init(&cxl_rr->endpoints); 725384e624bSDan Williams 726384e624bSDan Williams rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL); 727384e624bSDan Williams if (rc) { 728384e624bSDan Williams dev_dbg(&cxlr->dev, 729384e624bSDan Williams "%s: failed to track region reference: %d\n", 730384e624bSDan Williams dev_name(&port->dev), rc); 731384e624bSDan Williams kfree(cxl_rr); 732e29a8995SDan Williams return ERR_PTR(rc); 733384e624bSDan Williams } 734384e624bSDan Williams 735384e624bSDan Williams return cxl_rr; 736384e624bSDan Williams } 737384e624bSDan Williams 73871ee71d7SVishal Verma static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr) 739384e624bSDan Williams { 740384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region; 741384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 742384e624bSDan Williams 74371ee71d7SVishal Verma if (!cxld) 74471ee71d7SVishal Verma return; 74571ee71d7SVishal Verma 746384e624bSDan Williams dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n"); 747384e624bSDan Williams if (cxld->region == cxlr) { 748384e624bSDan Williams cxld->region = NULL; 749384e624bSDan Williams put_device(&cxlr->dev); 750384e624bSDan Williams } 75171ee71d7SVishal Verma } 752384e624bSDan Williams 75371ee71d7SVishal Verma static void free_region_ref(struct cxl_region_ref *cxl_rr) 75471ee71d7SVishal Verma { 75571ee71d7SVishal Verma struct cxl_port *port = cxl_rr->port; 75671ee71d7SVishal Verma struct cxl_region *cxlr = cxl_rr->region; 75771ee71d7SVishal Verma 75871ee71d7SVishal Verma cxl_rr_free_decoder(cxl_rr); 759384e624bSDan Williams xa_erase(&port->regions, (unsigned long)cxlr); 760384e624bSDan Williams xa_destroy(&cxl_rr->endpoints); 761384e624bSDan Williams kfree(cxl_rr); 762384e624bSDan Williams } 763384e624bSDan Williams 764384e624bSDan Williams static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr, 765384e624bSDan Williams struct cxl_endpoint_decoder *cxled) 766384e624bSDan Williams { 767384e624bSDan Williams int rc; 768384e624bSDan Williams struct cxl_port *port = cxl_rr->port; 769384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region; 770384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 771384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled)); 772384e624bSDan Williams 77327b3f8d1SDan Williams if (ep) { 774384e624bSDan Williams rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep, 775384e624bSDan Williams GFP_KERNEL); 776384e624bSDan Williams if (rc) 777384e624bSDan Williams return rc; 77827b3f8d1SDan Williams } 779384e624bSDan Williams cxl_rr->nr_eps++; 780384e624bSDan Williams 781384e624bSDan Williams if (!cxld->region) { 782384e624bSDan Williams cxld->region = cxlr; 783384e624bSDan Williams get_device(&cxlr->dev); 784384e624bSDan Williams } 785384e624bSDan Williams 786384e624bSDan Williams return 0; 787384e624bSDan Williams } 788384e624bSDan Williams 78971ee71d7SVishal Verma static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr, 79071ee71d7SVishal Verma struct cxl_endpoint_decoder *cxled, 79171ee71d7SVishal Verma struct cxl_region_ref *cxl_rr) 79271ee71d7SVishal Verma { 79371ee71d7SVishal Verma struct cxl_decoder *cxld; 79471ee71d7SVishal Verma 79571ee71d7SVishal Verma if (port == cxled_to_port(cxled)) 79671ee71d7SVishal Verma cxld = &cxled->cxld; 79771ee71d7SVishal Verma else 79871ee71d7SVishal Verma cxld = cxl_region_find_decoder(port, cxlr); 79971ee71d7SVishal Verma if (!cxld) { 80071ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: no decoder available\n", 80171ee71d7SVishal Verma dev_name(&port->dev)); 80271ee71d7SVishal Verma return -EBUSY; 80371ee71d7SVishal Verma } 80471ee71d7SVishal Verma 80571ee71d7SVishal Verma if (cxld->region) { 80671ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", 80771ee71d7SVishal Verma dev_name(&port->dev), dev_name(&cxld->dev), 80871ee71d7SVishal Verma dev_name(&cxld->region->dev)); 80971ee71d7SVishal Verma return -EBUSY; 81071ee71d7SVishal Verma } 81171ee71d7SVishal Verma 81271ee71d7SVishal Verma cxl_rr->decoder = cxld; 81371ee71d7SVishal Verma return 0; 81471ee71d7SVishal Verma } 81571ee71d7SVishal Verma 816384e624bSDan Williams /** 817384e624bSDan Williams * cxl_port_attach_region() - track a region's interest in a port by endpoint 818384e624bSDan Williams * @port: port to add a new region reference 'struct cxl_region_ref' 819384e624bSDan Williams * @cxlr: region to attach to @port 820384e624bSDan Williams * @cxled: endpoint decoder used to create or further pin a region reference 821384e624bSDan Williams * @pos: interleave position of @cxled in @cxlr 822384e624bSDan Williams * 823384e624bSDan Williams * The attach event is an opportunity to validate CXL decode setup 824384e624bSDan Williams * constraints and record metadata needed for programming HDM decoders, 825384e624bSDan Williams * in particular decoder target lists. 826384e624bSDan Williams * 827384e624bSDan Williams * The steps are: 828f13da0d9SBagas Sanjaya * 829384e624bSDan Williams * - validate that there are no other regions with a higher HPA already 830384e624bSDan Williams * associated with @port 831384e624bSDan Williams * - establish a region reference if one is not already present 832f13da0d9SBagas Sanjaya * 833384e624bSDan Williams * - additionally allocate a decoder instance that will host @cxlr on 834384e624bSDan Williams * @port 835f13da0d9SBagas Sanjaya * 836384e624bSDan Williams * - pin the region reference by the endpoint 837384e624bSDan Williams * - account for how many entries in @port's target list are needed to 838384e624bSDan Williams * cover all of the added endpoints. 839384e624bSDan Williams */ 840384e624bSDan Williams static int cxl_port_attach_region(struct cxl_port *port, 841384e624bSDan Williams struct cxl_region *cxlr, 842384e624bSDan Williams struct cxl_endpoint_decoder *cxled, int pos) 843384e624bSDan Williams { 844384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 845384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 846e29a8995SDan Williams struct cxl_region_ref *cxl_rr; 847e29a8995SDan Williams bool nr_targets_inc = false; 848e29a8995SDan Williams struct cxl_decoder *cxld; 849384e624bSDan Williams unsigned long index; 850384e624bSDan Williams int rc = -EBUSY; 851384e624bSDan Williams 852384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 853384e624bSDan Williams 854e29a8995SDan Williams cxl_rr = cxl_rr_load(port, cxlr); 855384e624bSDan Williams if (cxl_rr) { 856384e624bSDan Williams struct cxl_ep *ep_iter; 857384e624bSDan Williams int found = 0; 858384e624bSDan Williams 859e29a8995SDan Williams /* 860e29a8995SDan Williams * Walk the existing endpoints that have been attached to 861e29a8995SDan Williams * @cxlr at @port and see if they share the same 'next' port 862e29a8995SDan Williams * in the downstream direction. I.e. endpoints that share common 863e29a8995SDan Williams * upstream switch. 864e29a8995SDan Williams */ 865384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 866384e624bSDan Williams if (ep_iter == ep) 867384e624bSDan Williams continue; 868384e624bSDan Williams if (ep_iter->next == ep->next) { 869384e624bSDan Williams found++; 870384e624bSDan Williams break; 871384e624bSDan Williams } 872384e624bSDan Williams } 873384e624bSDan Williams 874384e624bSDan Williams /* 875e29a8995SDan Williams * New target port, or @port is an endpoint port that always 876e29a8995SDan Williams * accounts its own local decode as a target. 877384e624bSDan Williams */ 878e29a8995SDan Williams if (!found || !ep->next) { 879384e624bSDan Williams cxl_rr->nr_targets++; 880e29a8995SDan Williams nr_targets_inc = true; 881e29a8995SDan Williams } 882384e624bSDan Williams } else { 883384e624bSDan Williams cxl_rr = alloc_region_ref(port, cxlr); 884e29a8995SDan Williams if (IS_ERR(cxl_rr)) { 885384e624bSDan Williams dev_dbg(&cxlr->dev, 886384e624bSDan Williams "%s: failed to allocate region reference\n", 887384e624bSDan Williams dev_name(&port->dev)); 888e29a8995SDan Williams return PTR_ERR(cxl_rr); 889384e624bSDan Williams } 890e29a8995SDan Williams nr_targets_inc = true; 891384e624bSDan Williams 89271ee71d7SVishal Verma rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr); 89371ee71d7SVishal Verma if (rc) 894384e624bSDan Williams goto out_erase; 895384e624bSDan Williams } 89671ee71d7SVishal Verma cxld = cxl_rr->decoder; 897384e624bSDan Williams 898384e624bSDan Williams rc = cxl_rr_ep_add(cxl_rr, cxled); 899384e624bSDan Williams if (rc) { 900384e624bSDan Williams dev_dbg(&cxlr->dev, 901384e624bSDan Williams "%s: failed to track endpoint %s:%s reference\n", 902384e624bSDan Williams dev_name(&port->dev), dev_name(&cxlmd->dev), 903384e624bSDan Williams dev_name(&cxld->dev)); 904384e624bSDan Williams goto out_erase; 905384e624bSDan Williams } 906384e624bSDan Williams 90727b3f8d1SDan Williams dev_dbg(&cxlr->dev, 90827b3f8d1SDan Williams "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n", 90927b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 91027b3f8d1SDan Williams dev_name(&cxld->dev), dev_name(&cxlmd->dev), 91127b3f8d1SDan Williams dev_name(&cxled->cxld.dev), pos, 91227b3f8d1SDan Williams ep ? ep->next ? dev_name(ep->next->uport) : 91327b3f8d1SDan Williams dev_name(&cxlmd->dev) : 91427b3f8d1SDan Williams "none", 91527b3f8d1SDan Williams cxl_rr->nr_eps, cxl_rr->nr_targets); 91627b3f8d1SDan Williams 917384e624bSDan Williams return 0; 918384e624bSDan Williams out_erase: 919e29a8995SDan Williams if (nr_targets_inc) 920e29a8995SDan Williams cxl_rr->nr_targets--; 921384e624bSDan Williams if (cxl_rr->nr_eps == 0) 922384e624bSDan Williams free_region_ref(cxl_rr); 923384e624bSDan Williams return rc; 924384e624bSDan Williams } 925384e624bSDan Williams 926384e624bSDan Williams static void cxl_port_detach_region(struct cxl_port *port, 927384e624bSDan Williams struct cxl_region *cxlr, 928384e624bSDan Williams struct cxl_endpoint_decoder *cxled) 929384e624bSDan Williams { 930384e624bSDan Williams struct cxl_region_ref *cxl_rr; 93127b3f8d1SDan Williams struct cxl_ep *ep = NULL; 932384e624bSDan Williams 933384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 934384e624bSDan Williams 935384e624bSDan Williams cxl_rr = cxl_rr_load(port, cxlr); 936384e624bSDan Williams if (!cxl_rr) 937384e624bSDan Williams return; 938384e624bSDan Williams 93927b3f8d1SDan Williams /* 94027b3f8d1SDan Williams * Endpoint ports do not carry cxl_ep references, and they 94127b3f8d1SDan Williams * never target more than one endpoint by definition 94227b3f8d1SDan Williams */ 94327b3f8d1SDan Williams if (cxl_rr->decoder == &cxled->cxld) 94427b3f8d1SDan Williams cxl_rr->nr_eps--; 94527b3f8d1SDan Williams else 946384e624bSDan Williams ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled); 947384e624bSDan Williams if (ep) { 948384e624bSDan Williams struct cxl_ep *ep_iter; 949384e624bSDan Williams unsigned long index; 950384e624bSDan Williams int found = 0; 951384e624bSDan Williams 952384e624bSDan Williams cxl_rr->nr_eps--; 953384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 954384e624bSDan Williams if (ep_iter->next == ep->next) { 955384e624bSDan Williams found++; 956384e624bSDan Williams break; 957384e624bSDan Williams } 958384e624bSDan Williams } 959384e624bSDan Williams if (!found) 960384e624bSDan Williams cxl_rr->nr_targets--; 961384e624bSDan Williams } 962384e624bSDan Williams 963384e624bSDan Williams if (cxl_rr->nr_eps == 0) 964384e624bSDan Williams free_region_ref(cxl_rr); 965384e624bSDan Williams } 966384e624bSDan Williams 96727b3f8d1SDan Williams static int check_last_peer(struct cxl_endpoint_decoder *cxled, 96827b3f8d1SDan Williams struct cxl_ep *ep, struct cxl_region_ref *cxl_rr, 96927b3f8d1SDan Williams int distance) 97027b3f8d1SDan Williams { 97127b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 97227b3f8d1SDan Williams struct cxl_region *cxlr = cxl_rr->region; 97327b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 97427b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled_peer; 97527b3f8d1SDan Williams struct cxl_port *port = cxl_rr->port; 97627b3f8d1SDan Williams struct cxl_memdev *cxlmd_peer; 97727b3f8d1SDan Williams struct cxl_ep *ep_peer; 97827b3f8d1SDan Williams int pos = cxled->pos; 97927b3f8d1SDan Williams 98027b3f8d1SDan Williams /* 98127b3f8d1SDan Williams * If this position wants to share a dport with the last endpoint mapped 98227b3f8d1SDan Williams * then that endpoint, at index 'position - distance', must also be 98327b3f8d1SDan Williams * mapped by this dport. 98427b3f8d1SDan Williams */ 98527b3f8d1SDan Williams if (pos < distance) { 98627b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", 98727b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 98827b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 98927b3f8d1SDan Williams return -ENXIO; 99027b3f8d1SDan Williams } 99127b3f8d1SDan Williams cxled_peer = p->targets[pos - distance]; 99227b3f8d1SDan Williams cxlmd_peer = cxled_to_memdev(cxled_peer); 99327b3f8d1SDan Williams ep_peer = cxl_ep_load(port, cxlmd_peer); 99427b3f8d1SDan Williams if (ep->dport != ep_peer->dport) { 99527b3f8d1SDan Williams dev_dbg(&cxlr->dev, 99627b3f8d1SDan Williams "%s:%s: %s:%s pos %d mismatched peer %s:%s\n", 99727b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 99827b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, 99927b3f8d1SDan Williams dev_name(&cxlmd_peer->dev), 100027b3f8d1SDan Williams dev_name(&cxled_peer->cxld.dev)); 100127b3f8d1SDan Williams return -ENXIO; 100227b3f8d1SDan Williams } 100327b3f8d1SDan Williams 100427b3f8d1SDan Williams return 0; 100527b3f8d1SDan Williams } 100627b3f8d1SDan Williams 100727b3f8d1SDan Williams static int cxl_port_setup_targets(struct cxl_port *port, 100827b3f8d1SDan Williams struct cxl_region *cxlr, 100927b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled) 101027b3f8d1SDan Williams { 101127b3f8d1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 101227b3f8d1SDan Williams int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos; 101327b3f8d1SDan Williams struct cxl_port *parent_port = to_cxl_port(port->dev.parent); 101427b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 101527b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 101627b3f8d1SDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 101727b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 101827b3f8d1SDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 101927b3f8d1SDan Williams struct cxl_switch_decoder *cxlsd; 102027b3f8d1SDan Williams u16 eig, peig; 102127b3f8d1SDan Williams u8 eiw, peiw; 102227b3f8d1SDan Williams 102327b3f8d1SDan Williams /* 102427b3f8d1SDan Williams * While root level decoders support x3, x6, x12, switch level 102527b3f8d1SDan Williams * decoders only support powers of 2 up to x16. 102627b3f8d1SDan Williams */ 102727b3f8d1SDan Williams if (!is_power_of_2(cxl_rr->nr_targets)) { 102827b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n", 102927b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 103027b3f8d1SDan Williams cxl_rr->nr_targets); 103127b3f8d1SDan Williams return -EINVAL; 103227b3f8d1SDan Williams } 103327b3f8d1SDan Williams 103427b3f8d1SDan Williams cxlsd = to_cxl_switch_decoder(&cxld->dev); 103527b3f8d1SDan Williams if (cxl_rr->nr_targets_set) { 103627b3f8d1SDan Williams int i, distance; 103727b3f8d1SDan Williams 1038e4f6dfa9SDan Williams /* 1039711442e2SDan Williams * Passthrough decoders impose no distance requirements between 1040e4f6dfa9SDan Williams * peers 1041e4f6dfa9SDan Williams */ 1042711442e2SDan Williams if (cxl_rr->nr_targets == 1) 1043e4f6dfa9SDan Williams distance = 0; 1044e4f6dfa9SDan Williams else 104527b3f8d1SDan Williams distance = p->nr_targets / cxl_rr->nr_targets; 104627b3f8d1SDan Williams for (i = 0; i < cxl_rr->nr_targets_set; i++) 104727b3f8d1SDan Williams if (ep->dport == cxlsd->target[i]) { 104827b3f8d1SDan Williams rc = check_last_peer(cxled, ep, cxl_rr, 104927b3f8d1SDan Williams distance); 105027b3f8d1SDan Williams if (rc) 105127b3f8d1SDan Williams return rc; 105227b3f8d1SDan Williams goto out_target_set; 105327b3f8d1SDan Williams } 105427b3f8d1SDan Williams goto add_target; 105527b3f8d1SDan Williams } 105627b3f8d1SDan Williams 105727b3f8d1SDan Williams if (is_cxl_root(parent_port)) { 105827b3f8d1SDan Williams parent_ig = cxlrd->cxlsd.cxld.interleave_granularity; 105927b3f8d1SDan Williams parent_iw = cxlrd->cxlsd.cxld.interleave_ways; 106027b3f8d1SDan Williams /* 106127b3f8d1SDan Williams * For purposes of address bit routing, use power-of-2 math for 106227b3f8d1SDan Williams * switch ports. 106327b3f8d1SDan Williams */ 106427b3f8d1SDan Williams if (!is_power_of_2(parent_iw)) 106527b3f8d1SDan Williams parent_iw /= 3; 106627b3f8d1SDan Williams } else { 106727b3f8d1SDan Williams struct cxl_region_ref *parent_rr; 106827b3f8d1SDan Williams struct cxl_decoder *parent_cxld; 106927b3f8d1SDan Williams 107027b3f8d1SDan Williams parent_rr = cxl_rr_load(parent_port, cxlr); 107127b3f8d1SDan Williams parent_cxld = parent_rr->decoder; 107227b3f8d1SDan Williams parent_ig = parent_cxld->interleave_granularity; 107327b3f8d1SDan Williams parent_iw = parent_cxld->interleave_ways; 107427b3f8d1SDan Williams } 107527b3f8d1SDan Williams 107683351ddbSDave Jiang rc = granularity_to_eig(parent_ig, &peig); 10778d428542SDan Williams if (rc) { 10788d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n", 10798d428542SDan Williams dev_name(parent_port->uport), 10808d428542SDan Williams dev_name(&parent_port->dev), parent_ig); 10818d428542SDan Williams return rc; 10828d428542SDan Williams } 10838d428542SDan Williams 1084c99b2e8cSDave Jiang rc = ways_to_eiw(parent_iw, &peiw); 10858d428542SDan Williams if (rc) { 10868d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n", 10878d428542SDan Williams dev_name(parent_port->uport), 10888d428542SDan Williams dev_name(&parent_port->dev), parent_iw); 10898d428542SDan Williams return rc; 10908d428542SDan Williams } 109127b3f8d1SDan Williams 109227b3f8d1SDan Williams iw = cxl_rr->nr_targets; 1093c99b2e8cSDave Jiang rc = ways_to_eiw(iw, &eiw); 10948d428542SDan Williams if (rc) { 10958d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n", 10968d428542SDan Williams dev_name(port->uport), dev_name(&port->dev), iw); 10978d428542SDan Williams return rc; 10988d428542SDan Williams } 10998d428542SDan Williams 1100298d44d0SDan Williams /* 1101298d44d0SDan Williams * If @parent_port is masking address bits, pick the next unused address 1102298d44d0SDan Williams * bit to route @port's targets. 1103298d44d0SDan Williams */ 1104298d44d0SDan Williams if (parent_iw > 1 && cxl_rr->nr_targets > 1) { 110527b3f8d1SDan Williams u32 address_bit = max(peig + peiw, eiw + peig); 110627b3f8d1SDan Williams 110727b3f8d1SDan Williams eig = address_bit - eiw + 1; 110827b3f8d1SDan Williams } else { 110927b3f8d1SDan Williams eiw = peiw; 111027b3f8d1SDan Williams eig = peig; 111127b3f8d1SDan Williams } 111227b3f8d1SDan Williams 111383351ddbSDave Jiang rc = eig_to_granularity(eig, &ig); 111427b3f8d1SDan Williams if (rc) { 111527b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n", 111627b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 111727b3f8d1SDan Williams 256 << eig); 111827b3f8d1SDan Williams return rc; 111927b3f8d1SDan Williams } 112027b3f8d1SDan Williams 1121a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1122a32320b7SDan Williams if (cxld->interleave_ways != iw || 1123a32320b7SDan Williams cxld->interleave_granularity != ig || 1124a32320b7SDan Williams cxld->hpa_range.start != p->res->start || 1125a32320b7SDan Williams cxld->hpa_range.end != p->res->end || 1126a32320b7SDan Williams ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { 1127a32320b7SDan Williams dev_err(&cxlr->dev, 1128a32320b7SDan Williams "%s:%s %s expected iw: %d ig: %d %pr\n", 1129a32320b7SDan Williams dev_name(port->uport), dev_name(&port->dev), 1130a32320b7SDan Williams __func__, iw, ig, p->res); 1131a32320b7SDan Williams dev_err(&cxlr->dev, 1132a32320b7SDan Williams "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n", 1133a32320b7SDan Williams dev_name(port->uport), dev_name(&port->dev), 1134a32320b7SDan Williams __func__, cxld->interleave_ways, 1135a32320b7SDan Williams cxld->interleave_granularity, 1136a32320b7SDan Williams (cxld->flags & CXL_DECODER_F_ENABLE) ? 1137a32320b7SDan Williams "enabled" : 1138a32320b7SDan Williams "disabled", 1139a32320b7SDan Williams cxld->hpa_range.start, cxld->hpa_range.end); 1140a32320b7SDan Williams return -ENXIO; 1141a32320b7SDan Williams } 1142a32320b7SDan Williams } else { 114327b3f8d1SDan Williams cxld->interleave_ways = iw; 114427b3f8d1SDan Williams cxld->interleave_granularity = ig; 1145910bc55dSDan Williams cxld->hpa_range = (struct range) { 1146910bc55dSDan Williams .start = p->res->start, 1147910bc55dSDan Williams .end = p->res->end, 1148910bc55dSDan Williams }; 1149a32320b7SDan Williams } 115027b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport), 115127b3f8d1SDan Williams dev_name(&port->dev), iw, ig); 115227b3f8d1SDan Williams add_target: 115327b3f8d1SDan Williams if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) { 115427b3f8d1SDan Williams dev_dbg(&cxlr->dev, 115527b3f8d1SDan Williams "%s:%s: targets full trying to add %s:%s at %d\n", 115627b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 115727b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 115827b3f8d1SDan Williams return -ENXIO; 115927b3f8d1SDan Williams } 1160a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1161a32320b7SDan Williams if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) { 1162a32320b7SDan Williams dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n", 1163a32320b7SDan Williams dev_name(port->uport), dev_name(&port->dev), 1164a32320b7SDan Williams dev_name(&cxlsd->cxld.dev), 1165a32320b7SDan Williams dev_name(ep->dport->dport), 1166a32320b7SDan Williams cxl_rr->nr_targets_set); 1167a32320b7SDan Williams return -ENXIO; 1168a32320b7SDan Williams } 1169a32320b7SDan Williams } else 117027b3f8d1SDan Williams cxlsd->target[cxl_rr->nr_targets_set] = ep->dport; 117127b3f8d1SDan Williams inc = 1; 117227b3f8d1SDan Williams out_target_set: 117327b3f8d1SDan Williams cxl_rr->nr_targets_set += inc; 117427b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", 117527b3f8d1SDan Williams dev_name(port->uport), dev_name(&port->dev), 117627b3f8d1SDan Williams cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport), 117727b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 117827b3f8d1SDan Williams 117927b3f8d1SDan Williams return 0; 118027b3f8d1SDan Williams } 118127b3f8d1SDan Williams 118227b3f8d1SDan Williams static void cxl_port_reset_targets(struct cxl_port *port, 118327b3f8d1SDan Williams struct cxl_region *cxlr) 118427b3f8d1SDan Williams { 118527b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 1186910bc55dSDan Williams struct cxl_decoder *cxld; 118727b3f8d1SDan Williams 118827b3f8d1SDan Williams /* 118927b3f8d1SDan Williams * After the last endpoint has been detached the entire cxl_rr may now 119027b3f8d1SDan Williams * be gone. 119127b3f8d1SDan Williams */ 1192910bc55dSDan Williams if (!cxl_rr) 1193910bc55dSDan Williams return; 119427b3f8d1SDan Williams cxl_rr->nr_targets_set = 0; 1195910bc55dSDan Williams 1196910bc55dSDan Williams cxld = cxl_rr->decoder; 1197910bc55dSDan Williams cxld->hpa_range = (struct range) { 1198910bc55dSDan Williams .start = 0, 1199910bc55dSDan Williams .end = -1, 1200910bc55dSDan Williams }; 120127b3f8d1SDan Williams } 120227b3f8d1SDan Williams 120327b3f8d1SDan Williams static void cxl_region_teardown_targets(struct cxl_region *cxlr) 120427b3f8d1SDan Williams { 120527b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 120627b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled; 1207*030f8803SDan Williams struct cxl_dev_state *cxlds; 120827b3f8d1SDan Williams struct cxl_memdev *cxlmd; 120927b3f8d1SDan Williams struct cxl_port *iter; 121027b3f8d1SDan Williams struct cxl_ep *ep; 121127b3f8d1SDan Williams int i; 121227b3f8d1SDan Williams 1213a32320b7SDan Williams /* 1214a32320b7SDan Williams * In the auto-discovery case skip automatic teardown since the 1215a32320b7SDan Williams * address space is already active 1216a32320b7SDan Williams */ 1217a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) 1218a32320b7SDan Williams return; 1219a32320b7SDan Williams 122027b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) { 122127b3f8d1SDan Williams cxled = p->targets[i]; 122227b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled); 1223*030f8803SDan Williams cxlds = cxlmd->cxlds; 1224*030f8803SDan Williams 1225*030f8803SDan Williams if (cxlds->rcd) 1226*030f8803SDan Williams continue; 122727b3f8d1SDan Williams 122827b3f8d1SDan Williams iter = cxled_to_port(cxled); 122927b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 123027b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent); 123127b3f8d1SDan Williams 123227b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 123327b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) 123427b3f8d1SDan Williams cxl_port_reset_targets(iter, cxlr); 123527b3f8d1SDan Williams } 123627b3f8d1SDan Williams } 123727b3f8d1SDan Williams 123827b3f8d1SDan Williams static int cxl_region_setup_targets(struct cxl_region *cxlr) 123927b3f8d1SDan Williams { 124027b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 124127b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled; 1242*030f8803SDan Williams struct cxl_dev_state *cxlds; 1243*030f8803SDan Williams int i, rc, rch = 0, vh = 0; 124427b3f8d1SDan Williams struct cxl_memdev *cxlmd; 124527b3f8d1SDan Williams struct cxl_port *iter; 124627b3f8d1SDan Williams struct cxl_ep *ep; 124727b3f8d1SDan Williams 124827b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) { 124927b3f8d1SDan Williams cxled = p->targets[i]; 125027b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled); 1251*030f8803SDan Williams cxlds = cxlmd->cxlds; 1252*030f8803SDan Williams 1253*030f8803SDan Williams /* validate that all targets agree on topology */ 1254*030f8803SDan Williams if (!cxlds->rcd) { 1255*030f8803SDan Williams vh++; 1256*030f8803SDan Williams } else { 1257*030f8803SDan Williams rch++; 1258*030f8803SDan Williams continue; 1259*030f8803SDan Williams } 126027b3f8d1SDan Williams 126127b3f8d1SDan Williams iter = cxled_to_port(cxled); 126227b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 126327b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent); 126427b3f8d1SDan Williams 126527b3f8d1SDan Williams /* 1266a32320b7SDan Williams * Descend the topology tree programming / validating 1267a32320b7SDan Williams * targets while looking for conflicts. 126827b3f8d1SDan Williams */ 126927b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 127027b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 127127b3f8d1SDan Williams rc = cxl_port_setup_targets(iter, cxlr, cxled); 127227b3f8d1SDan Williams if (rc) { 127327b3f8d1SDan Williams cxl_region_teardown_targets(cxlr); 127427b3f8d1SDan Williams return rc; 127527b3f8d1SDan Williams } 127627b3f8d1SDan Williams } 127727b3f8d1SDan Williams } 127827b3f8d1SDan Williams 1279*030f8803SDan Williams if (rch && vh) { 1280*030f8803SDan Williams dev_err(&cxlr->dev, "mismatched CXL topologies detected\n"); 1281*030f8803SDan Williams cxl_region_teardown_targets(cxlr); 1282*030f8803SDan Williams return -ENXIO; 1283*030f8803SDan Williams } 1284*030f8803SDan Williams 128527b3f8d1SDan Williams return 0; 128627b3f8d1SDan Williams } 128727b3f8d1SDan Williams 12889995576cSDan Williams static int cxl_region_validate_position(struct cxl_region *cxlr, 12899995576cSDan Williams struct cxl_endpoint_decoder *cxled, 12909995576cSDan Williams int pos) 1291b9686e8cSDan Williams { 1292384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1293b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 12949995576cSDan Williams int i; 1295384e624bSDan Williams 1296384e624bSDan Williams if (pos < 0 || pos >= p->interleave_ways) { 1297b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1298b9686e8cSDan Williams p->interleave_ways); 1299b9686e8cSDan Williams return -ENXIO; 1300b9686e8cSDan Williams } 1301b9686e8cSDan Williams 1302b9686e8cSDan Williams if (p->targets[pos] == cxled) 1303b9686e8cSDan Williams return 0; 1304b9686e8cSDan Williams 1305b9686e8cSDan Williams if (p->targets[pos]) { 1306b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled_target = p->targets[pos]; 1307b9686e8cSDan Williams struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target); 1308b9686e8cSDan Williams 1309b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n", 1310b9686e8cSDan Williams pos, dev_name(&cxlmd_target->dev), 1311b9686e8cSDan Williams dev_name(&cxled_target->cxld.dev)); 1312b9686e8cSDan Williams return -EBUSY; 1313b9686e8cSDan Williams } 1314b9686e8cSDan Williams 1315384e624bSDan Williams for (i = 0; i < p->interleave_ways; i++) { 1316384e624bSDan Williams struct cxl_endpoint_decoder *cxled_target; 1317384e624bSDan Williams struct cxl_memdev *cxlmd_target; 1318384e624bSDan Williams 1319f04facfbSFan Ni cxled_target = p->targets[i]; 1320384e624bSDan Williams if (!cxled_target) 1321384e624bSDan Williams continue; 1322384e624bSDan Williams 1323384e624bSDan Williams cxlmd_target = cxled_to_memdev(cxled_target); 1324384e624bSDan Williams if (cxlmd_target == cxlmd) { 1325384e624bSDan Williams dev_dbg(&cxlr->dev, 1326384e624bSDan Williams "%s already specified at position %d via: %s\n", 1327384e624bSDan Williams dev_name(&cxlmd->dev), pos, 1328384e624bSDan Williams dev_name(&cxled_target->cxld.dev)); 1329384e624bSDan Williams return -EBUSY; 1330384e624bSDan Williams } 1331384e624bSDan Williams } 1332384e624bSDan Williams 13339995576cSDan Williams return 0; 13349995576cSDan Williams } 13359995576cSDan Williams 13369995576cSDan Williams static int cxl_region_attach_position(struct cxl_region *cxlr, 13379995576cSDan Williams struct cxl_root_decoder *cxlrd, 13389995576cSDan Williams struct cxl_endpoint_decoder *cxled, 13399995576cSDan Williams const struct cxl_dport *dport, int pos) 13409995576cSDan Williams { 13419995576cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 13429995576cSDan Williams struct cxl_port *iter; 13439995576cSDan Williams int rc; 13449995576cSDan Williams 13459995576cSDan Williams if (cxlrd->calc_hb(cxlrd, pos) != dport) { 13469995576cSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n", 13479995576cSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 13489995576cSDan Williams dev_name(&cxlrd->cxlsd.cxld.dev)); 13499995576cSDan Williams return -ENXIO; 13509995576cSDan Williams } 13519995576cSDan Williams 13529995576cSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 13539995576cSDan Williams iter = to_cxl_port(iter->dev.parent)) { 13549995576cSDan Williams rc = cxl_port_attach_region(iter, cxlr, cxled, pos); 13559995576cSDan Williams if (rc) 13569995576cSDan Williams goto err; 13579995576cSDan Williams } 13589995576cSDan Williams 13599995576cSDan Williams return 0; 13609995576cSDan Williams 13619995576cSDan Williams err: 13629995576cSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 13639995576cSDan Williams iter = to_cxl_port(iter->dev.parent)) 13649995576cSDan Williams cxl_port_detach_region(iter, cxlr, cxled); 13659995576cSDan Williams return rc; 13669995576cSDan Williams } 13679995576cSDan Williams 1368a32320b7SDan Williams static int cxl_region_attach_auto(struct cxl_region *cxlr, 1369a32320b7SDan Williams struct cxl_endpoint_decoder *cxled, int pos) 1370a32320b7SDan Williams { 1371a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params; 1372a32320b7SDan Williams 1373a32320b7SDan Williams if (cxled->state != CXL_DECODER_STATE_AUTO) { 1374a32320b7SDan Williams dev_err(&cxlr->dev, 1375a32320b7SDan Williams "%s: unable to add decoder to autodetected region\n", 1376a32320b7SDan Williams dev_name(&cxled->cxld.dev)); 1377a32320b7SDan Williams return -EINVAL; 1378a32320b7SDan Williams } 1379a32320b7SDan Williams 1380a32320b7SDan Williams if (pos >= 0) { 1381a32320b7SDan Williams dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n", 1382a32320b7SDan Williams dev_name(&cxled->cxld.dev), pos); 1383a32320b7SDan Williams return -EINVAL; 1384a32320b7SDan Williams } 1385a32320b7SDan Williams 1386a32320b7SDan Williams if (p->nr_targets >= p->interleave_ways) { 1387a32320b7SDan Williams dev_err(&cxlr->dev, "%s: no more target slots available\n", 1388a32320b7SDan Williams dev_name(&cxled->cxld.dev)); 1389a32320b7SDan Williams return -ENXIO; 1390a32320b7SDan Williams } 1391a32320b7SDan Williams 1392a32320b7SDan Williams /* 1393a32320b7SDan Williams * Temporarily record the endpoint decoder into the target array. Yes, 1394a32320b7SDan Williams * this means that userspace can view devices in the wrong position 1395a32320b7SDan Williams * before the region activates, and must be careful to understand when 1396a32320b7SDan Williams * it might be racing region autodiscovery. 1397a32320b7SDan Williams */ 1398a32320b7SDan Williams pos = p->nr_targets; 1399a32320b7SDan Williams p->targets[pos] = cxled; 1400a32320b7SDan Williams cxled->pos = pos; 1401a32320b7SDan Williams p->nr_targets++; 1402a32320b7SDan Williams 1403a32320b7SDan Williams return 0; 1404a32320b7SDan Williams } 1405a32320b7SDan Williams 1406a32320b7SDan Williams static struct cxl_port *next_port(struct cxl_port *port) 1407a32320b7SDan Williams { 1408a32320b7SDan Williams if (!port->parent_dport) 1409a32320b7SDan Williams return NULL; 1410a32320b7SDan Williams return port->parent_dport->port; 1411a32320b7SDan Williams } 1412a32320b7SDan Williams 1413a32320b7SDan Williams static int decoder_match_range(struct device *dev, void *data) 1414a32320b7SDan Williams { 1415a32320b7SDan Williams struct cxl_endpoint_decoder *cxled = data; 1416a32320b7SDan Williams struct cxl_switch_decoder *cxlsd; 1417a32320b7SDan Williams 1418a32320b7SDan Williams if (!is_switch_decoder(dev)) 1419a32320b7SDan Williams return 0; 1420a32320b7SDan Williams 1421a32320b7SDan Williams cxlsd = to_cxl_switch_decoder(dev); 1422a32320b7SDan Williams return range_contains(&cxlsd->cxld.hpa_range, &cxled->cxld.hpa_range); 1423a32320b7SDan Williams } 1424a32320b7SDan Williams 1425a32320b7SDan Williams static void find_positions(const struct cxl_switch_decoder *cxlsd, 1426a32320b7SDan Williams const struct cxl_port *iter_a, 1427a32320b7SDan Williams const struct cxl_port *iter_b, int *a_pos, 1428a32320b7SDan Williams int *b_pos) 1429a32320b7SDan Williams { 1430a32320b7SDan Williams int i; 1431a32320b7SDan Williams 1432a32320b7SDan Williams for (i = 0, *a_pos = -1, *b_pos = -1; i < cxlsd->nr_targets; i++) { 1433a32320b7SDan Williams if (cxlsd->target[i] == iter_a->parent_dport) 1434a32320b7SDan Williams *a_pos = i; 1435a32320b7SDan Williams else if (cxlsd->target[i] == iter_b->parent_dport) 1436a32320b7SDan Williams *b_pos = i; 1437a32320b7SDan Williams if (*a_pos >= 0 && *b_pos >= 0) 1438a32320b7SDan Williams break; 1439a32320b7SDan Williams } 1440a32320b7SDan Williams } 1441a32320b7SDan Williams 1442a32320b7SDan Williams static int cmp_decode_pos(const void *a, const void *b) 1443a32320b7SDan Williams { 1444a32320b7SDan Williams struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a; 1445a32320b7SDan Williams struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b; 1446a32320b7SDan Williams struct cxl_memdev *cxlmd_a = cxled_to_memdev(cxled_a); 1447a32320b7SDan Williams struct cxl_memdev *cxlmd_b = cxled_to_memdev(cxled_b); 1448a32320b7SDan Williams struct cxl_port *port_a = cxled_to_port(cxled_a); 1449a32320b7SDan Williams struct cxl_port *port_b = cxled_to_port(cxled_b); 1450a32320b7SDan Williams struct cxl_port *iter_a, *iter_b, *port = NULL; 1451a32320b7SDan Williams struct cxl_switch_decoder *cxlsd; 1452a32320b7SDan Williams struct device *dev; 1453a32320b7SDan Williams int a_pos, b_pos; 1454a32320b7SDan Williams unsigned int seq; 1455a32320b7SDan Williams 1456a32320b7SDan Williams /* Exit early if any prior sorting failed */ 1457a32320b7SDan Williams if (cxled_a->pos < 0 || cxled_b->pos < 0) 1458a32320b7SDan Williams return 0; 1459a32320b7SDan Williams 1460a32320b7SDan Williams /* 1461a32320b7SDan Williams * Walk up the hierarchy to find a shared port, find the decoder that 1462a32320b7SDan Williams * maps the range, compare the relative position of those dport 1463a32320b7SDan Williams * mappings. 1464a32320b7SDan Williams */ 1465a32320b7SDan Williams for (iter_a = port_a; iter_a; iter_a = next_port(iter_a)) { 1466a32320b7SDan Williams struct cxl_port *next_a, *next_b; 1467a32320b7SDan Williams 1468a32320b7SDan Williams next_a = next_port(iter_a); 1469a32320b7SDan Williams if (!next_a) 1470a32320b7SDan Williams break; 1471a32320b7SDan Williams 1472a32320b7SDan Williams for (iter_b = port_b; iter_b; iter_b = next_port(iter_b)) { 1473a32320b7SDan Williams next_b = next_port(iter_b); 1474a32320b7SDan Williams if (next_a != next_b) 1475a32320b7SDan Williams continue; 1476a32320b7SDan Williams port = next_a; 1477a32320b7SDan Williams break; 1478a32320b7SDan Williams } 1479a32320b7SDan Williams 1480a32320b7SDan Williams if (port) 1481a32320b7SDan Williams break; 1482a32320b7SDan Williams } 1483a32320b7SDan Williams 1484a32320b7SDan Williams if (!port) { 1485a32320b7SDan Williams dev_err(cxlmd_a->dev.parent, 1486a32320b7SDan Williams "failed to find shared port with %s\n", 1487a32320b7SDan Williams dev_name(cxlmd_b->dev.parent)); 1488a32320b7SDan Williams goto err; 1489a32320b7SDan Williams } 1490a32320b7SDan Williams 1491a32320b7SDan Williams dev = device_find_child(&port->dev, cxled_a, decoder_match_range); 1492a32320b7SDan Williams if (!dev) { 1493a32320b7SDan Williams struct range *range = &cxled_a->cxld.hpa_range; 1494a32320b7SDan Williams 1495a32320b7SDan Williams dev_err(port->uport, 1496a32320b7SDan Williams "failed to find decoder that maps %#llx-%#llx\n", 1497a32320b7SDan Williams range->start, range->end); 1498a32320b7SDan Williams goto err; 1499a32320b7SDan Williams } 1500a32320b7SDan Williams 1501a32320b7SDan Williams cxlsd = to_cxl_switch_decoder(dev); 1502a32320b7SDan Williams do { 1503a32320b7SDan Williams seq = read_seqbegin(&cxlsd->target_lock); 1504a32320b7SDan Williams find_positions(cxlsd, iter_a, iter_b, &a_pos, &b_pos); 1505a32320b7SDan Williams } while (read_seqretry(&cxlsd->target_lock, seq)); 1506a32320b7SDan Williams 1507a32320b7SDan Williams put_device(dev); 1508a32320b7SDan Williams 1509a32320b7SDan Williams if (a_pos < 0 || b_pos < 0) { 1510a32320b7SDan Williams dev_err(port->uport, 1511a32320b7SDan Williams "failed to find shared decoder for %s and %s\n", 1512a32320b7SDan Williams dev_name(cxlmd_a->dev.parent), 1513a32320b7SDan Williams dev_name(cxlmd_b->dev.parent)); 1514a32320b7SDan Williams goto err; 1515a32320b7SDan Williams } 1516a32320b7SDan Williams 1517a32320b7SDan Williams dev_dbg(port->uport, "%s comes %s %s\n", dev_name(cxlmd_a->dev.parent), 1518a32320b7SDan Williams a_pos - b_pos < 0 ? "before" : "after", 1519a32320b7SDan Williams dev_name(cxlmd_b->dev.parent)); 1520a32320b7SDan Williams 1521a32320b7SDan Williams return a_pos - b_pos; 1522a32320b7SDan Williams err: 1523a32320b7SDan Williams cxled_a->pos = -1; 1524a32320b7SDan Williams return 0; 1525a32320b7SDan Williams } 1526a32320b7SDan Williams 1527a32320b7SDan Williams static int cxl_region_sort_targets(struct cxl_region *cxlr) 1528a32320b7SDan Williams { 1529a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params; 1530a32320b7SDan Williams int i, rc = 0; 1531a32320b7SDan Williams 1532a32320b7SDan Williams sort(p->targets, p->nr_targets, sizeof(p->targets[0]), cmp_decode_pos, 1533a32320b7SDan Williams NULL); 1534a32320b7SDan Williams 1535a32320b7SDan Williams for (i = 0; i < p->nr_targets; i++) { 1536a32320b7SDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 1537a32320b7SDan Williams 1538a32320b7SDan Williams /* 1539a32320b7SDan Williams * Record that sorting failed, but still continue to restore 1540a32320b7SDan Williams * cxled->pos with its ->targets[] position so that follow-on 1541a32320b7SDan Williams * code paths can reliably do p->targets[cxled->pos] to 1542a32320b7SDan Williams * self-reference their entry. 1543a32320b7SDan Williams */ 1544a32320b7SDan Williams if (cxled->pos < 0) 1545a32320b7SDan Williams rc = -ENXIO; 1546a32320b7SDan Williams cxled->pos = i; 1547a32320b7SDan Williams } 1548a32320b7SDan Williams 1549a32320b7SDan Williams dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful"); 1550a32320b7SDan Williams return rc; 1551a32320b7SDan Williams } 1552a32320b7SDan Williams 15539995576cSDan Williams static int cxl_region_attach(struct cxl_region *cxlr, 15549995576cSDan Williams struct cxl_endpoint_decoder *cxled, int pos) 15559995576cSDan Williams { 15569995576cSDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 15579995576cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 15589995576cSDan Williams struct cxl_region_params *p = &cxlr->params; 15599995576cSDan Williams struct cxl_port *ep_port, *root_port; 15609995576cSDan Williams struct cxl_dport *dport; 15619995576cSDan Williams int rc = -ENXIO; 15629995576cSDan Williams 15639995576cSDan Williams if (cxled->mode != cxlr->mode) { 15649995576cSDan Williams dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n", 15659995576cSDan Williams dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode); 15669995576cSDan Williams return -EINVAL; 15679995576cSDan Williams } 15689995576cSDan Williams 15699995576cSDan Williams if (cxled->mode == CXL_DECODER_DEAD) { 15709995576cSDan Williams dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev)); 15719995576cSDan Williams return -ENODEV; 15729995576cSDan Williams } 15739995576cSDan Williams 15749995576cSDan Williams /* all full of members, or interleave config not established? */ 15759995576cSDan Williams if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) { 15769995576cSDan Williams dev_dbg(&cxlr->dev, "region already active\n"); 15779995576cSDan Williams return -EBUSY; 15789995576cSDan Williams } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) { 15799995576cSDan Williams dev_dbg(&cxlr->dev, "interleave config missing\n"); 15809995576cSDan Williams return -ENXIO; 15819995576cSDan Williams } 15829995576cSDan Williams 1583384e624bSDan Williams ep_port = cxled_to_port(cxled); 1584384e624bSDan Williams root_port = cxlrd_to_port(cxlrd); 1585384e624bSDan Williams dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge); 1586384e624bSDan Williams if (!dport) { 1587384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n", 1588384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1589384e624bSDan Williams dev_name(cxlr->dev.parent)); 1590384e624bSDan Williams return -ENXIO; 1591384e624bSDan Williams } 1592384e624bSDan Williams 1593384e624bSDan Williams if (cxled->cxld.target_type != cxlr->type) { 1594384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n", 1595384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1596384e624bSDan Williams cxled->cxld.target_type, cxlr->type); 1597384e624bSDan Williams return -ENXIO; 1598384e624bSDan Williams } 1599384e624bSDan Williams 1600384e624bSDan Williams if (!cxled->dpa_res) { 1601384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n", 1602384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev)); 1603384e624bSDan Williams return -ENXIO; 1604384e624bSDan Williams } 1605384e624bSDan Williams 1606384e624bSDan Williams if (resource_size(cxled->dpa_res) * p->interleave_ways != 1607384e624bSDan Williams resource_size(p->res)) { 1608384e624bSDan Williams dev_dbg(&cxlr->dev, 1609384e624bSDan Williams "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n", 1610384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1611384e624bSDan Williams (u64)resource_size(cxled->dpa_res), p->interleave_ways, 1612384e624bSDan Williams (u64)resource_size(p->res)); 1613384e624bSDan Williams return -EINVAL; 1614384e624bSDan Williams } 1615384e624bSDan Williams 1616a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1617a32320b7SDan Williams int i; 1618a32320b7SDan Williams 1619a32320b7SDan Williams rc = cxl_region_attach_auto(cxlr, cxled, pos); 1620384e624bSDan Williams if (rc) 1621a32320b7SDan Williams return rc; 1622a32320b7SDan Williams 1623a32320b7SDan Williams /* await more targets to arrive... */ 1624a32320b7SDan Williams if (p->nr_targets < p->interleave_ways) 1625a32320b7SDan Williams return 0; 1626a32320b7SDan Williams 1627a32320b7SDan Williams /* 1628a32320b7SDan Williams * All targets are here, which implies all PCI enumeration that 1629a32320b7SDan Williams * affects this region has been completed. Walk the topology to 1630a32320b7SDan Williams * sort the devices into their relative region decode position. 1631a32320b7SDan Williams */ 1632a32320b7SDan Williams rc = cxl_region_sort_targets(cxlr); 1633a32320b7SDan Williams if (rc) 1634a32320b7SDan Williams return rc; 1635a32320b7SDan Williams 1636a32320b7SDan Williams for (i = 0; i < p->nr_targets; i++) { 1637a32320b7SDan Williams cxled = p->targets[i]; 1638a32320b7SDan Williams ep_port = cxled_to_port(cxled); 1639a32320b7SDan Williams dport = cxl_find_dport_by_dev(root_port, 1640a32320b7SDan Williams ep_port->host_bridge); 1641a32320b7SDan Williams rc = cxl_region_attach_position(cxlr, cxlrd, cxled, 1642a32320b7SDan Williams dport, i); 1643a32320b7SDan Williams if (rc) 1644a32320b7SDan Williams return rc; 1645384e624bSDan Williams } 1646384e624bSDan Williams 1647a32320b7SDan Williams rc = cxl_region_setup_targets(cxlr); 1648a32320b7SDan Williams if (rc) 1649a32320b7SDan Williams return rc; 1650a32320b7SDan Williams 1651a32320b7SDan Williams /* 1652a32320b7SDan Williams * If target setup succeeds in the autodiscovery case 1653a32320b7SDan Williams * then the region is already committed. 1654a32320b7SDan Williams */ 1655a32320b7SDan Williams p->state = CXL_CONFIG_COMMIT; 1656a32320b7SDan Williams 1657a32320b7SDan Williams return 0; 1658a32320b7SDan Williams } 1659a32320b7SDan Williams 16609995576cSDan Williams rc = cxl_region_validate_position(cxlr, cxled, pos); 1661b9686e8cSDan Williams if (rc) 16629995576cSDan Williams return rc; 16639995576cSDan Williams 16649995576cSDan Williams rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos); 16659995576cSDan Williams if (rc) 16669995576cSDan Williams return rc; 1667b9686e8cSDan Williams 1668b9686e8cSDan Williams p->targets[pos] = cxled; 1669b9686e8cSDan Williams cxled->pos = pos; 1670b9686e8cSDan Williams p->nr_targets++; 1671b9686e8cSDan Williams 167227b3f8d1SDan Williams if (p->nr_targets == p->interleave_ways) { 167327b3f8d1SDan Williams rc = cxl_region_setup_targets(cxlr); 167427b3f8d1SDan Williams if (rc) 16755e42bcbcSDan Carpenter goto err_decrement; 1676384e624bSDan Williams p->state = CXL_CONFIG_ACTIVE; 167727b3f8d1SDan Williams } 1678384e624bSDan Williams 16792901c8bdSDan Williams cxled->cxld.interleave_ways = p->interleave_ways; 16802901c8bdSDan Williams cxled->cxld.interleave_granularity = p->interleave_granularity; 1681910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) { 1682910bc55dSDan Williams .start = p->res->start, 1683910bc55dSDan Williams .end = p->res->end, 1684910bc55dSDan Williams }; 16852901c8bdSDan Williams 1686b9686e8cSDan Williams return 0; 1687384e624bSDan Williams 16885e42bcbcSDan Carpenter err_decrement: 16895e42bcbcSDan Carpenter p->nr_targets--; 169086987c76SDan Williams cxled->pos = -1; 169186987c76SDan Williams p->targets[pos] = NULL; 1692384e624bSDan Williams return rc; 1693b9686e8cSDan Williams } 1694b9686e8cSDan Williams 1695176baefbSDan Williams static int cxl_region_detach(struct cxl_endpoint_decoder *cxled) 1696b9686e8cSDan Williams { 1697384e624bSDan Williams struct cxl_port *iter, *ep_port = cxled_to_port(cxled); 1698b9686e8cSDan Williams struct cxl_region *cxlr = cxled->cxld.region; 1699b9686e8cSDan Williams struct cxl_region_params *p; 1700176baefbSDan Williams int rc = 0; 1701b9686e8cSDan Williams 1702b9686e8cSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 1703b9686e8cSDan Williams 1704b9686e8cSDan Williams if (!cxlr) 1705176baefbSDan Williams return 0; 1706b9686e8cSDan Williams 1707b9686e8cSDan Williams p = &cxlr->params; 1708b9686e8cSDan Williams get_device(&cxlr->dev); 1709b9686e8cSDan Williams 1710176baefbSDan Williams if (p->state > CXL_CONFIG_ACTIVE) { 1711176baefbSDan Williams /* 1712176baefbSDan Williams * TODO: tear down all impacted regions if a device is 1713176baefbSDan Williams * removed out of order 1714176baefbSDan Williams */ 1715176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 1716176baefbSDan Williams if (rc) 1717176baefbSDan Williams goto out; 1718176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE; 1719176baefbSDan Williams } 1720176baefbSDan Williams 1721384e624bSDan Williams for (iter = ep_port; !is_cxl_root(iter); 1722384e624bSDan Williams iter = to_cxl_port(iter->dev.parent)) 1723384e624bSDan Williams cxl_port_detach_region(iter, cxlr, cxled); 1724384e624bSDan Williams 1725b9686e8cSDan Williams if (cxled->pos < 0 || cxled->pos >= p->interleave_ways || 1726b9686e8cSDan Williams p->targets[cxled->pos] != cxled) { 1727b9686e8cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1728b9686e8cSDan Williams 1729b9686e8cSDan Williams dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n", 1730b9686e8cSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1731b9686e8cSDan Williams cxled->pos); 1732b9686e8cSDan Williams goto out; 1733b9686e8cSDan Williams } 1734b9686e8cSDan Williams 173527b3f8d1SDan Williams if (p->state == CXL_CONFIG_ACTIVE) { 1736384e624bSDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 173727b3f8d1SDan Williams cxl_region_teardown_targets(cxlr); 173827b3f8d1SDan Williams } 1739b9686e8cSDan Williams p->targets[cxled->pos] = NULL; 1740b9686e8cSDan Williams p->nr_targets--; 1741910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) { 1742910bc55dSDan Williams .start = 0, 1743910bc55dSDan Williams .end = -1, 1744910bc55dSDan Williams }; 1745b9686e8cSDan Williams 1746384e624bSDan Williams /* notify the region driver that one of its targets has departed */ 1747b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1748b9686e8cSDan Williams device_release_driver(&cxlr->dev); 1749b9686e8cSDan Williams down_write(&cxl_region_rwsem); 1750b9686e8cSDan Williams out: 1751b9686e8cSDan Williams put_device(&cxlr->dev); 1752176baefbSDan Williams return rc; 1753b9686e8cSDan Williams } 1754b9686e8cSDan Williams 1755b9686e8cSDan Williams void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled) 1756b9686e8cSDan Williams { 1757b9686e8cSDan Williams down_write(&cxl_region_rwsem); 1758b9686e8cSDan Williams cxled->mode = CXL_DECODER_DEAD; 1759b9686e8cSDan Williams cxl_region_detach(cxled); 1760b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1761b9686e8cSDan Williams } 1762b9686e8cSDan Williams 17633528b1e1SDan Williams static int attach_target(struct cxl_region *cxlr, 17643528b1e1SDan Williams struct cxl_endpoint_decoder *cxled, int pos, 17653528b1e1SDan Williams unsigned int state) 1766b9686e8cSDan Williams { 17673528b1e1SDan Williams int rc = 0; 1768b9686e8cSDan Williams 17693528b1e1SDan Williams if (state == TASK_INTERRUPTIBLE) 1770b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem); 17713528b1e1SDan Williams else 17723528b1e1SDan Williams down_write(&cxl_region_rwsem); 1773b9686e8cSDan Williams if (rc) 17743528b1e1SDan Williams return rc; 17753528b1e1SDan Williams 1776b9686e8cSDan Williams down_read(&cxl_dpa_rwsem); 17773528b1e1SDan Williams rc = cxl_region_attach(cxlr, cxled, pos); 1778d18bc74aSDan Williams if (rc == 0) 1779d18bc74aSDan Williams set_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags); 1780b9686e8cSDan Williams up_read(&cxl_dpa_rwsem); 1781b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1782b9686e8cSDan Williams return rc; 1783b9686e8cSDan Williams } 1784b9686e8cSDan Williams 1785b9686e8cSDan Williams static int detach_target(struct cxl_region *cxlr, int pos) 1786b9686e8cSDan Williams { 1787b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 1788b9686e8cSDan Williams int rc; 1789b9686e8cSDan Williams 1790b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem); 1791b9686e8cSDan Williams if (rc) 1792b9686e8cSDan Williams return rc; 1793b9686e8cSDan Williams 1794b9686e8cSDan Williams if (pos >= p->interleave_ways) { 1795b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1796b9686e8cSDan Williams p->interleave_ways); 1797b9686e8cSDan Williams rc = -ENXIO; 1798b9686e8cSDan Williams goto out; 1799b9686e8cSDan Williams } 1800b9686e8cSDan Williams 1801b9686e8cSDan Williams if (!p->targets[pos]) { 1802b9686e8cSDan Williams rc = 0; 1803b9686e8cSDan Williams goto out; 1804b9686e8cSDan Williams } 1805b9686e8cSDan Williams 1806176baefbSDan Williams rc = cxl_region_detach(p->targets[pos]); 1807b9686e8cSDan Williams out: 1808b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1809b9686e8cSDan Williams return rc; 1810b9686e8cSDan Williams } 1811b9686e8cSDan Williams 1812b9686e8cSDan Williams static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos, 1813b9686e8cSDan Williams size_t len) 1814b9686e8cSDan Williams { 1815b9686e8cSDan Williams int rc; 1816b9686e8cSDan Williams 1817b9686e8cSDan Williams if (sysfs_streq(buf, "\n")) 1818b9686e8cSDan Williams rc = detach_target(cxlr, pos); 18193528b1e1SDan Williams else { 18203528b1e1SDan Williams struct device *dev; 18213528b1e1SDan Williams 18223528b1e1SDan Williams dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf); 18233528b1e1SDan Williams if (!dev) 18243528b1e1SDan Williams return -ENODEV; 18253528b1e1SDan Williams 18263528b1e1SDan Williams if (!is_endpoint_decoder(dev)) { 18273528b1e1SDan Williams rc = -EINVAL; 18283528b1e1SDan Williams goto out; 18293528b1e1SDan Williams } 18303528b1e1SDan Williams 18313528b1e1SDan Williams rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos, 18323528b1e1SDan Williams TASK_INTERRUPTIBLE); 18333528b1e1SDan Williams out: 18343528b1e1SDan Williams put_device(dev); 18353528b1e1SDan Williams } 1836b9686e8cSDan Williams 1837b9686e8cSDan Williams if (rc < 0) 1838b9686e8cSDan Williams return rc; 1839b9686e8cSDan Williams return len; 1840b9686e8cSDan Williams } 1841b9686e8cSDan Williams 1842b9686e8cSDan Williams #define TARGET_ATTR_RW(n) \ 1843b9686e8cSDan Williams static ssize_t target##n##_show( \ 1844b9686e8cSDan Williams struct device *dev, struct device_attribute *attr, char *buf) \ 1845b9686e8cSDan Williams { \ 1846b9686e8cSDan Williams return show_targetN(to_cxl_region(dev), buf, (n)); \ 1847b9686e8cSDan Williams } \ 1848b9686e8cSDan Williams static ssize_t target##n##_store(struct device *dev, \ 1849b9686e8cSDan Williams struct device_attribute *attr, \ 1850b9686e8cSDan Williams const char *buf, size_t len) \ 1851b9686e8cSDan Williams { \ 1852b9686e8cSDan Williams return store_targetN(to_cxl_region(dev), buf, (n), len); \ 1853b9686e8cSDan Williams } \ 1854b9686e8cSDan Williams static DEVICE_ATTR_RW(target##n) 1855b9686e8cSDan Williams 1856b9686e8cSDan Williams TARGET_ATTR_RW(0); 1857b9686e8cSDan Williams TARGET_ATTR_RW(1); 1858b9686e8cSDan Williams TARGET_ATTR_RW(2); 1859b9686e8cSDan Williams TARGET_ATTR_RW(3); 1860b9686e8cSDan Williams TARGET_ATTR_RW(4); 1861b9686e8cSDan Williams TARGET_ATTR_RW(5); 1862b9686e8cSDan Williams TARGET_ATTR_RW(6); 1863b9686e8cSDan Williams TARGET_ATTR_RW(7); 1864b9686e8cSDan Williams TARGET_ATTR_RW(8); 1865b9686e8cSDan Williams TARGET_ATTR_RW(9); 1866b9686e8cSDan Williams TARGET_ATTR_RW(10); 1867b9686e8cSDan Williams TARGET_ATTR_RW(11); 1868b9686e8cSDan Williams TARGET_ATTR_RW(12); 1869b9686e8cSDan Williams TARGET_ATTR_RW(13); 1870b9686e8cSDan Williams TARGET_ATTR_RW(14); 1871b9686e8cSDan Williams TARGET_ATTR_RW(15); 1872b9686e8cSDan Williams 1873b9686e8cSDan Williams static struct attribute *target_attrs[] = { 1874b9686e8cSDan Williams &dev_attr_target0.attr, 1875b9686e8cSDan Williams &dev_attr_target1.attr, 1876b9686e8cSDan Williams &dev_attr_target2.attr, 1877b9686e8cSDan Williams &dev_attr_target3.attr, 1878b9686e8cSDan Williams &dev_attr_target4.attr, 1879b9686e8cSDan Williams &dev_attr_target5.attr, 1880b9686e8cSDan Williams &dev_attr_target6.attr, 1881b9686e8cSDan Williams &dev_attr_target7.attr, 1882b9686e8cSDan Williams &dev_attr_target8.attr, 1883b9686e8cSDan Williams &dev_attr_target9.attr, 1884b9686e8cSDan Williams &dev_attr_target10.attr, 1885b9686e8cSDan Williams &dev_attr_target11.attr, 1886b9686e8cSDan Williams &dev_attr_target12.attr, 1887b9686e8cSDan Williams &dev_attr_target13.attr, 1888b9686e8cSDan Williams &dev_attr_target14.attr, 1889b9686e8cSDan Williams &dev_attr_target15.attr, 1890b9686e8cSDan Williams NULL, 1891b9686e8cSDan Williams }; 1892b9686e8cSDan Williams 1893b9686e8cSDan Williams static umode_t cxl_region_target_visible(struct kobject *kobj, 1894b9686e8cSDan Williams struct attribute *a, int n) 1895b9686e8cSDan Williams { 1896b9686e8cSDan Williams struct device *dev = kobj_to_dev(kobj); 1897b9686e8cSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 1898b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 1899b9686e8cSDan Williams 1900b9686e8cSDan Williams if (n < p->interleave_ways) 1901b9686e8cSDan Williams return a->mode; 1902b9686e8cSDan Williams return 0; 1903b9686e8cSDan Williams } 1904b9686e8cSDan Williams 1905b9686e8cSDan Williams static const struct attribute_group cxl_region_target_group = { 1906b9686e8cSDan Williams .attrs = target_attrs, 1907b9686e8cSDan Williams .is_visible = cxl_region_target_visible, 1908b9686e8cSDan Williams }; 1909b9686e8cSDan Williams 1910b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void) 1911b9686e8cSDan Williams { 1912b9686e8cSDan Williams return &cxl_region_target_group; 1913b9686e8cSDan Williams } 1914b9686e8cSDan Williams 1915dd5ba0ebSBen Widawsky static const struct attribute_group *region_groups[] = { 1916dd5ba0ebSBen Widawsky &cxl_base_attribute_group, 1917dd5ba0ebSBen Widawsky &cxl_region_group, 1918b9686e8cSDan Williams &cxl_region_target_group, 1919dd5ba0ebSBen Widawsky NULL, 1920dd5ba0ebSBen Widawsky }; 1921dd5ba0ebSBen Widawsky 1922779dd20cSBen Widawsky static void cxl_region_release(struct device *dev) 1923779dd20cSBen Widawsky { 19248f401ec1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 1925779dd20cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 19268f401ec1SDan Williams int id = atomic_read(&cxlrd->region_id); 19278f401ec1SDan Williams 19288f401ec1SDan Williams /* 19298f401ec1SDan Williams * Try to reuse the recently idled id rather than the cached 19308f401ec1SDan Williams * next id to prevent the region id space from increasing 19318f401ec1SDan Williams * unnecessarily. 19328f401ec1SDan Williams */ 19338f401ec1SDan Williams if (cxlr->id < id) 19348f401ec1SDan Williams if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) { 19358f401ec1SDan Williams memregion_free(id); 19368f401ec1SDan Williams goto out; 19378f401ec1SDan Williams } 1938779dd20cSBen Widawsky 1939779dd20cSBen Widawsky memregion_free(cxlr->id); 19408f401ec1SDan Williams out: 19418f401ec1SDan Williams put_device(dev->parent); 1942779dd20cSBen Widawsky kfree(cxlr); 1943779dd20cSBen Widawsky } 1944779dd20cSBen Widawsky 19458d48817dSDan Williams const struct device_type cxl_region_type = { 1946779dd20cSBen Widawsky .name = "cxl_region", 1947779dd20cSBen Widawsky .release = cxl_region_release, 1948dd5ba0ebSBen Widawsky .groups = region_groups 1949779dd20cSBen Widawsky }; 1950779dd20cSBen Widawsky 1951779dd20cSBen Widawsky bool is_cxl_region(struct device *dev) 1952779dd20cSBen Widawsky { 1953779dd20cSBen Widawsky return dev->type == &cxl_region_type; 1954779dd20cSBen Widawsky } 1955779dd20cSBen Widawsky EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL); 1956779dd20cSBen Widawsky 1957779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev) 1958779dd20cSBen Widawsky { 1959779dd20cSBen Widawsky if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type, 1960779dd20cSBen Widawsky "not a cxl_region device\n")) 1961779dd20cSBen Widawsky return NULL; 1962779dd20cSBen Widawsky 1963779dd20cSBen Widawsky return container_of(dev, struct cxl_region, dev); 1964779dd20cSBen Widawsky } 1965779dd20cSBen Widawsky 1966779dd20cSBen Widawsky static void unregister_region(void *dev) 1967779dd20cSBen Widawsky { 196823a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 19690d9e7340SDan Williams struct cxl_region_params *p = &cxlr->params; 19700d9e7340SDan Williams int i; 197123a22cd1SDan Williams 197223a22cd1SDan Williams device_del(dev); 19730d9e7340SDan Williams 19740d9e7340SDan Williams /* 19750d9e7340SDan Williams * Now that region sysfs is shutdown, the parameter block is now 19760d9e7340SDan Williams * read-only, so no need to hold the region rwsem to access the 19770d9e7340SDan Williams * region parameters. 19780d9e7340SDan Williams */ 19790d9e7340SDan Williams for (i = 0; i < p->interleave_ways; i++) 19800d9e7340SDan Williams detach_target(cxlr, i); 19810d9e7340SDan Williams 198223a22cd1SDan Williams cxl_region_iomem_release(cxlr); 198323a22cd1SDan Williams put_device(dev); 1984779dd20cSBen Widawsky } 1985779dd20cSBen Widawsky 1986779dd20cSBen Widawsky static struct lock_class_key cxl_region_key; 1987779dd20cSBen Widawsky 1988779dd20cSBen Widawsky static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id) 1989779dd20cSBen Widawsky { 1990779dd20cSBen Widawsky struct cxl_region *cxlr; 1991779dd20cSBen Widawsky struct device *dev; 1992779dd20cSBen Widawsky 1993779dd20cSBen Widawsky cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL); 1994779dd20cSBen Widawsky if (!cxlr) { 1995779dd20cSBen Widawsky memregion_free(id); 1996779dd20cSBen Widawsky return ERR_PTR(-ENOMEM); 1997779dd20cSBen Widawsky } 1998779dd20cSBen Widawsky 1999779dd20cSBen Widawsky dev = &cxlr->dev; 2000779dd20cSBen Widawsky device_initialize(dev); 2001779dd20cSBen Widawsky lockdep_set_class(&dev->mutex, &cxl_region_key); 2002779dd20cSBen Widawsky dev->parent = &cxlrd->cxlsd.cxld.dev; 20038f401ec1SDan Williams /* 20048f401ec1SDan Williams * Keep root decoder pinned through cxl_region_release to fixup 20058f401ec1SDan Williams * region id allocations 20068f401ec1SDan Williams */ 20078f401ec1SDan Williams get_device(dev->parent); 2008779dd20cSBen Widawsky device_set_pm_not_required(dev); 2009779dd20cSBen Widawsky dev->bus = &cxl_bus_type; 2010779dd20cSBen Widawsky dev->type = &cxl_region_type; 2011779dd20cSBen Widawsky cxlr->id = id; 2012779dd20cSBen Widawsky 2013779dd20cSBen Widawsky return cxlr; 2014779dd20cSBen Widawsky } 2015779dd20cSBen Widawsky 2016779dd20cSBen Widawsky /** 2017779dd20cSBen Widawsky * devm_cxl_add_region - Adds a region to a decoder 2018779dd20cSBen Widawsky * @cxlrd: root decoder 2019779dd20cSBen Widawsky * @id: memregion id to create, or memregion_free() on failure 2020779dd20cSBen Widawsky * @mode: mode for the endpoint decoders of this region 2021779dd20cSBen Widawsky * @type: select whether this is an expander or accelerator (type-2 or type-3) 2022779dd20cSBen Widawsky * 2023779dd20cSBen Widawsky * This is the second step of region initialization. Regions exist within an 2024779dd20cSBen Widawsky * address space which is mapped by a @cxlrd. 2025779dd20cSBen Widawsky * 2026779dd20cSBen Widawsky * Return: 0 if the region was added to the @cxlrd, else returns negative error 2027779dd20cSBen Widawsky * code. The region will be named "regionZ" where Z is the unique region number. 2028779dd20cSBen Widawsky */ 2029779dd20cSBen Widawsky static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, 2030779dd20cSBen Widawsky int id, 2031779dd20cSBen Widawsky enum cxl_decoder_mode mode, 2032779dd20cSBen Widawsky enum cxl_decoder_type type) 2033779dd20cSBen Widawsky { 2034779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent); 2035779dd20cSBen Widawsky struct cxl_region *cxlr; 2036779dd20cSBen Widawsky struct device *dev; 2037779dd20cSBen Widawsky int rc; 2038779dd20cSBen Widawsky 20396e099264SDan Williams switch (mode) { 20406e099264SDan Williams case CXL_DECODER_RAM: 20416e099264SDan Williams case CXL_DECODER_PMEM: 20426e099264SDan Williams break; 20436e099264SDan Williams default: 20446e099264SDan Williams dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode); 20456e099264SDan Williams return ERR_PTR(-EINVAL); 20466e099264SDan Williams } 20476e099264SDan Williams 2048779dd20cSBen Widawsky cxlr = cxl_region_alloc(cxlrd, id); 2049779dd20cSBen Widawsky if (IS_ERR(cxlr)) 2050779dd20cSBen Widawsky return cxlr; 2051779dd20cSBen Widawsky cxlr->mode = mode; 2052779dd20cSBen Widawsky cxlr->type = type; 2053779dd20cSBen Widawsky 2054779dd20cSBen Widawsky dev = &cxlr->dev; 2055779dd20cSBen Widawsky rc = dev_set_name(dev, "region%d", id); 2056779dd20cSBen Widawsky if (rc) 2057779dd20cSBen Widawsky goto err; 2058779dd20cSBen Widawsky 2059779dd20cSBen Widawsky rc = device_add(dev); 2060779dd20cSBen Widawsky if (rc) 2061779dd20cSBen Widawsky goto err; 2062779dd20cSBen Widawsky 2063779dd20cSBen Widawsky rc = devm_add_action_or_reset(port->uport, unregister_region, cxlr); 2064779dd20cSBen Widawsky if (rc) 2065779dd20cSBen Widawsky return ERR_PTR(rc); 2066779dd20cSBen Widawsky 2067779dd20cSBen Widawsky dev_dbg(port->uport, "%s: created %s\n", 2068779dd20cSBen Widawsky dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); 2069779dd20cSBen Widawsky return cxlr; 2070779dd20cSBen Widawsky 2071779dd20cSBen Widawsky err: 2072779dd20cSBen Widawsky put_device(dev); 2073779dd20cSBen Widawsky return ERR_PTR(rc); 2074779dd20cSBen Widawsky } 2075779dd20cSBen Widawsky 20766e099264SDan Williams static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf) 20776e099264SDan Williams { 20786e099264SDan Williams return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id)); 20796e099264SDan Williams } 20806e099264SDan Williams 2081779dd20cSBen Widawsky static ssize_t create_pmem_region_show(struct device *dev, 2082779dd20cSBen Widawsky struct device_attribute *attr, char *buf) 2083779dd20cSBen Widawsky { 20846e099264SDan Williams return __create_region_show(to_cxl_root_decoder(dev), buf); 20856e099264SDan Williams } 2086779dd20cSBen Widawsky 20876e099264SDan Williams static ssize_t create_ram_region_show(struct device *dev, 20886e099264SDan Williams struct device_attribute *attr, char *buf) 20896e099264SDan Williams { 20906e099264SDan Williams return __create_region_show(to_cxl_root_decoder(dev), buf); 20916e099264SDan Williams } 20926e099264SDan Williams 20936e099264SDan Williams static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd, 20946e099264SDan Williams enum cxl_decoder_mode mode, int id) 20956e099264SDan Williams { 20966e099264SDan Williams int rc; 20976e099264SDan Williams 20986e099264SDan Williams rc = memregion_alloc(GFP_KERNEL); 20996e099264SDan Williams if (rc < 0) 21006e099264SDan Williams return ERR_PTR(rc); 21016e099264SDan Williams 21026e099264SDan Williams if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) { 21036e099264SDan Williams memregion_free(rc); 21046e099264SDan Williams return ERR_PTR(-EBUSY); 21056e099264SDan Williams } 21066e099264SDan Williams 21076e099264SDan Williams return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_EXPANDER); 2108779dd20cSBen Widawsky } 2109779dd20cSBen Widawsky 2110779dd20cSBen Widawsky static ssize_t create_pmem_region_store(struct device *dev, 2111779dd20cSBen Widawsky struct device_attribute *attr, 2112779dd20cSBen Widawsky const char *buf, size_t len) 2113779dd20cSBen Widawsky { 2114779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2115779dd20cSBen Widawsky struct cxl_region *cxlr; 21166e099264SDan Williams int rc, id; 2117779dd20cSBen Widawsky 2118779dd20cSBen Widawsky rc = sscanf(buf, "region%d\n", &id); 2119779dd20cSBen Widawsky if (rc != 1) 2120779dd20cSBen Widawsky return -EINVAL; 2121779dd20cSBen Widawsky 21226e099264SDan Williams cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id); 2123779dd20cSBen Widawsky if (IS_ERR(cxlr)) 2124779dd20cSBen Widawsky return PTR_ERR(cxlr); 2125779dd20cSBen Widawsky 2126779dd20cSBen Widawsky return len; 2127779dd20cSBen Widawsky } 2128779dd20cSBen Widawsky DEVICE_ATTR_RW(create_pmem_region); 2129779dd20cSBen Widawsky 21306e099264SDan Williams static ssize_t create_ram_region_store(struct device *dev, 21316e099264SDan Williams struct device_attribute *attr, 21326e099264SDan Williams const char *buf, size_t len) 21336e099264SDan Williams { 21346e099264SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 21356e099264SDan Williams struct cxl_region *cxlr; 21366e099264SDan Williams int rc, id; 21376e099264SDan Williams 21386e099264SDan Williams rc = sscanf(buf, "region%d\n", &id); 21396e099264SDan Williams if (rc != 1) 21406e099264SDan Williams return -EINVAL; 21416e099264SDan Williams 21426e099264SDan Williams cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id); 21436e099264SDan Williams if (IS_ERR(cxlr)) 21446e099264SDan Williams return PTR_ERR(cxlr); 21456e099264SDan Williams 21466e099264SDan Williams return len; 21476e099264SDan Williams } 21486e099264SDan Williams DEVICE_ATTR_RW(create_ram_region); 21496e099264SDan Williams 2150b9686e8cSDan Williams static ssize_t region_show(struct device *dev, struct device_attribute *attr, 2151b9686e8cSDan Williams char *buf) 2152b9686e8cSDan Williams { 2153b9686e8cSDan Williams struct cxl_decoder *cxld = to_cxl_decoder(dev); 2154b9686e8cSDan Williams ssize_t rc; 2155b9686e8cSDan Williams 2156b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 2157b9686e8cSDan Williams if (rc) 2158b9686e8cSDan Williams return rc; 2159b9686e8cSDan Williams 2160b9686e8cSDan Williams if (cxld->region) 2161b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev)); 2162b9686e8cSDan Williams else 2163b9686e8cSDan Williams rc = sysfs_emit(buf, "\n"); 2164b9686e8cSDan Williams up_read(&cxl_region_rwsem); 2165b9686e8cSDan Williams 2166b9686e8cSDan Williams return rc; 2167b9686e8cSDan Williams } 2168b9686e8cSDan Williams DEVICE_ATTR_RO(region); 2169b9686e8cSDan Williams 2170779dd20cSBen Widawsky static struct cxl_region * 2171779dd20cSBen Widawsky cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name) 2172779dd20cSBen Widawsky { 2173779dd20cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 2174779dd20cSBen Widawsky struct device *region_dev; 2175779dd20cSBen Widawsky 2176779dd20cSBen Widawsky region_dev = device_find_child_by_name(&cxld->dev, name); 2177779dd20cSBen Widawsky if (!region_dev) 2178779dd20cSBen Widawsky return ERR_PTR(-ENODEV); 2179779dd20cSBen Widawsky 2180779dd20cSBen Widawsky return to_cxl_region(region_dev); 2181779dd20cSBen Widawsky } 2182779dd20cSBen Widawsky 2183779dd20cSBen Widawsky static ssize_t delete_region_store(struct device *dev, 2184779dd20cSBen Widawsky struct device_attribute *attr, 2185779dd20cSBen Widawsky const char *buf, size_t len) 2186779dd20cSBen Widawsky { 2187779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2188779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(dev->parent); 2189779dd20cSBen Widawsky struct cxl_region *cxlr; 2190779dd20cSBen Widawsky 2191779dd20cSBen Widawsky cxlr = cxl_find_region_by_name(cxlrd, buf); 2192779dd20cSBen Widawsky if (IS_ERR(cxlr)) 2193779dd20cSBen Widawsky return PTR_ERR(cxlr); 2194779dd20cSBen Widawsky 2195779dd20cSBen Widawsky devm_release_action(port->uport, unregister_region, cxlr); 2196779dd20cSBen Widawsky put_device(&cxlr->dev); 2197779dd20cSBen Widawsky 2198779dd20cSBen Widawsky return len; 2199779dd20cSBen Widawsky } 2200779dd20cSBen Widawsky DEVICE_ATTR_WO(delete_region); 220123a22cd1SDan Williams 220204ad63f0SDan Williams static void cxl_pmem_region_release(struct device *dev) 220304ad63f0SDan Williams { 220404ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); 220504ad63f0SDan Williams int i; 220604ad63f0SDan Williams 220704ad63f0SDan Williams for (i = 0; i < cxlr_pmem->nr_mappings; i++) { 220804ad63f0SDan Williams struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd; 220904ad63f0SDan Williams 221004ad63f0SDan Williams put_device(&cxlmd->dev); 221104ad63f0SDan Williams } 221204ad63f0SDan Williams 221304ad63f0SDan Williams kfree(cxlr_pmem); 221404ad63f0SDan Williams } 221504ad63f0SDan Williams 221604ad63f0SDan Williams static const struct attribute_group *cxl_pmem_region_attribute_groups[] = { 221704ad63f0SDan Williams &cxl_base_attribute_group, 221804ad63f0SDan Williams NULL, 221904ad63f0SDan Williams }; 222004ad63f0SDan Williams 222104ad63f0SDan Williams const struct device_type cxl_pmem_region_type = { 222204ad63f0SDan Williams .name = "cxl_pmem_region", 222304ad63f0SDan Williams .release = cxl_pmem_region_release, 222404ad63f0SDan Williams .groups = cxl_pmem_region_attribute_groups, 222504ad63f0SDan Williams }; 222604ad63f0SDan Williams 222704ad63f0SDan Williams bool is_cxl_pmem_region(struct device *dev) 222804ad63f0SDan Williams { 222904ad63f0SDan Williams return dev->type == &cxl_pmem_region_type; 223004ad63f0SDan Williams } 223104ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL); 223204ad63f0SDan Williams 223304ad63f0SDan Williams struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev) 223404ad63f0SDan Williams { 223504ad63f0SDan Williams if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev), 223604ad63f0SDan Williams "not a cxl_pmem_region device\n")) 223704ad63f0SDan Williams return NULL; 223804ad63f0SDan Williams return container_of(dev, struct cxl_pmem_region, dev); 223904ad63f0SDan Williams } 224004ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL); 224104ad63f0SDan Williams 224204ad63f0SDan Williams static struct lock_class_key cxl_pmem_region_key; 224304ad63f0SDan Williams 224404ad63f0SDan Williams static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr) 224504ad63f0SDan Williams { 224604ad63f0SDan Williams struct cxl_region_params *p = &cxlr->params; 2247f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb; 224804ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem; 224904ad63f0SDan Williams struct device *dev; 225004ad63f0SDan Williams int i; 225104ad63f0SDan Williams 225204ad63f0SDan Williams down_read(&cxl_region_rwsem); 225304ad63f0SDan Williams if (p->state != CXL_CONFIG_COMMIT) { 225404ad63f0SDan Williams cxlr_pmem = ERR_PTR(-ENXIO); 225504ad63f0SDan Williams goto out; 225604ad63f0SDan Williams } 225704ad63f0SDan Williams 225804ad63f0SDan Williams cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), 225904ad63f0SDan Williams GFP_KERNEL); 226004ad63f0SDan Williams if (!cxlr_pmem) { 226104ad63f0SDan Williams cxlr_pmem = ERR_PTR(-ENOMEM); 226204ad63f0SDan Williams goto out; 226304ad63f0SDan Williams } 226404ad63f0SDan Williams 226504ad63f0SDan Williams cxlr_pmem->hpa_range.start = p->res->start; 226604ad63f0SDan Williams cxlr_pmem->hpa_range.end = p->res->end; 226704ad63f0SDan Williams 226804ad63f0SDan Williams /* Snapshot the region configuration underneath the cxl_region_rwsem */ 226904ad63f0SDan Williams cxlr_pmem->nr_mappings = p->nr_targets; 227004ad63f0SDan Williams for (i = 0; i < p->nr_targets; i++) { 227104ad63f0SDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 227204ad63f0SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 227304ad63f0SDan Williams struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; 227404ad63f0SDan Williams 2275f17b558dSDan Williams /* 2276f17b558dSDan Williams * Regions never span CXL root devices, so by definition the 2277f17b558dSDan Williams * bridge for one device is the same for all. 2278f17b558dSDan Williams */ 2279f17b558dSDan Williams if (i == 0) { 2280d35b495dSDan Williams cxl_nvb = cxl_find_nvdimm_bridge(cxlmd); 2281f17b558dSDan Williams if (!cxl_nvb) { 2282f17b558dSDan Williams cxlr_pmem = ERR_PTR(-ENODEV); 2283f17b558dSDan Williams goto out; 2284f17b558dSDan Williams } 2285f17b558dSDan Williams cxlr->cxl_nvb = cxl_nvb; 2286f17b558dSDan Williams } 228704ad63f0SDan Williams m->cxlmd = cxlmd; 228804ad63f0SDan Williams get_device(&cxlmd->dev); 228904ad63f0SDan Williams m->start = cxled->dpa_res->start; 229004ad63f0SDan Williams m->size = resource_size(cxled->dpa_res); 229104ad63f0SDan Williams m->position = i; 229204ad63f0SDan Williams } 229304ad63f0SDan Williams 229404ad63f0SDan Williams dev = &cxlr_pmem->dev; 229504ad63f0SDan Williams cxlr_pmem->cxlr = cxlr; 2296f17b558dSDan Williams cxlr->cxlr_pmem = cxlr_pmem; 229704ad63f0SDan Williams device_initialize(dev); 229804ad63f0SDan Williams lockdep_set_class(&dev->mutex, &cxl_pmem_region_key); 229904ad63f0SDan Williams device_set_pm_not_required(dev); 230004ad63f0SDan Williams dev->parent = &cxlr->dev; 230104ad63f0SDan Williams dev->bus = &cxl_bus_type; 230204ad63f0SDan Williams dev->type = &cxl_pmem_region_type; 230304ad63f0SDan Williams out: 230404ad63f0SDan Williams up_read(&cxl_region_rwsem); 230504ad63f0SDan Williams 230604ad63f0SDan Williams return cxlr_pmem; 230704ad63f0SDan Williams } 230804ad63f0SDan Williams 230909d09e04SDan Williams static void cxl_dax_region_release(struct device *dev) 231009d09e04SDan Williams { 231109d09e04SDan Williams struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev); 231209d09e04SDan Williams 231309d09e04SDan Williams kfree(cxlr_dax); 231409d09e04SDan Williams } 231509d09e04SDan Williams 231609d09e04SDan Williams static const struct attribute_group *cxl_dax_region_attribute_groups[] = { 231709d09e04SDan Williams &cxl_base_attribute_group, 231809d09e04SDan Williams NULL, 231909d09e04SDan Williams }; 232009d09e04SDan Williams 232109d09e04SDan Williams const struct device_type cxl_dax_region_type = { 232209d09e04SDan Williams .name = "cxl_dax_region", 232309d09e04SDan Williams .release = cxl_dax_region_release, 232409d09e04SDan Williams .groups = cxl_dax_region_attribute_groups, 232509d09e04SDan Williams }; 232609d09e04SDan Williams 232709d09e04SDan Williams static bool is_cxl_dax_region(struct device *dev) 232809d09e04SDan Williams { 232909d09e04SDan Williams return dev->type == &cxl_dax_region_type; 233009d09e04SDan Williams } 233109d09e04SDan Williams 233209d09e04SDan Williams struct cxl_dax_region *to_cxl_dax_region(struct device *dev) 233309d09e04SDan Williams { 233409d09e04SDan Williams if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev), 233509d09e04SDan Williams "not a cxl_dax_region device\n")) 233609d09e04SDan Williams return NULL; 233709d09e04SDan Williams return container_of(dev, struct cxl_dax_region, dev); 233809d09e04SDan Williams } 233909d09e04SDan Williams EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL); 234009d09e04SDan Williams 234109d09e04SDan Williams static struct lock_class_key cxl_dax_region_key; 234209d09e04SDan Williams 234309d09e04SDan Williams static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr) 234409d09e04SDan Williams { 234509d09e04SDan Williams struct cxl_region_params *p = &cxlr->params; 234609d09e04SDan Williams struct cxl_dax_region *cxlr_dax; 234709d09e04SDan Williams struct device *dev; 234809d09e04SDan Williams 234909d09e04SDan Williams down_read(&cxl_region_rwsem); 235009d09e04SDan Williams if (p->state != CXL_CONFIG_COMMIT) { 235109d09e04SDan Williams cxlr_dax = ERR_PTR(-ENXIO); 235209d09e04SDan Williams goto out; 235309d09e04SDan Williams } 235409d09e04SDan Williams 235509d09e04SDan Williams cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL); 235609d09e04SDan Williams if (!cxlr_dax) { 235709d09e04SDan Williams cxlr_dax = ERR_PTR(-ENOMEM); 235809d09e04SDan Williams goto out; 235909d09e04SDan Williams } 236009d09e04SDan Williams 236109d09e04SDan Williams cxlr_dax->hpa_range.start = p->res->start; 236209d09e04SDan Williams cxlr_dax->hpa_range.end = p->res->end; 236309d09e04SDan Williams 236409d09e04SDan Williams dev = &cxlr_dax->dev; 236509d09e04SDan Williams cxlr_dax->cxlr = cxlr; 236609d09e04SDan Williams device_initialize(dev); 236709d09e04SDan Williams lockdep_set_class(&dev->mutex, &cxl_dax_region_key); 236809d09e04SDan Williams device_set_pm_not_required(dev); 236909d09e04SDan Williams dev->parent = &cxlr->dev; 237009d09e04SDan Williams dev->bus = &cxl_bus_type; 237109d09e04SDan Williams dev->type = &cxl_dax_region_type; 237209d09e04SDan Williams out: 237309d09e04SDan Williams up_read(&cxl_region_rwsem); 237409d09e04SDan Williams 237509d09e04SDan Williams return cxlr_dax; 237609d09e04SDan Williams } 237709d09e04SDan Williams 2378f17b558dSDan Williams static void cxlr_pmem_unregister(void *_cxlr_pmem) 237904ad63f0SDan Williams { 2380f17b558dSDan Williams struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem; 2381f17b558dSDan Williams struct cxl_region *cxlr = cxlr_pmem->cxlr; 2382f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 2383f17b558dSDan Williams 2384f17b558dSDan Williams /* 2385f17b558dSDan Williams * Either the bridge is in ->remove() context under the device_lock(), 2386f17b558dSDan Williams * or cxlr_release_nvdimm() is cancelling the bridge's release action 2387f17b558dSDan Williams * for @cxlr_pmem and doing it itself (while manually holding the bridge 2388f17b558dSDan Williams * lock). 2389f17b558dSDan Williams */ 2390f17b558dSDan Williams device_lock_assert(&cxl_nvb->dev); 2391f17b558dSDan Williams cxlr->cxlr_pmem = NULL; 2392f17b558dSDan Williams cxlr_pmem->cxlr = NULL; 2393f17b558dSDan Williams device_unregister(&cxlr_pmem->dev); 2394f17b558dSDan Williams } 2395f17b558dSDan Williams 2396f17b558dSDan Williams static void cxlr_release_nvdimm(void *_cxlr) 2397f17b558dSDan Williams { 2398f17b558dSDan Williams struct cxl_region *cxlr = _cxlr; 2399f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 2400f17b558dSDan Williams 2401f17b558dSDan Williams device_lock(&cxl_nvb->dev); 2402f17b558dSDan Williams if (cxlr->cxlr_pmem) 2403f17b558dSDan Williams devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister, 2404f17b558dSDan Williams cxlr->cxlr_pmem); 2405f17b558dSDan Williams device_unlock(&cxl_nvb->dev); 2406f17b558dSDan Williams cxlr->cxl_nvb = NULL; 2407f17b558dSDan Williams put_device(&cxl_nvb->dev); 240804ad63f0SDan Williams } 240904ad63f0SDan Williams 241004ad63f0SDan Williams /** 241104ad63f0SDan Williams * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge 241204ad63f0SDan Williams * @cxlr: parent CXL region for this pmem region bridge device 241304ad63f0SDan Williams * 241404ad63f0SDan Williams * Return: 0 on success negative error code on failure. 241504ad63f0SDan Williams */ 241604ad63f0SDan Williams static int devm_cxl_add_pmem_region(struct cxl_region *cxlr) 241704ad63f0SDan Williams { 241804ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem; 2419f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb; 242004ad63f0SDan Williams struct device *dev; 242104ad63f0SDan Williams int rc; 242204ad63f0SDan Williams 242304ad63f0SDan Williams cxlr_pmem = cxl_pmem_region_alloc(cxlr); 242404ad63f0SDan Williams if (IS_ERR(cxlr_pmem)) 242504ad63f0SDan Williams return PTR_ERR(cxlr_pmem); 2426f17b558dSDan Williams cxl_nvb = cxlr->cxl_nvb; 242704ad63f0SDan Williams 242804ad63f0SDan Williams dev = &cxlr_pmem->dev; 242904ad63f0SDan Williams rc = dev_set_name(dev, "pmem_region%d", cxlr->id); 243004ad63f0SDan Williams if (rc) 243104ad63f0SDan Williams goto err; 243204ad63f0SDan Williams 243304ad63f0SDan Williams rc = device_add(dev); 243404ad63f0SDan Williams if (rc) 243504ad63f0SDan Williams goto err; 243604ad63f0SDan Williams 243704ad63f0SDan Williams dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 243804ad63f0SDan Williams dev_name(dev)); 243904ad63f0SDan Williams 2440f17b558dSDan Williams device_lock(&cxl_nvb->dev); 2441f17b558dSDan Williams if (cxl_nvb->dev.driver) 2442f17b558dSDan Williams rc = devm_add_action_or_reset(&cxl_nvb->dev, 2443f17b558dSDan Williams cxlr_pmem_unregister, cxlr_pmem); 2444f17b558dSDan Williams else 2445f17b558dSDan Williams rc = -ENXIO; 2446f17b558dSDan Williams device_unlock(&cxl_nvb->dev); 2447f17b558dSDan Williams 2448f17b558dSDan Williams if (rc) 2449f17b558dSDan Williams goto err_bridge; 2450f17b558dSDan Williams 2451f17b558dSDan Williams /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */ 2452f17b558dSDan Williams return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr); 245304ad63f0SDan Williams 245404ad63f0SDan Williams err: 245504ad63f0SDan Williams put_device(dev); 2456f17b558dSDan Williams err_bridge: 2457f17b558dSDan Williams put_device(&cxl_nvb->dev); 2458f17b558dSDan Williams cxlr->cxl_nvb = NULL; 245904ad63f0SDan Williams return rc; 246004ad63f0SDan Williams } 246104ad63f0SDan Williams 246209d09e04SDan Williams static void cxlr_dax_unregister(void *_cxlr_dax) 246309d09e04SDan Williams { 246409d09e04SDan Williams struct cxl_dax_region *cxlr_dax = _cxlr_dax; 246509d09e04SDan Williams 246609d09e04SDan Williams device_unregister(&cxlr_dax->dev); 246709d09e04SDan Williams } 246809d09e04SDan Williams 246909d09e04SDan Williams static int devm_cxl_add_dax_region(struct cxl_region *cxlr) 247009d09e04SDan Williams { 247109d09e04SDan Williams struct cxl_dax_region *cxlr_dax; 247209d09e04SDan Williams struct device *dev; 247309d09e04SDan Williams int rc; 247409d09e04SDan Williams 247509d09e04SDan Williams cxlr_dax = cxl_dax_region_alloc(cxlr); 247609d09e04SDan Williams if (IS_ERR(cxlr_dax)) 247709d09e04SDan Williams return PTR_ERR(cxlr_dax); 247809d09e04SDan Williams 247909d09e04SDan Williams dev = &cxlr_dax->dev; 248009d09e04SDan Williams rc = dev_set_name(dev, "dax_region%d", cxlr->id); 248109d09e04SDan Williams if (rc) 248209d09e04SDan Williams goto err; 248309d09e04SDan Williams 248409d09e04SDan Williams rc = device_add(dev); 248509d09e04SDan Williams if (rc) 248609d09e04SDan Williams goto err; 248709d09e04SDan Williams 248809d09e04SDan Williams dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 248909d09e04SDan Williams dev_name(dev)); 249009d09e04SDan Williams 249109d09e04SDan Williams return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister, 249209d09e04SDan Williams cxlr_dax); 249309d09e04SDan Williams err: 249409d09e04SDan Williams put_device(dev); 249509d09e04SDan Williams return rc; 249609d09e04SDan Williams } 249709d09e04SDan Williams 2498a32320b7SDan Williams static int match_decoder_by_range(struct device *dev, void *data) 2499a32320b7SDan Williams { 2500a32320b7SDan Williams struct range *r1, *r2 = data; 2501a32320b7SDan Williams struct cxl_root_decoder *cxlrd; 2502a32320b7SDan Williams 2503a32320b7SDan Williams if (!is_root_decoder(dev)) 2504a32320b7SDan Williams return 0; 2505a32320b7SDan Williams 2506a32320b7SDan Williams cxlrd = to_cxl_root_decoder(dev); 2507a32320b7SDan Williams r1 = &cxlrd->cxlsd.cxld.hpa_range; 2508a32320b7SDan Williams return range_contains(r1, r2); 2509a32320b7SDan Williams } 2510a32320b7SDan Williams 2511a32320b7SDan Williams static int match_region_by_range(struct device *dev, void *data) 2512a32320b7SDan Williams { 2513a32320b7SDan Williams struct cxl_region_params *p; 2514a32320b7SDan Williams struct cxl_region *cxlr; 2515a32320b7SDan Williams struct range *r = data; 2516a32320b7SDan Williams int rc = 0; 2517a32320b7SDan Williams 2518a32320b7SDan Williams if (!is_cxl_region(dev)) 2519a32320b7SDan Williams return 0; 2520a32320b7SDan Williams 2521a32320b7SDan Williams cxlr = to_cxl_region(dev); 2522a32320b7SDan Williams p = &cxlr->params; 2523a32320b7SDan Williams 2524a32320b7SDan Williams down_read(&cxl_region_rwsem); 2525a32320b7SDan Williams if (p->res && p->res->start == r->start && p->res->end == r->end) 2526a32320b7SDan Williams rc = 1; 2527a32320b7SDan Williams up_read(&cxl_region_rwsem); 2528a32320b7SDan Williams 2529a32320b7SDan Williams return rc; 2530a32320b7SDan Williams } 2531a32320b7SDan Williams 2532a32320b7SDan Williams /* Establish an empty region covering the given HPA range */ 2533a32320b7SDan Williams static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, 2534a32320b7SDan Williams struct cxl_endpoint_decoder *cxled) 2535a32320b7SDan Williams { 2536a32320b7SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 2537a32320b7SDan Williams struct cxl_port *port = cxlrd_to_port(cxlrd); 2538a32320b7SDan Williams struct range *hpa = &cxled->cxld.hpa_range; 2539a32320b7SDan Williams struct cxl_region_params *p; 2540a32320b7SDan Williams struct cxl_region *cxlr; 2541a32320b7SDan Williams struct resource *res; 2542a32320b7SDan Williams int rc; 2543a32320b7SDan Williams 2544a32320b7SDan Williams do { 2545a32320b7SDan Williams cxlr = __create_region(cxlrd, cxled->mode, 2546a32320b7SDan Williams atomic_read(&cxlrd->region_id)); 2547a32320b7SDan Williams } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY); 2548a32320b7SDan Williams 2549a32320b7SDan Williams if (IS_ERR(cxlr)) { 2550a32320b7SDan Williams dev_err(cxlmd->dev.parent, 2551a32320b7SDan Williams "%s:%s: %s failed assign region: %ld\n", 2552a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 2553a32320b7SDan Williams __func__, PTR_ERR(cxlr)); 2554a32320b7SDan Williams return cxlr; 2555a32320b7SDan Williams } 2556a32320b7SDan Williams 2557a32320b7SDan Williams down_write(&cxl_region_rwsem); 2558a32320b7SDan Williams p = &cxlr->params; 2559a32320b7SDan Williams if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 2560a32320b7SDan Williams dev_err(cxlmd->dev.parent, 2561a32320b7SDan Williams "%s:%s: %s autodiscovery interrupted\n", 2562a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 2563a32320b7SDan Williams __func__); 2564a32320b7SDan Williams rc = -EBUSY; 2565a32320b7SDan Williams goto err; 2566a32320b7SDan Williams } 2567a32320b7SDan Williams 2568a32320b7SDan Williams set_bit(CXL_REGION_F_AUTO, &cxlr->flags); 2569a32320b7SDan Williams 2570a32320b7SDan Williams res = kmalloc(sizeof(*res), GFP_KERNEL); 2571a32320b7SDan Williams if (!res) { 2572a32320b7SDan Williams rc = -ENOMEM; 2573a32320b7SDan Williams goto err; 2574a32320b7SDan Williams } 2575a32320b7SDan Williams 2576a32320b7SDan Williams *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa), 2577a32320b7SDan Williams dev_name(&cxlr->dev)); 2578a32320b7SDan Williams rc = insert_resource(cxlrd->res, res); 2579a32320b7SDan Williams if (rc) { 2580a32320b7SDan Williams /* 2581a32320b7SDan Williams * Platform-firmware may not have split resources like "System 2582a32320b7SDan Williams * RAM" on CXL window boundaries see cxl_region_iomem_release() 2583a32320b7SDan Williams */ 2584a32320b7SDan Williams dev_warn(cxlmd->dev.parent, 2585a32320b7SDan Williams "%s:%s: %s %s cannot insert resource\n", 2586a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 2587a32320b7SDan Williams __func__, dev_name(&cxlr->dev)); 2588a32320b7SDan Williams } 2589a32320b7SDan Williams 2590a32320b7SDan Williams p->res = res; 2591a32320b7SDan Williams p->interleave_ways = cxled->cxld.interleave_ways; 2592a32320b7SDan Williams p->interleave_granularity = cxled->cxld.interleave_granularity; 2593a32320b7SDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 2594a32320b7SDan Williams 2595a32320b7SDan Williams rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 2596a32320b7SDan Williams if (rc) 2597a32320b7SDan Williams goto err; 2598a32320b7SDan Williams 2599a32320b7SDan Williams dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n", 2600a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__, 2601a32320b7SDan Williams dev_name(&cxlr->dev), p->res, p->interleave_ways, 2602a32320b7SDan Williams p->interleave_granularity); 2603a32320b7SDan Williams 2604a32320b7SDan Williams /* ...to match put_device() in cxl_add_to_region() */ 2605a32320b7SDan Williams get_device(&cxlr->dev); 2606a32320b7SDan Williams up_write(&cxl_region_rwsem); 2607a32320b7SDan Williams 2608a32320b7SDan Williams return cxlr; 2609a32320b7SDan Williams 2610a32320b7SDan Williams err: 2611a32320b7SDan Williams up_write(&cxl_region_rwsem); 2612a32320b7SDan Williams devm_release_action(port->uport, unregister_region, cxlr); 2613a32320b7SDan Williams return ERR_PTR(rc); 2614a32320b7SDan Williams } 2615a32320b7SDan Williams 2616a32320b7SDan Williams int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled) 2617a32320b7SDan Williams { 2618a32320b7SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 2619a32320b7SDan Williams struct range *hpa = &cxled->cxld.hpa_range; 2620a32320b7SDan Williams struct cxl_decoder *cxld = &cxled->cxld; 2621a32320b7SDan Williams struct device *cxlrd_dev, *region_dev; 2622a32320b7SDan Williams struct cxl_root_decoder *cxlrd; 2623a32320b7SDan Williams struct cxl_region_params *p; 2624a32320b7SDan Williams struct cxl_region *cxlr; 2625a32320b7SDan Williams bool attach = false; 2626a32320b7SDan Williams int rc; 2627a32320b7SDan Williams 2628a32320b7SDan Williams cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range, 2629a32320b7SDan Williams match_decoder_by_range); 2630a32320b7SDan Williams if (!cxlrd_dev) { 2631a32320b7SDan Williams dev_err(cxlmd->dev.parent, 2632a32320b7SDan Williams "%s:%s no CXL window for range %#llx:%#llx\n", 2633a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxld->dev), 2634a32320b7SDan Williams cxld->hpa_range.start, cxld->hpa_range.end); 2635a32320b7SDan Williams return -ENXIO; 2636a32320b7SDan Williams } 2637a32320b7SDan Williams 2638a32320b7SDan Williams cxlrd = to_cxl_root_decoder(cxlrd_dev); 2639a32320b7SDan Williams 2640a32320b7SDan Williams /* 2641a32320b7SDan Williams * Ensure that if multiple threads race to construct_region() for @hpa 2642a32320b7SDan Williams * one does the construction and the others add to that. 2643a32320b7SDan Williams */ 2644a32320b7SDan Williams mutex_lock(&cxlrd->range_lock); 2645a32320b7SDan Williams region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa, 2646a32320b7SDan Williams match_region_by_range); 2647a32320b7SDan Williams if (!region_dev) { 2648a32320b7SDan Williams cxlr = construct_region(cxlrd, cxled); 2649a32320b7SDan Williams region_dev = &cxlr->dev; 2650a32320b7SDan Williams } else 2651a32320b7SDan Williams cxlr = to_cxl_region(region_dev); 2652a32320b7SDan Williams mutex_unlock(&cxlrd->range_lock); 2653a32320b7SDan Williams 26547abcb0b1SArnd Bergmann rc = PTR_ERR_OR_ZERO(cxlr); 26557abcb0b1SArnd Bergmann if (rc) 2656a32320b7SDan Williams goto out; 2657a32320b7SDan Williams 2658a32320b7SDan Williams attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE); 2659a32320b7SDan Williams 2660a32320b7SDan Williams down_read(&cxl_region_rwsem); 2661a32320b7SDan Williams p = &cxlr->params; 2662a32320b7SDan Williams attach = p->state == CXL_CONFIG_COMMIT; 2663a32320b7SDan Williams up_read(&cxl_region_rwsem); 2664a32320b7SDan Williams 2665a32320b7SDan Williams if (attach) { 2666a32320b7SDan Williams /* 2667a32320b7SDan Williams * If device_attach() fails the range may still be active via 2668a32320b7SDan Williams * the platform-firmware memory map, otherwise the driver for 2669a32320b7SDan Williams * regions is local to this file, so driver matching can't fail. 2670a32320b7SDan Williams */ 2671a32320b7SDan Williams if (device_attach(&cxlr->dev) < 0) 2672a32320b7SDan Williams dev_err(&cxlr->dev, "failed to enable, range: %pr\n", 2673a32320b7SDan Williams p->res); 2674a32320b7SDan Williams } 2675a32320b7SDan Williams 2676a32320b7SDan Williams put_device(region_dev); 2677a32320b7SDan Williams out: 2678a32320b7SDan Williams put_device(cxlrd_dev); 2679a32320b7SDan Williams return rc; 2680a32320b7SDan Williams } 2681a32320b7SDan Williams EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL); 2682a32320b7SDan Williams 2683d18bc74aSDan Williams static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) 2684d18bc74aSDan Williams { 2685d18bc74aSDan Williams if (!test_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags)) 2686d18bc74aSDan Williams return 0; 2687d18bc74aSDan Williams 2688d18bc74aSDan Williams if (!cpu_cache_has_invalidate_memregion()) { 2689d18bc74aSDan Williams if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { 2690e520d52dSDavidlohr Bueso dev_warn_once( 2691d18bc74aSDan Williams &cxlr->dev, 2692cb4cdf74SColin Ian King "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); 2693d18bc74aSDan Williams clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags); 2694d18bc74aSDan Williams return 0; 2695d18bc74aSDan Williams } else { 2696d18bc74aSDan Williams dev_err(&cxlr->dev, 2697d18bc74aSDan Williams "Failed to synchronize CPU cache state\n"); 2698d18bc74aSDan Williams return -ENXIO; 2699d18bc74aSDan Williams } 2700d18bc74aSDan Williams } 2701d18bc74aSDan Williams 2702d18bc74aSDan Williams cpu_cache_invalidate_memregion(IORES_DESC_CXL); 2703d18bc74aSDan Williams clear_bit(CXL_REGION_F_INCOHERENT, &cxlr->flags); 2704d18bc74aSDan Williams return 0; 2705d18bc74aSDan Williams } 2706d18bc74aSDan Williams 2707a32320b7SDan Williams static int is_system_ram(struct resource *res, void *arg) 2708a32320b7SDan Williams { 2709a32320b7SDan Williams struct cxl_region *cxlr = arg; 2710a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params; 2711a32320b7SDan Williams 2712a32320b7SDan Williams dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res); 2713a32320b7SDan Williams return 1; 2714a32320b7SDan Williams } 2715a32320b7SDan Williams 27168d48817dSDan Williams static int cxl_region_probe(struct device *dev) 27178d48817dSDan Williams { 27188d48817dSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 27198d48817dSDan Williams struct cxl_region_params *p = &cxlr->params; 27208d48817dSDan Williams int rc; 27218d48817dSDan Williams 27228d48817dSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 27238d48817dSDan Williams if (rc) { 27248d48817dSDan Williams dev_dbg(&cxlr->dev, "probe interrupted\n"); 27258d48817dSDan Williams return rc; 27268d48817dSDan Williams } 27278d48817dSDan Williams 27288d48817dSDan Williams if (p->state < CXL_CONFIG_COMMIT) { 27298d48817dSDan Williams dev_dbg(&cxlr->dev, "config state: %d\n", p->state); 27308d48817dSDan Williams rc = -ENXIO; 2731d18bc74aSDan Williams goto out; 27328d48817dSDan Williams } 27338d48817dSDan Williams 2734d18bc74aSDan Williams rc = cxl_region_invalidate_memregion(cxlr); 2735d18bc74aSDan Williams 27368d48817dSDan Williams /* 27378d48817dSDan Williams * From this point on any path that changes the region's state away from 27388d48817dSDan Williams * CXL_CONFIG_COMMIT is also responsible for releasing the driver. 27398d48817dSDan Williams */ 2740d18bc74aSDan Williams out: 27418d48817dSDan Williams up_read(&cxl_region_rwsem); 27428d48817dSDan Williams 2743bf3e5da8SDan Williams if (rc) 2744bf3e5da8SDan Williams return rc; 2745bf3e5da8SDan Williams 274604ad63f0SDan Williams switch (cxlr->mode) { 274704ad63f0SDan Williams case CXL_DECODER_PMEM: 274804ad63f0SDan Williams return devm_cxl_add_pmem_region(cxlr); 2749a32320b7SDan Williams case CXL_DECODER_RAM: 2750a32320b7SDan Williams /* 2751a32320b7SDan Williams * The region can not be manged by CXL if any portion of 2752a32320b7SDan Williams * it is already online as 'System RAM' 2753a32320b7SDan Williams */ 2754a32320b7SDan Williams if (walk_iomem_res_desc(IORES_DESC_NONE, 2755a32320b7SDan Williams IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY, 2756a32320b7SDan Williams p->res->start, p->res->end, cxlr, 2757a32320b7SDan Williams is_system_ram) > 0) 2758a32320b7SDan Williams return 0; 275909d09e04SDan Williams return devm_cxl_add_dax_region(cxlr); 276004ad63f0SDan Williams default: 276104ad63f0SDan Williams dev_dbg(&cxlr->dev, "unsupported region mode: %d\n", 276204ad63f0SDan Williams cxlr->mode); 276304ad63f0SDan Williams return -ENXIO; 276404ad63f0SDan Williams } 27658d48817dSDan Williams } 27668d48817dSDan Williams 27678d48817dSDan Williams static struct cxl_driver cxl_region_driver = { 27688d48817dSDan Williams .name = "cxl_region", 27698d48817dSDan Williams .probe = cxl_region_probe, 27708d48817dSDan Williams .id = CXL_DEVICE_REGION, 27718d48817dSDan Williams }; 27728d48817dSDan Williams 27738d48817dSDan Williams int cxl_region_init(void) 27748d48817dSDan Williams { 27758d48817dSDan Williams return cxl_driver_register(&cxl_region_driver); 27768d48817dSDan Williams } 27778d48817dSDan Williams 27788d48817dSDan Williams void cxl_region_exit(void) 27798d48817dSDan Williams { 27808d48817dSDan Williams cxl_driver_unregister(&cxl_region_driver); 27818d48817dSDan Williams } 27828d48817dSDan Williams 278323a22cd1SDan Williams MODULE_IMPORT_NS(CXL); 2784d18bc74aSDan Williams MODULE_IMPORT_NS(DEVMEM); 27858d48817dSDan Williams MODULE_ALIAS_CXL(CXL_DEVICE_REGION); 2786