1779dd20cSBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 2779dd20cSBen Widawsky /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3779dd20cSBen Widawsky #include <linux/memregion.h> 4779dd20cSBen Widawsky #include <linux/genalloc.h> 5779dd20cSBen Widawsky #include <linux/device.h> 6779dd20cSBen Widawsky #include <linux/module.h> 7779dd20cSBen Widawsky #include <linux/slab.h> 8dd5ba0ebSBen Widawsky #include <linux/uuid.h> 9a32320b7SDan Williams #include <linux/sort.h> 10779dd20cSBen Widawsky #include <linux/idr.h> 1180d10a6cSBen Widawsky #include <cxlmem.h> 12779dd20cSBen Widawsky #include <cxl.h> 13779dd20cSBen Widawsky #include "core.h" 14779dd20cSBen Widawsky 15779dd20cSBen Widawsky /** 16779dd20cSBen Widawsky * DOC: cxl core region 17779dd20cSBen Widawsky * 18779dd20cSBen Widawsky * CXL Regions represent mapped memory capacity in system physical address 19779dd20cSBen Widawsky * space. Whereas the CXL Root Decoders identify the bounds of potential CXL 20779dd20cSBen Widawsky * Memory ranges, Regions represent the active mapped capacity by the HDM 21779dd20cSBen Widawsky * Decoder Capability structures throughout the Host Bridges, Switches, and 22779dd20cSBen Widawsky * Endpoints in the topology. 23dd5ba0ebSBen Widawsky * 24dd5ba0ebSBen Widawsky * Region configuration has ordering constraints. UUID may be set at any time 25dd5ba0ebSBen Widawsky * but is only visible for persistent regions. 2680d10a6cSBen Widawsky * 1. Interleave granularity 2780d10a6cSBen Widawsky * 2. Interleave size 28b9686e8cSDan Williams * 3. Decoder targets 29779dd20cSBen Widawsky */ 30779dd20cSBen Widawsky 31779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev); 32779dd20cSBen Widawsky 33dd5ba0ebSBen Widawsky static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, 34dd5ba0ebSBen Widawsky char *buf) 35dd5ba0ebSBen Widawsky { 36dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 37dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params; 38dd5ba0ebSBen Widawsky ssize_t rc; 39dd5ba0ebSBen Widawsky 40dd5ba0ebSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 41dd5ba0ebSBen Widawsky if (rc) 42dd5ba0ebSBen Widawsky return rc; 43a8e7d558SDan Williams if (cxlr->mode != CXL_DECODER_PMEM) 44a8e7d558SDan Williams rc = sysfs_emit(buf, "\n"); 45a8e7d558SDan Williams else 46dd5ba0ebSBen Widawsky rc = sysfs_emit(buf, "%pUb\n", &p->uuid); 47dd5ba0ebSBen Widawsky up_read(&cxl_region_rwsem); 48dd5ba0ebSBen Widawsky 49dd5ba0ebSBen Widawsky return rc; 50dd5ba0ebSBen Widawsky } 51dd5ba0ebSBen Widawsky 52dd5ba0ebSBen Widawsky static int is_dup(struct device *match, void *data) 53dd5ba0ebSBen Widawsky { 54dd5ba0ebSBen Widawsky struct cxl_region_params *p; 55dd5ba0ebSBen Widawsky struct cxl_region *cxlr; 56dd5ba0ebSBen Widawsky uuid_t *uuid = data; 57dd5ba0ebSBen Widawsky 58dd5ba0ebSBen Widawsky if (!is_cxl_region(match)) 59dd5ba0ebSBen Widawsky return 0; 60dd5ba0ebSBen Widawsky 61dd5ba0ebSBen Widawsky lockdep_assert_held(&cxl_region_rwsem); 62dd5ba0ebSBen Widawsky cxlr = to_cxl_region(match); 63dd5ba0ebSBen Widawsky p = &cxlr->params; 64dd5ba0ebSBen Widawsky 65dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, uuid)) { 66dd5ba0ebSBen Widawsky dev_dbg(match, "already has uuid: %pUb\n", uuid); 67dd5ba0ebSBen Widawsky return -EBUSY; 68dd5ba0ebSBen Widawsky } 69dd5ba0ebSBen Widawsky 70dd5ba0ebSBen Widawsky return 0; 71dd5ba0ebSBen Widawsky } 72dd5ba0ebSBen Widawsky 73dd5ba0ebSBen Widawsky static ssize_t uuid_store(struct device *dev, struct device_attribute *attr, 74dd5ba0ebSBen Widawsky const char *buf, size_t len) 75dd5ba0ebSBen Widawsky { 76dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 77dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params; 78dd5ba0ebSBen Widawsky uuid_t temp; 79dd5ba0ebSBen Widawsky ssize_t rc; 80dd5ba0ebSBen Widawsky 81dd5ba0ebSBen Widawsky if (len != UUID_STRING_LEN + 1) 82dd5ba0ebSBen Widawsky return -EINVAL; 83dd5ba0ebSBen Widawsky 84dd5ba0ebSBen Widawsky rc = uuid_parse(buf, &temp); 85dd5ba0ebSBen Widawsky if (rc) 86dd5ba0ebSBen Widawsky return rc; 87dd5ba0ebSBen Widawsky 88dd5ba0ebSBen Widawsky if (uuid_is_null(&temp)) 89dd5ba0ebSBen Widawsky return -EINVAL; 90dd5ba0ebSBen Widawsky 91dd5ba0ebSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 92dd5ba0ebSBen Widawsky if (rc) 93dd5ba0ebSBen Widawsky return rc; 94dd5ba0ebSBen Widawsky 95dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, &temp)) 96dd5ba0ebSBen Widawsky goto out; 97dd5ba0ebSBen Widawsky 98dd5ba0ebSBen Widawsky rc = -EBUSY; 99dd5ba0ebSBen Widawsky if (p->state >= CXL_CONFIG_ACTIVE) 100dd5ba0ebSBen Widawsky goto out; 101dd5ba0ebSBen Widawsky 102dd5ba0ebSBen Widawsky rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup); 103dd5ba0ebSBen Widawsky if (rc < 0) 104dd5ba0ebSBen Widawsky goto out; 105dd5ba0ebSBen Widawsky 106dd5ba0ebSBen Widawsky uuid_copy(&p->uuid, &temp); 107dd5ba0ebSBen Widawsky out: 108dd5ba0ebSBen Widawsky up_write(&cxl_region_rwsem); 109dd5ba0ebSBen Widawsky 110dd5ba0ebSBen Widawsky if (rc) 111dd5ba0ebSBen Widawsky return rc; 112dd5ba0ebSBen Widawsky return len; 113dd5ba0ebSBen Widawsky } 114dd5ba0ebSBen Widawsky static DEVICE_ATTR_RW(uuid); 115dd5ba0ebSBen Widawsky 116176baefbSDan Williams static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port, 117176baefbSDan Williams struct cxl_region *cxlr) 118176baefbSDan Williams { 119176baefbSDan Williams return xa_load(&port->regions, (unsigned long)cxlr); 120176baefbSDan Williams } 121176baefbSDan Williams 122d1257d09SDan Williams static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) 123d1257d09SDan Williams { 124d1257d09SDan Williams if (!cpu_cache_has_invalidate_memregion()) { 125d1257d09SDan Williams if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { 1267914992bSDan Williams dev_info_once( 127d1257d09SDan Williams &cxlr->dev, 128d1257d09SDan Williams "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); 129d1257d09SDan Williams return 0; 130d1257d09SDan Williams } else { 131d1257d09SDan Williams dev_err(&cxlr->dev, 132d1257d09SDan Williams "Failed to synchronize CPU cache state\n"); 133d1257d09SDan Williams return -ENXIO; 134d1257d09SDan Williams } 135d1257d09SDan Williams } 136d1257d09SDan Williams 137d1257d09SDan Williams cpu_cache_invalidate_memregion(IORES_DESC_CXL); 138d1257d09SDan Williams return 0; 139d1257d09SDan Williams } 140d1257d09SDan Williams 141176baefbSDan Williams static int cxl_region_decode_reset(struct cxl_region *cxlr, int count) 142176baefbSDan Williams { 143176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 144d1257d09SDan Williams int i, rc = 0; 145d1257d09SDan Williams 146d1257d09SDan Williams /* 147d1257d09SDan Williams * Before region teardown attempt to flush, and if the flush 148d1257d09SDan Williams * fails cancel the region teardown for data consistency 149d1257d09SDan Williams * concerns 150d1257d09SDan Williams */ 151d1257d09SDan Williams rc = cxl_region_invalidate_memregion(cxlr); 152d1257d09SDan Williams if (rc) 153d1257d09SDan Williams return rc; 154176baefbSDan Williams 155176baefbSDan Williams for (i = count - 1; i >= 0; i--) { 156176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 157176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 158176baefbSDan Williams struct cxl_port *iter = cxled_to_port(cxled); 159030f8803SDan Williams struct cxl_dev_state *cxlds = cxlmd->cxlds; 160176baefbSDan Williams struct cxl_ep *ep; 161176baefbSDan Williams 162030f8803SDan Williams if (cxlds->rcd) 163030f8803SDan Williams goto endpoint_reset; 164030f8803SDan Williams 165176baefbSDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 166176baefbSDan Williams iter = to_cxl_port(iter->dev.parent); 167176baefbSDan Williams 168176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 169176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 170176baefbSDan Williams struct cxl_region_ref *cxl_rr; 171176baefbSDan Williams struct cxl_decoder *cxld; 172176baefbSDan Williams 173176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 174176baefbSDan Williams cxld = cxl_rr->decoder; 1754fa4302dSFan Ni if (cxld->reset) 176176baefbSDan Williams rc = cxld->reset(cxld); 177176baefbSDan Williams if (rc) 178176baefbSDan Williams return rc; 1792ab47045SDan Williams set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 180176baefbSDan Williams } 181176baefbSDan Williams 182030f8803SDan Williams endpoint_reset: 183176baefbSDan Williams rc = cxled->cxld.reset(&cxled->cxld); 184176baefbSDan Williams if (rc) 185176baefbSDan Williams return rc; 1862ab47045SDan Williams set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 187176baefbSDan Williams } 188176baefbSDan Williams 1892ab47045SDan Williams /* all decoders associated with this region have been torn down */ 1902ab47045SDan Williams clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 1912ab47045SDan Williams 192176baefbSDan Williams return 0; 193176baefbSDan Williams } 194176baefbSDan Williams 195af3ea9abSDan Williams static int commit_decoder(struct cxl_decoder *cxld) 196af3ea9abSDan Williams { 197af3ea9abSDan Williams struct cxl_switch_decoder *cxlsd = NULL; 198af3ea9abSDan Williams 199af3ea9abSDan Williams if (cxld->commit) 200af3ea9abSDan Williams return cxld->commit(cxld); 201af3ea9abSDan Williams 202af3ea9abSDan Williams if (is_switch_decoder(&cxld->dev)) 203af3ea9abSDan Williams cxlsd = to_cxl_switch_decoder(&cxld->dev); 204af3ea9abSDan Williams 205af3ea9abSDan Williams if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1, 206af3ea9abSDan Williams "->commit() is required\n")) 207af3ea9abSDan Williams return -ENXIO; 208af3ea9abSDan Williams return 0; 209af3ea9abSDan Williams } 210af3ea9abSDan Williams 211176baefbSDan Williams static int cxl_region_decode_commit(struct cxl_region *cxlr) 212176baefbSDan Williams { 213176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 21469c99613SDan Williams int i, rc = 0; 215176baefbSDan Williams 216176baefbSDan Williams for (i = 0; i < p->nr_targets; i++) { 217176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 218176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 219176baefbSDan Williams struct cxl_region_ref *cxl_rr; 220176baefbSDan Williams struct cxl_decoder *cxld; 221176baefbSDan Williams struct cxl_port *iter; 222176baefbSDan Williams struct cxl_ep *ep; 223176baefbSDan Williams 224176baefbSDan Williams /* commit bottom up */ 225176baefbSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 226176baefbSDan Williams iter = to_cxl_port(iter->dev.parent)) { 227176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 228176baefbSDan Williams cxld = cxl_rr->decoder; 229af3ea9abSDan Williams rc = commit_decoder(cxld); 230176baefbSDan Williams if (rc) 231176baefbSDan Williams break; 232176baefbSDan Williams } 233176baefbSDan Williams 23469c99613SDan Williams if (rc) { 235176baefbSDan Williams /* programming @iter failed, teardown */ 236176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); ep && iter; 237176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 238176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 239176baefbSDan Williams cxld = cxl_rr->decoder; 2404fa4302dSFan Ni if (cxld->reset) 241176baefbSDan Williams cxld->reset(cxld); 242176baefbSDan Williams } 243176baefbSDan Williams 244176baefbSDan Williams cxled->cxld.reset(&cxled->cxld); 24569c99613SDan Williams goto err; 24669c99613SDan Williams } 247176baefbSDan Williams } 248176baefbSDan Williams 249176baefbSDan Williams return 0; 250176baefbSDan Williams 25169c99613SDan Williams err: 252176baefbSDan Williams /* undo the targets that were successfully committed */ 253176baefbSDan Williams cxl_region_decode_reset(cxlr, i); 254176baefbSDan Williams return rc; 255176baefbSDan Williams } 256176baefbSDan Williams 257176baefbSDan Williams static ssize_t commit_store(struct device *dev, struct device_attribute *attr, 258176baefbSDan Williams const char *buf, size_t len) 259176baefbSDan Williams { 260176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 261176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 262176baefbSDan Williams bool commit; 263176baefbSDan Williams ssize_t rc; 264176baefbSDan Williams 265176baefbSDan Williams rc = kstrtobool(buf, &commit); 266176baefbSDan Williams if (rc) 267176baefbSDan Williams return rc; 268176baefbSDan Williams 269176baefbSDan Williams rc = down_write_killable(&cxl_region_rwsem); 270176baefbSDan Williams if (rc) 271176baefbSDan Williams return rc; 272176baefbSDan Williams 273176baefbSDan Williams /* Already in the requested state? */ 274176baefbSDan Williams if (commit && p->state >= CXL_CONFIG_COMMIT) 275176baefbSDan Williams goto out; 276176baefbSDan Williams if (!commit && p->state < CXL_CONFIG_COMMIT) 277176baefbSDan Williams goto out; 278176baefbSDan Williams 279176baefbSDan Williams /* Not ready to commit? */ 280176baefbSDan Williams if (commit && p->state < CXL_CONFIG_ACTIVE) { 281176baefbSDan Williams rc = -ENXIO; 282176baefbSDan Williams goto out; 283176baefbSDan Williams } 284176baefbSDan Williams 285d1257d09SDan Williams /* 286d1257d09SDan Williams * Invalidate caches before region setup to drop any speculative 287d1257d09SDan Williams * consumption of this address space 288d1257d09SDan Williams */ 289d1257d09SDan Williams rc = cxl_region_invalidate_memregion(cxlr); 290d1257d09SDan Williams if (rc) 2913531b27fSLi Zhijian goto out; 292d1257d09SDan Williams 293adfe1973SDan Williams if (commit) { 294176baefbSDan Williams rc = cxl_region_decode_commit(cxlr); 295adfe1973SDan Williams if (rc == 0) 296adfe1973SDan Williams p->state = CXL_CONFIG_COMMIT; 297adfe1973SDan Williams } else { 298176baefbSDan Williams p->state = CXL_CONFIG_RESET_PENDING; 299176baefbSDan Williams up_write(&cxl_region_rwsem); 300176baefbSDan Williams device_release_driver(&cxlr->dev); 301176baefbSDan Williams down_write(&cxl_region_rwsem); 302176baefbSDan Williams 303176baefbSDan Williams /* 304176baefbSDan Williams * The lock was dropped, so need to revalidate that the reset is 305176baefbSDan Williams * still pending. 306176baefbSDan Williams */ 307adfe1973SDan Williams if (p->state == CXL_CONFIG_RESET_PENDING) { 308176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 309adfe1973SDan Williams /* 310adfe1973SDan Williams * Revert to committed since there may still be active 311adfe1973SDan Williams * decoders associated with this region, or move forward 312adfe1973SDan Williams * to active to mark the reset successful 313adfe1973SDan Williams */ 314176baefbSDan Williams if (rc) 315176baefbSDan Williams p->state = CXL_CONFIG_COMMIT; 316adfe1973SDan Williams else 317176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE; 318adfe1973SDan Williams } 319adfe1973SDan Williams } 320176baefbSDan Williams 321176baefbSDan Williams out: 322176baefbSDan Williams up_write(&cxl_region_rwsem); 323176baefbSDan Williams 324176baefbSDan Williams if (rc) 325176baefbSDan Williams return rc; 326176baefbSDan Williams return len; 327176baefbSDan Williams } 328176baefbSDan Williams 329176baefbSDan Williams static ssize_t commit_show(struct device *dev, struct device_attribute *attr, 330176baefbSDan Williams char *buf) 331176baefbSDan Williams { 332176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 333176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 334176baefbSDan Williams ssize_t rc; 335176baefbSDan Williams 336176baefbSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 337176baefbSDan Williams if (rc) 338176baefbSDan Williams return rc; 339176baefbSDan Williams rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT); 340176baefbSDan Williams up_read(&cxl_region_rwsem); 341176baefbSDan Williams 342176baefbSDan Williams return rc; 343176baefbSDan Williams } 344176baefbSDan Williams static DEVICE_ATTR_RW(commit); 345176baefbSDan Williams 346dd5ba0ebSBen Widawsky static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a, 347dd5ba0ebSBen Widawsky int n) 348dd5ba0ebSBen Widawsky { 349dd5ba0ebSBen Widawsky struct device *dev = kobj_to_dev(kobj); 350dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 351dd5ba0ebSBen Widawsky 352a8e7d558SDan Williams /* 353a8e7d558SDan Williams * Support tooling that expects to find a 'uuid' attribute for all 354a8e7d558SDan Williams * regions regardless of mode. 355a8e7d558SDan Williams */ 356dd5ba0ebSBen Widawsky if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM) 357a8e7d558SDan Williams return 0444; 358dd5ba0ebSBen Widawsky return a->mode; 359dd5ba0ebSBen Widawsky } 360dd5ba0ebSBen Widawsky 36180d10a6cSBen Widawsky static ssize_t interleave_ways_show(struct device *dev, 36280d10a6cSBen Widawsky struct device_attribute *attr, char *buf) 36380d10a6cSBen Widawsky { 36480d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 36580d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 36680d10a6cSBen Widawsky ssize_t rc; 36780d10a6cSBen Widawsky 36880d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 36980d10a6cSBen Widawsky if (rc) 37080d10a6cSBen Widawsky return rc; 37180d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_ways); 37280d10a6cSBen Widawsky up_read(&cxl_region_rwsem); 37380d10a6cSBen Widawsky 37480d10a6cSBen Widawsky return rc; 37580d10a6cSBen Widawsky } 37680d10a6cSBen Widawsky 377b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void); 378b9686e8cSDan Williams 37980d10a6cSBen Widawsky static ssize_t interleave_ways_store(struct device *dev, 38080d10a6cSBen Widawsky struct device_attribute *attr, 38180d10a6cSBen Widawsky const char *buf, size_t len) 38280d10a6cSBen Widawsky { 38380d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 38480d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 38580d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 38680d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 387c7e3548cSDan Carpenter unsigned int val, save; 388c7e3548cSDan Carpenter int rc; 38980d10a6cSBen Widawsky u8 iw; 39080d10a6cSBen Widawsky 391c7e3548cSDan Carpenter rc = kstrtouint(buf, 0, &val); 39280d10a6cSBen Widawsky if (rc) 39380d10a6cSBen Widawsky return rc; 39480d10a6cSBen Widawsky 395c99b2e8cSDave Jiang rc = ways_to_eiw(val, &iw); 39680d10a6cSBen Widawsky if (rc) 39780d10a6cSBen Widawsky return rc; 39880d10a6cSBen Widawsky 39980d10a6cSBen Widawsky /* 400*c7ad3dc3SJim Harris * Even for x3, x6, and x12 interleaves the region interleave must be a 40180d10a6cSBen Widawsky * power of 2 multiple of the host bridge interleave. 40280d10a6cSBen Widawsky */ 40380d10a6cSBen Widawsky if (!is_power_of_2(val / cxld->interleave_ways) || 40480d10a6cSBen Widawsky (val % cxld->interleave_ways)) { 40580d10a6cSBen Widawsky dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val); 40680d10a6cSBen Widawsky return -EINVAL; 40780d10a6cSBen Widawsky } 40880d10a6cSBen Widawsky 40980d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 41080d10a6cSBen Widawsky if (rc) 41180d10a6cSBen Widawsky return rc; 41280d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 41380d10a6cSBen Widawsky rc = -EBUSY; 41480d10a6cSBen Widawsky goto out; 41580d10a6cSBen Widawsky } 41680d10a6cSBen Widawsky 417b9686e8cSDan Williams save = p->interleave_ways; 41880d10a6cSBen Widawsky p->interleave_ways = val; 419b9686e8cSDan Williams rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 420b9686e8cSDan Williams if (rc) 421b9686e8cSDan Williams p->interleave_ways = save; 42280d10a6cSBen Widawsky out: 42380d10a6cSBen Widawsky up_write(&cxl_region_rwsem); 42480d10a6cSBen Widawsky if (rc) 42580d10a6cSBen Widawsky return rc; 42680d10a6cSBen Widawsky return len; 42780d10a6cSBen Widawsky } 42880d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_ways); 42980d10a6cSBen Widawsky 43080d10a6cSBen Widawsky static ssize_t interleave_granularity_show(struct device *dev, 43180d10a6cSBen Widawsky struct device_attribute *attr, 43280d10a6cSBen Widawsky char *buf) 43380d10a6cSBen Widawsky { 43480d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 43580d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 43680d10a6cSBen Widawsky ssize_t rc; 43780d10a6cSBen Widawsky 43880d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 43980d10a6cSBen Widawsky if (rc) 44080d10a6cSBen Widawsky return rc; 44180d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_granularity); 44280d10a6cSBen Widawsky up_read(&cxl_region_rwsem); 44380d10a6cSBen Widawsky 44480d10a6cSBen Widawsky return rc; 44580d10a6cSBen Widawsky } 44680d10a6cSBen Widawsky 44780d10a6cSBen Widawsky static ssize_t interleave_granularity_store(struct device *dev, 44880d10a6cSBen Widawsky struct device_attribute *attr, 44980d10a6cSBen Widawsky const char *buf, size_t len) 45080d10a6cSBen Widawsky { 45180d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 45280d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 45380d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 45480d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 45580d10a6cSBen Widawsky int rc, val; 45680d10a6cSBen Widawsky u16 ig; 45780d10a6cSBen Widawsky 45880d10a6cSBen Widawsky rc = kstrtoint(buf, 0, &val); 45980d10a6cSBen Widawsky if (rc) 46080d10a6cSBen Widawsky return rc; 46180d10a6cSBen Widawsky 46283351ddbSDave Jiang rc = granularity_to_eig(val, &ig); 46380d10a6cSBen Widawsky if (rc) 46480d10a6cSBen Widawsky return rc; 46580d10a6cSBen Widawsky 46680d10a6cSBen Widawsky /* 4674d8e4ea5SDan Williams * When the host-bridge is interleaved, disallow region granularity != 4684d8e4ea5SDan Williams * root granularity. Regions with a granularity less than the root 4694d8e4ea5SDan Williams * interleave result in needing multiple endpoints to support a single 470cbbd05d0SRandy Dunlap * slot in the interleave (possible to support in the future). Regions 4714d8e4ea5SDan Williams * with a granularity greater than the root interleave result in invalid 4724d8e4ea5SDan Williams * DPA translations (invalid to support). 47380d10a6cSBen Widawsky */ 4744d8e4ea5SDan Williams if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity) 47580d10a6cSBen Widawsky return -EINVAL; 47680d10a6cSBen Widawsky 47780d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 47880d10a6cSBen Widawsky if (rc) 47980d10a6cSBen Widawsky return rc; 48080d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 48180d10a6cSBen Widawsky rc = -EBUSY; 48280d10a6cSBen Widawsky goto out; 48380d10a6cSBen Widawsky } 48480d10a6cSBen Widawsky 48580d10a6cSBen Widawsky p->interleave_granularity = val; 48680d10a6cSBen Widawsky out: 48780d10a6cSBen Widawsky up_write(&cxl_region_rwsem); 48880d10a6cSBen Widawsky if (rc) 48980d10a6cSBen Widawsky return rc; 49080d10a6cSBen Widawsky return len; 49180d10a6cSBen Widawsky } 49280d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_granularity); 49380d10a6cSBen Widawsky 49423a22cd1SDan Williams static ssize_t resource_show(struct device *dev, struct device_attribute *attr, 49523a22cd1SDan Williams char *buf) 49623a22cd1SDan Williams { 49723a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 49823a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 49923a22cd1SDan Williams u64 resource = -1ULL; 50023a22cd1SDan Williams ssize_t rc; 50123a22cd1SDan Williams 50223a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 50323a22cd1SDan Williams if (rc) 50423a22cd1SDan Williams return rc; 50523a22cd1SDan Williams if (p->res) 50623a22cd1SDan Williams resource = p->res->start; 50723a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", resource); 50823a22cd1SDan Williams up_read(&cxl_region_rwsem); 50923a22cd1SDan Williams 51023a22cd1SDan Williams return rc; 51123a22cd1SDan Williams } 51223a22cd1SDan Williams static DEVICE_ATTR_RO(resource); 51323a22cd1SDan Williams 5147d505f98SDan Williams static ssize_t mode_show(struct device *dev, struct device_attribute *attr, 5157d505f98SDan Williams char *buf) 5167d505f98SDan Williams { 5177d505f98SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 5187d505f98SDan Williams 5197d505f98SDan Williams return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode)); 5207d505f98SDan Williams } 5217d505f98SDan Williams static DEVICE_ATTR_RO(mode); 5227d505f98SDan Williams 52323a22cd1SDan Williams static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) 52423a22cd1SDan Williams { 52523a22cd1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 52623a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 52723a22cd1SDan Williams struct resource *res; 52823a22cd1SDan Williams u32 remainder = 0; 52923a22cd1SDan Williams 53023a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 53123a22cd1SDan Williams 53223a22cd1SDan Williams /* Nothing to do... */ 53388ab1ddeSDan Carpenter if (p->res && resource_size(p->res) == size) 53423a22cd1SDan Williams return 0; 53523a22cd1SDan Williams 53623a22cd1SDan Williams /* To change size the old size must be freed first */ 53723a22cd1SDan Williams if (p->res) 53823a22cd1SDan Williams return -EBUSY; 53923a22cd1SDan Williams 54023a22cd1SDan Williams if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) 54123a22cd1SDan Williams return -EBUSY; 54223a22cd1SDan Williams 54323a22cd1SDan Williams /* ways, granularity and uuid (if PMEM) need to be set before HPA */ 54423a22cd1SDan Williams if (!p->interleave_ways || !p->interleave_granularity || 54523a22cd1SDan Williams (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid))) 54623a22cd1SDan Williams return -ENXIO; 54723a22cd1SDan Williams 54823a22cd1SDan Williams div_u64_rem(size, SZ_256M * p->interleave_ways, &remainder); 54923a22cd1SDan Williams if (remainder) 55023a22cd1SDan Williams return -EINVAL; 55123a22cd1SDan Williams 55223a22cd1SDan Williams res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, 55323a22cd1SDan Williams dev_name(&cxlr->dev)); 55423a22cd1SDan Williams if (IS_ERR(res)) { 55523a22cd1SDan Williams dev_dbg(&cxlr->dev, "failed to allocate HPA: %ld\n", 55623a22cd1SDan Williams PTR_ERR(res)); 55723a22cd1SDan Williams return PTR_ERR(res); 55823a22cd1SDan Williams } 55923a22cd1SDan Williams 56023a22cd1SDan Williams p->res = res; 56123a22cd1SDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 56223a22cd1SDan Williams 56323a22cd1SDan Williams return 0; 56423a22cd1SDan Williams } 56523a22cd1SDan Williams 56623a22cd1SDan Williams static void cxl_region_iomem_release(struct cxl_region *cxlr) 56723a22cd1SDan Williams { 56823a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 56923a22cd1SDan Williams 57023a22cd1SDan Williams if (device_is_registered(&cxlr->dev)) 57123a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 57223a22cd1SDan Williams if (p->res) { 573a32320b7SDan Williams /* 574a32320b7SDan Williams * Autodiscovered regions may not have been able to insert their 575a32320b7SDan Williams * resource. 576a32320b7SDan Williams */ 577a32320b7SDan Williams if (p->res->parent) 57823a22cd1SDan Williams remove_resource(p->res); 57923a22cd1SDan Williams kfree(p->res); 58023a22cd1SDan Williams p->res = NULL; 58123a22cd1SDan Williams } 58223a22cd1SDan Williams } 58323a22cd1SDan Williams 58423a22cd1SDan Williams static int free_hpa(struct cxl_region *cxlr) 58523a22cd1SDan Williams { 58623a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 58723a22cd1SDan Williams 58823a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 58923a22cd1SDan Williams 59023a22cd1SDan Williams if (!p->res) 59123a22cd1SDan Williams return 0; 59223a22cd1SDan Williams 59323a22cd1SDan Williams if (p->state >= CXL_CONFIG_ACTIVE) 59423a22cd1SDan Williams return -EBUSY; 59523a22cd1SDan Williams 59623a22cd1SDan Williams cxl_region_iomem_release(cxlr); 59723a22cd1SDan Williams p->state = CXL_CONFIG_IDLE; 59823a22cd1SDan Williams return 0; 59923a22cd1SDan Williams } 60023a22cd1SDan Williams 60123a22cd1SDan Williams static ssize_t size_store(struct device *dev, struct device_attribute *attr, 60223a22cd1SDan Williams const char *buf, size_t len) 60323a22cd1SDan Williams { 60423a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 60523a22cd1SDan Williams u64 val; 60623a22cd1SDan Williams int rc; 60723a22cd1SDan Williams 60823a22cd1SDan Williams rc = kstrtou64(buf, 0, &val); 60923a22cd1SDan Williams if (rc) 61023a22cd1SDan Williams return rc; 61123a22cd1SDan Williams 61223a22cd1SDan Williams rc = down_write_killable(&cxl_region_rwsem); 61323a22cd1SDan Williams if (rc) 61423a22cd1SDan Williams return rc; 61523a22cd1SDan Williams 61623a22cd1SDan Williams if (val) 61723a22cd1SDan Williams rc = alloc_hpa(cxlr, val); 61823a22cd1SDan Williams else 61923a22cd1SDan Williams rc = free_hpa(cxlr); 62023a22cd1SDan Williams up_write(&cxl_region_rwsem); 62123a22cd1SDan Williams 62223a22cd1SDan Williams if (rc) 62323a22cd1SDan Williams return rc; 62423a22cd1SDan Williams 62523a22cd1SDan Williams return len; 62623a22cd1SDan Williams } 62723a22cd1SDan Williams 62823a22cd1SDan Williams static ssize_t size_show(struct device *dev, struct device_attribute *attr, 62923a22cd1SDan Williams char *buf) 63023a22cd1SDan Williams { 63123a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 63223a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 63323a22cd1SDan Williams u64 size = 0; 63423a22cd1SDan Williams ssize_t rc; 63523a22cd1SDan Williams 63623a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 63723a22cd1SDan Williams if (rc) 63823a22cd1SDan Williams return rc; 63923a22cd1SDan Williams if (p->res) 64023a22cd1SDan Williams size = resource_size(p->res); 64123a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", size); 64223a22cd1SDan Williams up_read(&cxl_region_rwsem); 64323a22cd1SDan Williams 64423a22cd1SDan Williams return rc; 64523a22cd1SDan Williams } 64623a22cd1SDan Williams static DEVICE_ATTR_RW(size); 64723a22cd1SDan Williams 648dd5ba0ebSBen Widawsky static struct attribute *cxl_region_attrs[] = { 649dd5ba0ebSBen Widawsky &dev_attr_uuid.attr, 650176baefbSDan Williams &dev_attr_commit.attr, 65180d10a6cSBen Widawsky &dev_attr_interleave_ways.attr, 65280d10a6cSBen Widawsky &dev_attr_interleave_granularity.attr, 65323a22cd1SDan Williams &dev_attr_resource.attr, 65423a22cd1SDan Williams &dev_attr_size.attr, 6557d505f98SDan Williams &dev_attr_mode.attr, 656dd5ba0ebSBen Widawsky NULL, 657dd5ba0ebSBen Widawsky }; 658dd5ba0ebSBen Widawsky 659dd5ba0ebSBen Widawsky static const struct attribute_group cxl_region_group = { 660dd5ba0ebSBen Widawsky .attrs = cxl_region_attrs, 661dd5ba0ebSBen Widawsky .is_visible = cxl_region_visible, 662dd5ba0ebSBen Widawsky }; 663dd5ba0ebSBen Widawsky 664b9686e8cSDan Williams static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos) 665b9686e8cSDan Williams { 666b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 667b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled; 668b9686e8cSDan Williams int rc; 669b9686e8cSDan Williams 670b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 671b9686e8cSDan Williams if (rc) 672b9686e8cSDan Williams return rc; 673b9686e8cSDan Williams 674b9686e8cSDan Williams if (pos >= p->interleave_ways) { 675b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 676b9686e8cSDan Williams p->interleave_ways); 677b9686e8cSDan Williams rc = -ENXIO; 678b9686e8cSDan Williams goto out; 679b9686e8cSDan Williams } 680b9686e8cSDan Williams 681b9686e8cSDan Williams cxled = p->targets[pos]; 682b9686e8cSDan Williams if (!cxled) 683b9686e8cSDan Williams rc = sysfs_emit(buf, "\n"); 684b9686e8cSDan Williams else 685b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev)); 686b9686e8cSDan Williams out: 687b9686e8cSDan Williams up_read(&cxl_region_rwsem); 688b9686e8cSDan Williams 689b9686e8cSDan Williams return rc; 690b9686e8cSDan Williams } 691b9686e8cSDan Williams 692384e624bSDan Williams static int match_free_decoder(struct device *dev, void *data) 693384e624bSDan Williams { 694384e624bSDan Williams struct cxl_decoder *cxld; 695384e624bSDan Williams int *id = data; 696384e624bSDan Williams 697384e624bSDan Williams if (!is_switch_decoder(dev)) 698384e624bSDan Williams return 0; 699384e624bSDan Williams 700384e624bSDan Williams cxld = to_cxl_decoder(dev); 701384e624bSDan Williams 702384e624bSDan Williams /* enforce ordered allocation */ 703384e624bSDan Williams if (cxld->id != *id) 704384e624bSDan Williams return 0; 705384e624bSDan Williams 706384e624bSDan Williams if (!cxld->region) 707384e624bSDan Williams return 1; 708384e624bSDan Williams 709384e624bSDan Williams (*id)++; 710384e624bSDan Williams 711384e624bSDan Williams return 0; 712384e624bSDan Williams } 713384e624bSDan Williams 7149e4edf1aSAlison Schofield static int match_auto_decoder(struct device *dev, void *data) 7159e4edf1aSAlison Schofield { 7169e4edf1aSAlison Schofield struct cxl_region_params *p = data; 7179e4edf1aSAlison Schofield struct cxl_decoder *cxld; 7189e4edf1aSAlison Schofield struct range *r; 7199e4edf1aSAlison Schofield 7209e4edf1aSAlison Schofield if (!is_switch_decoder(dev)) 7219e4edf1aSAlison Schofield return 0; 7229e4edf1aSAlison Schofield 7239e4edf1aSAlison Schofield cxld = to_cxl_decoder(dev); 7249e4edf1aSAlison Schofield r = &cxld->hpa_range; 7259e4edf1aSAlison Schofield 7269e4edf1aSAlison Schofield if (p->res && p->res->start == r->start && p->res->end == r->end) 7279e4edf1aSAlison Schofield return 1; 7289e4edf1aSAlison Schofield 7299e4edf1aSAlison Schofield return 0; 7309e4edf1aSAlison Schofield } 7319e4edf1aSAlison Schofield 732384e624bSDan Williams static struct cxl_decoder *cxl_region_find_decoder(struct cxl_port *port, 733384e624bSDan Williams struct cxl_region *cxlr) 734384e624bSDan Williams { 735384e624bSDan Williams struct device *dev; 736384e624bSDan Williams int id = 0; 737384e624bSDan Williams 7389e4edf1aSAlison Schofield if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) 7399e4edf1aSAlison Schofield dev = device_find_child(&port->dev, &cxlr->params, 7409e4edf1aSAlison Schofield match_auto_decoder); 7419e4edf1aSAlison Schofield else 742384e624bSDan Williams dev = device_find_child(&port->dev, &id, match_free_decoder); 743384e624bSDan Williams if (!dev) 744384e624bSDan Williams return NULL; 745b9686e8cSDan Williams /* 746384e624bSDan Williams * This decoder is pinned registered as long as the endpoint decoder is 747384e624bSDan Williams * registered, and endpoint decoder unregistration holds the 748384e624bSDan Williams * cxl_region_rwsem over unregister events, so no need to hold on to 749384e624bSDan Williams * this extra reference. 750b9686e8cSDan Williams */ 751384e624bSDan Williams put_device(dev); 752384e624bSDan Williams return to_cxl_decoder(dev); 753384e624bSDan Williams } 754384e624bSDan Williams 755384e624bSDan Williams static struct cxl_region_ref *alloc_region_ref(struct cxl_port *port, 756384e624bSDan Williams struct cxl_region *cxlr) 757384e624bSDan Williams { 758e29a8995SDan Williams struct cxl_region_params *p = &cxlr->params; 759e29a8995SDan Williams struct cxl_region_ref *cxl_rr, *iter; 760e29a8995SDan Williams unsigned long index; 761384e624bSDan Williams int rc; 762384e624bSDan Williams 763e29a8995SDan Williams xa_for_each(&port->regions, index, iter) { 764e29a8995SDan Williams struct cxl_region_params *ip = &iter->region->params; 765e29a8995SDan Williams 766a90accb3SDan Williams if (!ip->res) 767a90accb3SDan Williams continue; 768a90accb3SDan Williams 769e29a8995SDan Williams if (ip->res->start > p->res->start) { 770e29a8995SDan Williams dev_dbg(&cxlr->dev, 771e29a8995SDan Williams "%s: HPA order violation %s:%pr vs %pr\n", 772e29a8995SDan Williams dev_name(&port->dev), 773e29a8995SDan Williams dev_name(&iter->region->dev), ip->res, p->res); 774e29a8995SDan Williams return ERR_PTR(-EBUSY); 775e29a8995SDan Williams } 776e29a8995SDan Williams } 777e29a8995SDan Williams 778384e624bSDan Williams cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL); 779384e624bSDan Williams if (!cxl_rr) 780e29a8995SDan Williams return ERR_PTR(-ENOMEM); 781384e624bSDan Williams cxl_rr->port = port; 782384e624bSDan Williams cxl_rr->region = cxlr; 78327b3f8d1SDan Williams cxl_rr->nr_targets = 1; 784384e624bSDan Williams xa_init(&cxl_rr->endpoints); 785384e624bSDan Williams 786384e624bSDan Williams rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL); 787384e624bSDan Williams if (rc) { 788384e624bSDan Williams dev_dbg(&cxlr->dev, 789384e624bSDan Williams "%s: failed to track region reference: %d\n", 790384e624bSDan Williams dev_name(&port->dev), rc); 791384e624bSDan Williams kfree(cxl_rr); 792e29a8995SDan Williams return ERR_PTR(rc); 793384e624bSDan Williams } 794384e624bSDan Williams 795384e624bSDan Williams return cxl_rr; 796384e624bSDan Williams } 797384e624bSDan Williams 79871ee71d7SVishal Verma static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr) 799384e624bSDan Williams { 800384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region; 801384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 802384e624bSDan Williams 80371ee71d7SVishal Verma if (!cxld) 80471ee71d7SVishal Verma return; 80571ee71d7SVishal Verma 806384e624bSDan Williams dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n"); 807384e624bSDan Williams if (cxld->region == cxlr) { 808384e624bSDan Williams cxld->region = NULL; 809384e624bSDan Williams put_device(&cxlr->dev); 810384e624bSDan Williams } 81171ee71d7SVishal Verma } 812384e624bSDan Williams 81371ee71d7SVishal Verma static void free_region_ref(struct cxl_region_ref *cxl_rr) 81471ee71d7SVishal Verma { 81571ee71d7SVishal Verma struct cxl_port *port = cxl_rr->port; 81671ee71d7SVishal Verma struct cxl_region *cxlr = cxl_rr->region; 81771ee71d7SVishal Verma 81871ee71d7SVishal Verma cxl_rr_free_decoder(cxl_rr); 819384e624bSDan Williams xa_erase(&port->regions, (unsigned long)cxlr); 820384e624bSDan Williams xa_destroy(&cxl_rr->endpoints); 821384e624bSDan Williams kfree(cxl_rr); 822384e624bSDan Williams } 823384e624bSDan Williams 824384e624bSDan Williams static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr, 825384e624bSDan Williams struct cxl_endpoint_decoder *cxled) 826384e624bSDan Williams { 827384e624bSDan Williams int rc; 828384e624bSDan Williams struct cxl_port *port = cxl_rr->port; 829384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region; 830384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 831384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled)); 832384e624bSDan Williams 83327b3f8d1SDan Williams if (ep) { 834384e624bSDan Williams rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep, 835384e624bSDan Williams GFP_KERNEL); 836384e624bSDan Williams if (rc) 837384e624bSDan Williams return rc; 83827b3f8d1SDan Williams } 839384e624bSDan Williams cxl_rr->nr_eps++; 840384e624bSDan Williams 841384e624bSDan Williams if (!cxld->region) { 842384e624bSDan Williams cxld->region = cxlr; 843384e624bSDan Williams get_device(&cxlr->dev); 844384e624bSDan Williams } 845384e624bSDan Williams 846384e624bSDan Williams return 0; 847384e624bSDan Williams } 848384e624bSDan Williams 84971ee71d7SVishal Verma static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr, 85071ee71d7SVishal Verma struct cxl_endpoint_decoder *cxled, 85171ee71d7SVishal Verma struct cxl_region_ref *cxl_rr) 85271ee71d7SVishal Verma { 85371ee71d7SVishal Verma struct cxl_decoder *cxld; 85471ee71d7SVishal Verma 85571ee71d7SVishal Verma if (port == cxled_to_port(cxled)) 85671ee71d7SVishal Verma cxld = &cxled->cxld; 85771ee71d7SVishal Verma else 85871ee71d7SVishal Verma cxld = cxl_region_find_decoder(port, cxlr); 85971ee71d7SVishal Verma if (!cxld) { 86071ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: no decoder available\n", 86171ee71d7SVishal Verma dev_name(&port->dev)); 86271ee71d7SVishal Verma return -EBUSY; 86371ee71d7SVishal Verma } 86471ee71d7SVishal Verma 86571ee71d7SVishal Verma if (cxld->region) { 86671ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", 86771ee71d7SVishal Verma dev_name(&port->dev), dev_name(&cxld->dev), 86871ee71d7SVishal Verma dev_name(&cxld->region->dev)); 86971ee71d7SVishal Verma return -EBUSY; 87071ee71d7SVishal Verma } 87171ee71d7SVishal Verma 8728c897b36SDan Williams /* 8738c897b36SDan Williams * Endpoints should already match the region type, but backstop that 8748c897b36SDan Williams * assumption with an assertion. Switch-decoders change mapping-type 8758c897b36SDan Williams * based on what is mapped when they are assigned to a region. 8768c897b36SDan Williams */ 8778c897b36SDan Williams dev_WARN_ONCE(&cxlr->dev, 8788c897b36SDan Williams port == cxled_to_port(cxled) && 8798c897b36SDan Williams cxld->target_type != cxlr->type, 8808c897b36SDan Williams "%s:%s mismatch decoder type %d -> %d\n", 8818c897b36SDan Williams dev_name(&cxled_to_memdev(cxled)->dev), 8828c897b36SDan Williams dev_name(&cxld->dev), cxld->target_type, cxlr->type); 8838c897b36SDan Williams cxld->target_type = cxlr->type; 88471ee71d7SVishal Verma cxl_rr->decoder = cxld; 88571ee71d7SVishal Verma return 0; 88671ee71d7SVishal Verma } 88771ee71d7SVishal Verma 888384e624bSDan Williams /** 889384e624bSDan Williams * cxl_port_attach_region() - track a region's interest in a port by endpoint 890384e624bSDan Williams * @port: port to add a new region reference 'struct cxl_region_ref' 891384e624bSDan Williams * @cxlr: region to attach to @port 892384e624bSDan Williams * @cxled: endpoint decoder used to create or further pin a region reference 893384e624bSDan Williams * @pos: interleave position of @cxled in @cxlr 894384e624bSDan Williams * 895384e624bSDan Williams * The attach event is an opportunity to validate CXL decode setup 896384e624bSDan Williams * constraints and record metadata needed for programming HDM decoders, 897384e624bSDan Williams * in particular decoder target lists. 898384e624bSDan Williams * 899384e624bSDan Williams * The steps are: 900f13da0d9SBagas Sanjaya * 901384e624bSDan Williams * - validate that there are no other regions with a higher HPA already 902384e624bSDan Williams * associated with @port 903384e624bSDan Williams * - establish a region reference if one is not already present 904f13da0d9SBagas Sanjaya * 905384e624bSDan Williams * - additionally allocate a decoder instance that will host @cxlr on 906384e624bSDan Williams * @port 907f13da0d9SBagas Sanjaya * 908384e624bSDan Williams * - pin the region reference by the endpoint 909384e624bSDan Williams * - account for how many entries in @port's target list are needed to 910384e624bSDan Williams * cover all of the added endpoints. 911384e624bSDan Williams */ 912384e624bSDan Williams static int cxl_port_attach_region(struct cxl_port *port, 913384e624bSDan Williams struct cxl_region *cxlr, 914384e624bSDan Williams struct cxl_endpoint_decoder *cxled, int pos) 915384e624bSDan Williams { 916384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 917384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 918e29a8995SDan Williams struct cxl_region_ref *cxl_rr; 919e29a8995SDan Williams bool nr_targets_inc = false; 920e29a8995SDan Williams struct cxl_decoder *cxld; 921384e624bSDan Williams unsigned long index; 922384e624bSDan Williams int rc = -EBUSY; 923384e624bSDan Williams 924384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 925384e624bSDan Williams 926e29a8995SDan Williams cxl_rr = cxl_rr_load(port, cxlr); 927384e624bSDan Williams if (cxl_rr) { 928384e624bSDan Williams struct cxl_ep *ep_iter; 929384e624bSDan Williams int found = 0; 930384e624bSDan Williams 931e29a8995SDan Williams /* 932e29a8995SDan Williams * Walk the existing endpoints that have been attached to 933e29a8995SDan Williams * @cxlr at @port and see if they share the same 'next' port 934e29a8995SDan Williams * in the downstream direction. I.e. endpoints that share common 935e29a8995SDan Williams * upstream switch. 936e29a8995SDan Williams */ 937384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 938384e624bSDan Williams if (ep_iter == ep) 939384e624bSDan Williams continue; 940384e624bSDan Williams if (ep_iter->next == ep->next) { 941384e624bSDan Williams found++; 942384e624bSDan Williams break; 943384e624bSDan Williams } 944384e624bSDan Williams } 945384e624bSDan Williams 946384e624bSDan Williams /* 947e29a8995SDan Williams * New target port, or @port is an endpoint port that always 948e29a8995SDan Williams * accounts its own local decode as a target. 949384e624bSDan Williams */ 950e29a8995SDan Williams if (!found || !ep->next) { 951384e624bSDan Williams cxl_rr->nr_targets++; 952e29a8995SDan Williams nr_targets_inc = true; 953e29a8995SDan Williams } 954384e624bSDan Williams } else { 955384e624bSDan Williams cxl_rr = alloc_region_ref(port, cxlr); 956e29a8995SDan Williams if (IS_ERR(cxl_rr)) { 957384e624bSDan Williams dev_dbg(&cxlr->dev, 958384e624bSDan Williams "%s: failed to allocate region reference\n", 959384e624bSDan Williams dev_name(&port->dev)); 960e29a8995SDan Williams return PTR_ERR(cxl_rr); 961384e624bSDan Williams } 962e29a8995SDan Williams nr_targets_inc = true; 963384e624bSDan Williams 96471ee71d7SVishal Verma rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr); 96571ee71d7SVishal Verma if (rc) 966384e624bSDan Williams goto out_erase; 967384e624bSDan Williams } 96871ee71d7SVishal Verma cxld = cxl_rr->decoder; 969384e624bSDan Williams 970384e624bSDan Williams rc = cxl_rr_ep_add(cxl_rr, cxled); 971384e624bSDan Williams if (rc) { 972384e624bSDan Williams dev_dbg(&cxlr->dev, 973384e624bSDan Williams "%s: failed to track endpoint %s:%s reference\n", 974384e624bSDan Williams dev_name(&port->dev), dev_name(&cxlmd->dev), 975384e624bSDan Williams dev_name(&cxld->dev)); 976384e624bSDan Williams goto out_erase; 977384e624bSDan Williams } 978384e624bSDan Williams 97927b3f8d1SDan Williams dev_dbg(&cxlr->dev, 98027b3f8d1SDan Williams "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n", 9817481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 98227b3f8d1SDan Williams dev_name(&cxld->dev), dev_name(&cxlmd->dev), 98327b3f8d1SDan Williams dev_name(&cxled->cxld.dev), pos, 9847481653dSDan Williams ep ? ep->next ? dev_name(ep->next->uport_dev) : 98527b3f8d1SDan Williams dev_name(&cxlmd->dev) : 98627b3f8d1SDan Williams "none", 98727b3f8d1SDan Williams cxl_rr->nr_eps, cxl_rr->nr_targets); 98827b3f8d1SDan Williams 989384e624bSDan Williams return 0; 990384e624bSDan Williams out_erase: 991e29a8995SDan Williams if (nr_targets_inc) 992e29a8995SDan Williams cxl_rr->nr_targets--; 993384e624bSDan Williams if (cxl_rr->nr_eps == 0) 994384e624bSDan Williams free_region_ref(cxl_rr); 995384e624bSDan Williams return rc; 996384e624bSDan Williams } 997384e624bSDan Williams 998384e624bSDan Williams static void cxl_port_detach_region(struct cxl_port *port, 999384e624bSDan Williams struct cxl_region *cxlr, 1000384e624bSDan Williams struct cxl_endpoint_decoder *cxled) 1001384e624bSDan Williams { 1002384e624bSDan Williams struct cxl_region_ref *cxl_rr; 100327b3f8d1SDan Williams struct cxl_ep *ep = NULL; 1004384e624bSDan Williams 1005384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 1006384e624bSDan Williams 1007384e624bSDan Williams cxl_rr = cxl_rr_load(port, cxlr); 1008384e624bSDan Williams if (!cxl_rr) 1009384e624bSDan Williams return; 1010384e624bSDan Williams 101127b3f8d1SDan Williams /* 101227b3f8d1SDan Williams * Endpoint ports do not carry cxl_ep references, and they 101327b3f8d1SDan Williams * never target more than one endpoint by definition 101427b3f8d1SDan Williams */ 101527b3f8d1SDan Williams if (cxl_rr->decoder == &cxled->cxld) 101627b3f8d1SDan Williams cxl_rr->nr_eps--; 101727b3f8d1SDan Williams else 1018384e624bSDan Williams ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled); 1019384e624bSDan Williams if (ep) { 1020384e624bSDan Williams struct cxl_ep *ep_iter; 1021384e624bSDan Williams unsigned long index; 1022384e624bSDan Williams int found = 0; 1023384e624bSDan Williams 1024384e624bSDan Williams cxl_rr->nr_eps--; 1025384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 1026384e624bSDan Williams if (ep_iter->next == ep->next) { 1027384e624bSDan Williams found++; 1028384e624bSDan Williams break; 1029384e624bSDan Williams } 1030384e624bSDan Williams } 1031384e624bSDan Williams if (!found) 1032384e624bSDan Williams cxl_rr->nr_targets--; 1033384e624bSDan Williams } 1034384e624bSDan Williams 1035384e624bSDan Williams if (cxl_rr->nr_eps == 0) 1036384e624bSDan Williams free_region_ref(cxl_rr); 1037384e624bSDan Williams } 1038384e624bSDan Williams 103927b3f8d1SDan Williams static int check_last_peer(struct cxl_endpoint_decoder *cxled, 104027b3f8d1SDan Williams struct cxl_ep *ep, struct cxl_region_ref *cxl_rr, 104127b3f8d1SDan Williams int distance) 104227b3f8d1SDan Williams { 104327b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 104427b3f8d1SDan Williams struct cxl_region *cxlr = cxl_rr->region; 104527b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 104627b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled_peer; 104727b3f8d1SDan Williams struct cxl_port *port = cxl_rr->port; 104827b3f8d1SDan Williams struct cxl_memdev *cxlmd_peer; 104927b3f8d1SDan Williams struct cxl_ep *ep_peer; 105027b3f8d1SDan Williams int pos = cxled->pos; 105127b3f8d1SDan Williams 105227b3f8d1SDan Williams /* 105327b3f8d1SDan Williams * If this position wants to share a dport with the last endpoint mapped 105427b3f8d1SDan Williams * then that endpoint, at index 'position - distance', must also be 105527b3f8d1SDan Williams * mapped by this dport. 105627b3f8d1SDan Williams */ 105727b3f8d1SDan Williams if (pos < distance) { 105827b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", 10597481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 106027b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 106127b3f8d1SDan Williams return -ENXIO; 106227b3f8d1SDan Williams } 106327b3f8d1SDan Williams cxled_peer = p->targets[pos - distance]; 106427b3f8d1SDan Williams cxlmd_peer = cxled_to_memdev(cxled_peer); 106527b3f8d1SDan Williams ep_peer = cxl_ep_load(port, cxlmd_peer); 106627b3f8d1SDan Williams if (ep->dport != ep_peer->dport) { 106727b3f8d1SDan Williams dev_dbg(&cxlr->dev, 106827b3f8d1SDan Williams "%s:%s: %s:%s pos %d mismatched peer %s:%s\n", 10697481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 107027b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, 107127b3f8d1SDan Williams dev_name(&cxlmd_peer->dev), 107227b3f8d1SDan Williams dev_name(&cxled_peer->cxld.dev)); 107327b3f8d1SDan Williams return -ENXIO; 107427b3f8d1SDan Williams } 107527b3f8d1SDan Williams 107627b3f8d1SDan Williams return 0; 107727b3f8d1SDan Williams } 107827b3f8d1SDan Williams 107927b3f8d1SDan Williams static int cxl_port_setup_targets(struct cxl_port *port, 108027b3f8d1SDan Williams struct cxl_region *cxlr, 108127b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled) 108227b3f8d1SDan Williams { 108327b3f8d1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 108427b3f8d1SDan Williams int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos; 108527b3f8d1SDan Williams struct cxl_port *parent_port = to_cxl_port(port->dev.parent); 108627b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 108727b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 108827b3f8d1SDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 108927b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 109027b3f8d1SDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 109127b3f8d1SDan Williams struct cxl_switch_decoder *cxlsd; 109227b3f8d1SDan Williams u16 eig, peig; 109327b3f8d1SDan Williams u8 eiw, peiw; 109427b3f8d1SDan Williams 109527b3f8d1SDan Williams /* 109627b3f8d1SDan Williams * While root level decoders support x3, x6, x12, switch level 109727b3f8d1SDan Williams * decoders only support powers of 2 up to x16. 109827b3f8d1SDan Williams */ 109927b3f8d1SDan Williams if (!is_power_of_2(cxl_rr->nr_targets)) { 110027b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n", 11017481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 110227b3f8d1SDan Williams cxl_rr->nr_targets); 110327b3f8d1SDan Williams return -EINVAL; 110427b3f8d1SDan Williams } 110527b3f8d1SDan Williams 110627b3f8d1SDan Williams cxlsd = to_cxl_switch_decoder(&cxld->dev); 110727b3f8d1SDan Williams if (cxl_rr->nr_targets_set) { 110827b3f8d1SDan Williams int i, distance; 110927b3f8d1SDan Williams 1110e4f6dfa9SDan Williams /* 1111711442e2SDan Williams * Passthrough decoders impose no distance requirements between 1112e4f6dfa9SDan Williams * peers 1113e4f6dfa9SDan Williams */ 1114711442e2SDan Williams if (cxl_rr->nr_targets == 1) 1115e4f6dfa9SDan Williams distance = 0; 1116e4f6dfa9SDan Williams else 111727b3f8d1SDan Williams distance = p->nr_targets / cxl_rr->nr_targets; 111827b3f8d1SDan Williams for (i = 0; i < cxl_rr->nr_targets_set; i++) 111927b3f8d1SDan Williams if (ep->dport == cxlsd->target[i]) { 112027b3f8d1SDan Williams rc = check_last_peer(cxled, ep, cxl_rr, 112127b3f8d1SDan Williams distance); 112227b3f8d1SDan Williams if (rc) 112327b3f8d1SDan Williams return rc; 112427b3f8d1SDan Williams goto out_target_set; 112527b3f8d1SDan Williams } 112627b3f8d1SDan Williams goto add_target; 112727b3f8d1SDan Williams } 112827b3f8d1SDan Williams 112927b3f8d1SDan Williams if (is_cxl_root(parent_port)) { 113098a04c7aSJim Harris /* 113198a04c7aSJim Harris * Root decoder IG is always set to value in CFMWS which 113298a04c7aSJim Harris * may be different than this region's IG. We can use the 113398a04c7aSJim Harris * region's IG here since interleave_granularity_store() 113498a04c7aSJim Harris * does not allow interleaved host-bridges with 113598a04c7aSJim Harris * root IG != region IG. 113698a04c7aSJim Harris */ 113798a04c7aSJim Harris parent_ig = p->interleave_granularity; 113827b3f8d1SDan Williams parent_iw = cxlrd->cxlsd.cxld.interleave_ways; 113927b3f8d1SDan Williams /* 114027b3f8d1SDan Williams * For purposes of address bit routing, use power-of-2 math for 114127b3f8d1SDan Williams * switch ports. 114227b3f8d1SDan Williams */ 114327b3f8d1SDan Williams if (!is_power_of_2(parent_iw)) 114427b3f8d1SDan Williams parent_iw /= 3; 114527b3f8d1SDan Williams } else { 114627b3f8d1SDan Williams struct cxl_region_ref *parent_rr; 114727b3f8d1SDan Williams struct cxl_decoder *parent_cxld; 114827b3f8d1SDan Williams 114927b3f8d1SDan Williams parent_rr = cxl_rr_load(parent_port, cxlr); 115027b3f8d1SDan Williams parent_cxld = parent_rr->decoder; 115127b3f8d1SDan Williams parent_ig = parent_cxld->interleave_granularity; 115227b3f8d1SDan Williams parent_iw = parent_cxld->interleave_ways; 115327b3f8d1SDan Williams } 115427b3f8d1SDan Williams 115583351ddbSDave Jiang rc = granularity_to_eig(parent_ig, &peig); 11568d428542SDan Williams if (rc) { 11578d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n", 11587481653dSDan Williams dev_name(parent_port->uport_dev), 11598d428542SDan Williams dev_name(&parent_port->dev), parent_ig); 11608d428542SDan Williams return rc; 11618d428542SDan Williams } 11628d428542SDan Williams 1163c99b2e8cSDave Jiang rc = ways_to_eiw(parent_iw, &peiw); 11648d428542SDan Williams if (rc) { 11658d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n", 11667481653dSDan Williams dev_name(parent_port->uport_dev), 11678d428542SDan Williams dev_name(&parent_port->dev), parent_iw); 11688d428542SDan Williams return rc; 11698d428542SDan Williams } 117027b3f8d1SDan Williams 117127b3f8d1SDan Williams iw = cxl_rr->nr_targets; 1172c99b2e8cSDave Jiang rc = ways_to_eiw(iw, &eiw); 11738d428542SDan Williams if (rc) { 11748d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n", 11757481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), iw); 11768d428542SDan Williams return rc; 11778d428542SDan Williams } 11788d428542SDan Williams 1179298d44d0SDan Williams /* 118018f35dc9SAlison Schofield * Interleave granularity is a multiple of @parent_port granularity. 118118f35dc9SAlison Schofield * Multiplier is the parent port interleave ways. 1182298d44d0SDan Williams */ 118318f35dc9SAlison Schofield rc = granularity_to_eig(parent_ig * parent_iw, &eig); 118418f35dc9SAlison Schofield if (rc) { 118518f35dc9SAlison Schofield dev_dbg(&cxlr->dev, 118618f35dc9SAlison Schofield "%s: invalid granularity calculation (%d * %d)\n", 118718f35dc9SAlison Schofield dev_name(&parent_port->dev), parent_ig, parent_iw); 118818f35dc9SAlison Schofield return rc; 118927b3f8d1SDan Williams } 119027b3f8d1SDan Williams 119183351ddbSDave Jiang rc = eig_to_granularity(eig, &ig); 119227b3f8d1SDan Williams if (rc) { 119327b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n", 11947481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 119527b3f8d1SDan Williams 256 << eig); 119627b3f8d1SDan Williams return rc; 119727b3f8d1SDan Williams } 119827b3f8d1SDan Williams 11995d09c63fSDan Williams if (iw > 8 || iw > cxlsd->nr_targets) { 12005d09c63fSDan Williams dev_dbg(&cxlr->dev, 12015d09c63fSDan Williams "%s:%s:%s: ways: %d overflows targets: %d\n", 12025d09c63fSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 12035d09c63fSDan Williams dev_name(&cxld->dev), iw, cxlsd->nr_targets); 12045d09c63fSDan Williams return -ENXIO; 12055d09c63fSDan Williams } 12065d09c63fSDan Williams 1207a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1208a32320b7SDan Williams if (cxld->interleave_ways != iw || 1209a32320b7SDan Williams cxld->interleave_granularity != ig || 1210a32320b7SDan Williams cxld->hpa_range.start != p->res->start || 1211a32320b7SDan Williams cxld->hpa_range.end != p->res->end || 1212a32320b7SDan Williams ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { 1213a32320b7SDan Williams dev_err(&cxlr->dev, 1214a32320b7SDan Williams "%s:%s %s expected iw: %d ig: %d %pr\n", 12157481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 1216a32320b7SDan Williams __func__, iw, ig, p->res); 1217a32320b7SDan Williams dev_err(&cxlr->dev, 1218a32320b7SDan Williams "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n", 12197481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 1220a32320b7SDan Williams __func__, cxld->interleave_ways, 1221a32320b7SDan Williams cxld->interleave_granularity, 1222a32320b7SDan Williams (cxld->flags & CXL_DECODER_F_ENABLE) ? 1223a32320b7SDan Williams "enabled" : 1224a32320b7SDan Williams "disabled", 1225a32320b7SDan Williams cxld->hpa_range.start, cxld->hpa_range.end); 1226a32320b7SDan Williams return -ENXIO; 1227a32320b7SDan Williams } 1228a32320b7SDan Williams } else { 122927b3f8d1SDan Williams cxld->interleave_ways = iw; 123027b3f8d1SDan Williams cxld->interleave_granularity = ig; 1231910bc55dSDan Williams cxld->hpa_range = (struct range) { 1232910bc55dSDan Williams .start = p->res->start, 1233910bc55dSDan Williams .end = p->res->end, 1234910bc55dSDan Williams }; 1235a32320b7SDan Williams } 12367481653dSDan Williams dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev), 123727b3f8d1SDan Williams dev_name(&port->dev), iw, ig); 123827b3f8d1SDan Williams add_target: 123927b3f8d1SDan Williams if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) { 124027b3f8d1SDan Williams dev_dbg(&cxlr->dev, 124127b3f8d1SDan Williams "%s:%s: targets full trying to add %s:%s at %d\n", 12427481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 124327b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 124427b3f8d1SDan Williams return -ENXIO; 124527b3f8d1SDan Williams } 1246a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1247a32320b7SDan Williams if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) { 1248a32320b7SDan Williams dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n", 12497481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 1250a32320b7SDan Williams dev_name(&cxlsd->cxld.dev), 1251227db574SRobert Richter dev_name(ep->dport->dport_dev), 1252a32320b7SDan Williams cxl_rr->nr_targets_set); 1253a32320b7SDan Williams return -ENXIO; 1254a32320b7SDan Williams } 1255a32320b7SDan Williams } else 125627b3f8d1SDan Williams cxlsd->target[cxl_rr->nr_targets_set] = ep->dport; 125727b3f8d1SDan Williams inc = 1; 125827b3f8d1SDan Williams out_target_set: 125927b3f8d1SDan Williams cxl_rr->nr_targets_set += inc; 126027b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", 12617481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 1262227db574SRobert Richter cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev), 126327b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 126427b3f8d1SDan Williams 126527b3f8d1SDan Williams return 0; 126627b3f8d1SDan Williams } 126727b3f8d1SDan Williams 126827b3f8d1SDan Williams static void cxl_port_reset_targets(struct cxl_port *port, 126927b3f8d1SDan Williams struct cxl_region *cxlr) 127027b3f8d1SDan Williams { 127127b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 1272910bc55dSDan Williams struct cxl_decoder *cxld; 127327b3f8d1SDan Williams 127427b3f8d1SDan Williams /* 127527b3f8d1SDan Williams * After the last endpoint has been detached the entire cxl_rr may now 127627b3f8d1SDan Williams * be gone. 127727b3f8d1SDan Williams */ 1278910bc55dSDan Williams if (!cxl_rr) 1279910bc55dSDan Williams return; 128027b3f8d1SDan Williams cxl_rr->nr_targets_set = 0; 1281910bc55dSDan Williams 1282910bc55dSDan Williams cxld = cxl_rr->decoder; 1283910bc55dSDan Williams cxld->hpa_range = (struct range) { 1284910bc55dSDan Williams .start = 0, 1285910bc55dSDan Williams .end = -1, 1286910bc55dSDan Williams }; 128727b3f8d1SDan Williams } 128827b3f8d1SDan Williams 128927b3f8d1SDan Williams static void cxl_region_teardown_targets(struct cxl_region *cxlr) 129027b3f8d1SDan Williams { 129127b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 129227b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled; 1293030f8803SDan Williams struct cxl_dev_state *cxlds; 129427b3f8d1SDan Williams struct cxl_memdev *cxlmd; 129527b3f8d1SDan Williams struct cxl_port *iter; 129627b3f8d1SDan Williams struct cxl_ep *ep; 129727b3f8d1SDan Williams int i; 129827b3f8d1SDan Williams 1299a32320b7SDan Williams /* 1300a32320b7SDan Williams * In the auto-discovery case skip automatic teardown since the 1301a32320b7SDan Williams * address space is already active 1302a32320b7SDan Williams */ 1303a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) 1304a32320b7SDan Williams return; 1305a32320b7SDan Williams 130627b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) { 130727b3f8d1SDan Williams cxled = p->targets[i]; 130827b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled); 1309030f8803SDan Williams cxlds = cxlmd->cxlds; 1310030f8803SDan Williams 1311030f8803SDan Williams if (cxlds->rcd) 1312030f8803SDan Williams continue; 131327b3f8d1SDan Williams 131427b3f8d1SDan Williams iter = cxled_to_port(cxled); 131527b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 131627b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent); 131727b3f8d1SDan Williams 131827b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 131927b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) 132027b3f8d1SDan Williams cxl_port_reset_targets(iter, cxlr); 132127b3f8d1SDan Williams } 132227b3f8d1SDan Williams } 132327b3f8d1SDan Williams 132427b3f8d1SDan Williams static int cxl_region_setup_targets(struct cxl_region *cxlr) 132527b3f8d1SDan Williams { 132627b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 132727b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled; 1328030f8803SDan Williams struct cxl_dev_state *cxlds; 1329030f8803SDan Williams int i, rc, rch = 0, vh = 0; 133027b3f8d1SDan Williams struct cxl_memdev *cxlmd; 133127b3f8d1SDan Williams struct cxl_port *iter; 133227b3f8d1SDan Williams struct cxl_ep *ep; 133327b3f8d1SDan Williams 133427b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) { 133527b3f8d1SDan Williams cxled = p->targets[i]; 133627b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled); 1337030f8803SDan Williams cxlds = cxlmd->cxlds; 1338030f8803SDan Williams 1339030f8803SDan Williams /* validate that all targets agree on topology */ 1340030f8803SDan Williams if (!cxlds->rcd) { 1341030f8803SDan Williams vh++; 1342030f8803SDan Williams } else { 1343030f8803SDan Williams rch++; 1344030f8803SDan Williams continue; 1345030f8803SDan Williams } 134627b3f8d1SDan Williams 134727b3f8d1SDan Williams iter = cxled_to_port(cxled); 134827b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 134927b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent); 135027b3f8d1SDan Williams 135127b3f8d1SDan Williams /* 1352a32320b7SDan Williams * Descend the topology tree programming / validating 1353a32320b7SDan Williams * targets while looking for conflicts. 135427b3f8d1SDan Williams */ 135527b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 135627b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 135727b3f8d1SDan Williams rc = cxl_port_setup_targets(iter, cxlr, cxled); 135827b3f8d1SDan Williams if (rc) { 135927b3f8d1SDan Williams cxl_region_teardown_targets(cxlr); 136027b3f8d1SDan Williams return rc; 136127b3f8d1SDan Williams } 136227b3f8d1SDan Williams } 136327b3f8d1SDan Williams } 136427b3f8d1SDan Williams 1365030f8803SDan Williams if (rch && vh) { 1366030f8803SDan Williams dev_err(&cxlr->dev, "mismatched CXL topologies detected\n"); 1367030f8803SDan Williams cxl_region_teardown_targets(cxlr); 1368030f8803SDan Williams return -ENXIO; 1369030f8803SDan Williams } 1370030f8803SDan Williams 137127b3f8d1SDan Williams return 0; 137227b3f8d1SDan Williams } 137327b3f8d1SDan Williams 13749995576cSDan Williams static int cxl_region_validate_position(struct cxl_region *cxlr, 13759995576cSDan Williams struct cxl_endpoint_decoder *cxled, 13769995576cSDan Williams int pos) 1377b9686e8cSDan Williams { 1378384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1379b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 13809995576cSDan Williams int i; 1381384e624bSDan Williams 1382384e624bSDan Williams if (pos < 0 || pos >= p->interleave_ways) { 1383b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1384b9686e8cSDan Williams p->interleave_ways); 1385b9686e8cSDan Williams return -ENXIO; 1386b9686e8cSDan Williams } 1387b9686e8cSDan Williams 1388b9686e8cSDan Williams if (p->targets[pos] == cxled) 1389b9686e8cSDan Williams return 0; 1390b9686e8cSDan Williams 1391b9686e8cSDan Williams if (p->targets[pos]) { 1392b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled_target = p->targets[pos]; 1393b9686e8cSDan Williams struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target); 1394b9686e8cSDan Williams 1395b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n", 1396b9686e8cSDan Williams pos, dev_name(&cxlmd_target->dev), 1397b9686e8cSDan Williams dev_name(&cxled_target->cxld.dev)); 1398b9686e8cSDan Williams return -EBUSY; 1399b9686e8cSDan Williams } 1400b9686e8cSDan Williams 1401384e624bSDan Williams for (i = 0; i < p->interleave_ways; i++) { 1402384e624bSDan Williams struct cxl_endpoint_decoder *cxled_target; 1403384e624bSDan Williams struct cxl_memdev *cxlmd_target; 1404384e624bSDan Williams 1405f04facfbSFan Ni cxled_target = p->targets[i]; 1406384e624bSDan Williams if (!cxled_target) 1407384e624bSDan Williams continue; 1408384e624bSDan Williams 1409384e624bSDan Williams cxlmd_target = cxled_to_memdev(cxled_target); 1410384e624bSDan Williams if (cxlmd_target == cxlmd) { 1411384e624bSDan Williams dev_dbg(&cxlr->dev, 1412384e624bSDan Williams "%s already specified at position %d via: %s\n", 1413384e624bSDan Williams dev_name(&cxlmd->dev), pos, 1414384e624bSDan Williams dev_name(&cxled_target->cxld.dev)); 1415384e624bSDan Williams return -EBUSY; 1416384e624bSDan Williams } 1417384e624bSDan Williams } 1418384e624bSDan Williams 14199995576cSDan Williams return 0; 14209995576cSDan Williams } 14219995576cSDan Williams 14229995576cSDan Williams static int cxl_region_attach_position(struct cxl_region *cxlr, 14239995576cSDan Williams struct cxl_root_decoder *cxlrd, 14249995576cSDan Williams struct cxl_endpoint_decoder *cxled, 14259995576cSDan Williams const struct cxl_dport *dport, int pos) 14269995576cSDan Williams { 14279995576cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 14289995576cSDan Williams struct cxl_port *iter; 14299995576cSDan Williams int rc; 14309995576cSDan Williams 14319995576cSDan Williams if (cxlrd->calc_hb(cxlrd, pos) != dport) { 14329995576cSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n", 14339995576cSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 14349995576cSDan Williams dev_name(&cxlrd->cxlsd.cxld.dev)); 14359995576cSDan Williams return -ENXIO; 14369995576cSDan Williams } 14379995576cSDan Williams 14389995576cSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 14399995576cSDan Williams iter = to_cxl_port(iter->dev.parent)) { 14409995576cSDan Williams rc = cxl_port_attach_region(iter, cxlr, cxled, pos); 14419995576cSDan Williams if (rc) 14429995576cSDan Williams goto err; 14439995576cSDan Williams } 14449995576cSDan Williams 14459995576cSDan Williams return 0; 14469995576cSDan Williams 14479995576cSDan Williams err: 14489995576cSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 14499995576cSDan Williams iter = to_cxl_port(iter->dev.parent)) 14509995576cSDan Williams cxl_port_detach_region(iter, cxlr, cxled); 14519995576cSDan Williams return rc; 14529995576cSDan Williams } 14539995576cSDan Williams 1454a32320b7SDan Williams static int cxl_region_attach_auto(struct cxl_region *cxlr, 1455a32320b7SDan Williams struct cxl_endpoint_decoder *cxled, int pos) 1456a32320b7SDan Williams { 1457a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params; 1458a32320b7SDan Williams 1459a32320b7SDan Williams if (cxled->state != CXL_DECODER_STATE_AUTO) { 1460a32320b7SDan Williams dev_err(&cxlr->dev, 1461a32320b7SDan Williams "%s: unable to add decoder to autodetected region\n", 1462a32320b7SDan Williams dev_name(&cxled->cxld.dev)); 1463a32320b7SDan Williams return -EINVAL; 1464a32320b7SDan Williams } 1465a32320b7SDan Williams 1466a32320b7SDan Williams if (pos >= 0) { 1467a32320b7SDan Williams dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n", 1468a32320b7SDan Williams dev_name(&cxled->cxld.dev), pos); 1469a32320b7SDan Williams return -EINVAL; 1470a32320b7SDan Williams } 1471a32320b7SDan Williams 1472a32320b7SDan Williams if (p->nr_targets >= p->interleave_ways) { 1473a32320b7SDan Williams dev_err(&cxlr->dev, "%s: no more target slots available\n", 1474a32320b7SDan Williams dev_name(&cxled->cxld.dev)); 1475a32320b7SDan Williams return -ENXIO; 1476a32320b7SDan Williams } 1477a32320b7SDan Williams 1478a32320b7SDan Williams /* 1479a32320b7SDan Williams * Temporarily record the endpoint decoder into the target array. Yes, 1480a32320b7SDan Williams * this means that userspace can view devices in the wrong position 1481a32320b7SDan Williams * before the region activates, and must be careful to understand when 1482a32320b7SDan Williams * it might be racing region autodiscovery. 1483a32320b7SDan Williams */ 1484a32320b7SDan Williams pos = p->nr_targets; 1485a32320b7SDan Williams p->targets[pos] = cxled; 1486a32320b7SDan Williams cxled->pos = pos; 1487a32320b7SDan Williams p->nr_targets++; 1488a32320b7SDan Williams 1489a32320b7SDan Williams return 0; 1490a32320b7SDan Williams } 1491a32320b7SDan Williams 14920cf36a85SAlison Schofield static int cmp_interleave_pos(const void *a, const void *b) 14930cf36a85SAlison Schofield { 14940cf36a85SAlison Schofield struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a; 14950cf36a85SAlison Schofield struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b; 14960cf36a85SAlison Schofield 14970cf36a85SAlison Schofield return cxled_a->pos - cxled_b->pos; 14980cf36a85SAlison Schofield } 14990cf36a85SAlison Schofield 1500a32320b7SDan Williams static struct cxl_port *next_port(struct cxl_port *port) 1501a32320b7SDan Williams { 1502a32320b7SDan Williams if (!port->parent_dport) 1503a32320b7SDan Williams return NULL; 1504a32320b7SDan Williams return port->parent_dport->port; 1505a32320b7SDan Williams } 1506a32320b7SDan Williams 150711105814SAlison Schofield static int match_switch_decoder_by_range(struct device *dev, void *data) 1508a32320b7SDan Williams { 1509a32320b7SDan Williams struct cxl_switch_decoder *cxlsd; 151011105814SAlison Schofield struct range *r1, *r2 = data; 1511a32320b7SDan Williams 1512a32320b7SDan Williams if (!is_switch_decoder(dev)) 1513a32320b7SDan Williams return 0; 1514a32320b7SDan Williams 1515a32320b7SDan Williams cxlsd = to_cxl_switch_decoder(dev); 151611105814SAlison Schofield r1 = &cxlsd->cxld.hpa_range; 151711105814SAlison Schofield 151811105814SAlison Schofield if (is_root_decoder(dev)) 151911105814SAlison Schofield return range_contains(r1, r2); 152011105814SAlison Schofield return (r1->start == r2->start && r1->end == r2->end); 1521a32320b7SDan Williams } 1522a32320b7SDan Williams 1523a3e00c96SAlison Schofield static int find_pos_and_ways(struct cxl_port *port, struct range *range, 1524a3e00c96SAlison Schofield int *pos, int *ways) 1525a3e00c96SAlison Schofield { 1526a3e00c96SAlison Schofield struct cxl_switch_decoder *cxlsd; 1527a3e00c96SAlison Schofield struct cxl_port *parent; 1528a3e00c96SAlison Schofield struct device *dev; 1529a3e00c96SAlison Schofield int rc = -ENXIO; 1530a3e00c96SAlison Schofield 1531a3e00c96SAlison Schofield parent = next_port(port); 1532a3e00c96SAlison Schofield if (!parent) 1533a3e00c96SAlison Schofield return rc; 1534a3e00c96SAlison Schofield 1535a3e00c96SAlison Schofield dev = device_find_child(&parent->dev, range, 1536a3e00c96SAlison Schofield match_switch_decoder_by_range); 1537a3e00c96SAlison Schofield if (!dev) { 1538a3e00c96SAlison Schofield dev_err(port->uport_dev, 1539a3e00c96SAlison Schofield "failed to find decoder mapping %#llx-%#llx\n", 1540a3e00c96SAlison Schofield range->start, range->end); 1541a3e00c96SAlison Schofield return rc; 1542a3e00c96SAlison Schofield } 1543a3e00c96SAlison Schofield cxlsd = to_cxl_switch_decoder(dev); 1544a3e00c96SAlison Schofield *ways = cxlsd->cxld.interleave_ways; 1545a3e00c96SAlison Schofield 1546a3e00c96SAlison Schofield for (int i = 0; i < *ways; i++) { 1547a3e00c96SAlison Schofield if (cxlsd->target[i] == port->parent_dport) { 1548a3e00c96SAlison Schofield *pos = i; 1549a3e00c96SAlison Schofield rc = 0; 1550a3e00c96SAlison Schofield break; 1551a3e00c96SAlison Schofield } 1552a3e00c96SAlison Schofield } 1553a3e00c96SAlison Schofield put_device(dev); 1554a3e00c96SAlison Schofield 1555a3e00c96SAlison Schofield return rc; 1556a3e00c96SAlison Schofield } 1557a3e00c96SAlison Schofield 1558a3e00c96SAlison Schofield /** 1559a3e00c96SAlison Schofield * cxl_calc_interleave_pos() - calculate an endpoint position in a region 1560a3e00c96SAlison Schofield * @cxled: endpoint decoder member of given region 1561a3e00c96SAlison Schofield * 1562a3e00c96SAlison Schofield * The endpoint position is calculated by traversing the topology from 1563a3e00c96SAlison Schofield * the endpoint to the root decoder and iteratively applying this 1564a3e00c96SAlison Schofield * calculation: 1565a3e00c96SAlison Schofield * 1566a3e00c96SAlison Schofield * position = position * parent_ways + parent_pos; 1567a3e00c96SAlison Schofield * 1568a3e00c96SAlison Schofield * ...where @position is inferred from switch and root decoder target lists. 1569a3e00c96SAlison Schofield * 1570a3e00c96SAlison Schofield * Return: position >= 0 on success 1571a3e00c96SAlison Schofield * -ENXIO on failure 1572a3e00c96SAlison Schofield */ 1573a3e00c96SAlison Schofield static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled) 1574a3e00c96SAlison Schofield { 1575a3e00c96SAlison Schofield struct cxl_port *iter, *port = cxled_to_port(cxled); 1576a3e00c96SAlison Schofield struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1577a3e00c96SAlison Schofield struct range *range = &cxled->cxld.hpa_range; 1578a3e00c96SAlison Schofield int parent_ways = 0, parent_pos = 0, pos = 0; 1579a3e00c96SAlison Schofield int rc; 1580a3e00c96SAlison Schofield 1581a3e00c96SAlison Schofield /* 1582a3e00c96SAlison Schofield * Example: the expected interleave order of the 4-way region shown 1583a3e00c96SAlison Schofield * below is: mem0, mem2, mem1, mem3 1584a3e00c96SAlison Schofield * 1585a3e00c96SAlison Schofield * root_port 1586a3e00c96SAlison Schofield * / \ 1587a3e00c96SAlison Schofield * host_bridge_0 host_bridge_1 1588a3e00c96SAlison Schofield * | | | | 1589a3e00c96SAlison Schofield * mem0 mem1 mem2 mem3 1590a3e00c96SAlison Schofield * 1591a3e00c96SAlison Schofield * In the example the calculator will iterate twice. The first iteration 1592a3e00c96SAlison Schofield * uses the mem position in the host-bridge and the ways of the host- 1593a3e00c96SAlison Schofield * bridge to generate the first, or local, position. The second 1594a3e00c96SAlison Schofield * iteration uses the host-bridge position in the root_port and the ways 1595a3e00c96SAlison Schofield * of the root_port to refine the position. 1596a3e00c96SAlison Schofield * 1597a3e00c96SAlison Schofield * A trace of the calculation per endpoint looks like this: 1598a3e00c96SAlison Schofield * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0 1599a3e00c96SAlison Schofield * pos = 0 * 2 + 0 pos = 0 * 2 + 1 1600a3e00c96SAlison Schofield * pos: 0 pos: 1 1601a3e00c96SAlison Schofield * 1602a3e00c96SAlison Schofield * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1 1603a3e00c96SAlison Schofield * pos = 1 * 2 + 0 pos = 1 * 2 + 1 1604a3e00c96SAlison Schofield * pos: 2 pos = 3 1605a3e00c96SAlison Schofield * 1606a3e00c96SAlison Schofield * Note that while this example is simple, the method applies to more 1607a3e00c96SAlison Schofield * complex topologies, including those with switches. 1608a3e00c96SAlison Schofield */ 1609a3e00c96SAlison Schofield 1610a3e00c96SAlison Schofield /* Iterate from endpoint to root_port refining the position */ 1611a3e00c96SAlison Schofield for (iter = port; iter; iter = next_port(iter)) { 1612a3e00c96SAlison Schofield if (is_cxl_root(iter)) 1613a3e00c96SAlison Schofield break; 1614a3e00c96SAlison Schofield 1615a3e00c96SAlison Schofield rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways); 1616a3e00c96SAlison Schofield if (rc) 1617a3e00c96SAlison Schofield return rc; 1618a3e00c96SAlison Schofield 1619a3e00c96SAlison Schofield pos = pos * parent_ways + parent_pos; 1620a3e00c96SAlison Schofield } 1621a3e00c96SAlison Schofield 1622a3e00c96SAlison Schofield dev_dbg(&cxlmd->dev, 1623a3e00c96SAlison Schofield "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n", 1624a3e00c96SAlison Schofield dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent), 1625a3e00c96SAlison Schofield dev_name(&port->dev), range->start, range->end, pos); 1626a3e00c96SAlison Schofield 1627a3e00c96SAlison Schofield return pos; 1628a3e00c96SAlison Schofield } 1629a3e00c96SAlison Schofield 1630a32320b7SDan Williams static int cxl_region_sort_targets(struct cxl_region *cxlr) 1631a32320b7SDan Williams { 1632a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params; 1633a32320b7SDan Williams int i, rc = 0; 1634a32320b7SDan Williams 1635a32320b7SDan Williams for (i = 0; i < p->nr_targets; i++) { 1636a32320b7SDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 1637a32320b7SDan Williams 16380cf36a85SAlison Schofield cxled->pos = cxl_calc_interleave_pos(cxled); 1639a32320b7SDan Williams /* 16400cf36a85SAlison Schofield * Record that sorting failed, but still continue to calc 16410cf36a85SAlison Schofield * cxled->pos so that follow-on code paths can reliably 16420cf36a85SAlison Schofield * do p->targets[cxled->pos] to self-reference their entry. 1643a32320b7SDan Williams */ 1644a32320b7SDan Williams if (cxled->pos < 0) 1645a32320b7SDan Williams rc = -ENXIO; 1646a32320b7SDan Williams } 16470cf36a85SAlison Schofield /* Keep the cxlr target list in interleave position order */ 16480cf36a85SAlison Schofield sort(p->targets, p->nr_targets, sizeof(p->targets[0]), 16490cf36a85SAlison Schofield cmp_interleave_pos, NULL); 1650a32320b7SDan Williams 1651a32320b7SDan Williams dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful"); 1652a32320b7SDan Williams return rc; 1653a32320b7SDan Williams } 1654a32320b7SDan Williams 16559995576cSDan Williams static int cxl_region_attach(struct cxl_region *cxlr, 16569995576cSDan Williams struct cxl_endpoint_decoder *cxled, int pos) 16579995576cSDan Williams { 16589995576cSDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 16599995576cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 16609995576cSDan Williams struct cxl_region_params *p = &cxlr->params; 16619995576cSDan Williams struct cxl_port *ep_port, *root_port; 16629995576cSDan Williams struct cxl_dport *dport; 16639995576cSDan Williams int rc = -ENXIO; 16649995576cSDan Williams 16659995576cSDan Williams if (cxled->mode != cxlr->mode) { 16669995576cSDan Williams dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n", 16679995576cSDan Williams dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode); 16689995576cSDan Williams return -EINVAL; 16699995576cSDan Williams } 16709995576cSDan Williams 16719995576cSDan Williams if (cxled->mode == CXL_DECODER_DEAD) { 16729995576cSDan Williams dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev)); 16739995576cSDan Williams return -ENODEV; 16749995576cSDan Williams } 16759995576cSDan Williams 16769995576cSDan Williams /* all full of members, or interleave config not established? */ 16779995576cSDan Williams if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) { 16789995576cSDan Williams dev_dbg(&cxlr->dev, "region already active\n"); 16799995576cSDan Williams return -EBUSY; 16809995576cSDan Williams } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) { 16819995576cSDan Williams dev_dbg(&cxlr->dev, "interleave config missing\n"); 16829995576cSDan Williams return -ENXIO; 16839995576cSDan Williams } 16849995576cSDan Williams 16850718588cSJim Harris if (p->nr_targets >= p->interleave_ways) { 16860718588cSJim Harris dev_dbg(&cxlr->dev, "region already has %d endpoints\n", 16870718588cSJim Harris p->nr_targets); 16880718588cSJim Harris return -EINVAL; 16890718588cSJim Harris } 16900718588cSJim Harris 1691384e624bSDan Williams ep_port = cxled_to_port(cxled); 1692384e624bSDan Williams root_port = cxlrd_to_port(cxlrd); 1693384e624bSDan Williams dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge); 1694384e624bSDan Williams if (!dport) { 1695384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n", 1696384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1697384e624bSDan Williams dev_name(cxlr->dev.parent)); 1698384e624bSDan Williams return -ENXIO; 1699384e624bSDan Williams } 1700384e624bSDan Williams 1701384e624bSDan Williams if (cxled->cxld.target_type != cxlr->type) { 1702384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n", 1703384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1704384e624bSDan Williams cxled->cxld.target_type, cxlr->type); 1705384e624bSDan Williams return -ENXIO; 1706384e624bSDan Williams } 1707384e624bSDan Williams 1708384e624bSDan Williams if (!cxled->dpa_res) { 1709384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n", 1710384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev)); 1711384e624bSDan Williams return -ENXIO; 1712384e624bSDan Williams } 1713384e624bSDan Williams 1714384e624bSDan Williams if (resource_size(cxled->dpa_res) * p->interleave_ways != 1715384e624bSDan Williams resource_size(p->res)) { 1716384e624bSDan Williams dev_dbg(&cxlr->dev, 1717384e624bSDan Williams "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n", 1718384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1719384e624bSDan Williams (u64)resource_size(cxled->dpa_res), p->interleave_ways, 1720384e624bSDan Williams (u64)resource_size(p->res)); 1721384e624bSDan Williams return -EINVAL; 1722384e624bSDan Williams } 1723384e624bSDan Williams 1724a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1725a32320b7SDan Williams int i; 1726a32320b7SDan Williams 1727a32320b7SDan Williams rc = cxl_region_attach_auto(cxlr, cxled, pos); 1728384e624bSDan Williams if (rc) 1729a32320b7SDan Williams return rc; 1730a32320b7SDan Williams 1731a32320b7SDan Williams /* await more targets to arrive... */ 1732a32320b7SDan Williams if (p->nr_targets < p->interleave_ways) 1733a32320b7SDan Williams return 0; 1734a32320b7SDan Williams 1735a32320b7SDan Williams /* 1736a32320b7SDan Williams * All targets are here, which implies all PCI enumeration that 1737a32320b7SDan Williams * affects this region has been completed. Walk the topology to 1738a32320b7SDan Williams * sort the devices into their relative region decode position. 1739a32320b7SDan Williams */ 1740a32320b7SDan Williams rc = cxl_region_sort_targets(cxlr); 1741a32320b7SDan Williams if (rc) 1742a32320b7SDan Williams return rc; 1743a32320b7SDan Williams 1744a32320b7SDan Williams for (i = 0; i < p->nr_targets; i++) { 1745a32320b7SDan Williams cxled = p->targets[i]; 1746a32320b7SDan Williams ep_port = cxled_to_port(cxled); 1747a32320b7SDan Williams dport = cxl_find_dport_by_dev(root_port, 1748a32320b7SDan Williams ep_port->host_bridge); 1749a32320b7SDan Williams rc = cxl_region_attach_position(cxlr, cxlrd, cxled, 1750a32320b7SDan Williams dport, i); 1751a32320b7SDan Williams if (rc) 1752a32320b7SDan Williams return rc; 1753384e624bSDan Williams } 1754384e624bSDan Williams 1755a32320b7SDan Williams rc = cxl_region_setup_targets(cxlr); 1756a32320b7SDan Williams if (rc) 1757a32320b7SDan Williams return rc; 1758a32320b7SDan Williams 1759a32320b7SDan Williams /* 1760a32320b7SDan Williams * If target setup succeeds in the autodiscovery case 1761a32320b7SDan Williams * then the region is already committed. 1762a32320b7SDan Williams */ 1763a32320b7SDan Williams p->state = CXL_CONFIG_COMMIT; 1764a32320b7SDan Williams 1765a32320b7SDan Williams return 0; 1766a32320b7SDan Williams } 1767a32320b7SDan Williams 17689995576cSDan Williams rc = cxl_region_validate_position(cxlr, cxled, pos); 1769b9686e8cSDan Williams if (rc) 17709995576cSDan Williams return rc; 17719995576cSDan Williams 17729995576cSDan Williams rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos); 17739995576cSDan Williams if (rc) 17749995576cSDan Williams return rc; 1775b9686e8cSDan Williams 1776b9686e8cSDan Williams p->targets[pos] = cxled; 1777b9686e8cSDan Williams cxled->pos = pos; 1778b9686e8cSDan Williams p->nr_targets++; 1779b9686e8cSDan Williams 178027b3f8d1SDan Williams if (p->nr_targets == p->interleave_ways) { 178127b3f8d1SDan Williams rc = cxl_region_setup_targets(cxlr); 178227b3f8d1SDan Williams if (rc) 17830718588cSJim Harris return rc; 1784384e624bSDan Williams p->state = CXL_CONFIG_ACTIVE; 178527b3f8d1SDan Williams } 1786384e624bSDan Williams 17872901c8bdSDan Williams cxled->cxld.interleave_ways = p->interleave_ways; 17882901c8bdSDan Williams cxled->cxld.interleave_granularity = p->interleave_granularity; 1789910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) { 1790910bc55dSDan Williams .start = p->res->start, 1791910bc55dSDan Williams .end = p->res->end, 1792910bc55dSDan Williams }; 17932901c8bdSDan Williams 1794a3e00c96SAlison Schofield if (p->nr_targets != p->interleave_ways) 1795a3e00c96SAlison Schofield return 0; 1796a3e00c96SAlison Schofield 1797a3e00c96SAlison Schofield /* 1798a3e00c96SAlison Schofield * Test the auto-discovery position calculator function 1799a3e00c96SAlison Schofield * against this successfully created user-defined region. 1800a3e00c96SAlison Schofield * A fail message here means that this interleave config 1801a3e00c96SAlison Schofield * will fail when presented as CXL_REGION_F_AUTO. 1802a3e00c96SAlison Schofield */ 1803a3e00c96SAlison Schofield for (int i = 0; i < p->nr_targets; i++) { 1804a3e00c96SAlison Schofield struct cxl_endpoint_decoder *cxled = p->targets[i]; 1805a3e00c96SAlison Schofield int test_pos; 1806a3e00c96SAlison Schofield 1807a3e00c96SAlison Schofield test_pos = cxl_calc_interleave_pos(cxled); 1808a3e00c96SAlison Schofield dev_dbg(&cxled->cxld.dev, 1809a3e00c96SAlison Schofield "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n", 1810a3e00c96SAlison Schofield (test_pos == cxled->pos) ? "success" : "fail", 1811a3e00c96SAlison Schofield test_pos, cxled->pos); 1812a3e00c96SAlison Schofield } 1813a3e00c96SAlison Schofield 1814b9686e8cSDan Williams return 0; 1815b9686e8cSDan Williams } 1816b9686e8cSDan Williams 1817176baefbSDan Williams static int cxl_region_detach(struct cxl_endpoint_decoder *cxled) 1818b9686e8cSDan Williams { 1819384e624bSDan Williams struct cxl_port *iter, *ep_port = cxled_to_port(cxled); 1820b9686e8cSDan Williams struct cxl_region *cxlr = cxled->cxld.region; 1821b9686e8cSDan Williams struct cxl_region_params *p; 1822176baefbSDan Williams int rc = 0; 1823b9686e8cSDan Williams 1824b9686e8cSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 1825b9686e8cSDan Williams 1826b9686e8cSDan Williams if (!cxlr) 1827176baefbSDan Williams return 0; 1828b9686e8cSDan Williams 1829b9686e8cSDan Williams p = &cxlr->params; 1830b9686e8cSDan Williams get_device(&cxlr->dev); 1831b9686e8cSDan Williams 1832176baefbSDan Williams if (p->state > CXL_CONFIG_ACTIVE) { 1833176baefbSDan Williams /* 1834176baefbSDan Williams * TODO: tear down all impacted regions if a device is 1835176baefbSDan Williams * removed out of order 1836176baefbSDan Williams */ 1837176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 1838176baefbSDan Williams if (rc) 1839176baefbSDan Williams goto out; 1840176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE; 1841176baefbSDan Williams } 1842176baefbSDan Williams 1843384e624bSDan Williams for (iter = ep_port; !is_cxl_root(iter); 1844384e624bSDan Williams iter = to_cxl_port(iter->dev.parent)) 1845384e624bSDan Williams cxl_port_detach_region(iter, cxlr, cxled); 1846384e624bSDan Williams 1847b9686e8cSDan Williams if (cxled->pos < 0 || cxled->pos >= p->interleave_ways || 1848b9686e8cSDan Williams p->targets[cxled->pos] != cxled) { 1849b9686e8cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1850b9686e8cSDan Williams 1851b9686e8cSDan Williams dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n", 1852b9686e8cSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1853b9686e8cSDan Williams cxled->pos); 1854b9686e8cSDan Williams goto out; 1855b9686e8cSDan Williams } 1856b9686e8cSDan Williams 185727b3f8d1SDan Williams if (p->state == CXL_CONFIG_ACTIVE) { 1858384e624bSDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 185927b3f8d1SDan Williams cxl_region_teardown_targets(cxlr); 186027b3f8d1SDan Williams } 1861b9686e8cSDan Williams p->targets[cxled->pos] = NULL; 1862b9686e8cSDan Williams p->nr_targets--; 1863910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) { 1864910bc55dSDan Williams .start = 0, 1865910bc55dSDan Williams .end = -1, 1866910bc55dSDan Williams }; 1867b9686e8cSDan Williams 1868384e624bSDan Williams /* notify the region driver that one of its targets has departed */ 1869b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1870b9686e8cSDan Williams device_release_driver(&cxlr->dev); 1871b9686e8cSDan Williams down_write(&cxl_region_rwsem); 1872b9686e8cSDan Williams out: 1873b9686e8cSDan Williams put_device(&cxlr->dev); 1874176baefbSDan Williams return rc; 1875b9686e8cSDan Williams } 1876b9686e8cSDan Williams 1877b9686e8cSDan Williams void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled) 1878b9686e8cSDan Williams { 1879b9686e8cSDan Williams down_write(&cxl_region_rwsem); 1880b9686e8cSDan Williams cxled->mode = CXL_DECODER_DEAD; 1881b9686e8cSDan Williams cxl_region_detach(cxled); 1882b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1883b9686e8cSDan Williams } 1884b9686e8cSDan Williams 18853528b1e1SDan Williams static int attach_target(struct cxl_region *cxlr, 18863528b1e1SDan Williams struct cxl_endpoint_decoder *cxled, int pos, 18873528b1e1SDan Williams unsigned int state) 1888b9686e8cSDan Williams { 18893528b1e1SDan Williams int rc = 0; 1890b9686e8cSDan Williams 18913528b1e1SDan Williams if (state == TASK_INTERRUPTIBLE) 1892b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem); 18933528b1e1SDan Williams else 18943528b1e1SDan Williams down_write(&cxl_region_rwsem); 1895b9686e8cSDan Williams if (rc) 18963528b1e1SDan Williams return rc; 18973528b1e1SDan Williams 1898b9686e8cSDan Williams down_read(&cxl_dpa_rwsem); 18993528b1e1SDan Williams rc = cxl_region_attach(cxlr, cxled, pos); 1900b9686e8cSDan Williams up_read(&cxl_dpa_rwsem); 1901b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1902b9686e8cSDan Williams return rc; 1903b9686e8cSDan Williams } 1904b9686e8cSDan Williams 1905b9686e8cSDan Williams static int detach_target(struct cxl_region *cxlr, int pos) 1906b9686e8cSDan Williams { 1907b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 1908b9686e8cSDan Williams int rc; 1909b9686e8cSDan Williams 1910b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem); 1911b9686e8cSDan Williams if (rc) 1912b9686e8cSDan Williams return rc; 1913b9686e8cSDan Williams 1914b9686e8cSDan Williams if (pos >= p->interleave_ways) { 1915b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1916b9686e8cSDan Williams p->interleave_ways); 1917b9686e8cSDan Williams rc = -ENXIO; 1918b9686e8cSDan Williams goto out; 1919b9686e8cSDan Williams } 1920b9686e8cSDan Williams 1921b9686e8cSDan Williams if (!p->targets[pos]) { 1922b9686e8cSDan Williams rc = 0; 1923b9686e8cSDan Williams goto out; 1924b9686e8cSDan Williams } 1925b9686e8cSDan Williams 1926176baefbSDan Williams rc = cxl_region_detach(p->targets[pos]); 1927b9686e8cSDan Williams out: 1928b9686e8cSDan Williams up_write(&cxl_region_rwsem); 1929b9686e8cSDan Williams return rc; 1930b9686e8cSDan Williams } 1931b9686e8cSDan Williams 1932b9686e8cSDan Williams static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos, 1933b9686e8cSDan Williams size_t len) 1934b9686e8cSDan Williams { 1935b9686e8cSDan Williams int rc; 1936b9686e8cSDan Williams 1937b9686e8cSDan Williams if (sysfs_streq(buf, "\n")) 1938b9686e8cSDan Williams rc = detach_target(cxlr, pos); 19393528b1e1SDan Williams else { 19403528b1e1SDan Williams struct device *dev; 19413528b1e1SDan Williams 19423528b1e1SDan Williams dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf); 19433528b1e1SDan Williams if (!dev) 19443528b1e1SDan Williams return -ENODEV; 19453528b1e1SDan Williams 19463528b1e1SDan Williams if (!is_endpoint_decoder(dev)) { 19473528b1e1SDan Williams rc = -EINVAL; 19483528b1e1SDan Williams goto out; 19493528b1e1SDan Williams } 19503528b1e1SDan Williams 19513528b1e1SDan Williams rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos, 19523528b1e1SDan Williams TASK_INTERRUPTIBLE); 19533528b1e1SDan Williams out: 19543528b1e1SDan Williams put_device(dev); 19553528b1e1SDan Williams } 1956b9686e8cSDan Williams 1957b9686e8cSDan Williams if (rc < 0) 1958b9686e8cSDan Williams return rc; 1959b9686e8cSDan Williams return len; 1960b9686e8cSDan Williams } 1961b9686e8cSDan Williams 1962b9686e8cSDan Williams #define TARGET_ATTR_RW(n) \ 1963b9686e8cSDan Williams static ssize_t target##n##_show( \ 1964b9686e8cSDan Williams struct device *dev, struct device_attribute *attr, char *buf) \ 1965b9686e8cSDan Williams { \ 1966b9686e8cSDan Williams return show_targetN(to_cxl_region(dev), buf, (n)); \ 1967b9686e8cSDan Williams } \ 1968b9686e8cSDan Williams static ssize_t target##n##_store(struct device *dev, \ 1969b9686e8cSDan Williams struct device_attribute *attr, \ 1970b9686e8cSDan Williams const char *buf, size_t len) \ 1971b9686e8cSDan Williams { \ 1972b9686e8cSDan Williams return store_targetN(to_cxl_region(dev), buf, (n), len); \ 1973b9686e8cSDan Williams } \ 1974b9686e8cSDan Williams static DEVICE_ATTR_RW(target##n) 1975b9686e8cSDan Williams 1976b9686e8cSDan Williams TARGET_ATTR_RW(0); 1977b9686e8cSDan Williams TARGET_ATTR_RW(1); 1978b9686e8cSDan Williams TARGET_ATTR_RW(2); 1979b9686e8cSDan Williams TARGET_ATTR_RW(3); 1980b9686e8cSDan Williams TARGET_ATTR_RW(4); 1981b9686e8cSDan Williams TARGET_ATTR_RW(5); 1982b9686e8cSDan Williams TARGET_ATTR_RW(6); 1983b9686e8cSDan Williams TARGET_ATTR_RW(7); 1984b9686e8cSDan Williams TARGET_ATTR_RW(8); 1985b9686e8cSDan Williams TARGET_ATTR_RW(9); 1986b9686e8cSDan Williams TARGET_ATTR_RW(10); 1987b9686e8cSDan Williams TARGET_ATTR_RW(11); 1988b9686e8cSDan Williams TARGET_ATTR_RW(12); 1989b9686e8cSDan Williams TARGET_ATTR_RW(13); 1990b9686e8cSDan Williams TARGET_ATTR_RW(14); 1991b9686e8cSDan Williams TARGET_ATTR_RW(15); 1992b9686e8cSDan Williams 1993b9686e8cSDan Williams static struct attribute *target_attrs[] = { 1994b9686e8cSDan Williams &dev_attr_target0.attr, 1995b9686e8cSDan Williams &dev_attr_target1.attr, 1996b9686e8cSDan Williams &dev_attr_target2.attr, 1997b9686e8cSDan Williams &dev_attr_target3.attr, 1998b9686e8cSDan Williams &dev_attr_target4.attr, 1999b9686e8cSDan Williams &dev_attr_target5.attr, 2000b9686e8cSDan Williams &dev_attr_target6.attr, 2001b9686e8cSDan Williams &dev_attr_target7.attr, 2002b9686e8cSDan Williams &dev_attr_target8.attr, 2003b9686e8cSDan Williams &dev_attr_target9.attr, 2004b9686e8cSDan Williams &dev_attr_target10.attr, 2005b9686e8cSDan Williams &dev_attr_target11.attr, 2006b9686e8cSDan Williams &dev_attr_target12.attr, 2007b9686e8cSDan Williams &dev_attr_target13.attr, 2008b9686e8cSDan Williams &dev_attr_target14.attr, 2009b9686e8cSDan Williams &dev_attr_target15.attr, 2010b9686e8cSDan Williams NULL, 2011b9686e8cSDan Williams }; 2012b9686e8cSDan Williams 2013b9686e8cSDan Williams static umode_t cxl_region_target_visible(struct kobject *kobj, 2014b9686e8cSDan Williams struct attribute *a, int n) 2015b9686e8cSDan Williams { 2016b9686e8cSDan Williams struct device *dev = kobj_to_dev(kobj); 2017b9686e8cSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 2018b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 2019b9686e8cSDan Williams 2020b9686e8cSDan Williams if (n < p->interleave_ways) 2021b9686e8cSDan Williams return a->mode; 2022b9686e8cSDan Williams return 0; 2023b9686e8cSDan Williams } 2024b9686e8cSDan Williams 2025b9686e8cSDan Williams static const struct attribute_group cxl_region_target_group = { 2026b9686e8cSDan Williams .attrs = target_attrs, 2027b9686e8cSDan Williams .is_visible = cxl_region_target_visible, 2028b9686e8cSDan Williams }; 2029b9686e8cSDan Williams 2030b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void) 2031b9686e8cSDan Williams { 2032b9686e8cSDan Williams return &cxl_region_target_group; 2033b9686e8cSDan Williams } 2034b9686e8cSDan Williams 2035dd5ba0ebSBen Widawsky static const struct attribute_group *region_groups[] = { 2036dd5ba0ebSBen Widawsky &cxl_base_attribute_group, 2037dd5ba0ebSBen Widawsky &cxl_region_group, 2038b9686e8cSDan Williams &cxl_region_target_group, 2039dd5ba0ebSBen Widawsky NULL, 2040dd5ba0ebSBen Widawsky }; 2041dd5ba0ebSBen Widawsky 2042779dd20cSBen Widawsky static void cxl_region_release(struct device *dev) 2043779dd20cSBen Widawsky { 20448f401ec1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 2045779dd20cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 20468f401ec1SDan Williams int id = atomic_read(&cxlrd->region_id); 20478f401ec1SDan Williams 20488f401ec1SDan Williams /* 20498f401ec1SDan Williams * Try to reuse the recently idled id rather than the cached 20508f401ec1SDan Williams * next id to prevent the region id space from increasing 20518f401ec1SDan Williams * unnecessarily. 20528f401ec1SDan Williams */ 20538f401ec1SDan Williams if (cxlr->id < id) 20548f401ec1SDan Williams if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) { 20558f401ec1SDan Williams memregion_free(id); 20568f401ec1SDan Williams goto out; 20578f401ec1SDan Williams } 2058779dd20cSBen Widawsky 2059779dd20cSBen Widawsky memregion_free(cxlr->id); 20608f401ec1SDan Williams out: 20618f401ec1SDan Williams put_device(dev->parent); 2062779dd20cSBen Widawsky kfree(cxlr); 2063779dd20cSBen Widawsky } 2064779dd20cSBen Widawsky 20658d48817dSDan Williams const struct device_type cxl_region_type = { 2066779dd20cSBen Widawsky .name = "cxl_region", 2067779dd20cSBen Widawsky .release = cxl_region_release, 2068dd5ba0ebSBen Widawsky .groups = region_groups 2069779dd20cSBen Widawsky }; 2070779dd20cSBen Widawsky 2071779dd20cSBen Widawsky bool is_cxl_region(struct device *dev) 2072779dd20cSBen Widawsky { 2073779dd20cSBen Widawsky return dev->type == &cxl_region_type; 2074779dd20cSBen Widawsky } 2075779dd20cSBen Widawsky EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL); 2076779dd20cSBen Widawsky 2077779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev) 2078779dd20cSBen Widawsky { 2079779dd20cSBen Widawsky if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type, 2080779dd20cSBen Widawsky "not a cxl_region device\n")) 2081779dd20cSBen Widawsky return NULL; 2082779dd20cSBen Widawsky 2083779dd20cSBen Widawsky return container_of(dev, struct cxl_region, dev); 2084779dd20cSBen Widawsky } 2085779dd20cSBen Widawsky 2086779dd20cSBen Widawsky static void unregister_region(void *dev) 2087779dd20cSBen Widawsky { 208823a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 20890d9e7340SDan Williams struct cxl_region_params *p = &cxlr->params; 20900d9e7340SDan Williams int i; 209123a22cd1SDan Williams 209223a22cd1SDan Williams device_del(dev); 20930d9e7340SDan Williams 20940d9e7340SDan Williams /* 20950d9e7340SDan Williams * Now that region sysfs is shutdown, the parameter block is now 20960d9e7340SDan Williams * read-only, so no need to hold the region rwsem to access the 20970d9e7340SDan Williams * region parameters. 20980d9e7340SDan Williams */ 20990d9e7340SDan Williams for (i = 0; i < p->interleave_ways; i++) 21000d9e7340SDan Williams detach_target(cxlr, i); 21010d9e7340SDan Williams 210223a22cd1SDan Williams cxl_region_iomem_release(cxlr); 210323a22cd1SDan Williams put_device(dev); 2104779dd20cSBen Widawsky } 2105779dd20cSBen Widawsky 2106779dd20cSBen Widawsky static struct lock_class_key cxl_region_key; 2107779dd20cSBen Widawsky 2108779dd20cSBen Widawsky static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id) 2109779dd20cSBen Widawsky { 2110779dd20cSBen Widawsky struct cxl_region *cxlr; 2111779dd20cSBen Widawsky struct device *dev; 2112779dd20cSBen Widawsky 2113779dd20cSBen Widawsky cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL); 2114779dd20cSBen Widawsky if (!cxlr) { 2115779dd20cSBen Widawsky memregion_free(id); 2116779dd20cSBen Widawsky return ERR_PTR(-ENOMEM); 2117779dd20cSBen Widawsky } 2118779dd20cSBen Widawsky 2119779dd20cSBen Widawsky dev = &cxlr->dev; 2120779dd20cSBen Widawsky device_initialize(dev); 2121779dd20cSBen Widawsky lockdep_set_class(&dev->mutex, &cxl_region_key); 2122779dd20cSBen Widawsky dev->parent = &cxlrd->cxlsd.cxld.dev; 21238f401ec1SDan Williams /* 21248f401ec1SDan Williams * Keep root decoder pinned through cxl_region_release to fixup 21258f401ec1SDan Williams * region id allocations 21268f401ec1SDan Williams */ 21278f401ec1SDan Williams get_device(dev->parent); 2128779dd20cSBen Widawsky device_set_pm_not_required(dev); 2129779dd20cSBen Widawsky dev->bus = &cxl_bus_type; 2130779dd20cSBen Widawsky dev->type = &cxl_region_type; 2131779dd20cSBen Widawsky cxlr->id = id; 2132779dd20cSBen Widawsky 2133779dd20cSBen Widawsky return cxlr; 2134779dd20cSBen Widawsky } 2135779dd20cSBen Widawsky 2136779dd20cSBen Widawsky /** 2137779dd20cSBen Widawsky * devm_cxl_add_region - Adds a region to a decoder 2138779dd20cSBen Widawsky * @cxlrd: root decoder 2139779dd20cSBen Widawsky * @id: memregion id to create, or memregion_free() on failure 2140779dd20cSBen Widawsky * @mode: mode for the endpoint decoders of this region 2141779dd20cSBen Widawsky * @type: select whether this is an expander or accelerator (type-2 or type-3) 2142779dd20cSBen Widawsky * 2143779dd20cSBen Widawsky * This is the second step of region initialization. Regions exist within an 2144779dd20cSBen Widawsky * address space which is mapped by a @cxlrd. 2145779dd20cSBen Widawsky * 2146779dd20cSBen Widawsky * Return: 0 if the region was added to the @cxlrd, else returns negative error 2147779dd20cSBen Widawsky * code. The region will be named "regionZ" where Z is the unique region number. 2148779dd20cSBen Widawsky */ 2149779dd20cSBen Widawsky static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, 2150779dd20cSBen Widawsky int id, 2151779dd20cSBen Widawsky enum cxl_decoder_mode mode, 2152779dd20cSBen Widawsky enum cxl_decoder_type type) 2153779dd20cSBen Widawsky { 2154779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent); 2155779dd20cSBen Widawsky struct cxl_region *cxlr; 2156779dd20cSBen Widawsky struct device *dev; 2157779dd20cSBen Widawsky int rc; 2158779dd20cSBen Widawsky 21596e099264SDan Williams switch (mode) { 21606e099264SDan Williams case CXL_DECODER_RAM: 21616e099264SDan Williams case CXL_DECODER_PMEM: 21626e099264SDan Williams break; 21636e099264SDan Williams default: 21646e099264SDan Williams dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode); 21656e099264SDan Williams return ERR_PTR(-EINVAL); 21666e099264SDan Williams } 21676e099264SDan Williams 2168779dd20cSBen Widawsky cxlr = cxl_region_alloc(cxlrd, id); 2169779dd20cSBen Widawsky if (IS_ERR(cxlr)) 2170779dd20cSBen Widawsky return cxlr; 2171779dd20cSBen Widawsky cxlr->mode = mode; 2172779dd20cSBen Widawsky cxlr->type = type; 2173779dd20cSBen Widawsky 2174779dd20cSBen Widawsky dev = &cxlr->dev; 2175779dd20cSBen Widawsky rc = dev_set_name(dev, "region%d", id); 2176779dd20cSBen Widawsky if (rc) 2177779dd20cSBen Widawsky goto err; 2178779dd20cSBen Widawsky 2179779dd20cSBen Widawsky rc = device_add(dev); 2180779dd20cSBen Widawsky if (rc) 2181779dd20cSBen Widawsky goto err; 2182779dd20cSBen Widawsky 21837481653dSDan Williams rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); 2184779dd20cSBen Widawsky if (rc) 2185779dd20cSBen Widawsky return ERR_PTR(rc); 2186779dd20cSBen Widawsky 21877481653dSDan Williams dev_dbg(port->uport_dev, "%s: created %s\n", 2188779dd20cSBen Widawsky dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); 2189779dd20cSBen Widawsky return cxlr; 2190779dd20cSBen Widawsky 2191779dd20cSBen Widawsky err: 2192779dd20cSBen Widawsky put_device(dev); 2193779dd20cSBen Widawsky return ERR_PTR(rc); 2194779dd20cSBen Widawsky } 2195779dd20cSBen Widawsky 21966e099264SDan Williams static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf) 21976e099264SDan Williams { 21986e099264SDan Williams return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id)); 21996e099264SDan Williams } 22006e099264SDan Williams 2201779dd20cSBen Widawsky static ssize_t create_pmem_region_show(struct device *dev, 2202779dd20cSBen Widawsky struct device_attribute *attr, char *buf) 2203779dd20cSBen Widawsky { 22046e099264SDan Williams return __create_region_show(to_cxl_root_decoder(dev), buf); 22056e099264SDan Williams } 2206779dd20cSBen Widawsky 22076e099264SDan Williams static ssize_t create_ram_region_show(struct device *dev, 22086e099264SDan Williams struct device_attribute *attr, char *buf) 22096e099264SDan Williams { 22106e099264SDan Williams return __create_region_show(to_cxl_root_decoder(dev), buf); 22116e099264SDan Williams } 22126e099264SDan Williams 22136e099264SDan Williams static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd, 22146e099264SDan Williams enum cxl_decoder_mode mode, int id) 22156e099264SDan Williams { 22166e099264SDan Williams int rc; 22176e099264SDan Williams 22186e099264SDan Williams rc = memregion_alloc(GFP_KERNEL); 22196e099264SDan Williams if (rc < 0) 22206e099264SDan Williams return ERR_PTR(rc); 22216e099264SDan Williams 22226e099264SDan Williams if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) { 22236e099264SDan Williams memregion_free(rc); 22246e099264SDan Williams return ERR_PTR(-EBUSY); 22256e099264SDan Williams } 22266e099264SDan Williams 22275aa39a91SDan Williams return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM); 2228779dd20cSBen Widawsky } 2229779dd20cSBen Widawsky 2230779dd20cSBen Widawsky static ssize_t create_pmem_region_store(struct device *dev, 2231779dd20cSBen Widawsky struct device_attribute *attr, 2232779dd20cSBen Widawsky const char *buf, size_t len) 2233779dd20cSBen Widawsky { 2234779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2235779dd20cSBen Widawsky struct cxl_region *cxlr; 22366e099264SDan Williams int rc, id; 2237779dd20cSBen Widawsky 2238779dd20cSBen Widawsky rc = sscanf(buf, "region%d\n", &id); 2239779dd20cSBen Widawsky if (rc != 1) 2240779dd20cSBen Widawsky return -EINVAL; 2241779dd20cSBen Widawsky 22426e099264SDan Williams cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id); 2243779dd20cSBen Widawsky if (IS_ERR(cxlr)) 2244779dd20cSBen Widawsky return PTR_ERR(cxlr); 2245779dd20cSBen Widawsky 2246779dd20cSBen Widawsky return len; 2247779dd20cSBen Widawsky } 2248779dd20cSBen Widawsky DEVICE_ATTR_RW(create_pmem_region); 2249779dd20cSBen Widawsky 22506e099264SDan Williams static ssize_t create_ram_region_store(struct device *dev, 22516e099264SDan Williams struct device_attribute *attr, 22526e099264SDan Williams const char *buf, size_t len) 22536e099264SDan Williams { 22546e099264SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 22556e099264SDan Williams struct cxl_region *cxlr; 22566e099264SDan Williams int rc, id; 22576e099264SDan Williams 22586e099264SDan Williams rc = sscanf(buf, "region%d\n", &id); 22596e099264SDan Williams if (rc != 1) 22606e099264SDan Williams return -EINVAL; 22616e099264SDan Williams 22626e099264SDan Williams cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id); 22636e099264SDan Williams if (IS_ERR(cxlr)) 22646e099264SDan Williams return PTR_ERR(cxlr); 22656e099264SDan Williams 22666e099264SDan Williams return len; 22676e099264SDan Williams } 22686e099264SDan Williams DEVICE_ATTR_RW(create_ram_region); 22696e099264SDan Williams 2270b9686e8cSDan Williams static ssize_t region_show(struct device *dev, struct device_attribute *attr, 2271b9686e8cSDan Williams char *buf) 2272b9686e8cSDan Williams { 2273b9686e8cSDan Williams struct cxl_decoder *cxld = to_cxl_decoder(dev); 2274b9686e8cSDan Williams ssize_t rc; 2275b9686e8cSDan Williams 2276b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 2277b9686e8cSDan Williams if (rc) 2278b9686e8cSDan Williams return rc; 2279b9686e8cSDan Williams 2280b9686e8cSDan Williams if (cxld->region) 2281b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev)); 2282b9686e8cSDan Williams else 2283b9686e8cSDan Williams rc = sysfs_emit(buf, "\n"); 2284b9686e8cSDan Williams up_read(&cxl_region_rwsem); 2285b9686e8cSDan Williams 2286b9686e8cSDan Williams return rc; 2287b9686e8cSDan Williams } 2288b9686e8cSDan Williams DEVICE_ATTR_RO(region); 2289b9686e8cSDan Williams 2290779dd20cSBen Widawsky static struct cxl_region * 2291779dd20cSBen Widawsky cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name) 2292779dd20cSBen Widawsky { 2293779dd20cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 2294779dd20cSBen Widawsky struct device *region_dev; 2295779dd20cSBen Widawsky 2296779dd20cSBen Widawsky region_dev = device_find_child_by_name(&cxld->dev, name); 2297779dd20cSBen Widawsky if (!region_dev) 2298779dd20cSBen Widawsky return ERR_PTR(-ENODEV); 2299779dd20cSBen Widawsky 2300779dd20cSBen Widawsky return to_cxl_region(region_dev); 2301779dd20cSBen Widawsky } 2302779dd20cSBen Widawsky 2303779dd20cSBen Widawsky static ssize_t delete_region_store(struct device *dev, 2304779dd20cSBen Widawsky struct device_attribute *attr, 2305779dd20cSBen Widawsky const char *buf, size_t len) 2306779dd20cSBen Widawsky { 2307779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2308779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(dev->parent); 2309779dd20cSBen Widawsky struct cxl_region *cxlr; 2310779dd20cSBen Widawsky 2311779dd20cSBen Widawsky cxlr = cxl_find_region_by_name(cxlrd, buf); 2312779dd20cSBen Widawsky if (IS_ERR(cxlr)) 2313779dd20cSBen Widawsky return PTR_ERR(cxlr); 2314779dd20cSBen Widawsky 23157481653dSDan Williams devm_release_action(port->uport_dev, unregister_region, cxlr); 2316779dd20cSBen Widawsky put_device(&cxlr->dev); 2317779dd20cSBen Widawsky 2318779dd20cSBen Widawsky return len; 2319779dd20cSBen Widawsky } 2320779dd20cSBen Widawsky DEVICE_ATTR_WO(delete_region); 232123a22cd1SDan Williams 232204ad63f0SDan Williams static void cxl_pmem_region_release(struct device *dev) 232304ad63f0SDan Williams { 232404ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); 232504ad63f0SDan Williams int i; 232604ad63f0SDan Williams 232704ad63f0SDan Williams for (i = 0; i < cxlr_pmem->nr_mappings; i++) { 232804ad63f0SDan Williams struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd; 232904ad63f0SDan Williams 233004ad63f0SDan Williams put_device(&cxlmd->dev); 233104ad63f0SDan Williams } 233204ad63f0SDan Williams 233304ad63f0SDan Williams kfree(cxlr_pmem); 233404ad63f0SDan Williams } 233504ad63f0SDan Williams 233604ad63f0SDan Williams static const struct attribute_group *cxl_pmem_region_attribute_groups[] = { 233704ad63f0SDan Williams &cxl_base_attribute_group, 233804ad63f0SDan Williams NULL, 233904ad63f0SDan Williams }; 234004ad63f0SDan Williams 234104ad63f0SDan Williams const struct device_type cxl_pmem_region_type = { 234204ad63f0SDan Williams .name = "cxl_pmem_region", 234304ad63f0SDan Williams .release = cxl_pmem_region_release, 234404ad63f0SDan Williams .groups = cxl_pmem_region_attribute_groups, 234504ad63f0SDan Williams }; 234604ad63f0SDan Williams 234704ad63f0SDan Williams bool is_cxl_pmem_region(struct device *dev) 234804ad63f0SDan Williams { 234904ad63f0SDan Williams return dev->type == &cxl_pmem_region_type; 235004ad63f0SDan Williams } 235104ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL); 235204ad63f0SDan Williams 235304ad63f0SDan Williams struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev) 235404ad63f0SDan Williams { 235504ad63f0SDan Williams if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev), 235604ad63f0SDan Williams "not a cxl_pmem_region device\n")) 235704ad63f0SDan Williams return NULL; 235804ad63f0SDan Williams return container_of(dev, struct cxl_pmem_region, dev); 235904ad63f0SDan Williams } 236004ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL); 236104ad63f0SDan Williams 2362f0832a58SAlison Schofield struct cxl_poison_context { 2363f0832a58SAlison Schofield struct cxl_port *port; 2364f0832a58SAlison Schofield enum cxl_decoder_mode mode; 2365f0832a58SAlison Schofield u64 offset; 2366f0832a58SAlison Schofield }; 2367f0832a58SAlison Schofield 2368f0832a58SAlison Schofield static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd, 2369f0832a58SAlison Schofield struct cxl_poison_context *ctx) 2370f0832a58SAlison Schofield { 2371f0832a58SAlison Schofield struct cxl_dev_state *cxlds = cxlmd->cxlds; 2372f0832a58SAlison Schofield u64 offset, length; 2373f0832a58SAlison Schofield int rc = 0; 2374f0832a58SAlison Schofield 2375f0832a58SAlison Schofield /* 2376f0832a58SAlison Schofield * Collect poison for the remaining unmapped resources 2377f0832a58SAlison Schofield * after poison is collected by committed endpoints. 2378f0832a58SAlison Schofield * 2379f0832a58SAlison Schofield * Knowing that PMEM must always follow RAM, get poison 2380f0832a58SAlison Schofield * for unmapped resources based on the last decoder's mode: 2381f0832a58SAlison Schofield * ram: scan remains of ram range, then any pmem range 2382f0832a58SAlison Schofield * pmem: scan remains of pmem range 2383f0832a58SAlison Schofield */ 2384f0832a58SAlison Schofield 2385f0832a58SAlison Schofield if (ctx->mode == CXL_DECODER_RAM) { 2386f0832a58SAlison Schofield offset = ctx->offset; 2387f0832a58SAlison Schofield length = resource_size(&cxlds->ram_res) - offset; 2388f0832a58SAlison Schofield rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 2389f0832a58SAlison Schofield if (rc == -EFAULT) 2390f0832a58SAlison Schofield rc = 0; 2391f0832a58SAlison Schofield if (rc) 2392f0832a58SAlison Schofield return rc; 2393f0832a58SAlison Schofield } 2394f0832a58SAlison Schofield if (ctx->mode == CXL_DECODER_PMEM) { 2395f0832a58SAlison Schofield offset = ctx->offset; 2396f0832a58SAlison Schofield length = resource_size(&cxlds->dpa_res) - offset; 2397f0832a58SAlison Schofield if (!length) 2398f0832a58SAlison Schofield return 0; 2399f0832a58SAlison Schofield } else if (resource_size(&cxlds->pmem_res)) { 2400f0832a58SAlison Schofield offset = cxlds->pmem_res.start; 2401f0832a58SAlison Schofield length = resource_size(&cxlds->pmem_res); 2402f0832a58SAlison Schofield } else { 2403f0832a58SAlison Schofield return 0; 2404f0832a58SAlison Schofield } 2405f0832a58SAlison Schofield 2406f0832a58SAlison Schofield return cxl_mem_get_poison(cxlmd, offset, length, NULL); 2407f0832a58SAlison Schofield } 2408f0832a58SAlison Schofield 2409f0832a58SAlison Schofield static int poison_by_decoder(struct device *dev, void *arg) 2410f0832a58SAlison Schofield { 2411f0832a58SAlison Schofield struct cxl_poison_context *ctx = arg; 2412f0832a58SAlison Schofield struct cxl_endpoint_decoder *cxled; 2413f0832a58SAlison Schofield struct cxl_memdev *cxlmd; 2414f0832a58SAlison Schofield u64 offset, length; 2415f0832a58SAlison Schofield int rc = 0; 2416f0832a58SAlison Schofield 2417f0832a58SAlison Schofield if (!is_endpoint_decoder(dev)) 2418f0832a58SAlison Schofield return rc; 2419f0832a58SAlison Schofield 2420f0832a58SAlison Schofield cxled = to_cxl_endpoint_decoder(dev); 2421f0832a58SAlison Schofield if (!cxled->dpa_res || !resource_size(cxled->dpa_res)) 2422f0832a58SAlison Schofield return rc; 2423f0832a58SAlison Schofield 2424f0832a58SAlison Schofield /* 2425f0832a58SAlison Schofield * Regions are only created with single mode decoders: pmem or ram. 2426f0832a58SAlison Schofield * Linux does not support mixed mode decoders. This means that 2427f0832a58SAlison Schofield * reading poison per endpoint decoder adheres to the requirement 2428f0832a58SAlison Schofield * that poison reads of pmem and ram must be separated. 2429f0832a58SAlison Schofield * CXL 3.0 Spec 8.2.9.8.4.1 2430f0832a58SAlison Schofield */ 2431f0832a58SAlison Schofield if (cxled->mode == CXL_DECODER_MIXED) { 2432f0832a58SAlison Schofield dev_dbg(dev, "poison list read unsupported in mixed mode\n"); 2433f0832a58SAlison Schofield return rc; 2434f0832a58SAlison Schofield } 2435f0832a58SAlison Schofield 2436f0832a58SAlison Schofield cxlmd = cxled_to_memdev(cxled); 2437f0832a58SAlison Schofield if (cxled->skip) { 2438f0832a58SAlison Schofield offset = cxled->dpa_res->start - cxled->skip; 2439f0832a58SAlison Schofield length = cxled->skip; 2440f0832a58SAlison Schofield rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 2441f0832a58SAlison Schofield if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM) 2442f0832a58SAlison Schofield rc = 0; 2443f0832a58SAlison Schofield if (rc) 2444f0832a58SAlison Schofield return rc; 2445f0832a58SAlison Schofield } 2446f0832a58SAlison Schofield 2447f0832a58SAlison Schofield offset = cxled->dpa_res->start; 2448f0832a58SAlison Schofield length = cxled->dpa_res->end - offset + 1; 2449f0832a58SAlison Schofield rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region); 2450f0832a58SAlison Schofield if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM) 2451f0832a58SAlison Schofield rc = 0; 2452f0832a58SAlison Schofield if (rc) 2453f0832a58SAlison Schofield return rc; 2454f0832a58SAlison Schofield 2455f0832a58SAlison Schofield /* Iterate until commit_end is reached */ 2456f0832a58SAlison Schofield if (cxled->cxld.id == ctx->port->commit_end) { 2457f0832a58SAlison Schofield ctx->offset = cxled->dpa_res->end + 1; 2458f0832a58SAlison Schofield ctx->mode = cxled->mode; 2459f0832a58SAlison Schofield return 1; 2460f0832a58SAlison Schofield } 2461f0832a58SAlison Schofield 2462f0832a58SAlison Schofield return 0; 2463f0832a58SAlison Schofield } 2464f0832a58SAlison Schofield 2465f0832a58SAlison Schofield int cxl_get_poison_by_endpoint(struct cxl_port *port) 2466f0832a58SAlison Schofield { 2467f0832a58SAlison Schofield struct cxl_poison_context ctx; 2468f0832a58SAlison Schofield int rc = 0; 2469f0832a58SAlison Schofield 2470f0832a58SAlison Schofield ctx = (struct cxl_poison_context) { 2471f0832a58SAlison Schofield .port = port 2472f0832a58SAlison Schofield }; 2473f0832a58SAlison Schofield 2474f0832a58SAlison Schofield rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder); 2475f0832a58SAlison Schofield if (rc == 1) 24767481653dSDan Williams rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev), 24777481653dSDan Williams &ctx); 2478f0832a58SAlison Schofield 2479f0832a58SAlison Schofield return rc; 2480f0832a58SAlison Schofield } 2481f0832a58SAlison Schofield 248204ad63f0SDan Williams static struct lock_class_key cxl_pmem_region_key; 248304ad63f0SDan Williams 248404ad63f0SDan Williams static struct cxl_pmem_region *cxl_pmem_region_alloc(struct cxl_region *cxlr) 248504ad63f0SDan Williams { 248604ad63f0SDan Williams struct cxl_region_params *p = &cxlr->params; 2487f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb; 248804ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem; 248904ad63f0SDan Williams struct device *dev; 249004ad63f0SDan Williams int i; 249104ad63f0SDan Williams 249204ad63f0SDan Williams down_read(&cxl_region_rwsem); 249304ad63f0SDan Williams if (p->state != CXL_CONFIG_COMMIT) { 249404ad63f0SDan Williams cxlr_pmem = ERR_PTR(-ENXIO); 249504ad63f0SDan Williams goto out; 249604ad63f0SDan Williams } 249704ad63f0SDan Williams 249804ad63f0SDan Williams cxlr_pmem = kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), 249904ad63f0SDan Williams GFP_KERNEL); 250004ad63f0SDan Williams if (!cxlr_pmem) { 250104ad63f0SDan Williams cxlr_pmem = ERR_PTR(-ENOMEM); 250204ad63f0SDan Williams goto out; 250304ad63f0SDan Williams } 250404ad63f0SDan Williams 250504ad63f0SDan Williams cxlr_pmem->hpa_range.start = p->res->start; 250604ad63f0SDan Williams cxlr_pmem->hpa_range.end = p->res->end; 250704ad63f0SDan Williams 250804ad63f0SDan Williams /* Snapshot the region configuration underneath the cxl_region_rwsem */ 250904ad63f0SDan Williams cxlr_pmem->nr_mappings = p->nr_targets; 251004ad63f0SDan Williams for (i = 0; i < p->nr_targets; i++) { 251104ad63f0SDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 251204ad63f0SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 251304ad63f0SDan Williams struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; 251404ad63f0SDan Williams 2515f17b558dSDan Williams /* 2516f17b558dSDan Williams * Regions never span CXL root devices, so by definition the 2517f17b558dSDan Williams * bridge for one device is the same for all. 2518f17b558dSDan Williams */ 2519f17b558dSDan Williams if (i == 0) { 2520d35b495dSDan Williams cxl_nvb = cxl_find_nvdimm_bridge(cxlmd); 2521f17b558dSDan Williams if (!cxl_nvb) { 2522f17b558dSDan Williams cxlr_pmem = ERR_PTR(-ENODEV); 2523f17b558dSDan Williams goto out; 2524f17b558dSDan Williams } 2525f17b558dSDan Williams cxlr->cxl_nvb = cxl_nvb; 2526f17b558dSDan Williams } 252704ad63f0SDan Williams m->cxlmd = cxlmd; 252804ad63f0SDan Williams get_device(&cxlmd->dev); 252904ad63f0SDan Williams m->start = cxled->dpa_res->start; 253004ad63f0SDan Williams m->size = resource_size(cxled->dpa_res); 253104ad63f0SDan Williams m->position = i; 253204ad63f0SDan Williams } 253304ad63f0SDan Williams 253404ad63f0SDan Williams dev = &cxlr_pmem->dev; 253504ad63f0SDan Williams cxlr_pmem->cxlr = cxlr; 2536f17b558dSDan Williams cxlr->cxlr_pmem = cxlr_pmem; 253704ad63f0SDan Williams device_initialize(dev); 253804ad63f0SDan Williams lockdep_set_class(&dev->mutex, &cxl_pmem_region_key); 253904ad63f0SDan Williams device_set_pm_not_required(dev); 254004ad63f0SDan Williams dev->parent = &cxlr->dev; 254104ad63f0SDan Williams dev->bus = &cxl_bus_type; 254204ad63f0SDan Williams dev->type = &cxl_pmem_region_type; 254304ad63f0SDan Williams out: 254404ad63f0SDan Williams up_read(&cxl_region_rwsem); 254504ad63f0SDan Williams 254604ad63f0SDan Williams return cxlr_pmem; 254704ad63f0SDan Williams } 254804ad63f0SDan Williams 254909d09e04SDan Williams static void cxl_dax_region_release(struct device *dev) 255009d09e04SDan Williams { 255109d09e04SDan Williams struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev); 255209d09e04SDan Williams 255309d09e04SDan Williams kfree(cxlr_dax); 255409d09e04SDan Williams } 255509d09e04SDan Williams 255609d09e04SDan Williams static const struct attribute_group *cxl_dax_region_attribute_groups[] = { 255709d09e04SDan Williams &cxl_base_attribute_group, 255809d09e04SDan Williams NULL, 255909d09e04SDan Williams }; 256009d09e04SDan Williams 256109d09e04SDan Williams const struct device_type cxl_dax_region_type = { 256209d09e04SDan Williams .name = "cxl_dax_region", 256309d09e04SDan Williams .release = cxl_dax_region_release, 256409d09e04SDan Williams .groups = cxl_dax_region_attribute_groups, 256509d09e04SDan Williams }; 256609d09e04SDan Williams 256709d09e04SDan Williams static bool is_cxl_dax_region(struct device *dev) 256809d09e04SDan Williams { 256909d09e04SDan Williams return dev->type == &cxl_dax_region_type; 257009d09e04SDan Williams } 257109d09e04SDan Williams 257209d09e04SDan Williams struct cxl_dax_region *to_cxl_dax_region(struct device *dev) 257309d09e04SDan Williams { 257409d09e04SDan Williams if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev), 257509d09e04SDan Williams "not a cxl_dax_region device\n")) 257609d09e04SDan Williams return NULL; 257709d09e04SDan Williams return container_of(dev, struct cxl_dax_region, dev); 257809d09e04SDan Williams } 257909d09e04SDan Williams EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL); 258009d09e04SDan Williams 258109d09e04SDan Williams static struct lock_class_key cxl_dax_region_key; 258209d09e04SDan Williams 258309d09e04SDan Williams static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr) 258409d09e04SDan Williams { 258509d09e04SDan Williams struct cxl_region_params *p = &cxlr->params; 258609d09e04SDan Williams struct cxl_dax_region *cxlr_dax; 258709d09e04SDan Williams struct device *dev; 258809d09e04SDan Williams 258909d09e04SDan Williams down_read(&cxl_region_rwsem); 259009d09e04SDan Williams if (p->state != CXL_CONFIG_COMMIT) { 259109d09e04SDan Williams cxlr_dax = ERR_PTR(-ENXIO); 259209d09e04SDan Williams goto out; 259309d09e04SDan Williams } 259409d09e04SDan Williams 259509d09e04SDan Williams cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL); 259609d09e04SDan Williams if (!cxlr_dax) { 259709d09e04SDan Williams cxlr_dax = ERR_PTR(-ENOMEM); 259809d09e04SDan Williams goto out; 259909d09e04SDan Williams } 260009d09e04SDan Williams 260109d09e04SDan Williams cxlr_dax->hpa_range.start = p->res->start; 260209d09e04SDan Williams cxlr_dax->hpa_range.end = p->res->end; 260309d09e04SDan Williams 260409d09e04SDan Williams dev = &cxlr_dax->dev; 260509d09e04SDan Williams cxlr_dax->cxlr = cxlr; 260609d09e04SDan Williams device_initialize(dev); 260709d09e04SDan Williams lockdep_set_class(&dev->mutex, &cxl_dax_region_key); 260809d09e04SDan Williams device_set_pm_not_required(dev); 260909d09e04SDan Williams dev->parent = &cxlr->dev; 261009d09e04SDan Williams dev->bus = &cxl_bus_type; 261109d09e04SDan Williams dev->type = &cxl_dax_region_type; 261209d09e04SDan Williams out: 261309d09e04SDan Williams up_read(&cxl_region_rwsem); 261409d09e04SDan Williams 261509d09e04SDan Williams return cxlr_dax; 261609d09e04SDan Williams } 261709d09e04SDan Williams 2618f17b558dSDan Williams static void cxlr_pmem_unregister(void *_cxlr_pmem) 261904ad63f0SDan Williams { 2620f17b558dSDan Williams struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem; 2621f17b558dSDan Williams struct cxl_region *cxlr = cxlr_pmem->cxlr; 2622f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 2623f17b558dSDan Williams 2624f17b558dSDan Williams /* 2625f17b558dSDan Williams * Either the bridge is in ->remove() context under the device_lock(), 2626f17b558dSDan Williams * or cxlr_release_nvdimm() is cancelling the bridge's release action 2627f17b558dSDan Williams * for @cxlr_pmem and doing it itself (while manually holding the bridge 2628f17b558dSDan Williams * lock). 2629f17b558dSDan Williams */ 2630f17b558dSDan Williams device_lock_assert(&cxl_nvb->dev); 2631f17b558dSDan Williams cxlr->cxlr_pmem = NULL; 2632f17b558dSDan Williams cxlr_pmem->cxlr = NULL; 2633f17b558dSDan Williams device_unregister(&cxlr_pmem->dev); 2634f17b558dSDan Williams } 2635f17b558dSDan Williams 2636f17b558dSDan Williams static void cxlr_release_nvdimm(void *_cxlr) 2637f17b558dSDan Williams { 2638f17b558dSDan Williams struct cxl_region *cxlr = _cxlr; 2639f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 2640f17b558dSDan Williams 2641f17b558dSDan Williams device_lock(&cxl_nvb->dev); 2642f17b558dSDan Williams if (cxlr->cxlr_pmem) 2643f17b558dSDan Williams devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister, 2644f17b558dSDan Williams cxlr->cxlr_pmem); 2645f17b558dSDan Williams device_unlock(&cxl_nvb->dev); 2646f17b558dSDan Williams cxlr->cxl_nvb = NULL; 2647f17b558dSDan Williams put_device(&cxl_nvb->dev); 264804ad63f0SDan Williams } 264904ad63f0SDan Williams 265004ad63f0SDan Williams /** 265104ad63f0SDan Williams * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge 265204ad63f0SDan Williams * @cxlr: parent CXL region for this pmem region bridge device 265304ad63f0SDan Williams * 265404ad63f0SDan Williams * Return: 0 on success negative error code on failure. 265504ad63f0SDan Williams */ 265604ad63f0SDan Williams static int devm_cxl_add_pmem_region(struct cxl_region *cxlr) 265704ad63f0SDan Williams { 265804ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem; 2659f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb; 266004ad63f0SDan Williams struct device *dev; 266104ad63f0SDan Williams int rc; 266204ad63f0SDan Williams 266304ad63f0SDan Williams cxlr_pmem = cxl_pmem_region_alloc(cxlr); 266404ad63f0SDan Williams if (IS_ERR(cxlr_pmem)) 266504ad63f0SDan Williams return PTR_ERR(cxlr_pmem); 2666f17b558dSDan Williams cxl_nvb = cxlr->cxl_nvb; 266704ad63f0SDan Williams 266804ad63f0SDan Williams dev = &cxlr_pmem->dev; 266904ad63f0SDan Williams rc = dev_set_name(dev, "pmem_region%d", cxlr->id); 267004ad63f0SDan Williams if (rc) 267104ad63f0SDan Williams goto err; 267204ad63f0SDan Williams 267304ad63f0SDan Williams rc = device_add(dev); 267404ad63f0SDan Williams if (rc) 267504ad63f0SDan Williams goto err; 267604ad63f0SDan Williams 267704ad63f0SDan Williams dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 267804ad63f0SDan Williams dev_name(dev)); 267904ad63f0SDan Williams 2680f17b558dSDan Williams device_lock(&cxl_nvb->dev); 2681f17b558dSDan Williams if (cxl_nvb->dev.driver) 2682f17b558dSDan Williams rc = devm_add_action_or_reset(&cxl_nvb->dev, 2683f17b558dSDan Williams cxlr_pmem_unregister, cxlr_pmem); 2684f17b558dSDan Williams else 2685f17b558dSDan Williams rc = -ENXIO; 2686f17b558dSDan Williams device_unlock(&cxl_nvb->dev); 2687f17b558dSDan Williams 2688f17b558dSDan Williams if (rc) 2689f17b558dSDan Williams goto err_bridge; 2690f17b558dSDan Williams 2691f17b558dSDan Williams /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */ 2692f17b558dSDan Williams return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr); 269304ad63f0SDan Williams 269404ad63f0SDan Williams err: 269504ad63f0SDan Williams put_device(dev); 2696f17b558dSDan Williams err_bridge: 2697f17b558dSDan Williams put_device(&cxl_nvb->dev); 2698f17b558dSDan Williams cxlr->cxl_nvb = NULL; 269904ad63f0SDan Williams return rc; 270004ad63f0SDan Williams } 270104ad63f0SDan Williams 270209d09e04SDan Williams static void cxlr_dax_unregister(void *_cxlr_dax) 270309d09e04SDan Williams { 270409d09e04SDan Williams struct cxl_dax_region *cxlr_dax = _cxlr_dax; 270509d09e04SDan Williams 270609d09e04SDan Williams device_unregister(&cxlr_dax->dev); 270709d09e04SDan Williams } 270809d09e04SDan Williams 270909d09e04SDan Williams static int devm_cxl_add_dax_region(struct cxl_region *cxlr) 271009d09e04SDan Williams { 271109d09e04SDan Williams struct cxl_dax_region *cxlr_dax; 271209d09e04SDan Williams struct device *dev; 271309d09e04SDan Williams int rc; 271409d09e04SDan Williams 271509d09e04SDan Williams cxlr_dax = cxl_dax_region_alloc(cxlr); 271609d09e04SDan Williams if (IS_ERR(cxlr_dax)) 271709d09e04SDan Williams return PTR_ERR(cxlr_dax); 271809d09e04SDan Williams 271909d09e04SDan Williams dev = &cxlr_dax->dev; 272009d09e04SDan Williams rc = dev_set_name(dev, "dax_region%d", cxlr->id); 272109d09e04SDan Williams if (rc) 272209d09e04SDan Williams goto err; 272309d09e04SDan Williams 272409d09e04SDan Williams rc = device_add(dev); 272509d09e04SDan Williams if (rc) 272609d09e04SDan Williams goto err; 272709d09e04SDan Williams 272809d09e04SDan Williams dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 272909d09e04SDan Williams dev_name(dev)); 273009d09e04SDan Williams 273109d09e04SDan Williams return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister, 273209d09e04SDan Williams cxlr_dax); 273309d09e04SDan Williams err: 273409d09e04SDan Williams put_device(dev); 273509d09e04SDan Williams return rc; 273609d09e04SDan Williams } 273709d09e04SDan Williams 273811105814SAlison Schofield static int match_root_decoder_by_range(struct device *dev, void *data) 2739a32320b7SDan Williams { 2740a32320b7SDan Williams struct range *r1, *r2 = data; 2741a32320b7SDan Williams struct cxl_root_decoder *cxlrd; 2742a32320b7SDan Williams 2743a32320b7SDan Williams if (!is_root_decoder(dev)) 2744a32320b7SDan Williams return 0; 2745a32320b7SDan Williams 2746a32320b7SDan Williams cxlrd = to_cxl_root_decoder(dev); 2747a32320b7SDan Williams r1 = &cxlrd->cxlsd.cxld.hpa_range; 2748a32320b7SDan Williams return range_contains(r1, r2); 2749a32320b7SDan Williams } 2750a32320b7SDan Williams 2751a32320b7SDan Williams static int match_region_by_range(struct device *dev, void *data) 2752a32320b7SDan Williams { 2753a32320b7SDan Williams struct cxl_region_params *p; 2754a32320b7SDan Williams struct cxl_region *cxlr; 2755a32320b7SDan Williams struct range *r = data; 2756a32320b7SDan Williams int rc = 0; 2757a32320b7SDan Williams 2758a32320b7SDan Williams if (!is_cxl_region(dev)) 2759a32320b7SDan Williams return 0; 2760a32320b7SDan Williams 2761a32320b7SDan Williams cxlr = to_cxl_region(dev); 2762a32320b7SDan Williams p = &cxlr->params; 2763a32320b7SDan Williams 2764a32320b7SDan Williams down_read(&cxl_region_rwsem); 2765a32320b7SDan Williams if (p->res && p->res->start == r->start && p->res->end == r->end) 2766a32320b7SDan Williams rc = 1; 2767a32320b7SDan Williams up_read(&cxl_region_rwsem); 2768a32320b7SDan Williams 2769a32320b7SDan Williams return rc; 2770a32320b7SDan Williams } 2771a32320b7SDan Williams 2772a32320b7SDan Williams /* Establish an empty region covering the given HPA range */ 2773a32320b7SDan Williams static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, 2774a32320b7SDan Williams struct cxl_endpoint_decoder *cxled) 2775a32320b7SDan Williams { 2776a32320b7SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 2777a32320b7SDan Williams struct cxl_port *port = cxlrd_to_port(cxlrd); 2778a32320b7SDan Williams struct range *hpa = &cxled->cxld.hpa_range; 2779a32320b7SDan Williams struct cxl_region_params *p; 2780a32320b7SDan Williams struct cxl_region *cxlr; 2781a32320b7SDan Williams struct resource *res; 2782a32320b7SDan Williams int rc; 2783a32320b7SDan Williams 2784a32320b7SDan Williams do { 2785a32320b7SDan Williams cxlr = __create_region(cxlrd, cxled->mode, 2786a32320b7SDan Williams atomic_read(&cxlrd->region_id)); 2787a32320b7SDan Williams } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY); 2788a32320b7SDan Williams 2789a32320b7SDan Williams if (IS_ERR(cxlr)) { 2790a32320b7SDan Williams dev_err(cxlmd->dev.parent, 2791a32320b7SDan Williams "%s:%s: %s failed assign region: %ld\n", 2792a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 2793a32320b7SDan Williams __func__, PTR_ERR(cxlr)); 2794a32320b7SDan Williams return cxlr; 2795a32320b7SDan Williams } 2796a32320b7SDan Williams 2797a32320b7SDan Williams down_write(&cxl_region_rwsem); 2798a32320b7SDan Williams p = &cxlr->params; 2799a32320b7SDan Williams if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 2800a32320b7SDan Williams dev_err(cxlmd->dev.parent, 2801a32320b7SDan Williams "%s:%s: %s autodiscovery interrupted\n", 2802a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 2803a32320b7SDan Williams __func__); 2804a32320b7SDan Williams rc = -EBUSY; 2805a32320b7SDan Williams goto err; 2806a32320b7SDan Williams } 2807a32320b7SDan Williams 2808a32320b7SDan Williams set_bit(CXL_REGION_F_AUTO, &cxlr->flags); 2809a32320b7SDan Williams 2810a32320b7SDan Williams res = kmalloc(sizeof(*res), GFP_KERNEL); 2811a32320b7SDan Williams if (!res) { 2812a32320b7SDan Williams rc = -ENOMEM; 2813a32320b7SDan Williams goto err; 2814a32320b7SDan Williams } 2815a32320b7SDan Williams 2816a32320b7SDan Williams *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa), 2817a32320b7SDan Williams dev_name(&cxlr->dev)); 2818a32320b7SDan Williams rc = insert_resource(cxlrd->res, res); 2819a32320b7SDan Williams if (rc) { 2820a32320b7SDan Williams /* 2821a32320b7SDan Williams * Platform-firmware may not have split resources like "System 2822a32320b7SDan Williams * RAM" on CXL window boundaries see cxl_region_iomem_release() 2823a32320b7SDan Williams */ 2824a32320b7SDan Williams dev_warn(cxlmd->dev.parent, 2825a32320b7SDan Williams "%s:%s: %s %s cannot insert resource\n", 2826a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 2827a32320b7SDan Williams __func__, dev_name(&cxlr->dev)); 2828a32320b7SDan Williams } 2829a32320b7SDan Williams 2830a32320b7SDan Williams p->res = res; 2831a32320b7SDan Williams p->interleave_ways = cxled->cxld.interleave_ways; 2832a32320b7SDan Williams p->interleave_granularity = cxled->cxld.interleave_granularity; 2833a32320b7SDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 2834a32320b7SDan Williams 2835a32320b7SDan Williams rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 2836a32320b7SDan Williams if (rc) 2837a32320b7SDan Williams goto err; 2838a32320b7SDan Williams 2839a32320b7SDan Williams dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n", 2840a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__, 2841a32320b7SDan Williams dev_name(&cxlr->dev), p->res, p->interleave_ways, 2842a32320b7SDan Williams p->interleave_granularity); 2843a32320b7SDan Williams 2844a32320b7SDan Williams /* ...to match put_device() in cxl_add_to_region() */ 2845a32320b7SDan Williams get_device(&cxlr->dev); 2846a32320b7SDan Williams up_write(&cxl_region_rwsem); 2847a32320b7SDan Williams 2848a32320b7SDan Williams return cxlr; 2849a32320b7SDan Williams 2850a32320b7SDan Williams err: 2851a32320b7SDan Williams up_write(&cxl_region_rwsem); 28527481653dSDan Williams devm_release_action(port->uport_dev, unregister_region, cxlr); 2853a32320b7SDan Williams return ERR_PTR(rc); 2854a32320b7SDan Williams } 2855a32320b7SDan Williams 2856a32320b7SDan Williams int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled) 2857a32320b7SDan Williams { 2858a32320b7SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 2859a32320b7SDan Williams struct range *hpa = &cxled->cxld.hpa_range; 2860a32320b7SDan Williams struct cxl_decoder *cxld = &cxled->cxld; 2861a32320b7SDan Williams struct device *cxlrd_dev, *region_dev; 2862a32320b7SDan Williams struct cxl_root_decoder *cxlrd; 2863a32320b7SDan Williams struct cxl_region_params *p; 2864a32320b7SDan Williams struct cxl_region *cxlr; 2865a32320b7SDan Williams bool attach = false; 2866a32320b7SDan Williams int rc; 2867a32320b7SDan Williams 2868a32320b7SDan Williams cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range, 286911105814SAlison Schofield match_root_decoder_by_range); 2870a32320b7SDan Williams if (!cxlrd_dev) { 2871a32320b7SDan Williams dev_err(cxlmd->dev.parent, 2872a32320b7SDan Williams "%s:%s no CXL window for range %#llx:%#llx\n", 2873a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxld->dev), 2874a32320b7SDan Williams cxld->hpa_range.start, cxld->hpa_range.end); 2875a32320b7SDan Williams return -ENXIO; 2876a32320b7SDan Williams } 2877a32320b7SDan Williams 2878a32320b7SDan Williams cxlrd = to_cxl_root_decoder(cxlrd_dev); 2879a32320b7SDan Williams 2880a32320b7SDan Williams /* 2881a32320b7SDan Williams * Ensure that if multiple threads race to construct_region() for @hpa 2882a32320b7SDan Williams * one does the construction and the others add to that. 2883a32320b7SDan Williams */ 2884a32320b7SDan Williams mutex_lock(&cxlrd->range_lock); 2885a32320b7SDan Williams region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa, 2886a32320b7SDan Williams match_region_by_range); 2887a32320b7SDan Williams if (!region_dev) { 2888a32320b7SDan Williams cxlr = construct_region(cxlrd, cxled); 2889a32320b7SDan Williams region_dev = &cxlr->dev; 2890a32320b7SDan Williams } else 2891a32320b7SDan Williams cxlr = to_cxl_region(region_dev); 2892a32320b7SDan Williams mutex_unlock(&cxlrd->range_lock); 2893a32320b7SDan Williams 28947abcb0b1SArnd Bergmann rc = PTR_ERR_OR_ZERO(cxlr); 28957abcb0b1SArnd Bergmann if (rc) 2896a32320b7SDan Williams goto out; 2897a32320b7SDan Williams 2898a32320b7SDan Williams attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE); 2899a32320b7SDan Williams 2900a32320b7SDan Williams down_read(&cxl_region_rwsem); 2901a32320b7SDan Williams p = &cxlr->params; 2902a32320b7SDan Williams attach = p->state == CXL_CONFIG_COMMIT; 2903a32320b7SDan Williams up_read(&cxl_region_rwsem); 2904a32320b7SDan Williams 2905a32320b7SDan Williams if (attach) { 2906a32320b7SDan Williams /* 2907a32320b7SDan Williams * If device_attach() fails the range may still be active via 2908a32320b7SDan Williams * the platform-firmware memory map, otherwise the driver for 2909a32320b7SDan Williams * regions is local to this file, so driver matching can't fail. 2910a32320b7SDan Williams */ 2911a32320b7SDan Williams if (device_attach(&cxlr->dev) < 0) 2912a32320b7SDan Williams dev_err(&cxlr->dev, "failed to enable, range: %pr\n", 2913a32320b7SDan Williams p->res); 2914a32320b7SDan Williams } 2915a32320b7SDan Williams 2916a32320b7SDan Williams put_device(region_dev); 2917a32320b7SDan Williams out: 2918a32320b7SDan Williams put_device(cxlrd_dev); 2919a32320b7SDan Williams return rc; 2920a32320b7SDan Williams } 2921a32320b7SDan Williams EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL); 2922a32320b7SDan Williams 2923a32320b7SDan Williams static int is_system_ram(struct resource *res, void *arg) 2924a32320b7SDan Williams { 2925a32320b7SDan Williams struct cxl_region *cxlr = arg; 2926a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params; 2927a32320b7SDan Williams 2928a32320b7SDan Williams dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res); 2929a32320b7SDan Williams return 1; 2930a32320b7SDan Williams } 2931a32320b7SDan Williams 29328d48817dSDan Williams static int cxl_region_probe(struct device *dev) 29338d48817dSDan Williams { 29348d48817dSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 29358d48817dSDan Williams struct cxl_region_params *p = &cxlr->params; 29368d48817dSDan Williams int rc; 29378d48817dSDan Williams 29388d48817dSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 29398d48817dSDan Williams if (rc) { 29408d48817dSDan Williams dev_dbg(&cxlr->dev, "probe interrupted\n"); 29418d48817dSDan Williams return rc; 29428d48817dSDan Williams } 29438d48817dSDan Williams 29448d48817dSDan Williams if (p->state < CXL_CONFIG_COMMIT) { 29458d48817dSDan Williams dev_dbg(&cxlr->dev, "config state: %d\n", p->state); 29468d48817dSDan Williams rc = -ENXIO; 2947d18bc74aSDan Williams goto out; 29488d48817dSDan Williams } 29498d48817dSDan Williams 29502ab47045SDan Williams if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) { 29512ab47045SDan Williams dev_err(&cxlr->dev, 29522ab47045SDan Williams "failed to activate, re-commit region and retry\n"); 29532ab47045SDan Williams rc = -ENXIO; 29542ab47045SDan Williams goto out; 29552ab47045SDan Williams } 2956d18bc74aSDan Williams 29578d48817dSDan Williams /* 29588d48817dSDan Williams * From this point on any path that changes the region's state away from 29598d48817dSDan Williams * CXL_CONFIG_COMMIT is also responsible for releasing the driver. 29608d48817dSDan Williams */ 2961d18bc74aSDan Williams out: 29628d48817dSDan Williams up_read(&cxl_region_rwsem); 29638d48817dSDan Williams 2964bf3e5da8SDan Williams if (rc) 2965bf3e5da8SDan Williams return rc; 2966bf3e5da8SDan Williams 296704ad63f0SDan Williams switch (cxlr->mode) { 296804ad63f0SDan Williams case CXL_DECODER_PMEM: 296904ad63f0SDan Williams return devm_cxl_add_pmem_region(cxlr); 2970a32320b7SDan Williams case CXL_DECODER_RAM: 2971a32320b7SDan Williams /* 2972a32320b7SDan Williams * The region can not be manged by CXL if any portion of 2973a32320b7SDan Williams * it is already online as 'System RAM' 2974a32320b7SDan Williams */ 2975a32320b7SDan Williams if (walk_iomem_res_desc(IORES_DESC_NONE, 2976a32320b7SDan Williams IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY, 2977a32320b7SDan Williams p->res->start, p->res->end, cxlr, 2978a32320b7SDan Williams is_system_ram) > 0) 2979a32320b7SDan Williams return 0; 298009d09e04SDan Williams return devm_cxl_add_dax_region(cxlr); 298104ad63f0SDan Williams default: 298204ad63f0SDan Williams dev_dbg(&cxlr->dev, "unsupported region mode: %d\n", 298304ad63f0SDan Williams cxlr->mode); 298404ad63f0SDan Williams return -ENXIO; 298504ad63f0SDan Williams } 29868d48817dSDan Williams } 29878d48817dSDan Williams 29888d48817dSDan Williams static struct cxl_driver cxl_region_driver = { 29898d48817dSDan Williams .name = "cxl_region", 29908d48817dSDan Williams .probe = cxl_region_probe, 29918d48817dSDan Williams .id = CXL_DEVICE_REGION, 29928d48817dSDan Williams }; 29938d48817dSDan Williams 29948d48817dSDan Williams int cxl_region_init(void) 29958d48817dSDan Williams { 29968d48817dSDan Williams return cxl_driver_register(&cxl_region_driver); 29978d48817dSDan Williams } 29988d48817dSDan Williams 29998d48817dSDan Williams void cxl_region_exit(void) 30008d48817dSDan Williams { 30018d48817dSDan Williams cxl_driver_unregister(&cxl_region_driver); 30028d48817dSDan Williams } 30038d48817dSDan Williams 300423a22cd1SDan Williams MODULE_IMPORT_NS(CXL); 3005d18bc74aSDan Williams MODULE_IMPORT_NS(DEVMEM); 30068d48817dSDan Williams MODULE_ALIAS_CXL(CXL_DEVICE_REGION); 3007