1779dd20cSBen Widawsky // SPDX-License-Identifier: GPL-2.0-only 2779dd20cSBen Widawsky /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3779dd20cSBen Widawsky #include <linux/memregion.h> 4779dd20cSBen Widawsky #include <linux/genalloc.h> 5779dd20cSBen Widawsky #include <linux/device.h> 6779dd20cSBen Widawsky #include <linux/module.h> 7067353a4SDave Jiang #include <linux/memory.h> 8779dd20cSBen Widawsky #include <linux/slab.h> 9dd5ba0ebSBen Widawsky #include <linux/uuid.h> 10a32320b7SDan Williams #include <linux/sort.h> 11779dd20cSBen Widawsky #include <linux/idr.h> 12643e8e3eSHuang Ying #include <linux/memory-tiers.h> 1380d10a6cSBen Widawsky #include <cxlmem.h> 14779dd20cSBen Widawsky #include <cxl.h> 15779dd20cSBen Widawsky #include "core.h" 16779dd20cSBen Widawsky 17779dd20cSBen Widawsky /** 18779dd20cSBen Widawsky * DOC: cxl core region 19779dd20cSBen Widawsky * 20779dd20cSBen Widawsky * CXL Regions represent mapped memory capacity in system physical address 21779dd20cSBen Widawsky * space. Whereas the CXL Root Decoders identify the bounds of potential CXL 22779dd20cSBen Widawsky * Memory ranges, Regions represent the active mapped capacity by the HDM 23779dd20cSBen Widawsky * Decoder Capability structures throughout the Host Bridges, Switches, and 24779dd20cSBen Widawsky * Endpoints in the topology. 25dd5ba0ebSBen Widawsky * 26dd5ba0ebSBen Widawsky * Region configuration has ordering constraints. UUID may be set at any time 27dd5ba0ebSBen Widawsky * but is only visible for persistent regions. 2880d10a6cSBen Widawsky * 1. Interleave granularity 2980d10a6cSBen Widawsky * 2. Interleave size 30b9686e8cSDan Williams * 3. Decoder targets 31779dd20cSBen Widawsky */ 32779dd20cSBen Widawsky 33779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev); 34779dd20cSBen Widawsky 35c20eaf44SDave Jiang #define __ACCESS_ATTR_RO(_level, _name) { \ 36c20eaf44SDave Jiang .attr = { .name = __stringify(_name), .mode = 0444 }, \ 37c20eaf44SDave Jiang .show = _name##_access##_level##_show, \ 38c20eaf44SDave Jiang } 39c20eaf44SDave Jiang 40c20eaf44SDave Jiang #define ACCESS_DEVICE_ATTR_RO(level, name) \ 41c20eaf44SDave Jiang struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name) 42c20eaf44SDave Jiang 43c20eaf44SDave Jiang #define ACCESS_ATTR_RO(level, attrib) \ 44c20eaf44SDave Jiang static ssize_t attrib##_access##level##_show(struct device *dev, \ 45c20eaf44SDave Jiang struct device_attribute *attr, \ 46c20eaf44SDave Jiang char *buf) \ 47c20eaf44SDave Jiang { \ 48c20eaf44SDave Jiang struct cxl_region *cxlr = to_cxl_region(dev); \ 49c20eaf44SDave Jiang \ 50c20eaf44SDave Jiang if (cxlr->coord[level].attrib == 0) \ 51c20eaf44SDave Jiang return -ENOENT; \ 52c20eaf44SDave Jiang \ 53c20eaf44SDave Jiang return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \ 54c20eaf44SDave Jiang } \ 55c20eaf44SDave Jiang static ACCESS_DEVICE_ATTR_RO(level, attrib) 56c20eaf44SDave Jiang 57c20eaf44SDave Jiang ACCESS_ATTR_RO(0, read_bandwidth); 58c20eaf44SDave Jiang ACCESS_ATTR_RO(0, read_latency); 59c20eaf44SDave Jiang ACCESS_ATTR_RO(0, write_bandwidth); 60c20eaf44SDave Jiang ACCESS_ATTR_RO(0, write_latency); 61c20eaf44SDave Jiang 62c20eaf44SDave Jiang #define ACCESS_ATTR_DECLARE(level, attrib) \ 63c20eaf44SDave Jiang (&dev_attr_access##level##_##attrib.attr) 64c20eaf44SDave Jiang 65c20eaf44SDave Jiang static struct attribute *access0_coordinate_attrs[] = { 66c20eaf44SDave Jiang ACCESS_ATTR_DECLARE(0, read_bandwidth), 67c20eaf44SDave Jiang ACCESS_ATTR_DECLARE(0, write_bandwidth), 68c20eaf44SDave Jiang ACCESS_ATTR_DECLARE(0, read_latency), 69c20eaf44SDave Jiang ACCESS_ATTR_DECLARE(0, write_latency), 70c20eaf44SDave Jiang NULL 71c20eaf44SDave Jiang }; 72c20eaf44SDave Jiang 73c20eaf44SDave Jiang ACCESS_ATTR_RO(1, read_bandwidth); 74c20eaf44SDave Jiang ACCESS_ATTR_RO(1, read_latency); 75c20eaf44SDave Jiang ACCESS_ATTR_RO(1, write_bandwidth); 76c20eaf44SDave Jiang ACCESS_ATTR_RO(1, write_latency); 77c20eaf44SDave Jiang 78c20eaf44SDave Jiang static struct attribute *access1_coordinate_attrs[] = { 79c20eaf44SDave Jiang ACCESS_ATTR_DECLARE(1, read_bandwidth), 80c20eaf44SDave Jiang ACCESS_ATTR_DECLARE(1, write_bandwidth), 81c20eaf44SDave Jiang ACCESS_ATTR_DECLARE(1, read_latency), 82c20eaf44SDave Jiang ACCESS_ATTR_DECLARE(1, write_latency), 83c20eaf44SDave Jiang NULL 84c20eaf44SDave Jiang }; 85c20eaf44SDave Jiang 86c20eaf44SDave Jiang #define ACCESS_VISIBLE(level) \ 87c20eaf44SDave Jiang static umode_t cxl_region_access##level##_coordinate_visible( \ 88c20eaf44SDave Jiang struct kobject *kobj, struct attribute *a, int n) \ 89c20eaf44SDave Jiang { \ 90c20eaf44SDave Jiang struct device *dev = kobj_to_dev(kobj); \ 91c20eaf44SDave Jiang struct cxl_region *cxlr = to_cxl_region(dev); \ 92c20eaf44SDave Jiang \ 93c20eaf44SDave Jiang if (a == &dev_attr_access##level##_read_latency.attr && \ 94c20eaf44SDave Jiang cxlr->coord[level].read_latency == 0) \ 95c20eaf44SDave Jiang return 0; \ 96c20eaf44SDave Jiang \ 97c20eaf44SDave Jiang if (a == &dev_attr_access##level##_write_latency.attr && \ 98c20eaf44SDave Jiang cxlr->coord[level].write_latency == 0) \ 99c20eaf44SDave Jiang return 0; \ 100c20eaf44SDave Jiang \ 101c20eaf44SDave Jiang if (a == &dev_attr_access##level##_read_bandwidth.attr && \ 102c20eaf44SDave Jiang cxlr->coord[level].read_bandwidth == 0) \ 103c20eaf44SDave Jiang return 0; \ 104c20eaf44SDave Jiang \ 105c20eaf44SDave Jiang if (a == &dev_attr_access##level##_write_bandwidth.attr && \ 106c20eaf44SDave Jiang cxlr->coord[level].write_bandwidth == 0) \ 107c20eaf44SDave Jiang return 0; \ 108c20eaf44SDave Jiang \ 109c20eaf44SDave Jiang return a->mode; \ 110c20eaf44SDave Jiang } 111c20eaf44SDave Jiang 112c20eaf44SDave Jiang ACCESS_VISIBLE(0); 113c20eaf44SDave Jiang ACCESS_VISIBLE(1); 114c20eaf44SDave Jiang 115c20eaf44SDave Jiang static const struct attribute_group cxl_region_access0_coordinate_group = { 116c20eaf44SDave Jiang .name = "access0", 117c20eaf44SDave Jiang .attrs = access0_coordinate_attrs, 118c20eaf44SDave Jiang .is_visible = cxl_region_access0_coordinate_visible, 119c20eaf44SDave Jiang }; 120c20eaf44SDave Jiang 121067353a4SDave Jiang static const struct attribute_group *get_cxl_region_access0_group(void) 122067353a4SDave Jiang { 123067353a4SDave Jiang return &cxl_region_access0_coordinate_group; 124067353a4SDave Jiang } 125067353a4SDave Jiang 126c20eaf44SDave Jiang static const struct attribute_group cxl_region_access1_coordinate_group = { 127c20eaf44SDave Jiang .name = "access1", 128c20eaf44SDave Jiang .attrs = access1_coordinate_attrs, 129c20eaf44SDave Jiang .is_visible = cxl_region_access1_coordinate_visible, 130c20eaf44SDave Jiang }; 131c20eaf44SDave Jiang 132067353a4SDave Jiang static const struct attribute_group *get_cxl_region_access1_group(void) 133067353a4SDave Jiang { 134067353a4SDave Jiang return &cxl_region_access1_coordinate_group; 135067353a4SDave Jiang } 136067353a4SDave Jiang 137dd5ba0ebSBen Widawsky static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, 138dd5ba0ebSBen Widawsky char *buf) 139dd5ba0ebSBen Widawsky { 140dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 141dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params; 142dd5ba0ebSBen Widawsky ssize_t rc; 143dd5ba0ebSBen Widawsky 144dd5ba0ebSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 145dd5ba0ebSBen Widawsky if (rc) 146dd5ba0ebSBen Widawsky return rc; 147a8e7d558SDan Williams if (cxlr->mode != CXL_DECODER_PMEM) 148a8e7d558SDan Williams rc = sysfs_emit(buf, "\n"); 149a8e7d558SDan Williams else 150dd5ba0ebSBen Widawsky rc = sysfs_emit(buf, "%pUb\n", &p->uuid); 151dd5ba0ebSBen Widawsky up_read(&cxl_region_rwsem); 152dd5ba0ebSBen Widawsky 153dd5ba0ebSBen Widawsky return rc; 154dd5ba0ebSBen Widawsky } 155dd5ba0ebSBen Widawsky 156dd5ba0ebSBen Widawsky static int is_dup(struct device *match, void *data) 157dd5ba0ebSBen Widawsky { 158dd5ba0ebSBen Widawsky struct cxl_region_params *p; 159dd5ba0ebSBen Widawsky struct cxl_region *cxlr; 160dd5ba0ebSBen Widawsky uuid_t *uuid = data; 161dd5ba0ebSBen Widawsky 162dd5ba0ebSBen Widawsky if (!is_cxl_region(match)) 163dd5ba0ebSBen Widawsky return 0; 164dd5ba0ebSBen Widawsky 165dd5ba0ebSBen Widawsky lockdep_assert_held(&cxl_region_rwsem); 166dd5ba0ebSBen Widawsky cxlr = to_cxl_region(match); 167dd5ba0ebSBen Widawsky p = &cxlr->params; 168dd5ba0ebSBen Widawsky 169dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, uuid)) { 170dd5ba0ebSBen Widawsky dev_dbg(match, "already has uuid: %pUb\n", uuid); 171dd5ba0ebSBen Widawsky return -EBUSY; 172dd5ba0ebSBen Widawsky } 173dd5ba0ebSBen Widawsky 174dd5ba0ebSBen Widawsky return 0; 175dd5ba0ebSBen Widawsky } 176dd5ba0ebSBen Widawsky 177dd5ba0ebSBen Widawsky static ssize_t uuid_store(struct device *dev, struct device_attribute *attr, 178dd5ba0ebSBen Widawsky const char *buf, size_t len) 179dd5ba0ebSBen Widawsky { 180dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 181dd5ba0ebSBen Widawsky struct cxl_region_params *p = &cxlr->params; 182dd5ba0ebSBen Widawsky uuid_t temp; 183dd5ba0ebSBen Widawsky ssize_t rc; 184dd5ba0ebSBen Widawsky 185dd5ba0ebSBen Widawsky if (len != UUID_STRING_LEN + 1) 186dd5ba0ebSBen Widawsky return -EINVAL; 187dd5ba0ebSBen Widawsky 188dd5ba0ebSBen Widawsky rc = uuid_parse(buf, &temp); 189dd5ba0ebSBen Widawsky if (rc) 190dd5ba0ebSBen Widawsky return rc; 191dd5ba0ebSBen Widawsky 192dd5ba0ebSBen Widawsky if (uuid_is_null(&temp)) 193dd5ba0ebSBen Widawsky return -EINVAL; 194dd5ba0ebSBen Widawsky 195dd5ba0ebSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 196dd5ba0ebSBen Widawsky if (rc) 197dd5ba0ebSBen Widawsky return rc; 198dd5ba0ebSBen Widawsky 199dd5ba0ebSBen Widawsky if (uuid_equal(&p->uuid, &temp)) 200dd5ba0ebSBen Widawsky goto out; 201dd5ba0ebSBen Widawsky 202dd5ba0ebSBen Widawsky rc = -EBUSY; 203dd5ba0ebSBen Widawsky if (p->state >= CXL_CONFIG_ACTIVE) 204dd5ba0ebSBen Widawsky goto out; 205dd5ba0ebSBen Widawsky 206dd5ba0ebSBen Widawsky rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup); 207dd5ba0ebSBen Widawsky if (rc < 0) 208dd5ba0ebSBen Widawsky goto out; 209dd5ba0ebSBen Widawsky 210dd5ba0ebSBen Widawsky uuid_copy(&p->uuid, &temp); 211dd5ba0ebSBen Widawsky out: 212dd5ba0ebSBen Widawsky up_write(&cxl_region_rwsem); 213dd5ba0ebSBen Widawsky 214dd5ba0ebSBen Widawsky if (rc) 215dd5ba0ebSBen Widawsky return rc; 216dd5ba0ebSBen Widawsky return len; 217dd5ba0ebSBen Widawsky } 218dd5ba0ebSBen Widawsky static DEVICE_ATTR_RW(uuid); 219dd5ba0ebSBen Widawsky 220176baefbSDan Williams static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port, 221176baefbSDan Williams struct cxl_region *cxlr) 222176baefbSDan Williams { 223176baefbSDan Williams return xa_load(&port->regions, (unsigned long)cxlr); 224176baefbSDan Williams } 225176baefbSDan Williams 226d1257d09SDan Williams static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) 227d1257d09SDan Williams { 228d1257d09SDan Williams if (!cpu_cache_has_invalidate_memregion()) { 229d1257d09SDan Williams if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { 2307914992bSDan Williams dev_info_once( 231d1257d09SDan Williams &cxlr->dev, 232d1257d09SDan Williams "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); 233d1257d09SDan Williams return 0; 234d1257d09SDan Williams } else { 235d1257d09SDan Williams dev_err(&cxlr->dev, 236d1257d09SDan Williams "Failed to synchronize CPU cache state\n"); 237d1257d09SDan Williams return -ENXIO; 238d1257d09SDan Williams } 239d1257d09SDan Williams } 240d1257d09SDan Williams 241d1257d09SDan Williams cpu_cache_invalidate_memregion(IORES_DESC_CXL); 242d1257d09SDan Williams return 0; 243d1257d09SDan Williams } 244d1257d09SDan Williams 245176baefbSDan Williams static int cxl_region_decode_reset(struct cxl_region *cxlr, int count) 246176baefbSDan Williams { 247176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 248d1257d09SDan Williams int i, rc = 0; 249d1257d09SDan Williams 250d1257d09SDan Williams /* 251d1257d09SDan Williams * Before region teardown attempt to flush, and if the flush 252d1257d09SDan Williams * fails cancel the region teardown for data consistency 253d1257d09SDan Williams * concerns 254d1257d09SDan Williams */ 255d1257d09SDan Williams rc = cxl_region_invalidate_memregion(cxlr); 256d1257d09SDan Williams if (rc) 257d1257d09SDan Williams return rc; 258176baefbSDan Williams 259176baefbSDan Williams for (i = count - 1; i >= 0; i--) { 260176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 261176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 262176baefbSDan Williams struct cxl_port *iter = cxled_to_port(cxled); 263030f8803SDan Williams struct cxl_dev_state *cxlds = cxlmd->cxlds; 264176baefbSDan Williams struct cxl_ep *ep; 265176baefbSDan Williams 266030f8803SDan Williams if (cxlds->rcd) 267030f8803SDan Williams goto endpoint_reset; 268030f8803SDan Williams 269176baefbSDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 270176baefbSDan Williams iter = to_cxl_port(iter->dev.parent); 271176baefbSDan Williams 272176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 273176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 274176baefbSDan Williams struct cxl_region_ref *cxl_rr; 275176baefbSDan Williams struct cxl_decoder *cxld; 276176baefbSDan Williams 277176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 278176baefbSDan Williams cxld = cxl_rr->decoder; 2794fa4302dSFan Ni if (cxld->reset) 280176baefbSDan Williams rc = cxld->reset(cxld); 281176baefbSDan Williams if (rc) 282176baefbSDan Williams return rc; 2832ab47045SDan Williams set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 284176baefbSDan Williams } 285176baefbSDan Williams 286030f8803SDan Williams endpoint_reset: 287176baefbSDan Williams rc = cxled->cxld.reset(&cxled->cxld); 288176baefbSDan Williams if (rc) 289176baefbSDan Williams return rc; 2902ab47045SDan Williams set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 291176baefbSDan Williams } 292176baefbSDan Williams 2932ab47045SDan Williams /* all decoders associated with this region have been torn down */ 2942ab47045SDan Williams clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 2952ab47045SDan Williams 296176baefbSDan Williams return 0; 297176baefbSDan Williams } 298176baefbSDan Williams 299af3ea9abSDan Williams static int commit_decoder(struct cxl_decoder *cxld) 300af3ea9abSDan Williams { 301af3ea9abSDan Williams struct cxl_switch_decoder *cxlsd = NULL; 302af3ea9abSDan Williams 303af3ea9abSDan Williams if (cxld->commit) 304af3ea9abSDan Williams return cxld->commit(cxld); 305af3ea9abSDan Williams 306af3ea9abSDan Williams if (is_switch_decoder(&cxld->dev)) 307af3ea9abSDan Williams cxlsd = to_cxl_switch_decoder(&cxld->dev); 308af3ea9abSDan Williams 309af3ea9abSDan Williams if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1, 310af3ea9abSDan Williams "->commit() is required\n")) 311af3ea9abSDan Williams return -ENXIO; 312af3ea9abSDan Williams return 0; 313af3ea9abSDan Williams } 314af3ea9abSDan Williams 315176baefbSDan Williams static int cxl_region_decode_commit(struct cxl_region *cxlr) 316176baefbSDan Williams { 317176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 31869c99613SDan Williams int i, rc = 0; 319176baefbSDan Williams 320176baefbSDan Williams for (i = 0; i < p->nr_targets; i++) { 321176baefbSDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 322176baefbSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 323176baefbSDan Williams struct cxl_region_ref *cxl_rr; 324176baefbSDan Williams struct cxl_decoder *cxld; 325176baefbSDan Williams struct cxl_port *iter; 326176baefbSDan Williams struct cxl_ep *ep; 327176baefbSDan Williams 328176baefbSDan Williams /* commit bottom up */ 329176baefbSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 330176baefbSDan Williams iter = to_cxl_port(iter->dev.parent)) { 331176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 332176baefbSDan Williams cxld = cxl_rr->decoder; 333af3ea9abSDan Williams rc = commit_decoder(cxld); 334176baefbSDan Williams if (rc) 335176baefbSDan Williams break; 336176baefbSDan Williams } 337176baefbSDan Williams 33869c99613SDan Williams if (rc) { 339176baefbSDan Williams /* programming @iter failed, teardown */ 340176baefbSDan Williams for (ep = cxl_ep_load(iter, cxlmd); ep && iter; 341176baefbSDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 342176baefbSDan Williams cxl_rr = cxl_rr_load(iter, cxlr); 343176baefbSDan Williams cxld = cxl_rr->decoder; 3444fa4302dSFan Ni if (cxld->reset) 345176baefbSDan Williams cxld->reset(cxld); 346176baefbSDan Williams } 347176baefbSDan Williams 348176baefbSDan Williams cxled->cxld.reset(&cxled->cxld); 34969c99613SDan Williams goto err; 35069c99613SDan Williams } 351176baefbSDan Williams } 352176baefbSDan Williams 353176baefbSDan Williams return 0; 354176baefbSDan Williams 35569c99613SDan Williams err: 356176baefbSDan Williams /* undo the targets that were successfully committed */ 357176baefbSDan Williams cxl_region_decode_reset(cxlr, i); 358176baefbSDan Williams return rc; 359176baefbSDan Williams } 360176baefbSDan Williams 361176baefbSDan Williams static ssize_t commit_store(struct device *dev, struct device_attribute *attr, 362176baefbSDan Williams const char *buf, size_t len) 363176baefbSDan Williams { 364176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 365176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 366176baefbSDan Williams bool commit; 367176baefbSDan Williams ssize_t rc; 368176baefbSDan Williams 369176baefbSDan Williams rc = kstrtobool(buf, &commit); 370176baefbSDan Williams if (rc) 371176baefbSDan Williams return rc; 372176baefbSDan Williams 373176baefbSDan Williams rc = down_write_killable(&cxl_region_rwsem); 374176baefbSDan Williams if (rc) 375176baefbSDan Williams return rc; 376176baefbSDan Williams 377176baefbSDan Williams /* Already in the requested state? */ 378176baefbSDan Williams if (commit && p->state >= CXL_CONFIG_COMMIT) 379176baefbSDan Williams goto out; 380176baefbSDan Williams if (!commit && p->state < CXL_CONFIG_COMMIT) 381176baefbSDan Williams goto out; 382176baefbSDan Williams 383176baefbSDan Williams /* Not ready to commit? */ 384176baefbSDan Williams if (commit && p->state < CXL_CONFIG_ACTIVE) { 385176baefbSDan Williams rc = -ENXIO; 386176baefbSDan Williams goto out; 387176baefbSDan Williams } 388176baefbSDan Williams 389d1257d09SDan Williams /* 390d1257d09SDan Williams * Invalidate caches before region setup to drop any speculative 391d1257d09SDan Williams * consumption of this address space 392d1257d09SDan Williams */ 393d1257d09SDan Williams rc = cxl_region_invalidate_memregion(cxlr); 394d1257d09SDan Williams if (rc) 3953531b27fSLi Zhijian goto out; 396d1257d09SDan Williams 397adfe1973SDan Williams if (commit) { 398176baefbSDan Williams rc = cxl_region_decode_commit(cxlr); 399adfe1973SDan Williams if (rc == 0) 400adfe1973SDan Williams p->state = CXL_CONFIG_COMMIT; 401adfe1973SDan Williams } else { 402176baefbSDan Williams p->state = CXL_CONFIG_RESET_PENDING; 403176baefbSDan Williams up_write(&cxl_region_rwsem); 404176baefbSDan Williams device_release_driver(&cxlr->dev); 405176baefbSDan Williams down_write(&cxl_region_rwsem); 406176baefbSDan Williams 407176baefbSDan Williams /* 408176baefbSDan Williams * The lock was dropped, so need to revalidate that the reset is 409176baefbSDan Williams * still pending. 410176baefbSDan Williams */ 411adfe1973SDan Williams if (p->state == CXL_CONFIG_RESET_PENDING) { 412176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 413adfe1973SDan Williams /* 414adfe1973SDan Williams * Revert to committed since there may still be active 415adfe1973SDan Williams * decoders associated with this region, or move forward 416adfe1973SDan Williams * to active to mark the reset successful 417adfe1973SDan Williams */ 418176baefbSDan Williams if (rc) 419176baefbSDan Williams p->state = CXL_CONFIG_COMMIT; 420adfe1973SDan Williams else 421176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE; 422adfe1973SDan Williams } 423adfe1973SDan Williams } 424176baefbSDan Williams 425176baefbSDan Williams out: 426176baefbSDan Williams up_write(&cxl_region_rwsem); 427176baefbSDan Williams 428176baefbSDan Williams if (rc) 429176baefbSDan Williams return rc; 430176baefbSDan Williams return len; 431176baefbSDan Williams } 432176baefbSDan Williams 433176baefbSDan Williams static ssize_t commit_show(struct device *dev, struct device_attribute *attr, 434176baefbSDan Williams char *buf) 435176baefbSDan Williams { 436176baefbSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 437176baefbSDan Williams struct cxl_region_params *p = &cxlr->params; 438176baefbSDan Williams ssize_t rc; 439176baefbSDan Williams 440176baefbSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 441176baefbSDan Williams if (rc) 442176baefbSDan Williams return rc; 443176baefbSDan Williams rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT); 444176baefbSDan Williams up_read(&cxl_region_rwsem); 445176baefbSDan Williams 446176baefbSDan Williams return rc; 447176baefbSDan Williams } 448176baefbSDan Williams static DEVICE_ATTR_RW(commit); 449176baefbSDan Williams 450dd5ba0ebSBen Widawsky static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a, 451dd5ba0ebSBen Widawsky int n) 452dd5ba0ebSBen Widawsky { 453dd5ba0ebSBen Widawsky struct device *dev = kobj_to_dev(kobj); 454dd5ba0ebSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 455dd5ba0ebSBen Widawsky 456a8e7d558SDan Williams /* 457a8e7d558SDan Williams * Support tooling that expects to find a 'uuid' attribute for all 458a8e7d558SDan Williams * regions regardless of mode. 459a8e7d558SDan Williams */ 460dd5ba0ebSBen Widawsky if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM) 461a8e7d558SDan Williams return 0444; 462dd5ba0ebSBen Widawsky return a->mode; 463dd5ba0ebSBen Widawsky } 464dd5ba0ebSBen Widawsky 46580d10a6cSBen Widawsky static ssize_t interleave_ways_show(struct device *dev, 46680d10a6cSBen Widawsky struct device_attribute *attr, char *buf) 46780d10a6cSBen Widawsky { 46880d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 46980d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 47080d10a6cSBen Widawsky ssize_t rc; 47180d10a6cSBen Widawsky 47280d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 47380d10a6cSBen Widawsky if (rc) 47480d10a6cSBen Widawsky return rc; 47580d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_ways); 47680d10a6cSBen Widawsky up_read(&cxl_region_rwsem); 47780d10a6cSBen Widawsky 47880d10a6cSBen Widawsky return rc; 47980d10a6cSBen Widawsky } 48080d10a6cSBen Widawsky 481b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void); 482b9686e8cSDan Williams 48380d10a6cSBen Widawsky static ssize_t interleave_ways_store(struct device *dev, 48480d10a6cSBen Widawsky struct device_attribute *attr, 48580d10a6cSBen Widawsky const char *buf, size_t len) 48680d10a6cSBen Widawsky { 48780d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 48880d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 48980d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 49080d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 491c7e3548cSDan Carpenter unsigned int val, save; 492c7e3548cSDan Carpenter int rc; 49380d10a6cSBen Widawsky u8 iw; 49480d10a6cSBen Widawsky 495c7e3548cSDan Carpenter rc = kstrtouint(buf, 0, &val); 49680d10a6cSBen Widawsky if (rc) 49780d10a6cSBen Widawsky return rc; 49880d10a6cSBen Widawsky 499c99b2e8cSDave Jiang rc = ways_to_eiw(val, &iw); 50080d10a6cSBen Widawsky if (rc) 50180d10a6cSBen Widawsky return rc; 50280d10a6cSBen Widawsky 50380d10a6cSBen Widawsky /* 504c7ad3dc3SJim Harris * Even for x3, x6, and x12 interleaves the region interleave must be a 50580d10a6cSBen Widawsky * power of 2 multiple of the host bridge interleave. 50680d10a6cSBen Widawsky */ 50780d10a6cSBen Widawsky if (!is_power_of_2(val / cxld->interleave_ways) || 50880d10a6cSBen Widawsky (val % cxld->interleave_ways)) { 50980d10a6cSBen Widawsky dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val); 51080d10a6cSBen Widawsky return -EINVAL; 51180d10a6cSBen Widawsky } 51280d10a6cSBen Widawsky 51380d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 51480d10a6cSBen Widawsky if (rc) 51580d10a6cSBen Widawsky return rc; 51680d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 51780d10a6cSBen Widawsky rc = -EBUSY; 51880d10a6cSBen Widawsky goto out; 51980d10a6cSBen Widawsky } 52080d10a6cSBen Widawsky 521b9686e8cSDan Williams save = p->interleave_ways; 52280d10a6cSBen Widawsky p->interleave_ways = val; 523b9686e8cSDan Williams rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 524b9686e8cSDan Williams if (rc) 525b9686e8cSDan Williams p->interleave_ways = save; 52680d10a6cSBen Widawsky out: 52780d10a6cSBen Widawsky up_write(&cxl_region_rwsem); 52880d10a6cSBen Widawsky if (rc) 52980d10a6cSBen Widawsky return rc; 53080d10a6cSBen Widawsky return len; 53180d10a6cSBen Widawsky } 53280d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_ways); 53380d10a6cSBen Widawsky 53480d10a6cSBen Widawsky static ssize_t interleave_granularity_show(struct device *dev, 53580d10a6cSBen Widawsky struct device_attribute *attr, 53680d10a6cSBen Widawsky char *buf) 53780d10a6cSBen Widawsky { 53880d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 53980d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 54080d10a6cSBen Widawsky ssize_t rc; 54180d10a6cSBen Widawsky 54280d10a6cSBen Widawsky rc = down_read_interruptible(&cxl_region_rwsem); 54380d10a6cSBen Widawsky if (rc) 54480d10a6cSBen Widawsky return rc; 54580d10a6cSBen Widawsky rc = sysfs_emit(buf, "%d\n", p->interleave_granularity); 54680d10a6cSBen Widawsky up_read(&cxl_region_rwsem); 54780d10a6cSBen Widawsky 54880d10a6cSBen Widawsky return rc; 54980d10a6cSBen Widawsky } 55080d10a6cSBen Widawsky 55180d10a6cSBen Widawsky static ssize_t interleave_granularity_store(struct device *dev, 55280d10a6cSBen Widawsky struct device_attribute *attr, 55380d10a6cSBen Widawsky const char *buf, size_t len) 55480d10a6cSBen Widawsky { 55580d10a6cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 55680d10a6cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 55780d10a6cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 55880d10a6cSBen Widawsky struct cxl_region_params *p = &cxlr->params; 55980d10a6cSBen Widawsky int rc, val; 56080d10a6cSBen Widawsky u16 ig; 56180d10a6cSBen Widawsky 56280d10a6cSBen Widawsky rc = kstrtoint(buf, 0, &val); 56380d10a6cSBen Widawsky if (rc) 56480d10a6cSBen Widawsky return rc; 56580d10a6cSBen Widawsky 56683351ddbSDave Jiang rc = granularity_to_eig(val, &ig); 56780d10a6cSBen Widawsky if (rc) 56880d10a6cSBen Widawsky return rc; 56980d10a6cSBen Widawsky 57080d10a6cSBen Widawsky /* 5714d8e4ea5SDan Williams * When the host-bridge is interleaved, disallow region granularity != 5724d8e4ea5SDan Williams * root granularity. Regions with a granularity less than the root 5734d8e4ea5SDan Williams * interleave result in needing multiple endpoints to support a single 574cbbd05d0SRandy Dunlap * slot in the interleave (possible to support in the future). Regions 5754d8e4ea5SDan Williams * with a granularity greater than the root interleave result in invalid 5764d8e4ea5SDan Williams * DPA translations (invalid to support). 57780d10a6cSBen Widawsky */ 5784d8e4ea5SDan Williams if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity) 57980d10a6cSBen Widawsky return -EINVAL; 58080d10a6cSBen Widawsky 58180d10a6cSBen Widawsky rc = down_write_killable(&cxl_region_rwsem); 58280d10a6cSBen Widawsky if (rc) 58380d10a6cSBen Widawsky return rc; 58480d10a6cSBen Widawsky if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 58580d10a6cSBen Widawsky rc = -EBUSY; 58680d10a6cSBen Widawsky goto out; 58780d10a6cSBen Widawsky } 58880d10a6cSBen Widawsky 58980d10a6cSBen Widawsky p->interleave_granularity = val; 59080d10a6cSBen Widawsky out: 59180d10a6cSBen Widawsky up_write(&cxl_region_rwsem); 59280d10a6cSBen Widawsky if (rc) 59380d10a6cSBen Widawsky return rc; 59480d10a6cSBen Widawsky return len; 59580d10a6cSBen Widawsky } 59680d10a6cSBen Widawsky static DEVICE_ATTR_RW(interleave_granularity); 59780d10a6cSBen Widawsky 59823a22cd1SDan Williams static ssize_t resource_show(struct device *dev, struct device_attribute *attr, 59923a22cd1SDan Williams char *buf) 60023a22cd1SDan Williams { 60123a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 60223a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 60323a22cd1SDan Williams u64 resource = -1ULL; 60423a22cd1SDan Williams ssize_t rc; 60523a22cd1SDan Williams 60623a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 60723a22cd1SDan Williams if (rc) 60823a22cd1SDan Williams return rc; 60923a22cd1SDan Williams if (p->res) 61023a22cd1SDan Williams resource = p->res->start; 61123a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", resource); 61223a22cd1SDan Williams up_read(&cxl_region_rwsem); 61323a22cd1SDan Williams 61423a22cd1SDan Williams return rc; 61523a22cd1SDan Williams } 61623a22cd1SDan Williams static DEVICE_ATTR_RO(resource); 61723a22cd1SDan Williams 6187d505f98SDan Williams static ssize_t mode_show(struct device *dev, struct device_attribute *attr, 6197d505f98SDan Williams char *buf) 6207d505f98SDan Williams { 6217d505f98SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 6227d505f98SDan Williams 6237d505f98SDan Williams return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode)); 6247d505f98SDan Williams } 6257d505f98SDan Williams static DEVICE_ATTR_RO(mode); 6267d505f98SDan Williams 62723a22cd1SDan Williams static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) 62823a22cd1SDan Williams { 62923a22cd1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 63023a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 63123a22cd1SDan Williams struct resource *res; 632d76779ddSQuanquan Cao u64 remainder = 0; 63323a22cd1SDan Williams 63423a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 63523a22cd1SDan Williams 63623a22cd1SDan Williams /* Nothing to do... */ 63788ab1ddeSDan Carpenter if (p->res && resource_size(p->res) == size) 63823a22cd1SDan Williams return 0; 63923a22cd1SDan Williams 64023a22cd1SDan Williams /* To change size the old size must be freed first */ 64123a22cd1SDan Williams if (p->res) 64223a22cd1SDan Williams return -EBUSY; 64323a22cd1SDan Williams 64423a22cd1SDan Williams if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) 64523a22cd1SDan Williams return -EBUSY; 64623a22cd1SDan Williams 64723a22cd1SDan Williams /* ways, granularity and uuid (if PMEM) need to be set before HPA */ 64823a22cd1SDan Williams if (!p->interleave_ways || !p->interleave_granularity || 64923a22cd1SDan Williams (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid))) 65023a22cd1SDan Williams return -ENXIO; 65123a22cd1SDan Williams 652d76779ddSQuanquan Cao div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder); 65323a22cd1SDan Williams if (remainder) 65423a22cd1SDan Williams return -EINVAL; 65523a22cd1SDan Williams 65623a22cd1SDan Williams res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, 65723a22cd1SDan Williams dev_name(&cxlr->dev)); 65823a22cd1SDan Williams if (IS_ERR(res)) { 6597984d22fSAlison Schofield dev_dbg(&cxlr->dev, 66058f1e9d3SRandy Dunlap "HPA allocation error (%ld) for size:%pap in %s %pr\n", 66158f1e9d3SRandy Dunlap PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res); 66223a22cd1SDan Williams return PTR_ERR(res); 66323a22cd1SDan Williams } 66423a22cd1SDan Williams 66523a22cd1SDan Williams p->res = res; 66623a22cd1SDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 66723a22cd1SDan Williams 66823a22cd1SDan Williams return 0; 66923a22cd1SDan Williams } 67023a22cd1SDan Williams 67123a22cd1SDan Williams static void cxl_region_iomem_release(struct cxl_region *cxlr) 67223a22cd1SDan Williams { 67323a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 67423a22cd1SDan Williams 67523a22cd1SDan Williams if (device_is_registered(&cxlr->dev)) 67623a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 67723a22cd1SDan Williams if (p->res) { 678a32320b7SDan Williams /* 679a32320b7SDan Williams * Autodiscovered regions may not have been able to insert their 680a32320b7SDan Williams * resource. 681a32320b7SDan Williams */ 682a32320b7SDan Williams if (p->res->parent) 68323a22cd1SDan Williams remove_resource(p->res); 68423a22cd1SDan Williams kfree(p->res); 68523a22cd1SDan Williams p->res = NULL; 68623a22cd1SDan Williams } 68723a22cd1SDan Williams } 68823a22cd1SDan Williams 68923a22cd1SDan Williams static int free_hpa(struct cxl_region *cxlr) 69023a22cd1SDan Williams { 69123a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 69223a22cd1SDan Williams 69323a22cd1SDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 69423a22cd1SDan Williams 69523a22cd1SDan Williams if (!p->res) 69623a22cd1SDan Williams return 0; 69723a22cd1SDan Williams 69823a22cd1SDan Williams if (p->state >= CXL_CONFIG_ACTIVE) 69923a22cd1SDan Williams return -EBUSY; 70023a22cd1SDan Williams 70123a22cd1SDan Williams cxl_region_iomem_release(cxlr); 70223a22cd1SDan Williams p->state = CXL_CONFIG_IDLE; 70323a22cd1SDan Williams return 0; 70423a22cd1SDan Williams } 70523a22cd1SDan Williams 70623a22cd1SDan Williams static ssize_t size_store(struct device *dev, struct device_attribute *attr, 70723a22cd1SDan Williams const char *buf, size_t len) 70823a22cd1SDan Williams { 70923a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 71023a22cd1SDan Williams u64 val; 71123a22cd1SDan Williams int rc; 71223a22cd1SDan Williams 71323a22cd1SDan Williams rc = kstrtou64(buf, 0, &val); 71423a22cd1SDan Williams if (rc) 71523a22cd1SDan Williams return rc; 71623a22cd1SDan Williams 71723a22cd1SDan Williams rc = down_write_killable(&cxl_region_rwsem); 71823a22cd1SDan Williams if (rc) 71923a22cd1SDan Williams return rc; 72023a22cd1SDan Williams 72123a22cd1SDan Williams if (val) 72223a22cd1SDan Williams rc = alloc_hpa(cxlr, val); 72323a22cd1SDan Williams else 72423a22cd1SDan Williams rc = free_hpa(cxlr); 72523a22cd1SDan Williams up_write(&cxl_region_rwsem); 72623a22cd1SDan Williams 72723a22cd1SDan Williams if (rc) 72823a22cd1SDan Williams return rc; 72923a22cd1SDan Williams 73023a22cd1SDan Williams return len; 73123a22cd1SDan Williams } 73223a22cd1SDan Williams 73323a22cd1SDan Williams static ssize_t size_show(struct device *dev, struct device_attribute *attr, 73423a22cd1SDan Williams char *buf) 73523a22cd1SDan Williams { 73623a22cd1SDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 73723a22cd1SDan Williams struct cxl_region_params *p = &cxlr->params; 73823a22cd1SDan Williams u64 size = 0; 73923a22cd1SDan Williams ssize_t rc; 74023a22cd1SDan Williams 74123a22cd1SDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 74223a22cd1SDan Williams if (rc) 74323a22cd1SDan Williams return rc; 74423a22cd1SDan Williams if (p->res) 74523a22cd1SDan Williams size = resource_size(p->res); 74623a22cd1SDan Williams rc = sysfs_emit(buf, "%#llx\n", size); 74723a22cd1SDan Williams up_read(&cxl_region_rwsem); 74823a22cd1SDan Williams 74923a22cd1SDan Williams return rc; 75023a22cd1SDan Williams } 75123a22cd1SDan Williams static DEVICE_ATTR_RW(size); 75223a22cd1SDan Williams 753dd5ba0ebSBen Widawsky static struct attribute *cxl_region_attrs[] = { 754dd5ba0ebSBen Widawsky &dev_attr_uuid.attr, 755176baefbSDan Williams &dev_attr_commit.attr, 75680d10a6cSBen Widawsky &dev_attr_interleave_ways.attr, 75780d10a6cSBen Widawsky &dev_attr_interleave_granularity.attr, 75823a22cd1SDan Williams &dev_attr_resource.attr, 75923a22cd1SDan Williams &dev_attr_size.attr, 7607d505f98SDan Williams &dev_attr_mode.attr, 761dd5ba0ebSBen Widawsky NULL, 762dd5ba0ebSBen Widawsky }; 763dd5ba0ebSBen Widawsky 764dd5ba0ebSBen Widawsky static const struct attribute_group cxl_region_group = { 765dd5ba0ebSBen Widawsky .attrs = cxl_region_attrs, 766dd5ba0ebSBen Widawsky .is_visible = cxl_region_visible, 767dd5ba0ebSBen Widawsky }; 768dd5ba0ebSBen Widawsky 769b9686e8cSDan Williams static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos) 770b9686e8cSDan Williams { 771b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 772b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled; 773b9686e8cSDan Williams int rc; 774b9686e8cSDan Williams 775b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 776b9686e8cSDan Williams if (rc) 777b9686e8cSDan Williams return rc; 778b9686e8cSDan Williams 779b9686e8cSDan Williams if (pos >= p->interleave_ways) { 780b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 781b9686e8cSDan Williams p->interleave_ways); 782b9686e8cSDan Williams rc = -ENXIO; 783b9686e8cSDan Williams goto out; 784b9686e8cSDan Williams } 785b9686e8cSDan Williams 786b9686e8cSDan Williams cxled = p->targets[pos]; 787b9686e8cSDan Williams if (!cxled) 788b9686e8cSDan Williams rc = sysfs_emit(buf, "\n"); 789b9686e8cSDan Williams else 790b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev)); 791b9686e8cSDan Williams out: 792b9686e8cSDan Williams up_read(&cxl_region_rwsem); 793b9686e8cSDan Williams 794b9686e8cSDan Williams return rc; 795b9686e8cSDan Williams } 796b9686e8cSDan Williams 797384e624bSDan Williams static int match_free_decoder(struct device *dev, void *data) 798384e624bSDan Williams { 799384e624bSDan Williams struct cxl_decoder *cxld; 800384e624bSDan Williams int *id = data; 801384e624bSDan Williams 802384e624bSDan Williams if (!is_switch_decoder(dev)) 803384e624bSDan Williams return 0; 804384e624bSDan Williams 805384e624bSDan Williams cxld = to_cxl_decoder(dev); 806384e624bSDan Williams 807384e624bSDan Williams /* enforce ordered allocation */ 808384e624bSDan Williams if (cxld->id != *id) 809384e624bSDan Williams return 0; 810384e624bSDan Williams 811384e624bSDan Williams if (!cxld->region) 812384e624bSDan Williams return 1; 813384e624bSDan Williams 814384e624bSDan Williams (*id)++; 815384e624bSDan Williams 816384e624bSDan Williams return 0; 817384e624bSDan Williams } 818384e624bSDan Williams 8199e4edf1aSAlison Schofield static int match_auto_decoder(struct device *dev, void *data) 8209e4edf1aSAlison Schofield { 8219e4edf1aSAlison Schofield struct cxl_region_params *p = data; 8229e4edf1aSAlison Schofield struct cxl_decoder *cxld; 8239e4edf1aSAlison Schofield struct range *r; 8249e4edf1aSAlison Schofield 8259e4edf1aSAlison Schofield if (!is_switch_decoder(dev)) 8269e4edf1aSAlison Schofield return 0; 8279e4edf1aSAlison Schofield 8289e4edf1aSAlison Schofield cxld = to_cxl_decoder(dev); 8299e4edf1aSAlison Schofield r = &cxld->hpa_range; 8309e4edf1aSAlison Schofield 8319e4edf1aSAlison Schofield if (p->res && p->res->start == r->start && p->res->end == r->end) 8329e4edf1aSAlison Schofield return 1; 8339e4edf1aSAlison Schofield 8349e4edf1aSAlison Schofield return 0; 8359e4edf1aSAlison Schofield } 8369e4edf1aSAlison Schofield 837453a7fdeSAlison Schofield static struct cxl_decoder * 838453a7fdeSAlison Schofield cxl_region_find_decoder(struct cxl_port *port, 839453a7fdeSAlison Schofield struct cxl_endpoint_decoder *cxled, 840384e624bSDan Williams struct cxl_region *cxlr) 841384e624bSDan Williams { 842384e624bSDan Williams struct device *dev; 843384e624bSDan Williams int id = 0; 844384e624bSDan Williams 845453a7fdeSAlison Schofield if (port == cxled_to_port(cxled)) 846453a7fdeSAlison Schofield return &cxled->cxld; 847453a7fdeSAlison Schofield 8489e4edf1aSAlison Schofield if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) 8499e4edf1aSAlison Schofield dev = device_find_child(&port->dev, &cxlr->params, 8509e4edf1aSAlison Schofield match_auto_decoder); 8519e4edf1aSAlison Schofield else 852384e624bSDan Williams dev = device_find_child(&port->dev, &id, match_free_decoder); 853384e624bSDan Williams if (!dev) 854384e624bSDan Williams return NULL; 855b9686e8cSDan Williams /* 856384e624bSDan Williams * This decoder is pinned registered as long as the endpoint decoder is 857384e624bSDan Williams * registered, and endpoint decoder unregistration holds the 858384e624bSDan Williams * cxl_region_rwsem over unregister events, so no need to hold on to 859384e624bSDan Williams * this extra reference. 860b9686e8cSDan Williams */ 861384e624bSDan Williams put_device(dev); 862384e624bSDan Williams return to_cxl_decoder(dev); 863384e624bSDan Williams } 864384e624bSDan Williams 865cb66b1d6SAlison Schofield static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter, 866cb66b1d6SAlison Schofield struct cxl_decoder *cxld) 867cb66b1d6SAlison Schofield { 868cb66b1d6SAlison Schofield struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter); 869cb66b1d6SAlison Schofield struct cxl_decoder *cxld_iter = rr->decoder; 870cb66b1d6SAlison Schofield 871cb66b1d6SAlison Schofield /* 872cb66b1d6SAlison Schofield * Allow the out of order assembly of auto-discovered regions. 873cb66b1d6SAlison Schofield * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders 874cb66b1d6SAlison Schofield * in HPA order. Confirm that the decoder with the lesser HPA 875cb66b1d6SAlison Schofield * starting address has the lesser id. 876cb66b1d6SAlison Schofield */ 877cb66b1d6SAlison Schofield dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n", 878cb66b1d6SAlison Schofield dev_name(&cxld->dev), cxld->id, 879cb66b1d6SAlison Schofield dev_name(&cxld_iter->dev), cxld_iter->id); 880cb66b1d6SAlison Schofield 881cb66b1d6SAlison Schofield if (cxld_iter->id > cxld->id) 882cb66b1d6SAlison Schofield return true; 883cb66b1d6SAlison Schofield 884cb66b1d6SAlison Schofield return false; 885cb66b1d6SAlison Schofield } 886cb66b1d6SAlison Schofield 887cb66b1d6SAlison Schofield static struct cxl_region_ref * 888cb66b1d6SAlison Schofield alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr, 889cb66b1d6SAlison Schofield struct cxl_endpoint_decoder *cxled) 890384e624bSDan Williams { 891e29a8995SDan Williams struct cxl_region_params *p = &cxlr->params; 892e29a8995SDan Williams struct cxl_region_ref *cxl_rr, *iter; 893e29a8995SDan Williams unsigned long index; 894384e624bSDan Williams int rc; 895384e624bSDan Williams 896e29a8995SDan Williams xa_for_each(&port->regions, index, iter) { 897e29a8995SDan Williams struct cxl_region_params *ip = &iter->region->params; 898e29a8995SDan Williams 899cb66b1d6SAlison Schofield if (!ip->res || ip->res->start < p->res->start) 900a90accb3SDan Williams continue; 901a90accb3SDan Williams 902cb66b1d6SAlison Schofield if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 903cb66b1d6SAlison Schofield struct cxl_decoder *cxld; 904cb66b1d6SAlison Schofield 905cb66b1d6SAlison Schofield cxld = cxl_region_find_decoder(port, cxled, cxlr); 906cb66b1d6SAlison Schofield if (auto_order_ok(port, iter->region, cxld)) 907cb66b1d6SAlison Schofield continue; 908cb66b1d6SAlison Schofield } 909cb66b1d6SAlison Schofield dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n", 910e29a8995SDan Williams dev_name(&port->dev), 911e29a8995SDan Williams dev_name(&iter->region->dev), ip->res, p->res); 912cb66b1d6SAlison Schofield 913e29a8995SDan Williams return ERR_PTR(-EBUSY); 914e29a8995SDan Williams } 915e29a8995SDan Williams 916384e624bSDan Williams cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL); 917384e624bSDan Williams if (!cxl_rr) 918e29a8995SDan Williams return ERR_PTR(-ENOMEM); 919384e624bSDan Williams cxl_rr->port = port; 920384e624bSDan Williams cxl_rr->region = cxlr; 92127b3f8d1SDan Williams cxl_rr->nr_targets = 1; 922384e624bSDan Williams xa_init(&cxl_rr->endpoints); 923384e624bSDan Williams 924384e624bSDan Williams rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL); 925384e624bSDan Williams if (rc) { 926384e624bSDan Williams dev_dbg(&cxlr->dev, 927384e624bSDan Williams "%s: failed to track region reference: %d\n", 928384e624bSDan Williams dev_name(&port->dev), rc); 929384e624bSDan Williams kfree(cxl_rr); 930e29a8995SDan Williams return ERR_PTR(rc); 931384e624bSDan Williams } 932384e624bSDan Williams 933384e624bSDan Williams return cxl_rr; 934384e624bSDan Williams } 935384e624bSDan Williams 93671ee71d7SVishal Verma static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr) 937384e624bSDan Williams { 938384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region; 939384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 940384e624bSDan Williams 94171ee71d7SVishal Verma if (!cxld) 94271ee71d7SVishal Verma return; 94371ee71d7SVishal Verma 944384e624bSDan Williams dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n"); 945384e624bSDan Williams if (cxld->region == cxlr) { 946384e624bSDan Williams cxld->region = NULL; 947384e624bSDan Williams put_device(&cxlr->dev); 948384e624bSDan Williams } 94971ee71d7SVishal Verma } 950384e624bSDan Williams 95171ee71d7SVishal Verma static void free_region_ref(struct cxl_region_ref *cxl_rr) 95271ee71d7SVishal Verma { 95371ee71d7SVishal Verma struct cxl_port *port = cxl_rr->port; 95471ee71d7SVishal Verma struct cxl_region *cxlr = cxl_rr->region; 95571ee71d7SVishal Verma 95671ee71d7SVishal Verma cxl_rr_free_decoder(cxl_rr); 957384e624bSDan Williams xa_erase(&port->regions, (unsigned long)cxlr); 958384e624bSDan Williams xa_destroy(&cxl_rr->endpoints); 959384e624bSDan Williams kfree(cxl_rr); 960384e624bSDan Williams } 961384e624bSDan Williams 962384e624bSDan Williams static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr, 963384e624bSDan Williams struct cxl_endpoint_decoder *cxled) 964384e624bSDan Williams { 965384e624bSDan Williams int rc; 966384e624bSDan Williams struct cxl_port *port = cxl_rr->port; 967384e624bSDan Williams struct cxl_region *cxlr = cxl_rr->region; 968384e624bSDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 969384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled)); 970384e624bSDan Williams 97127b3f8d1SDan Williams if (ep) { 972384e624bSDan Williams rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep, 973384e624bSDan Williams GFP_KERNEL); 974384e624bSDan Williams if (rc) 975384e624bSDan Williams return rc; 97627b3f8d1SDan Williams } 977384e624bSDan Williams cxl_rr->nr_eps++; 978384e624bSDan Williams 979384e624bSDan Williams if (!cxld->region) { 980384e624bSDan Williams cxld->region = cxlr; 981384e624bSDan Williams get_device(&cxlr->dev); 982384e624bSDan Williams } 983384e624bSDan Williams 984384e624bSDan Williams return 0; 985384e624bSDan Williams } 986384e624bSDan Williams 98771ee71d7SVishal Verma static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr, 98871ee71d7SVishal Verma struct cxl_endpoint_decoder *cxled, 98971ee71d7SVishal Verma struct cxl_region_ref *cxl_rr) 99071ee71d7SVishal Verma { 99171ee71d7SVishal Verma struct cxl_decoder *cxld; 99271ee71d7SVishal Verma 993453a7fdeSAlison Schofield cxld = cxl_region_find_decoder(port, cxled, cxlr); 99471ee71d7SVishal Verma if (!cxld) { 99571ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: no decoder available\n", 99671ee71d7SVishal Verma dev_name(&port->dev)); 99771ee71d7SVishal Verma return -EBUSY; 99871ee71d7SVishal Verma } 99971ee71d7SVishal Verma 100071ee71d7SVishal Verma if (cxld->region) { 100171ee71d7SVishal Verma dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", 100271ee71d7SVishal Verma dev_name(&port->dev), dev_name(&cxld->dev), 100371ee71d7SVishal Verma dev_name(&cxld->region->dev)); 100471ee71d7SVishal Verma return -EBUSY; 100571ee71d7SVishal Verma } 100671ee71d7SVishal Verma 10078c897b36SDan Williams /* 10088c897b36SDan Williams * Endpoints should already match the region type, but backstop that 10098c897b36SDan Williams * assumption with an assertion. Switch-decoders change mapping-type 10108c897b36SDan Williams * based on what is mapped when they are assigned to a region. 10118c897b36SDan Williams */ 10128c897b36SDan Williams dev_WARN_ONCE(&cxlr->dev, 10138c897b36SDan Williams port == cxled_to_port(cxled) && 10148c897b36SDan Williams cxld->target_type != cxlr->type, 10158c897b36SDan Williams "%s:%s mismatch decoder type %d -> %d\n", 10168c897b36SDan Williams dev_name(&cxled_to_memdev(cxled)->dev), 10178c897b36SDan Williams dev_name(&cxld->dev), cxld->target_type, cxlr->type); 10188c897b36SDan Williams cxld->target_type = cxlr->type; 101971ee71d7SVishal Verma cxl_rr->decoder = cxld; 102071ee71d7SVishal Verma return 0; 102171ee71d7SVishal Verma } 102271ee71d7SVishal Verma 1023384e624bSDan Williams /** 1024384e624bSDan Williams * cxl_port_attach_region() - track a region's interest in a port by endpoint 1025384e624bSDan Williams * @port: port to add a new region reference 'struct cxl_region_ref' 1026384e624bSDan Williams * @cxlr: region to attach to @port 1027384e624bSDan Williams * @cxled: endpoint decoder used to create or further pin a region reference 1028384e624bSDan Williams * @pos: interleave position of @cxled in @cxlr 1029384e624bSDan Williams * 1030384e624bSDan Williams * The attach event is an opportunity to validate CXL decode setup 1031384e624bSDan Williams * constraints and record metadata needed for programming HDM decoders, 1032384e624bSDan Williams * in particular decoder target lists. 1033384e624bSDan Williams * 1034384e624bSDan Williams * The steps are: 1035f13da0d9SBagas Sanjaya * 1036384e624bSDan Williams * - validate that there are no other regions with a higher HPA already 1037384e624bSDan Williams * associated with @port 1038384e624bSDan Williams * - establish a region reference if one is not already present 1039f13da0d9SBagas Sanjaya * 1040384e624bSDan Williams * - additionally allocate a decoder instance that will host @cxlr on 1041384e624bSDan Williams * @port 1042f13da0d9SBagas Sanjaya * 1043384e624bSDan Williams * - pin the region reference by the endpoint 1044384e624bSDan Williams * - account for how many entries in @port's target list are needed to 1045384e624bSDan Williams * cover all of the added endpoints. 1046384e624bSDan Williams */ 1047384e624bSDan Williams static int cxl_port_attach_region(struct cxl_port *port, 1048384e624bSDan Williams struct cxl_region *cxlr, 1049384e624bSDan Williams struct cxl_endpoint_decoder *cxled, int pos) 1050384e624bSDan Williams { 1051384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1052384e624bSDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 1053e29a8995SDan Williams struct cxl_region_ref *cxl_rr; 1054e29a8995SDan Williams bool nr_targets_inc = false; 1055e29a8995SDan Williams struct cxl_decoder *cxld; 1056384e624bSDan Williams unsigned long index; 1057384e624bSDan Williams int rc = -EBUSY; 1058384e624bSDan Williams 1059384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 1060384e624bSDan Williams 1061e29a8995SDan Williams cxl_rr = cxl_rr_load(port, cxlr); 1062384e624bSDan Williams if (cxl_rr) { 1063384e624bSDan Williams struct cxl_ep *ep_iter; 1064384e624bSDan Williams int found = 0; 1065384e624bSDan Williams 1066e29a8995SDan Williams /* 1067e29a8995SDan Williams * Walk the existing endpoints that have been attached to 1068e29a8995SDan Williams * @cxlr at @port and see if they share the same 'next' port 1069e29a8995SDan Williams * in the downstream direction. I.e. endpoints that share common 1070e29a8995SDan Williams * upstream switch. 1071e29a8995SDan Williams */ 1072384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 1073384e624bSDan Williams if (ep_iter == ep) 1074384e624bSDan Williams continue; 1075384e624bSDan Williams if (ep_iter->next == ep->next) { 1076384e624bSDan Williams found++; 1077384e624bSDan Williams break; 1078384e624bSDan Williams } 1079384e624bSDan Williams } 1080384e624bSDan Williams 1081384e624bSDan Williams /* 1082e29a8995SDan Williams * New target port, or @port is an endpoint port that always 1083e29a8995SDan Williams * accounts its own local decode as a target. 1084384e624bSDan Williams */ 1085e29a8995SDan Williams if (!found || !ep->next) { 1086384e624bSDan Williams cxl_rr->nr_targets++; 1087e29a8995SDan Williams nr_targets_inc = true; 1088e29a8995SDan Williams } 1089384e624bSDan Williams } else { 1090cb66b1d6SAlison Schofield cxl_rr = alloc_region_ref(port, cxlr, cxled); 1091e29a8995SDan Williams if (IS_ERR(cxl_rr)) { 1092384e624bSDan Williams dev_dbg(&cxlr->dev, 1093384e624bSDan Williams "%s: failed to allocate region reference\n", 1094384e624bSDan Williams dev_name(&port->dev)); 1095e29a8995SDan Williams return PTR_ERR(cxl_rr); 1096384e624bSDan Williams } 1097e29a8995SDan Williams nr_targets_inc = true; 1098384e624bSDan Williams 109971ee71d7SVishal Verma rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr); 110071ee71d7SVishal Verma if (rc) 1101384e624bSDan Williams goto out_erase; 1102384e624bSDan Williams } 110371ee71d7SVishal Verma cxld = cxl_rr->decoder; 1104384e624bSDan Williams 110584328c5aSYao Xingtao /* 110684328c5aSYao Xingtao * the number of targets should not exceed the target_count 110784328c5aSYao Xingtao * of the decoder 110884328c5aSYao Xingtao */ 110984328c5aSYao Xingtao if (is_switch_decoder(&cxld->dev)) { 111084328c5aSYao Xingtao struct cxl_switch_decoder *cxlsd; 111184328c5aSYao Xingtao 111284328c5aSYao Xingtao cxlsd = to_cxl_switch_decoder(&cxld->dev); 111384328c5aSYao Xingtao if (cxl_rr->nr_targets > cxlsd->nr_targets) { 111484328c5aSYao Xingtao dev_dbg(&cxlr->dev, 111584328c5aSYao Xingtao "%s:%s %s add: %s:%s @ %d overflows targets: %d\n", 111684328c5aSYao Xingtao dev_name(port->uport_dev), dev_name(&port->dev), 111784328c5aSYao Xingtao dev_name(&cxld->dev), dev_name(&cxlmd->dev), 111884328c5aSYao Xingtao dev_name(&cxled->cxld.dev), pos, 111984328c5aSYao Xingtao cxlsd->nr_targets); 112084328c5aSYao Xingtao rc = -ENXIO; 112184328c5aSYao Xingtao goto out_erase; 112284328c5aSYao Xingtao } 112384328c5aSYao Xingtao } 112484328c5aSYao Xingtao 1125384e624bSDan Williams rc = cxl_rr_ep_add(cxl_rr, cxled); 1126384e624bSDan Williams if (rc) { 1127384e624bSDan Williams dev_dbg(&cxlr->dev, 1128384e624bSDan Williams "%s: failed to track endpoint %s:%s reference\n", 1129384e624bSDan Williams dev_name(&port->dev), dev_name(&cxlmd->dev), 1130384e624bSDan Williams dev_name(&cxld->dev)); 1131384e624bSDan Williams goto out_erase; 1132384e624bSDan Williams } 1133384e624bSDan Williams 113427b3f8d1SDan Williams dev_dbg(&cxlr->dev, 113527b3f8d1SDan Williams "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n", 11367481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 113727b3f8d1SDan Williams dev_name(&cxld->dev), dev_name(&cxlmd->dev), 113827b3f8d1SDan Williams dev_name(&cxled->cxld.dev), pos, 11397481653dSDan Williams ep ? ep->next ? dev_name(ep->next->uport_dev) : 114027b3f8d1SDan Williams dev_name(&cxlmd->dev) : 114127b3f8d1SDan Williams "none", 114227b3f8d1SDan Williams cxl_rr->nr_eps, cxl_rr->nr_targets); 114327b3f8d1SDan Williams 1144384e624bSDan Williams return 0; 1145384e624bSDan Williams out_erase: 1146e29a8995SDan Williams if (nr_targets_inc) 1147e29a8995SDan Williams cxl_rr->nr_targets--; 1148384e624bSDan Williams if (cxl_rr->nr_eps == 0) 1149384e624bSDan Williams free_region_ref(cxl_rr); 1150384e624bSDan Williams return rc; 1151384e624bSDan Williams } 1152384e624bSDan Williams 1153384e624bSDan Williams static void cxl_port_detach_region(struct cxl_port *port, 1154384e624bSDan Williams struct cxl_region *cxlr, 1155384e624bSDan Williams struct cxl_endpoint_decoder *cxled) 1156384e624bSDan Williams { 1157384e624bSDan Williams struct cxl_region_ref *cxl_rr; 115827b3f8d1SDan Williams struct cxl_ep *ep = NULL; 1159384e624bSDan Williams 1160384e624bSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 1161384e624bSDan Williams 1162384e624bSDan Williams cxl_rr = cxl_rr_load(port, cxlr); 1163384e624bSDan Williams if (!cxl_rr) 1164384e624bSDan Williams return; 1165384e624bSDan Williams 116627b3f8d1SDan Williams /* 116727b3f8d1SDan Williams * Endpoint ports do not carry cxl_ep references, and they 116827b3f8d1SDan Williams * never target more than one endpoint by definition 116927b3f8d1SDan Williams */ 117027b3f8d1SDan Williams if (cxl_rr->decoder == &cxled->cxld) 117127b3f8d1SDan Williams cxl_rr->nr_eps--; 117227b3f8d1SDan Williams else 1173384e624bSDan Williams ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled); 1174384e624bSDan Williams if (ep) { 1175384e624bSDan Williams struct cxl_ep *ep_iter; 1176384e624bSDan Williams unsigned long index; 1177384e624bSDan Williams int found = 0; 1178384e624bSDan Williams 1179384e624bSDan Williams cxl_rr->nr_eps--; 1180384e624bSDan Williams xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 1181384e624bSDan Williams if (ep_iter->next == ep->next) { 1182384e624bSDan Williams found++; 1183384e624bSDan Williams break; 1184384e624bSDan Williams } 1185384e624bSDan Williams } 1186384e624bSDan Williams if (!found) 1187384e624bSDan Williams cxl_rr->nr_targets--; 1188384e624bSDan Williams } 1189384e624bSDan Williams 1190384e624bSDan Williams if (cxl_rr->nr_eps == 0) 1191384e624bSDan Williams free_region_ref(cxl_rr); 1192384e624bSDan Williams } 1193384e624bSDan Williams 119427b3f8d1SDan Williams static int check_last_peer(struct cxl_endpoint_decoder *cxled, 119527b3f8d1SDan Williams struct cxl_ep *ep, struct cxl_region_ref *cxl_rr, 119627b3f8d1SDan Williams int distance) 119727b3f8d1SDan Williams { 119827b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 119927b3f8d1SDan Williams struct cxl_region *cxlr = cxl_rr->region; 120027b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 120127b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled_peer; 120227b3f8d1SDan Williams struct cxl_port *port = cxl_rr->port; 120327b3f8d1SDan Williams struct cxl_memdev *cxlmd_peer; 120427b3f8d1SDan Williams struct cxl_ep *ep_peer; 120527b3f8d1SDan Williams int pos = cxled->pos; 120627b3f8d1SDan Williams 120727b3f8d1SDan Williams /* 120827b3f8d1SDan Williams * If this position wants to share a dport with the last endpoint mapped 120927b3f8d1SDan Williams * then that endpoint, at index 'position - distance', must also be 121027b3f8d1SDan Williams * mapped by this dport. 121127b3f8d1SDan Williams */ 121227b3f8d1SDan Williams if (pos < distance) { 121327b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", 12147481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 121527b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 121627b3f8d1SDan Williams return -ENXIO; 121727b3f8d1SDan Williams } 121827b3f8d1SDan Williams cxled_peer = p->targets[pos - distance]; 121927b3f8d1SDan Williams cxlmd_peer = cxled_to_memdev(cxled_peer); 122027b3f8d1SDan Williams ep_peer = cxl_ep_load(port, cxlmd_peer); 122127b3f8d1SDan Williams if (ep->dport != ep_peer->dport) { 122227b3f8d1SDan Williams dev_dbg(&cxlr->dev, 122327b3f8d1SDan Williams "%s:%s: %s:%s pos %d mismatched peer %s:%s\n", 12247481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 122527b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, 122627b3f8d1SDan Williams dev_name(&cxlmd_peer->dev), 122727b3f8d1SDan Williams dev_name(&cxled_peer->cxld.dev)); 122827b3f8d1SDan Williams return -ENXIO; 122927b3f8d1SDan Williams } 123027b3f8d1SDan Williams 123127b3f8d1SDan Williams return 0; 123227b3f8d1SDan Williams } 123327b3f8d1SDan Williams 123484328c5aSYao Xingtao static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig) 123584328c5aSYao Xingtao { 123684328c5aSYao Xingtao struct cxl_port *port = to_cxl_port(cxld->dev.parent); 123784328c5aSYao Xingtao struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 123884328c5aSYao Xingtao unsigned int interleave_mask; 123984328c5aSYao Xingtao u8 eiw; 124084328c5aSYao Xingtao u16 eig; 124184328c5aSYao Xingtao int high_pos, low_pos; 124284328c5aSYao Xingtao 124384328c5aSYao Xingtao if (!test_bit(iw, &cxlhdm->iw_cap_mask)) 124484328c5aSYao Xingtao return -ENXIO; 124584328c5aSYao Xingtao /* 124684328c5aSYao Xingtao * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection), 124784328c5aSYao Xingtao * if eiw < 8: 124884328c5aSYao Xingtao * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw] 124984328c5aSYao Xingtao * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0] 125084328c5aSYao Xingtao * 125184328c5aSYao Xingtao * when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the 125284328c5aSYao Xingtao * interleave bits are none. 125384328c5aSYao Xingtao * 125484328c5aSYao Xingtao * if eiw >= 8: 125584328c5aSYao Xingtao * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3 125684328c5aSYao Xingtao * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0] 125784328c5aSYao Xingtao * 125884328c5aSYao Xingtao * when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the 125984328c5aSYao Xingtao * interleave bits are none. 126084328c5aSYao Xingtao */ 126184328c5aSYao Xingtao ways_to_eiw(iw, &eiw); 126284328c5aSYao Xingtao if (eiw == 0 || eiw == 8) 126384328c5aSYao Xingtao return 0; 126484328c5aSYao Xingtao 126584328c5aSYao Xingtao granularity_to_eig(ig, &eig); 126684328c5aSYao Xingtao if (eiw > 8) 126784328c5aSYao Xingtao high_pos = eiw + eig - 1; 126884328c5aSYao Xingtao else 126984328c5aSYao Xingtao high_pos = eiw + eig + 7; 127084328c5aSYao Xingtao low_pos = eig + 8; 127184328c5aSYao Xingtao interleave_mask = GENMASK(high_pos, low_pos); 127284328c5aSYao Xingtao if (interleave_mask & ~cxlhdm->interleave_mask) 127384328c5aSYao Xingtao return -ENXIO; 127484328c5aSYao Xingtao 127584328c5aSYao Xingtao return 0; 127684328c5aSYao Xingtao } 127784328c5aSYao Xingtao 127827b3f8d1SDan Williams static int cxl_port_setup_targets(struct cxl_port *port, 127927b3f8d1SDan Williams struct cxl_region *cxlr, 128027b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled) 128127b3f8d1SDan Williams { 128227b3f8d1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 128327b3f8d1SDan Williams int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos; 128427b3f8d1SDan Williams struct cxl_port *parent_port = to_cxl_port(port->dev.parent); 128527b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 128627b3f8d1SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 128727b3f8d1SDan Williams struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 128827b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 128927b3f8d1SDan Williams struct cxl_decoder *cxld = cxl_rr->decoder; 129027b3f8d1SDan Williams struct cxl_switch_decoder *cxlsd; 129127b3f8d1SDan Williams u16 eig, peig; 129227b3f8d1SDan Williams u8 eiw, peiw; 129327b3f8d1SDan Williams 129427b3f8d1SDan Williams /* 129527b3f8d1SDan Williams * While root level decoders support x3, x6, x12, switch level 129627b3f8d1SDan Williams * decoders only support powers of 2 up to x16. 129727b3f8d1SDan Williams */ 129827b3f8d1SDan Williams if (!is_power_of_2(cxl_rr->nr_targets)) { 129927b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n", 13007481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 130127b3f8d1SDan Williams cxl_rr->nr_targets); 130227b3f8d1SDan Williams return -EINVAL; 130327b3f8d1SDan Williams } 130427b3f8d1SDan Williams 130527b3f8d1SDan Williams cxlsd = to_cxl_switch_decoder(&cxld->dev); 130627b3f8d1SDan Williams if (cxl_rr->nr_targets_set) { 130727b3f8d1SDan Williams int i, distance; 130827b3f8d1SDan Williams 1309e4f6dfa9SDan Williams /* 1310711442e2SDan Williams * Passthrough decoders impose no distance requirements between 1311e4f6dfa9SDan Williams * peers 1312e4f6dfa9SDan Williams */ 1313711442e2SDan Williams if (cxl_rr->nr_targets == 1) 1314e4f6dfa9SDan Williams distance = 0; 1315e4f6dfa9SDan Williams else 131627b3f8d1SDan Williams distance = p->nr_targets / cxl_rr->nr_targets; 131727b3f8d1SDan Williams for (i = 0; i < cxl_rr->nr_targets_set; i++) 131827b3f8d1SDan Williams if (ep->dport == cxlsd->target[i]) { 131927b3f8d1SDan Williams rc = check_last_peer(cxled, ep, cxl_rr, 132027b3f8d1SDan Williams distance); 132127b3f8d1SDan Williams if (rc) 132227b3f8d1SDan Williams return rc; 132327b3f8d1SDan Williams goto out_target_set; 132427b3f8d1SDan Williams } 132527b3f8d1SDan Williams goto add_target; 132627b3f8d1SDan Williams } 132727b3f8d1SDan Williams 132827b3f8d1SDan Williams if (is_cxl_root(parent_port)) { 132998a04c7aSJim Harris /* 133098a04c7aSJim Harris * Root decoder IG is always set to value in CFMWS which 133198a04c7aSJim Harris * may be different than this region's IG. We can use the 133298a04c7aSJim Harris * region's IG here since interleave_granularity_store() 133398a04c7aSJim Harris * does not allow interleaved host-bridges with 133498a04c7aSJim Harris * root IG != region IG. 133598a04c7aSJim Harris */ 133698a04c7aSJim Harris parent_ig = p->interleave_granularity; 133727b3f8d1SDan Williams parent_iw = cxlrd->cxlsd.cxld.interleave_ways; 133827b3f8d1SDan Williams /* 133927b3f8d1SDan Williams * For purposes of address bit routing, use power-of-2 math for 134027b3f8d1SDan Williams * switch ports. 134127b3f8d1SDan Williams */ 134227b3f8d1SDan Williams if (!is_power_of_2(parent_iw)) 134327b3f8d1SDan Williams parent_iw /= 3; 134427b3f8d1SDan Williams } else { 134527b3f8d1SDan Williams struct cxl_region_ref *parent_rr; 134627b3f8d1SDan Williams struct cxl_decoder *parent_cxld; 134727b3f8d1SDan Williams 134827b3f8d1SDan Williams parent_rr = cxl_rr_load(parent_port, cxlr); 134927b3f8d1SDan Williams parent_cxld = parent_rr->decoder; 135027b3f8d1SDan Williams parent_ig = parent_cxld->interleave_granularity; 135127b3f8d1SDan Williams parent_iw = parent_cxld->interleave_ways; 135227b3f8d1SDan Williams } 135327b3f8d1SDan Williams 135483351ddbSDave Jiang rc = granularity_to_eig(parent_ig, &peig); 13558d428542SDan Williams if (rc) { 13568d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n", 13577481653dSDan Williams dev_name(parent_port->uport_dev), 13588d428542SDan Williams dev_name(&parent_port->dev), parent_ig); 13598d428542SDan Williams return rc; 13608d428542SDan Williams } 13618d428542SDan Williams 1362c99b2e8cSDave Jiang rc = ways_to_eiw(parent_iw, &peiw); 13638d428542SDan Williams if (rc) { 13648d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n", 13657481653dSDan Williams dev_name(parent_port->uport_dev), 13668d428542SDan Williams dev_name(&parent_port->dev), parent_iw); 13678d428542SDan Williams return rc; 13688d428542SDan Williams } 136927b3f8d1SDan Williams 137027b3f8d1SDan Williams iw = cxl_rr->nr_targets; 1371c99b2e8cSDave Jiang rc = ways_to_eiw(iw, &eiw); 13728d428542SDan Williams if (rc) { 13738d428542SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n", 13747481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), iw); 13758d428542SDan Williams return rc; 13768d428542SDan Williams } 13778d428542SDan Williams 1378298d44d0SDan Williams /* 137918f35dc9SAlison Schofield * Interleave granularity is a multiple of @parent_port granularity. 138018f35dc9SAlison Schofield * Multiplier is the parent port interleave ways. 1381298d44d0SDan Williams */ 138218f35dc9SAlison Schofield rc = granularity_to_eig(parent_ig * parent_iw, &eig); 138318f35dc9SAlison Schofield if (rc) { 138418f35dc9SAlison Schofield dev_dbg(&cxlr->dev, 138518f35dc9SAlison Schofield "%s: invalid granularity calculation (%d * %d)\n", 138618f35dc9SAlison Schofield dev_name(&parent_port->dev), parent_ig, parent_iw); 138718f35dc9SAlison Schofield return rc; 138827b3f8d1SDan Williams } 138927b3f8d1SDan Williams 139083351ddbSDave Jiang rc = eig_to_granularity(eig, &ig); 139127b3f8d1SDan Williams if (rc) { 139227b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n", 13937481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 139427b3f8d1SDan Williams 256 << eig); 139527b3f8d1SDan Williams return rc; 139627b3f8d1SDan Williams } 139727b3f8d1SDan Williams 13985d09c63fSDan Williams if (iw > 8 || iw > cxlsd->nr_targets) { 13995d09c63fSDan Williams dev_dbg(&cxlr->dev, 14005d09c63fSDan Williams "%s:%s:%s: ways: %d overflows targets: %d\n", 14015d09c63fSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 14025d09c63fSDan Williams dev_name(&cxld->dev), iw, cxlsd->nr_targets); 14035d09c63fSDan Williams return -ENXIO; 14045d09c63fSDan Williams } 14055d09c63fSDan Williams 1406a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1407a32320b7SDan Williams if (cxld->interleave_ways != iw || 1408a32320b7SDan Williams cxld->interleave_granularity != ig || 1409a32320b7SDan Williams cxld->hpa_range.start != p->res->start || 1410a32320b7SDan Williams cxld->hpa_range.end != p->res->end || 1411a32320b7SDan Williams ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { 1412a32320b7SDan Williams dev_err(&cxlr->dev, 1413a32320b7SDan Williams "%s:%s %s expected iw: %d ig: %d %pr\n", 14147481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 1415a32320b7SDan Williams __func__, iw, ig, p->res); 1416a32320b7SDan Williams dev_err(&cxlr->dev, 1417a32320b7SDan Williams "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n", 14187481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 1419a32320b7SDan Williams __func__, cxld->interleave_ways, 1420a32320b7SDan Williams cxld->interleave_granularity, 1421a32320b7SDan Williams (cxld->flags & CXL_DECODER_F_ENABLE) ? 1422a32320b7SDan Williams "enabled" : 1423a32320b7SDan Williams "disabled", 1424a32320b7SDan Williams cxld->hpa_range.start, cxld->hpa_range.end); 1425a32320b7SDan Williams return -ENXIO; 1426a32320b7SDan Williams } 1427a32320b7SDan Williams } else { 142884328c5aSYao Xingtao rc = check_interleave_cap(cxld, iw, ig); 142984328c5aSYao Xingtao if (rc) { 143084328c5aSYao Xingtao dev_dbg(&cxlr->dev, 143184328c5aSYao Xingtao "%s:%s iw: %d ig: %d is not supported\n", 143284328c5aSYao Xingtao dev_name(port->uport_dev), 143384328c5aSYao Xingtao dev_name(&port->dev), iw, ig); 143484328c5aSYao Xingtao return rc; 143584328c5aSYao Xingtao } 143684328c5aSYao Xingtao 143727b3f8d1SDan Williams cxld->interleave_ways = iw; 143827b3f8d1SDan Williams cxld->interleave_granularity = ig; 1439910bc55dSDan Williams cxld->hpa_range = (struct range) { 1440910bc55dSDan Williams .start = p->res->start, 1441910bc55dSDan Williams .end = p->res->end, 1442910bc55dSDan Williams }; 1443a32320b7SDan Williams } 14447481653dSDan Williams dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev), 144527b3f8d1SDan Williams dev_name(&port->dev), iw, ig); 144627b3f8d1SDan Williams add_target: 144727b3f8d1SDan Williams if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) { 144827b3f8d1SDan Williams dev_dbg(&cxlr->dev, 144927b3f8d1SDan Williams "%s:%s: targets full trying to add %s:%s at %d\n", 14507481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 145127b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 145227b3f8d1SDan Williams return -ENXIO; 145327b3f8d1SDan Williams } 1454a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1455a32320b7SDan Williams if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) { 1456a32320b7SDan Williams dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n", 14577481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 1458a32320b7SDan Williams dev_name(&cxlsd->cxld.dev), 1459227db574SRobert Richter dev_name(ep->dport->dport_dev), 1460a32320b7SDan Williams cxl_rr->nr_targets_set); 1461a32320b7SDan Williams return -ENXIO; 1462a32320b7SDan Williams } 1463a32320b7SDan Williams } else 146427b3f8d1SDan Williams cxlsd->target[cxl_rr->nr_targets_set] = ep->dport; 146527b3f8d1SDan Williams inc = 1; 146627b3f8d1SDan Williams out_target_set: 146727b3f8d1SDan Williams cxl_rr->nr_targets_set += inc; 146827b3f8d1SDan Williams dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", 14697481653dSDan Williams dev_name(port->uport_dev), dev_name(&port->dev), 1470227db574SRobert Richter cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev), 147127b3f8d1SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 147227b3f8d1SDan Williams 147327b3f8d1SDan Williams return 0; 147427b3f8d1SDan Williams } 147527b3f8d1SDan Williams 147627b3f8d1SDan Williams static void cxl_port_reset_targets(struct cxl_port *port, 147727b3f8d1SDan Williams struct cxl_region *cxlr) 147827b3f8d1SDan Williams { 147927b3f8d1SDan Williams struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 1480910bc55dSDan Williams struct cxl_decoder *cxld; 148127b3f8d1SDan Williams 148227b3f8d1SDan Williams /* 148327b3f8d1SDan Williams * After the last endpoint has been detached the entire cxl_rr may now 148427b3f8d1SDan Williams * be gone. 148527b3f8d1SDan Williams */ 1486910bc55dSDan Williams if (!cxl_rr) 1487910bc55dSDan Williams return; 148827b3f8d1SDan Williams cxl_rr->nr_targets_set = 0; 1489910bc55dSDan Williams 1490910bc55dSDan Williams cxld = cxl_rr->decoder; 1491910bc55dSDan Williams cxld->hpa_range = (struct range) { 1492910bc55dSDan Williams .start = 0, 1493910bc55dSDan Williams .end = -1, 1494910bc55dSDan Williams }; 149527b3f8d1SDan Williams } 149627b3f8d1SDan Williams 149727b3f8d1SDan Williams static void cxl_region_teardown_targets(struct cxl_region *cxlr) 149827b3f8d1SDan Williams { 149927b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 150027b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled; 1501030f8803SDan Williams struct cxl_dev_state *cxlds; 150227b3f8d1SDan Williams struct cxl_memdev *cxlmd; 150327b3f8d1SDan Williams struct cxl_port *iter; 150427b3f8d1SDan Williams struct cxl_ep *ep; 150527b3f8d1SDan Williams int i; 150627b3f8d1SDan Williams 1507a32320b7SDan Williams /* 1508a32320b7SDan Williams * In the auto-discovery case skip automatic teardown since the 1509a32320b7SDan Williams * address space is already active 1510a32320b7SDan Williams */ 1511a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) 1512a32320b7SDan Williams return; 1513a32320b7SDan Williams 151427b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) { 151527b3f8d1SDan Williams cxled = p->targets[i]; 151627b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled); 1517030f8803SDan Williams cxlds = cxlmd->cxlds; 1518030f8803SDan Williams 1519030f8803SDan Williams if (cxlds->rcd) 1520030f8803SDan Williams continue; 152127b3f8d1SDan Williams 152227b3f8d1SDan Williams iter = cxled_to_port(cxled); 152327b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 152427b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent); 152527b3f8d1SDan Williams 152627b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 152727b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) 152827b3f8d1SDan Williams cxl_port_reset_targets(iter, cxlr); 152927b3f8d1SDan Williams } 153027b3f8d1SDan Williams } 153127b3f8d1SDan Williams 153227b3f8d1SDan Williams static int cxl_region_setup_targets(struct cxl_region *cxlr) 153327b3f8d1SDan Williams { 153427b3f8d1SDan Williams struct cxl_region_params *p = &cxlr->params; 153527b3f8d1SDan Williams struct cxl_endpoint_decoder *cxled; 1536030f8803SDan Williams struct cxl_dev_state *cxlds; 1537030f8803SDan Williams int i, rc, rch = 0, vh = 0; 153827b3f8d1SDan Williams struct cxl_memdev *cxlmd; 153927b3f8d1SDan Williams struct cxl_port *iter; 154027b3f8d1SDan Williams struct cxl_ep *ep; 154127b3f8d1SDan Williams 154227b3f8d1SDan Williams for (i = 0; i < p->nr_targets; i++) { 154327b3f8d1SDan Williams cxled = p->targets[i]; 154427b3f8d1SDan Williams cxlmd = cxled_to_memdev(cxled); 1545030f8803SDan Williams cxlds = cxlmd->cxlds; 1546030f8803SDan Williams 1547030f8803SDan Williams /* validate that all targets agree on topology */ 1548030f8803SDan Williams if (!cxlds->rcd) { 1549030f8803SDan Williams vh++; 1550030f8803SDan Williams } else { 1551030f8803SDan Williams rch++; 1552030f8803SDan Williams continue; 1553030f8803SDan Williams } 155427b3f8d1SDan Williams 155527b3f8d1SDan Williams iter = cxled_to_port(cxled); 155627b3f8d1SDan Williams while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 155727b3f8d1SDan Williams iter = to_cxl_port(iter->dev.parent); 155827b3f8d1SDan Williams 155927b3f8d1SDan Williams /* 1560a32320b7SDan Williams * Descend the topology tree programming / validating 1561a32320b7SDan Williams * targets while looking for conflicts. 156227b3f8d1SDan Williams */ 156327b3f8d1SDan Williams for (ep = cxl_ep_load(iter, cxlmd); iter; 156427b3f8d1SDan Williams iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 156527b3f8d1SDan Williams rc = cxl_port_setup_targets(iter, cxlr, cxled); 156627b3f8d1SDan Williams if (rc) { 156727b3f8d1SDan Williams cxl_region_teardown_targets(cxlr); 156827b3f8d1SDan Williams return rc; 156927b3f8d1SDan Williams } 157027b3f8d1SDan Williams } 157127b3f8d1SDan Williams } 157227b3f8d1SDan Williams 1573030f8803SDan Williams if (rch && vh) { 1574030f8803SDan Williams dev_err(&cxlr->dev, "mismatched CXL topologies detected\n"); 1575030f8803SDan Williams cxl_region_teardown_targets(cxlr); 1576030f8803SDan Williams return -ENXIO; 1577030f8803SDan Williams } 1578030f8803SDan Williams 157927b3f8d1SDan Williams return 0; 158027b3f8d1SDan Williams } 158127b3f8d1SDan Williams 15829995576cSDan Williams static int cxl_region_validate_position(struct cxl_region *cxlr, 15839995576cSDan Williams struct cxl_endpoint_decoder *cxled, 15849995576cSDan Williams int pos) 1585b9686e8cSDan Williams { 1586384e624bSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1587b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 15889995576cSDan Williams int i; 1589384e624bSDan Williams 1590384e624bSDan Williams if (pos < 0 || pos >= p->interleave_ways) { 1591b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1592b9686e8cSDan Williams p->interleave_ways); 1593b9686e8cSDan Williams return -ENXIO; 1594b9686e8cSDan Williams } 1595b9686e8cSDan Williams 1596b9686e8cSDan Williams if (p->targets[pos] == cxled) 1597b9686e8cSDan Williams return 0; 1598b9686e8cSDan Williams 1599b9686e8cSDan Williams if (p->targets[pos]) { 1600b9686e8cSDan Williams struct cxl_endpoint_decoder *cxled_target = p->targets[pos]; 1601b9686e8cSDan Williams struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target); 1602b9686e8cSDan Williams 1603b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n", 1604b9686e8cSDan Williams pos, dev_name(&cxlmd_target->dev), 1605b9686e8cSDan Williams dev_name(&cxled_target->cxld.dev)); 1606b9686e8cSDan Williams return -EBUSY; 1607b9686e8cSDan Williams } 1608b9686e8cSDan Williams 1609384e624bSDan Williams for (i = 0; i < p->interleave_ways; i++) { 1610384e624bSDan Williams struct cxl_endpoint_decoder *cxled_target; 1611384e624bSDan Williams struct cxl_memdev *cxlmd_target; 1612384e624bSDan Williams 1613f04facfbSFan Ni cxled_target = p->targets[i]; 1614384e624bSDan Williams if (!cxled_target) 1615384e624bSDan Williams continue; 1616384e624bSDan Williams 1617384e624bSDan Williams cxlmd_target = cxled_to_memdev(cxled_target); 1618384e624bSDan Williams if (cxlmd_target == cxlmd) { 1619384e624bSDan Williams dev_dbg(&cxlr->dev, 1620384e624bSDan Williams "%s already specified at position %d via: %s\n", 1621384e624bSDan Williams dev_name(&cxlmd->dev), pos, 1622384e624bSDan Williams dev_name(&cxled_target->cxld.dev)); 1623384e624bSDan Williams return -EBUSY; 1624384e624bSDan Williams } 1625384e624bSDan Williams } 1626384e624bSDan Williams 16279995576cSDan Williams return 0; 16289995576cSDan Williams } 16299995576cSDan Williams 16309995576cSDan Williams static int cxl_region_attach_position(struct cxl_region *cxlr, 16319995576cSDan Williams struct cxl_root_decoder *cxlrd, 16329995576cSDan Williams struct cxl_endpoint_decoder *cxled, 16339995576cSDan Williams const struct cxl_dport *dport, int pos) 16349995576cSDan Williams { 16359995576cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 163682a3e3a2SAlison Schofield struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd; 163782a3e3a2SAlison Schofield struct cxl_decoder *cxld = &cxlsd->cxld; 163882a3e3a2SAlison Schofield int iw = cxld->interleave_ways; 16399995576cSDan Williams struct cxl_port *iter; 16409995576cSDan Williams int rc; 16419995576cSDan Williams 164282a3e3a2SAlison Schofield if (dport != cxlrd->cxlsd.target[pos % iw]) { 16439995576cSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n", 16449995576cSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 16459995576cSDan Williams dev_name(&cxlrd->cxlsd.cxld.dev)); 16469995576cSDan Williams return -ENXIO; 16479995576cSDan Williams } 16489995576cSDan Williams 16499995576cSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 16509995576cSDan Williams iter = to_cxl_port(iter->dev.parent)) { 16519995576cSDan Williams rc = cxl_port_attach_region(iter, cxlr, cxled, pos); 16529995576cSDan Williams if (rc) 16539995576cSDan Williams goto err; 16549995576cSDan Williams } 16559995576cSDan Williams 16569995576cSDan Williams return 0; 16579995576cSDan Williams 16589995576cSDan Williams err: 16599995576cSDan Williams for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 16609995576cSDan Williams iter = to_cxl_port(iter->dev.parent)) 16619995576cSDan Williams cxl_port_detach_region(iter, cxlr, cxled); 16629995576cSDan Williams return rc; 16639995576cSDan Williams } 16649995576cSDan Williams 1665a32320b7SDan Williams static int cxl_region_attach_auto(struct cxl_region *cxlr, 1666a32320b7SDan Williams struct cxl_endpoint_decoder *cxled, int pos) 1667a32320b7SDan Williams { 1668a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params; 1669a32320b7SDan Williams 1670a32320b7SDan Williams if (cxled->state != CXL_DECODER_STATE_AUTO) { 1671a32320b7SDan Williams dev_err(&cxlr->dev, 1672a32320b7SDan Williams "%s: unable to add decoder to autodetected region\n", 1673a32320b7SDan Williams dev_name(&cxled->cxld.dev)); 1674a32320b7SDan Williams return -EINVAL; 1675a32320b7SDan Williams } 1676a32320b7SDan Williams 1677a32320b7SDan Williams if (pos >= 0) { 1678a32320b7SDan Williams dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n", 1679a32320b7SDan Williams dev_name(&cxled->cxld.dev), pos); 1680a32320b7SDan Williams return -EINVAL; 1681a32320b7SDan Williams } 1682a32320b7SDan Williams 1683a32320b7SDan Williams if (p->nr_targets >= p->interleave_ways) { 1684a32320b7SDan Williams dev_err(&cxlr->dev, "%s: no more target slots available\n", 1685a32320b7SDan Williams dev_name(&cxled->cxld.dev)); 1686a32320b7SDan Williams return -ENXIO; 1687a32320b7SDan Williams } 1688a32320b7SDan Williams 1689a32320b7SDan Williams /* 1690a32320b7SDan Williams * Temporarily record the endpoint decoder into the target array. Yes, 1691a32320b7SDan Williams * this means that userspace can view devices in the wrong position 1692a32320b7SDan Williams * before the region activates, and must be careful to understand when 1693a32320b7SDan Williams * it might be racing region autodiscovery. 1694a32320b7SDan Williams */ 1695a32320b7SDan Williams pos = p->nr_targets; 1696a32320b7SDan Williams p->targets[pos] = cxled; 1697a32320b7SDan Williams cxled->pos = pos; 1698a32320b7SDan Williams p->nr_targets++; 1699a32320b7SDan Williams 1700a32320b7SDan Williams return 0; 1701a32320b7SDan Williams } 1702a32320b7SDan Williams 17030cf36a85SAlison Schofield static int cmp_interleave_pos(const void *a, const void *b) 17040cf36a85SAlison Schofield { 17050cf36a85SAlison Schofield struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a; 17060cf36a85SAlison Schofield struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b; 17070cf36a85SAlison Schofield 17080cf36a85SAlison Schofield return cxled_a->pos - cxled_b->pos; 17090cf36a85SAlison Schofield } 17100cf36a85SAlison Schofield 1711a32320b7SDan Williams static struct cxl_port *next_port(struct cxl_port *port) 1712a32320b7SDan Williams { 1713a32320b7SDan Williams if (!port->parent_dport) 1714a32320b7SDan Williams return NULL; 1715a32320b7SDan Williams return port->parent_dport->port; 1716a32320b7SDan Williams } 1717a32320b7SDan Williams 171811105814SAlison Schofield static int match_switch_decoder_by_range(struct device *dev, void *data) 1719a32320b7SDan Williams { 1720a32320b7SDan Williams struct cxl_switch_decoder *cxlsd; 172111105814SAlison Schofield struct range *r1, *r2 = data; 1722a32320b7SDan Williams 1723a32320b7SDan Williams if (!is_switch_decoder(dev)) 1724a32320b7SDan Williams return 0; 1725a32320b7SDan Williams 1726a32320b7SDan Williams cxlsd = to_cxl_switch_decoder(dev); 172711105814SAlison Schofield r1 = &cxlsd->cxld.hpa_range; 172811105814SAlison Schofield 172911105814SAlison Schofield if (is_root_decoder(dev)) 173011105814SAlison Schofield return range_contains(r1, r2); 173111105814SAlison Schofield return (r1->start == r2->start && r1->end == r2->end); 1732a32320b7SDan Williams } 1733a32320b7SDan Williams 1734a3e00c96SAlison Schofield static int find_pos_and_ways(struct cxl_port *port, struct range *range, 1735a3e00c96SAlison Schofield int *pos, int *ways) 1736a3e00c96SAlison Schofield { 1737a3e00c96SAlison Schofield struct cxl_switch_decoder *cxlsd; 1738a3e00c96SAlison Schofield struct cxl_port *parent; 1739a3e00c96SAlison Schofield struct device *dev; 1740a3e00c96SAlison Schofield int rc = -ENXIO; 1741a3e00c96SAlison Schofield 1742a3e00c96SAlison Schofield parent = next_port(port); 1743a3e00c96SAlison Schofield if (!parent) 1744a3e00c96SAlison Schofield return rc; 1745a3e00c96SAlison Schofield 1746a3e00c96SAlison Schofield dev = device_find_child(&parent->dev, range, 1747a3e00c96SAlison Schofield match_switch_decoder_by_range); 1748a3e00c96SAlison Schofield if (!dev) { 1749a3e00c96SAlison Schofield dev_err(port->uport_dev, 1750a3e00c96SAlison Schofield "failed to find decoder mapping %#llx-%#llx\n", 1751a3e00c96SAlison Schofield range->start, range->end); 1752a3e00c96SAlison Schofield return rc; 1753a3e00c96SAlison Schofield } 1754a3e00c96SAlison Schofield cxlsd = to_cxl_switch_decoder(dev); 1755a3e00c96SAlison Schofield *ways = cxlsd->cxld.interleave_ways; 1756a3e00c96SAlison Schofield 1757a3e00c96SAlison Schofield for (int i = 0; i < *ways; i++) { 1758a3e00c96SAlison Schofield if (cxlsd->target[i] == port->parent_dport) { 1759a3e00c96SAlison Schofield *pos = i; 1760a3e00c96SAlison Schofield rc = 0; 1761a3e00c96SAlison Schofield break; 1762a3e00c96SAlison Schofield } 1763a3e00c96SAlison Schofield } 1764a3e00c96SAlison Schofield put_device(dev); 1765a3e00c96SAlison Schofield 1766a3e00c96SAlison Schofield return rc; 1767a3e00c96SAlison Schofield } 1768a3e00c96SAlison Schofield 1769a3e00c96SAlison Schofield /** 1770a3e00c96SAlison Schofield * cxl_calc_interleave_pos() - calculate an endpoint position in a region 1771a3e00c96SAlison Schofield * @cxled: endpoint decoder member of given region 1772a3e00c96SAlison Schofield * 1773a3e00c96SAlison Schofield * The endpoint position is calculated by traversing the topology from 1774a3e00c96SAlison Schofield * the endpoint to the root decoder and iteratively applying this 1775a3e00c96SAlison Schofield * calculation: 1776a3e00c96SAlison Schofield * 1777a3e00c96SAlison Schofield * position = position * parent_ways + parent_pos; 1778a3e00c96SAlison Schofield * 1779a3e00c96SAlison Schofield * ...where @position is inferred from switch and root decoder target lists. 1780a3e00c96SAlison Schofield * 1781a3e00c96SAlison Schofield * Return: position >= 0 on success 1782a3e00c96SAlison Schofield * -ENXIO on failure 1783a3e00c96SAlison Schofield */ 1784a3e00c96SAlison Schofield static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled) 1785a3e00c96SAlison Schofield { 1786a3e00c96SAlison Schofield struct cxl_port *iter, *port = cxled_to_port(cxled); 1787a3e00c96SAlison Schofield struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1788a3e00c96SAlison Schofield struct range *range = &cxled->cxld.hpa_range; 1789a3e00c96SAlison Schofield int parent_ways = 0, parent_pos = 0, pos = 0; 1790a3e00c96SAlison Schofield int rc; 1791a3e00c96SAlison Schofield 1792a3e00c96SAlison Schofield /* 1793a3e00c96SAlison Schofield * Example: the expected interleave order of the 4-way region shown 1794a3e00c96SAlison Schofield * below is: mem0, mem2, mem1, mem3 1795a3e00c96SAlison Schofield * 1796a3e00c96SAlison Schofield * root_port 1797a3e00c96SAlison Schofield * / \ 1798a3e00c96SAlison Schofield * host_bridge_0 host_bridge_1 1799a3e00c96SAlison Schofield * | | | | 1800a3e00c96SAlison Schofield * mem0 mem1 mem2 mem3 1801a3e00c96SAlison Schofield * 1802a3e00c96SAlison Schofield * In the example the calculator will iterate twice. The first iteration 1803a3e00c96SAlison Schofield * uses the mem position in the host-bridge and the ways of the host- 1804a3e00c96SAlison Schofield * bridge to generate the first, or local, position. The second 1805a3e00c96SAlison Schofield * iteration uses the host-bridge position in the root_port and the ways 1806a3e00c96SAlison Schofield * of the root_port to refine the position. 1807a3e00c96SAlison Schofield * 1808a3e00c96SAlison Schofield * A trace of the calculation per endpoint looks like this: 1809a3e00c96SAlison Schofield * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0 1810a3e00c96SAlison Schofield * pos = 0 * 2 + 0 pos = 0 * 2 + 1 1811a3e00c96SAlison Schofield * pos: 0 pos: 1 1812a3e00c96SAlison Schofield * 1813a3e00c96SAlison Schofield * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1 1814a3e00c96SAlison Schofield * pos = 1 * 2 + 0 pos = 1 * 2 + 1 1815a3e00c96SAlison Schofield * pos: 2 pos = 3 1816a3e00c96SAlison Schofield * 1817a3e00c96SAlison Schofield * Note that while this example is simple, the method applies to more 1818a3e00c96SAlison Schofield * complex topologies, including those with switches. 1819a3e00c96SAlison Schofield */ 1820a3e00c96SAlison Schofield 1821a3e00c96SAlison Schofield /* Iterate from endpoint to root_port refining the position */ 1822a3e00c96SAlison Schofield for (iter = port; iter; iter = next_port(iter)) { 1823a3e00c96SAlison Schofield if (is_cxl_root(iter)) 1824a3e00c96SAlison Schofield break; 1825a3e00c96SAlison Schofield 1826a3e00c96SAlison Schofield rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways); 1827a3e00c96SAlison Schofield if (rc) 1828a3e00c96SAlison Schofield return rc; 1829a3e00c96SAlison Schofield 1830a3e00c96SAlison Schofield pos = pos * parent_ways + parent_pos; 1831a3e00c96SAlison Schofield } 1832a3e00c96SAlison Schofield 1833a3e00c96SAlison Schofield dev_dbg(&cxlmd->dev, 1834a3e00c96SAlison Schofield "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n", 1835a3e00c96SAlison Schofield dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent), 1836a3e00c96SAlison Schofield dev_name(&port->dev), range->start, range->end, pos); 1837a3e00c96SAlison Schofield 1838a3e00c96SAlison Schofield return pos; 1839a3e00c96SAlison Schofield } 1840a3e00c96SAlison Schofield 1841a32320b7SDan Williams static int cxl_region_sort_targets(struct cxl_region *cxlr) 1842a32320b7SDan Williams { 1843a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params; 1844a32320b7SDan Williams int i, rc = 0; 1845a32320b7SDan Williams 1846a32320b7SDan Williams for (i = 0; i < p->nr_targets; i++) { 1847a32320b7SDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 1848a32320b7SDan Williams 18490cf36a85SAlison Schofield cxled->pos = cxl_calc_interleave_pos(cxled); 1850a32320b7SDan Williams /* 18510cf36a85SAlison Schofield * Record that sorting failed, but still continue to calc 18520cf36a85SAlison Schofield * cxled->pos so that follow-on code paths can reliably 18530cf36a85SAlison Schofield * do p->targets[cxled->pos] to self-reference their entry. 1854a32320b7SDan Williams */ 1855a32320b7SDan Williams if (cxled->pos < 0) 1856a32320b7SDan Williams rc = -ENXIO; 1857a32320b7SDan Williams } 18580cf36a85SAlison Schofield /* Keep the cxlr target list in interleave position order */ 18590cf36a85SAlison Schofield sort(p->targets, p->nr_targets, sizeof(p->targets[0]), 18600cf36a85SAlison Schofield cmp_interleave_pos, NULL); 1861a32320b7SDan Williams 1862a32320b7SDan Williams dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful"); 1863a32320b7SDan Williams return rc; 1864a32320b7SDan Williams } 1865a32320b7SDan Williams 18669995576cSDan Williams static int cxl_region_attach(struct cxl_region *cxlr, 18679995576cSDan Williams struct cxl_endpoint_decoder *cxled, int pos) 18689995576cSDan Williams { 18699995576cSDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 18709995576cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 18719995576cSDan Williams struct cxl_region_params *p = &cxlr->params; 18729995576cSDan Williams struct cxl_port *ep_port, *root_port; 18739995576cSDan Williams struct cxl_dport *dport; 18749995576cSDan Williams int rc = -ENXIO; 18759995576cSDan Williams 187684328c5aSYao Xingtao rc = check_interleave_cap(&cxled->cxld, p->interleave_ways, 187784328c5aSYao Xingtao p->interleave_granularity); 187884328c5aSYao Xingtao if (rc) { 187984328c5aSYao Xingtao dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n", 188084328c5aSYao Xingtao dev_name(&cxled->cxld.dev), p->interleave_ways, 188184328c5aSYao Xingtao p->interleave_granularity); 188284328c5aSYao Xingtao return rc; 188384328c5aSYao Xingtao } 188484328c5aSYao Xingtao 18859995576cSDan Williams if (cxled->mode != cxlr->mode) { 18869995576cSDan Williams dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n", 18879995576cSDan Williams dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode); 18889995576cSDan Williams return -EINVAL; 18899995576cSDan Williams } 18909995576cSDan Williams 18919995576cSDan Williams if (cxled->mode == CXL_DECODER_DEAD) { 18929995576cSDan Williams dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev)); 18939995576cSDan Williams return -ENODEV; 18949995576cSDan Williams } 18959995576cSDan Williams 18969995576cSDan Williams /* all full of members, or interleave config not established? */ 18979995576cSDan Williams if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) { 18989995576cSDan Williams dev_dbg(&cxlr->dev, "region already active\n"); 18999995576cSDan Williams return -EBUSY; 19009995576cSDan Williams } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) { 19019995576cSDan Williams dev_dbg(&cxlr->dev, "interleave config missing\n"); 19029995576cSDan Williams return -ENXIO; 19039995576cSDan Williams } 19049995576cSDan Williams 19050718588cSJim Harris if (p->nr_targets >= p->interleave_ways) { 19060718588cSJim Harris dev_dbg(&cxlr->dev, "region already has %d endpoints\n", 19070718588cSJim Harris p->nr_targets); 19080718588cSJim Harris return -EINVAL; 19090718588cSJim Harris } 19100718588cSJim Harris 1911384e624bSDan Williams ep_port = cxled_to_port(cxled); 1912384e624bSDan Williams root_port = cxlrd_to_port(cxlrd); 1913384e624bSDan Williams dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge); 1914384e624bSDan Williams if (!dport) { 1915384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n", 1916384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1917384e624bSDan Williams dev_name(cxlr->dev.parent)); 1918384e624bSDan Williams return -ENXIO; 1919384e624bSDan Williams } 1920384e624bSDan Williams 1921384e624bSDan Williams if (cxled->cxld.target_type != cxlr->type) { 1922384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n", 1923384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1924384e624bSDan Williams cxled->cxld.target_type, cxlr->type); 1925384e624bSDan Williams return -ENXIO; 1926384e624bSDan Williams } 1927384e624bSDan Williams 1928384e624bSDan Williams if (!cxled->dpa_res) { 1929384e624bSDan Williams dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n", 1930384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev)); 1931384e624bSDan Williams return -ENXIO; 1932384e624bSDan Williams } 1933384e624bSDan Williams 1934384e624bSDan Williams if (resource_size(cxled->dpa_res) * p->interleave_ways != 1935384e624bSDan Williams resource_size(p->res)) { 1936384e624bSDan Williams dev_dbg(&cxlr->dev, 1937384e624bSDan Williams "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n", 1938384e624bSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1939384e624bSDan Williams (u64)resource_size(cxled->dpa_res), p->interleave_ways, 1940384e624bSDan Williams (u64)resource_size(p->res)); 1941384e624bSDan Williams return -EINVAL; 1942384e624bSDan Williams } 1943384e624bSDan Williams 19443d9f4a19SDave Jiang cxl_region_perf_data_calculate(cxlr, cxled); 19453d9f4a19SDave Jiang 1946a32320b7SDan Williams if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1947a32320b7SDan Williams int i; 1948a32320b7SDan Williams 1949a32320b7SDan Williams rc = cxl_region_attach_auto(cxlr, cxled, pos); 1950384e624bSDan Williams if (rc) 1951a32320b7SDan Williams return rc; 1952a32320b7SDan Williams 1953a32320b7SDan Williams /* await more targets to arrive... */ 1954a32320b7SDan Williams if (p->nr_targets < p->interleave_ways) 1955a32320b7SDan Williams return 0; 1956a32320b7SDan Williams 1957a32320b7SDan Williams /* 1958a32320b7SDan Williams * All targets are here, which implies all PCI enumeration that 1959a32320b7SDan Williams * affects this region has been completed. Walk the topology to 1960a32320b7SDan Williams * sort the devices into their relative region decode position. 1961a32320b7SDan Williams */ 1962a32320b7SDan Williams rc = cxl_region_sort_targets(cxlr); 1963a32320b7SDan Williams if (rc) 1964a32320b7SDan Williams return rc; 1965a32320b7SDan Williams 1966a32320b7SDan Williams for (i = 0; i < p->nr_targets; i++) { 1967a32320b7SDan Williams cxled = p->targets[i]; 1968a32320b7SDan Williams ep_port = cxled_to_port(cxled); 1969a32320b7SDan Williams dport = cxl_find_dport_by_dev(root_port, 1970a32320b7SDan Williams ep_port->host_bridge); 1971a32320b7SDan Williams rc = cxl_region_attach_position(cxlr, cxlrd, cxled, 1972a32320b7SDan Williams dport, i); 1973a32320b7SDan Williams if (rc) 1974a32320b7SDan Williams return rc; 1975384e624bSDan Williams } 1976384e624bSDan Williams 1977a32320b7SDan Williams rc = cxl_region_setup_targets(cxlr); 1978a32320b7SDan Williams if (rc) 1979a32320b7SDan Williams return rc; 1980a32320b7SDan Williams 1981a32320b7SDan Williams /* 1982a32320b7SDan Williams * If target setup succeeds in the autodiscovery case 1983a32320b7SDan Williams * then the region is already committed. 1984a32320b7SDan Williams */ 1985a32320b7SDan Williams p->state = CXL_CONFIG_COMMIT; 1986*a5ab0de0SDave Jiang cxl_region_shared_upstream_bandwidth_update(cxlr); 1987a32320b7SDan Williams 1988a32320b7SDan Williams return 0; 1989a32320b7SDan Williams } 1990a32320b7SDan Williams 19919995576cSDan Williams rc = cxl_region_validate_position(cxlr, cxled, pos); 1992b9686e8cSDan Williams if (rc) 19939995576cSDan Williams return rc; 19949995576cSDan Williams 19959995576cSDan Williams rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos); 19969995576cSDan Williams if (rc) 19979995576cSDan Williams return rc; 1998b9686e8cSDan Williams 1999b9686e8cSDan Williams p->targets[pos] = cxled; 2000b9686e8cSDan Williams cxled->pos = pos; 2001b9686e8cSDan Williams p->nr_targets++; 2002b9686e8cSDan Williams 200327b3f8d1SDan Williams if (p->nr_targets == p->interleave_ways) { 200427b3f8d1SDan Williams rc = cxl_region_setup_targets(cxlr); 200527b3f8d1SDan Williams if (rc) 20060718588cSJim Harris return rc; 2007384e624bSDan Williams p->state = CXL_CONFIG_ACTIVE; 2008*a5ab0de0SDave Jiang cxl_region_shared_upstream_bandwidth_update(cxlr); 200927b3f8d1SDan Williams } 2010384e624bSDan Williams 20112901c8bdSDan Williams cxled->cxld.interleave_ways = p->interleave_ways; 20122901c8bdSDan Williams cxled->cxld.interleave_granularity = p->interleave_granularity; 2013910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) { 2014910bc55dSDan Williams .start = p->res->start, 2015910bc55dSDan Williams .end = p->res->end, 2016910bc55dSDan Williams }; 20172901c8bdSDan Williams 2018a3e00c96SAlison Schofield if (p->nr_targets != p->interleave_ways) 2019a3e00c96SAlison Schofield return 0; 2020a3e00c96SAlison Schofield 2021a3e00c96SAlison Schofield /* 2022a3e00c96SAlison Schofield * Test the auto-discovery position calculator function 2023a3e00c96SAlison Schofield * against this successfully created user-defined region. 2024a3e00c96SAlison Schofield * A fail message here means that this interleave config 2025a3e00c96SAlison Schofield * will fail when presented as CXL_REGION_F_AUTO. 2026a3e00c96SAlison Schofield */ 2027a3e00c96SAlison Schofield for (int i = 0; i < p->nr_targets; i++) { 2028a3e00c96SAlison Schofield struct cxl_endpoint_decoder *cxled = p->targets[i]; 2029a3e00c96SAlison Schofield int test_pos; 2030a3e00c96SAlison Schofield 2031a3e00c96SAlison Schofield test_pos = cxl_calc_interleave_pos(cxled); 2032a3e00c96SAlison Schofield dev_dbg(&cxled->cxld.dev, 2033a3e00c96SAlison Schofield "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n", 2034a3e00c96SAlison Schofield (test_pos == cxled->pos) ? "success" : "fail", 2035a3e00c96SAlison Schofield test_pos, cxled->pos); 2036a3e00c96SAlison Schofield } 2037a3e00c96SAlison Schofield 2038b9686e8cSDan Williams return 0; 2039b9686e8cSDan Williams } 2040b9686e8cSDan Williams 2041176baefbSDan Williams static int cxl_region_detach(struct cxl_endpoint_decoder *cxled) 2042b9686e8cSDan Williams { 2043384e624bSDan Williams struct cxl_port *iter, *ep_port = cxled_to_port(cxled); 2044b9686e8cSDan Williams struct cxl_region *cxlr = cxled->cxld.region; 2045b9686e8cSDan Williams struct cxl_region_params *p; 2046176baefbSDan Williams int rc = 0; 2047b9686e8cSDan Williams 2048b9686e8cSDan Williams lockdep_assert_held_write(&cxl_region_rwsem); 2049b9686e8cSDan Williams 2050b9686e8cSDan Williams if (!cxlr) 2051176baefbSDan Williams return 0; 2052b9686e8cSDan Williams 2053b9686e8cSDan Williams p = &cxlr->params; 2054b9686e8cSDan Williams get_device(&cxlr->dev); 2055b9686e8cSDan Williams 2056176baefbSDan Williams if (p->state > CXL_CONFIG_ACTIVE) { 2057176baefbSDan Williams /* 2058176baefbSDan Williams * TODO: tear down all impacted regions if a device is 2059176baefbSDan Williams * removed out of order 2060176baefbSDan Williams */ 2061176baefbSDan Williams rc = cxl_region_decode_reset(cxlr, p->interleave_ways); 2062176baefbSDan Williams if (rc) 2063176baefbSDan Williams goto out; 2064176baefbSDan Williams p->state = CXL_CONFIG_ACTIVE; 2065176baefbSDan Williams } 2066176baefbSDan Williams 2067384e624bSDan Williams for (iter = ep_port; !is_cxl_root(iter); 2068384e624bSDan Williams iter = to_cxl_port(iter->dev.parent)) 2069384e624bSDan Williams cxl_port_detach_region(iter, cxlr, cxled); 2070384e624bSDan Williams 2071b9686e8cSDan Williams if (cxled->pos < 0 || cxled->pos >= p->interleave_ways || 2072b9686e8cSDan Williams p->targets[cxled->pos] != cxled) { 2073b9686e8cSDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 2074b9686e8cSDan Williams 2075b9686e8cSDan Williams dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n", 2076b9686e8cSDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 2077b9686e8cSDan Williams cxled->pos); 2078b9686e8cSDan Williams goto out; 2079b9686e8cSDan Williams } 2080b9686e8cSDan Williams 208127b3f8d1SDan Williams if (p->state == CXL_CONFIG_ACTIVE) { 2082384e624bSDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 208327b3f8d1SDan Williams cxl_region_teardown_targets(cxlr); 208427b3f8d1SDan Williams } 2085b9686e8cSDan Williams p->targets[cxled->pos] = NULL; 2086b9686e8cSDan Williams p->nr_targets--; 2087910bc55dSDan Williams cxled->cxld.hpa_range = (struct range) { 2088910bc55dSDan Williams .start = 0, 2089910bc55dSDan Williams .end = -1, 2090910bc55dSDan Williams }; 2091b9686e8cSDan Williams 2092384e624bSDan Williams /* notify the region driver that one of its targets has departed */ 2093b9686e8cSDan Williams up_write(&cxl_region_rwsem); 2094b9686e8cSDan Williams device_release_driver(&cxlr->dev); 2095b9686e8cSDan Williams down_write(&cxl_region_rwsem); 2096b9686e8cSDan Williams out: 2097b9686e8cSDan Williams put_device(&cxlr->dev); 2098176baefbSDan Williams return rc; 2099b9686e8cSDan Williams } 2100b9686e8cSDan Williams 2101b9686e8cSDan Williams void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled) 2102b9686e8cSDan Williams { 2103b9686e8cSDan Williams down_write(&cxl_region_rwsem); 2104b9686e8cSDan Williams cxled->mode = CXL_DECODER_DEAD; 2105b9686e8cSDan Williams cxl_region_detach(cxled); 2106b9686e8cSDan Williams up_write(&cxl_region_rwsem); 2107b9686e8cSDan Williams } 2108b9686e8cSDan Williams 21093528b1e1SDan Williams static int attach_target(struct cxl_region *cxlr, 21103528b1e1SDan Williams struct cxl_endpoint_decoder *cxled, int pos, 21113528b1e1SDan Williams unsigned int state) 2112b9686e8cSDan Williams { 21133528b1e1SDan Williams int rc = 0; 2114b9686e8cSDan Williams 21153528b1e1SDan Williams if (state == TASK_INTERRUPTIBLE) 2116b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem); 21173528b1e1SDan Williams else 21183528b1e1SDan Williams down_write(&cxl_region_rwsem); 2119b9686e8cSDan Williams if (rc) 21203528b1e1SDan Williams return rc; 21213528b1e1SDan Williams 2122b9686e8cSDan Williams down_read(&cxl_dpa_rwsem); 21233528b1e1SDan Williams rc = cxl_region_attach(cxlr, cxled, pos); 2124b9686e8cSDan Williams up_read(&cxl_dpa_rwsem); 2125b9686e8cSDan Williams up_write(&cxl_region_rwsem); 2126b9686e8cSDan Williams return rc; 2127b9686e8cSDan Williams } 2128b9686e8cSDan Williams 2129b9686e8cSDan Williams static int detach_target(struct cxl_region *cxlr, int pos) 2130b9686e8cSDan Williams { 2131b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 2132b9686e8cSDan Williams int rc; 2133b9686e8cSDan Williams 2134b9686e8cSDan Williams rc = down_write_killable(&cxl_region_rwsem); 2135b9686e8cSDan Williams if (rc) 2136b9686e8cSDan Williams return rc; 2137b9686e8cSDan Williams 2138b9686e8cSDan Williams if (pos >= p->interleave_ways) { 2139b9686e8cSDan Williams dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 2140b9686e8cSDan Williams p->interleave_ways); 2141b9686e8cSDan Williams rc = -ENXIO; 2142b9686e8cSDan Williams goto out; 2143b9686e8cSDan Williams } 2144b9686e8cSDan Williams 2145b9686e8cSDan Williams if (!p->targets[pos]) { 2146b9686e8cSDan Williams rc = 0; 2147b9686e8cSDan Williams goto out; 2148b9686e8cSDan Williams } 2149b9686e8cSDan Williams 2150176baefbSDan Williams rc = cxl_region_detach(p->targets[pos]); 2151b9686e8cSDan Williams out: 2152b9686e8cSDan Williams up_write(&cxl_region_rwsem); 2153b9686e8cSDan Williams return rc; 2154b9686e8cSDan Williams } 2155b9686e8cSDan Williams 2156b9686e8cSDan Williams static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos, 2157b9686e8cSDan Williams size_t len) 2158b9686e8cSDan Williams { 2159b9686e8cSDan Williams int rc; 2160b9686e8cSDan Williams 2161b9686e8cSDan Williams if (sysfs_streq(buf, "\n")) 2162b9686e8cSDan Williams rc = detach_target(cxlr, pos); 21633528b1e1SDan Williams else { 21643528b1e1SDan Williams struct device *dev; 21653528b1e1SDan Williams 21663528b1e1SDan Williams dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf); 21673528b1e1SDan Williams if (!dev) 21683528b1e1SDan Williams return -ENODEV; 21693528b1e1SDan Williams 21703528b1e1SDan Williams if (!is_endpoint_decoder(dev)) { 21713528b1e1SDan Williams rc = -EINVAL; 21723528b1e1SDan Williams goto out; 21733528b1e1SDan Williams } 21743528b1e1SDan Williams 21753528b1e1SDan Williams rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos, 21763528b1e1SDan Williams TASK_INTERRUPTIBLE); 21773528b1e1SDan Williams out: 21783528b1e1SDan Williams put_device(dev); 21793528b1e1SDan Williams } 2180b9686e8cSDan Williams 2181b9686e8cSDan Williams if (rc < 0) 2182b9686e8cSDan Williams return rc; 2183b9686e8cSDan Williams return len; 2184b9686e8cSDan Williams } 2185b9686e8cSDan Williams 2186b9686e8cSDan Williams #define TARGET_ATTR_RW(n) \ 2187b9686e8cSDan Williams static ssize_t target##n##_show( \ 2188b9686e8cSDan Williams struct device *dev, struct device_attribute *attr, char *buf) \ 2189b9686e8cSDan Williams { \ 2190b9686e8cSDan Williams return show_targetN(to_cxl_region(dev), buf, (n)); \ 2191b9686e8cSDan Williams } \ 2192b9686e8cSDan Williams static ssize_t target##n##_store(struct device *dev, \ 2193b9686e8cSDan Williams struct device_attribute *attr, \ 2194b9686e8cSDan Williams const char *buf, size_t len) \ 2195b9686e8cSDan Williams { \ 2196b9686e8cSDan Williams return store_targetN(to_cxl_region(dev), buf, (n), len); \ 2197b9686e8cSDan Williams } \ 2198b9686e8cSDan Williams static DEVICE_ATTR_RW(target##n) 2199b9686e8cSDan Williams 2200b9686e8cSDan Williams TARGET_ATTR_RW(0); 2201b9686e8cSDan Williams TARGET_ATTR_RW(1); 2202b9686e8cSDan Williams TARGET_ATTR_RW(2); 2203b9686e8cSDan Williams TARGET_ATTR_RW(3); 2204b9686e8cSDan Williams TARGET_ATTR_RW(4); 2205b9686e8cSDan Williams TARGET_ATTR_RW(5); 2206b9686e8cSDan Williams TARGET_ATTR_RW(6); 2207b9686e8cSDan Williams TARGET_ATTR_RW(7); 2208b9686e8cSDan Williams TARGET_ATTR_RW(8); 2209b9686e8cSDan Williams TARGET_ATTR_RW(9); 2210b9686e8cSDan Williams TARGET_ATTR_RW(10); 2211b9686e8cSDan Williams TARGET_ATTR_RW(11); 2212b9686e8cSDan Williams TARGET_ATTR_RW(12); 2213b9686e8cSDan Williams TARGET_ATTR_RW(13); 2214b9686e8cSDan Williams TARGET_ATTR_RW(14); 2215b9686e8cSDan Williams TARGET_ATTR_RW(15); 2216b9686e8cSDan Williams 2217b9686e8cSDan Williams static struct attribute *target_attrs[] = { 2218b9686e8cSDan Williams &dev_attr_target0.attr, 2219b9686e8cSDan Williams &dev_attr_target1.attr, 2220b9686e8cSDan Williams &dev_attr_target2.attr, 2221b9686e8cSDan Williams &dev_attr_target3.attr, 2222b9686e8cSDan Williams &dev_attr_target4.attr, 2223b9686e8cSDan Williams &dev_attr_target5.attr, 2224b9686e8cSDan Williams &dev_attr_target6.attr, 2225b9686e8cSDan Williams &dev_attr_target7.attr, 2226b9686e8cSDan Williams &dev_attr_target8.attr, 2227b9686e8cSDan Williams &dev_attr_target9.attr, 2228b9686e8cSDan Williams &dev_attr_target10.attr, 2229b9686e8cSDan Williams &dev_attr_target11.attr, 2230b9686e8cSDan Williams &dev_attr_target12.attr, 2231b9686e8cSDan Williams &dev_attr_target13.attr, 2232b9686e8cSDan Williams &dev_attr_target14.attr, 2233b9686e8cSDan Williams &dev_attr_target15.attr, 2234b9686e8cSDan Williams NULL, 2235b9686e8cSDan Williams }; 2236b9686e8cSDan Williams 2237b9686e8cSDan Williams static umode_t cxl_region_target_visible(struct kobject *kobj, 2238b9686e8cSDan Williams struct attribute *a, int n) 2239b9686e8cSDan Williams { 2240b9686e8cSDan Williams struct device *dev = kobj_to_dev(kobj); 2241b9686e8cSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 2242b9686e8cSDan Williams struct cxl_region_params *p = &cxlr->params; 2243b9686e8cSDan Williams 2244b9686e8cSDan Williams if (n < p->interleave_ways) 2245b9686e8cSDan Williams return a->mode; 2246b9686e8cSDan Williams return 0; 2247b9686e8cSDan Williams } 2248b9686e8cSDan Williams 2249b9686e8cSDan Williams static const struct attribute_group cxl_region_target_group = { 2250b9686e8cSDan Williams .attrs = target_attrs, 2251b9686e8cSDan Williams .is_visible = cxl_region_target_visible, 2252b9686e8cSDan Williams }; 2253b9686e8cSDan Williams 2254b9686e8cSDan Williams static const struct attribute_group *get_cxl_region_target_group(void) 2255b9686e8cSDan Williams { 2256b9686e8cSDan Williams return &cxl_region_target_group; 2257b9686e8cSDan Williams } 2258b9686e8cSDan Williams 2259dd5ba0ebSBen Widawsky static const struct attribute_group *region_groups[] = { 2260dd5ba0ebSBen Widawsky &cxl_base_attribute_group, 2261dd5ba0ebSBen Widawsky &cxl_region_group, 2262b9686e8cSDan Williams &cxl_region_target_group, 2263c20eaf44SDave Jiang &cxl_region_access0_coordinate_group, 2264c20eaf44SDave Jiang &cxl_region_access1_coordinate_group, 2265dd5ba0ebSBen Widawsky NULL, 2266dd5ba0ebSBen Widawsky }; 2267dd5ba0ebSBen Widawsky 2268779dd20cSBen Widawsky static void cxl_region_release(struct device *dev) 2269779dd20cSBen Widawsky { 22708f401ec1SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 2271779dd20cSBen Widawsky struct cxl_region *cxlr = to_cxl_region(dev); 22728f401ec1SDan Williams int id = atomic_read(&cxlrd->region_id); 22738f401ec1SDan Williams 22748f401ec1SDan Williams /* 22758f401ec1SDan Williams * Try to reuse the recently idled id rather than the cached 22768f401ec1SDan Williams * next id to prevent the region id space from increasing 22778f401ec1SDan Williams * unnecessarily. 22788f401ec1SDan Williams */ 22798f401ec1SDan Williams if (cxlr->id < id) 22808f401ec1SDan Williams if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) { 22818f401ec1SDan Williams memregion_free(id); 22828f401ec1SDan Williams goto out; 22838f401ec1SDan Williams } 2284779dd20cSBen Widawsky 2285779dd20cSBen Widawsky memregion_free(cxlr->id); 22868f401ec1SDan Williams out: 22878f401ec1SDan Williams put_device(dev->parent); 2288779dd20cSBen Widawsky kfree(cxlr); 2289779dd20cSBen Widawsky } 2290779dd20cSBen Widawsky 22918d48817dSDan Williams const struct device_type cxl_region_type = { 2292779dd20cSBen Widawsky .name = "cxl_region", 2293779dd20cSBen Widawsky .release = cxl_region_release, 2294dd5ba0ebSBen Widawsky .groups = region_groups 2295779dd20cSBen Widawsky }; 2296779dd20cSBen Widawsky 2297779dd20cSBen Widawsky bool is_cxl_region(struct device *dev) 2298779dd20cSBen Widawsky { 2299779dd20cSBen Widawsky return dev->type == &cxl_region_type; 2300779dd20cSBen Widawsky } 2301779dd20cSBen Widawsky EXPORT_SYMBOL_NS_GPL(is_cxl_region, CXL); 2302779dd20cSBen Widawsky 2303779dd20cSBen Widawsky static struct cxl_region *to_cxl_region(struct device *dev) 2304779dd20cSBen Widawsky { 2305779dd20cSBen Widawsky if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type, 2306779dd20cSBen Widawsky "not a cxl_region device\n")) 2307779dd20cSBen Widawsky return NULL; 2308779dd20cSBen Widawsky 2309779dd20cSBen Widawsky return container_of(dev, struct cxl_region, dev); 2310779dd20cSBen Widawsky } 2311779dd20cSBen Widawsky 2312ace196deSDave Jiang static void unregister_region(void *_cxlr) 2313779dd20cSBen Widawsky { 2314ace196deSDave Jiang struct cxl_region *cxlr = _cxlr; 23150d9e7340SDan Williams struct cxl_region_params *p = &cxlr->params; 23160d9e7340SDan Williams int i; 231723a22cd1SDan Williams 2318ace196deSDave Jiang device_del(&cxlr->dev); 23190d9e7340SDan Williams 23200d9e7340SDan Williams /* 23210d9e7340SDan Williams * Now that region sysfs is shutdown, the parameter block is now 23220d9e7340SDan Williams * read-only, so no need to hold the region rwsem to access the 23230d9e7340SDan Williams * region parameters. 23240d9e7340SDan Williams */ 23250d9e7340SDan Williams for (i = 0; i < p->interleave_ways; i++) 23260d9e7340SDan Williams detach_target(cxlr, i); 23270d9e7340SDan Williams 232823a22cd1SDan Williams cxl_region_iomem_release(cxlr); 2329ace196deSDave Jiang put_device(&cxlr->dev); 2330779dd20cSBen Widawsky } 2331779dd20cSBen Widawsky 2332779dd20cSBen Widawsky static struct lock_class_key cxl_region_key; 2333779dd20cSBen Widawsky 2334779dd20cSBen Widawsky static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id) 2335779dd20cSBen Widawsky { 2336779dd20cSBen Widawsky struct cxl_region *cxlr; 2337779dd20cSBen Widawsky struct device *dev; 2338779dd20cSBen Widawsky 2339779dd20cSBen Widawsky cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL); 2340779dd20cSBen Widawsky if (!cxlr) { 2341779dd20cSBen Widawsky memregion_free(id); 2342779dd20cSBen Widawsky return ERR_PTR(-ENOMEM); 2343779dd20cSBen Widawsky } 2344779dd20cSBen Widawsky 2345779dd20cSBen Widawsky dev = &cxlr->dev; 2346779dd20cSBen Widawsky device_initialize(dev); 2347779dd20cSBen Widawsky lockdep_set_class(&dev->mutex, &cxl_region_key); 2348779dd20cSBen Widawsky dev->parent = &cxlrd->cxlsd.cxld.dev; 23498f401ec1SDan Williams /* 23508f401ec1SDan Williams * Keep root decoder pinned through cxl_region_release to fixup 23518f401ec1SDan Williams * region id allocations 23528f401ec1SDan Williams */ 23538f401ec1SDan Williams get_device(dev->parent); 2354779dd20cSBen Widawsky device_set_pm_not_required(dev); 2355779dd20cSBen Widawsky dev->bus = &cxl_bus_type; 2356779dd20cSBen Widawsky dev->type = &cxl_region_type; 2357779dd20cSBen Widawsky cxlr->id = id; 2358779dd20cSBen Widawsky 2359779dd20cSBen Widawsky return cxlr; 2360779dd20cSBen Widawsky } 2361779dd20cSBen Widawsky 2362067353a4SDave Jiang static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) 2363067353a4SDave Jiang { 2364067353a4SDave Jiang int cset = 0; 2365067353a4SDave Jiang int rc; 2366067353a4SDave Jiang 2367067353a4SDave Jiang for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { 2368067353a4SDave Jiang if (cxlr->coord[i].read_bandwidth) { 2369debdce20SDave Jiang rc = 0; 2370debdce20SDave Jiang if (cxl_need_node_perf_attrs_update(nid)) 2371debdce20SDave Jiang node_set_perf_attrs(nid, &cxlr->coord[i], i); 2372debdce20SDave Jiang else 2373067353a4SDave Jiang rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); 2374debdce20SDave Jiang 2375067353a4SDave Jiang if (rc == 0) 2376067353a4SDave Jiang cset++; 2377067353a4SDave Jiang } 2378067353a4SDave Jiang } 2379067353a4SDave Jiang 2380067353a4SDave Jiang if (!cset) 2381067353a4SDave Jiang return false; 2382067353a4SDave Jiang 2383067353a4SDave Jiang rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group()); 2384067353a4SDave Jiang if (rc) 2385067353a4SDave Jiang dev_dbg(&cxlr->dev, "Failed to update access0 group\n"); 2386067353a4SDave Jiang 2387067353a4SDave Jiang rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group()); 2388067353a4SDave Jiang if (rc) 2389067353a4SDave Jiang dev_dbg(&cxlr->dev, "Failed to update access1 group\n"); 2390067353a4SDave Jiang 2391067353a4SDave Jiang return true; 2392067353a4SDave Jiang } 2393067353a4SDave Jiang 2394067353a4SDave Jiang static int cxl_region_perf_attrs_callback(struct notifier_block *nb, 2395067353a4SDave Jiang unsigned long action, void *arg) 2396067353a4SDave Jiang { 2397067353a4SDave Jiang struct cxl_region *cxlr = container_of(nb, struct cxl_region, 2398067353a4SDave Jiang memory_notifier); 2399067353a4SDave Jiang struct memory_notify *mnb = arg; 2400067353a4SDave Jiang int nid = mnb->status_change_nid; 2401067353a4SDave Jiang int region_nid; 2402067353a4SDave Jiang 2403067353a4SDave Jiang if (nid == NUMA_NO_NODE || action != MEM_ONLINE) 2404067353a4SDave Jiang return NOTIFY_DONE; 2405067353a4SDave Jiang 2406d9a476c8SIra Weiny /* 2407d9a476c8SIra Weiny * No need to hold cxl_region_rwsem; region parameters are stable 2408d9a476c8SIra Weiny * within the cxl_region driver. 2409d9a476c8SIra Weiny */ 2410d9a476c8SIra Weiny region_nid = phys_to_target_node(cxlr->params.res->start); 2411067353a4SDave Jiang if (nid != region_nid) 2412067353a4SDave Jiang return NOTIFY_DONE; 2413067353a4SDave Jiang 2414067353a4SDave Jiang if (!cxl_region_update_coordinates(cxlr, nid)) 2415067353a4SDave Jiang return NOTIFY_DONE; 2416067353a4SDave Jiang 2417067353a4SDave Jiang return NOTIFY_OK; 2418067353a4SDave Jiang } 2419067353a4SDave Jiang 2420643e8e3eSHuang Ying static int cxl_region_calculate_adistance(struct notifier_block *nb, 2421643e8e3eSHuang Ying unsigned long nid, void *data) 2422643e8e3eSHuang Ying { 2423643e8e3eSHuang Ying struct cxl_region *cxlr = container_of(nb, struct cxl_region, 2424643e8e3eSHuang Ying adist_notifier); 2425643e8e3eSHuang Ying struct access_coordinate *perf; 2426643e8e3eSHuang Ying int *adist = data; 2427643e8e3eSHuang Ying int region_nid; 2428643e8e3eSHuang Ying 2429d9a476c8SIra Weiny /* 2430d9a476c8SIra Weiny * No need to hold cxl_region_rwsem; region parameters are stable 2431d9a476c8SIra Weiny * within the cxl_region driver. 2432d9a476c8SIra Weiny */ 2433d9a476c8SIra Weiny region_nid = phys_to_target_node(cxlr->params.res->start); 2434643e8e3eSHuang Ying if (nid != region_nid) 2435643e8e3eSHuang Ying return NOTIFY_OK; 2436643e8e3eSHuang Ying 2437643e8e3eSHuang Ying perf = &cxlr->coord[ACCESS_COORDINATE_CPU]; 2438643e8e3eSHuang Ying 2439643e8e3eSHuang Ying if (mt_perf_to_adistance(perf, adist)) 2440643e8e3eSHuang Ying return NOTIFY_OK; 2441643e8e3eSHuang Ying 2442643e8e3eSHuang Ying return NOTIFY_STOP; 2443643e8e3eSHuang Ying } 2444643e8e3eSHuang Ying 2445779dd20cSBen Widawsky /** 2446779dd20cSBen Widawsky * devm_cxl_add_region - Adds a region to a decoder 2447779dd20cSBen Widawsky * @cxlrd: root decoder 2448779dd20cSBen Widawsky * @id: memregion id to create, or memregion_free() on failure 2449779dd20cSBen Widawsky * @mode: mode for the endpoint decoders of this region 2450779dd20cSBen Widawsky * @type: select whether this is an expander or accelerator (type-2 or type-3) 2451779dd20cSBen Widawsky * 2452779dd20cSBen Widawsky * This is the second step of region initialization. Regions exist within an 2453779dd20cSBen Widawsky * address space which is mapped by a @cxlrd. 2454779dd20cSBen Widawsky * 2455779dd20cSBen Widawsky * Return: 0 if the region was added to the @cxlrd, else returns negative error 2456779dd20cSBen Widawsky * code. The region will be named "regionZ" where Z is the unique region number. 2457779dd20cSBen Widawsky */ 2458779dd20cSBen Widawsky static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, 2459779dd20cSBen Widawsky int id, 2460779dd20cSBen Widawsky enum cxl_decoder_mode mode, 2461779dd20cSBen Widawsky enum cxl_decoder_type type) 2462779dd20cSBen Widawsky { 2463779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent); 2464779dd20cSBen Widawsky struct cxl_region *cxlr; 2465779dd20cSBen Widawsky struct device *dev; 2466779dd20cSBen Widawsky int rc; 2467779dd20cSBen Widawsky 2468779dd20cSBen Widawsky cxlr = cxl_region_alloc(cxlrd, id); 2469779dd20cSBen Widawsky if (IS_ERR(cxlr)) 2470779dd20cSBen Widawsky return cxlr; 2471779dd20cSBen Widawsky cxlr->mode = mode; 2472779dd20cSBen Widawsky cxlr->type = type; 2473779dd20cSBen Widawsky 2474779dd20cSBen Widawsky dev = &cxlr->dev; 2475779dd20cSBen Widawsky rc = dev_set_name(dev, "region%d", id); 2476779dd20cSBen Widawsky if (rc) 2477779dd20cSBen Widawsky goto err; 2478779dd20cSBen Widawsky 2479779dd20cSBen Widawsky rc = device_add(dev); 2480779dd20cSBen Widawsky if (rc) 2481779dd20cSBen Widawsky goto err; 2482779dd20cSBen Widawsky 24837481653dSDan Williams rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); 2484779dd20cSBen Widawsky if (rc) 2485779dd20cSBen Widawsky return ERR_PTR(rc); 2486779dd20cSBen Widawsky 24877481653dSDan Williams dev_dbg(port->uport_dev, "%s: created %s\n", 2488779dd20cSBen Widawsky dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); 2489779dd20cSBen Widawsky return cxlr; 2490779dd20cSBen Widawsky 2491779dd20cSBen Widawsky err: 2492779dd20cSBen Widawsky put_device(dev); 2493779dd20cSBen Widawsky return ERR_PTR(rc); 2494779dd20cSBen Widawsky } 2495779dd20cSBen Widawsky 24966e099264SDan Williams static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf) 24976e099264SDan Williams { 24986e099264SDan Williams return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id)); 24996e099264SDan Williams } 25006e099264SDan Williams 2501779dd20cSBen Widawsky static ssize_t create_pmem_region_show(struct device *dev, 2502779dd20cSBen Widawsky struct device_attribute *attr, char *buf) 2503779dd20cSBen Widawsky { 25046e099264SDan Williams return __create_region_show(to_cxl_root_decoder(dev), buf); 25056e099264SDan Williams } 2506779dd20cSBen Widawsky 25076e099264SDan Williams static ssize_t create_ram_region_show(struct device *dev, 25086e099264SDan Williams struct device_attribute *attr, char *buf) 25096e099264SDan Williams { 25106e099264SDan Williams return __create_region_show(to_cxl_root_decoder(dev), buf); 25116e099264SDan Williams } 25126e099264SDan Williams 25136e099264SDan Williams static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd, 25146e099264SDan Williams enum cxl_decoder_mode mode, int id) 25156e099264SDan Williams { 25166e099264SDan Williams int rc; 25176e099264SDan Williams 251849ba7b51SLi Zhijian switch (mode) { 251949ba7b51SLi Zhijian case CXL_DECODER_RAM: 252049ba7b51SLi Zhijian case CXL_DECODER_PMEM: 252149ba7b51SLi Zhijian break; 252249ba7b51SLi Zhijian default: 252349ba7b51SLi Zhijian dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode); 252449ba7b51SLi Zhijian return ERR_PTR(-EINVAL); 252549ba7b51SLi Zhijian } 252649ba7b51SLi Zhijian 25276e099264SDan Williams rc = memregion_alloc(GFP_KERNEL); 25286e099264SDan Williams if (rc < 0) 25296e099264SDan Williams return ERR_PTR(rc); 25306e099264SDan Williams 25316e099264SDan Williams if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) { 25326e099264SDan Williams memregion_free(rc); 25336e099264SDan Williams return ERR_PTR(-EBUSY); 25346e099264SDan Williams } 25356e099264SDan Williams 25365aa39a91SDan Williams return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM); 2537779dd20cSBen Widawsky } 2538779dd20cSBen Widawsky 2539779dd20cSBen Widawsky static ssize_t create_pmem_region_store(struct device *dev, 2540779dd20cSBen Widawsky struct device_attribute *attr, 2541779dd20cSBen Widawsky const char *buf, size_t len) 2542779dd20cSBen Widawsky { 2543779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2544779dd20cSBen Widawsky struct cxl_region *cxlr; 25456e099264SDan Williams int rc, id; 2546779dd20cSBen Widawsky 2547779dd20cSBen Widawsky rc = sscanf(buf, "region%d\n", &id); 2548779dd20cSBen Widawsky if (rc != 1) 2549779dd20cSBen Widawsky return -EINVAL; 2550779dd20cSBen Widawsky 25516e099264SDan Williams cxlr = __create_region(cxlrd, CXL_DECODER_PMEM, id); 2552779dd20cSBen Widawsky if (IS_ERR(cxlr)) 2553779dd20cSBen Widawsky return PTR_ERR(cxlr); 2554779dd20cSBen Widawsky 2555779dd20cSBen Widawsky return len; 2556779dd20cSBen Widawsky } 2557779dd20cSBen Widawsky DEVICE_ATTR_RW(create_pmem_region); 2558779dd20cSBen Widawsky 25596e099264SDan Williams static ssize_t create_ram_region_store(struct device *dev, 25606e099264SDan Williams struct device_attribute *attr, 25616e099264SDan Williams const char *buf, size_t len) 25626e099264SDan Williams { 25636e099264SDan Williams struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 25646e099264SDan Williams struct cxl_region *cxlr; 25656e099264SDan Williams int rc, id; 25666e099264SDan Williams 25676e099264SDan Williams rc = sscanf(buf, "region%d\n", &id); 25686e099264SDan Williams if (rc != 1) 25696e099264SDan Williams return -EINVAL; 25706e099264SDan Williams 25716e099264SDan Williams cxlr = __create_region(cxlrd, CXL_DECODER_RAM, id); 25726e099264SDan Williams if (IS_ERR(cxlr)) 25736e099264SDan Williams return PTR_ERR(cxlr); 25746e099264SDan Williams 25756e099264SDan Williams return len; 25766e099264SDan Williams } 25776e099264SDan Williams DEVICE_ATTR_RW(create_ram_region); 25786e099264SDan Williams 2579b9686e8cSDan Williams static ssize_t region_show(struct device *dev, struct device_attribute *attr, 2580b9686e8cSDan Williams char *buf) 2581b9686e8cSDan Williams { 2582b9686e8cSDan Williams struct cxl_decoder *cxld = to_cxl_decoder(dev); 2583b9686e8cSDan Williams ssize_t rc; 2584b9686e8cSDan Williams 2585b9686e8cSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 2586b9686e8cSDan Williams if (rc) 2587b9686e8cSDan Williams return rc; 2588b9686e8cSDan Williams 2589b9686e8cSDan Williams if (cxld->region) 2590b9686e8cSDan Williams rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev)); 2591b9686e8cSDan Williams else 2592b9686e8cSDan Williams rc = sysfs_emit(buf, "\n"); 2593b9686e8cSDan Williams up_read(&cxl_region_rwsem); 2594b9686e8cSDan Williams 2595b9686e8cSDan Williams return rc; 2596b9686e8cSDan Williams } 2597b9686e8cSDan Williams DEVICE_ATTR_RO(region); 2598b9686e8cSDan Williams 2599779dd20cSBen Widawsky static struct cxl_region * 2600779dd20cSBen Widawsky cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name) 2601779dd20cSBen Widawsky { 2602779dd20cSBen Widawsky struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 2603779dd20cSBen Widawsky struct device *region_dev; 2604779dd20cSBen Widawsky 2605779dd20cSBen Widawsky region_dev = device_find_child_by_name(&cxld->dev, name); 2606779dd20cSBen Widawsky if (!region_dev) 2607779dd20cSBen Widawsky return ERR_PTR(-ENODEV); 2608779dd20cSBen Widawsky 2609779dd20cSBen Widawsky return to_cxl_region(region_dev); 2610779dd20cSBen Widawsky } 2611779dd20cSBen Widawsky 2612779dd20cSBen Widawsky static ssize_t delete_region_store(struct device *dev, 2613779dd20cSBen Widawsky struct device_attribute *attr, 2614779dd20cSBen Widawsky const char *buf, size_t len) 2615779dd20cSBen Widawsky { 2616779dd20cSBen Widawsky struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2617779dd20cSBen Widawsky struct cxl_port *port = to_cxl_port(dev->parent); 2618779dd20cSBen Widawsky struct cxl_region *cxlr; 2619779dd20cSBen Widawsky 2620779dd20cSBen Widawsky cxlr = cxl_find_region_by_name(cxlrd, buf); 2621779dd20cSBen Widawsky if (IS_ERR(cxlr)) 2622779dd20cSBen Widawsky return PTR_ERR(cxlr); 2623779dd20cSBen Widawsky 26247481653dSDan Williams devm_release_action(port->uport_dev, unregister_region, cxlr); 2625779dd20cSBen Widawsky put_device(&cxlr->dev); 2626779dd20cSBen Widawsky 2627779dd20cSBen Widawsky return len; 2628779dd20cSBen Widawsky } 2629779dd20cSBen Widawsky DEVICE_ATTR_WO(delete_region); 263023a22cd1SDan Williams 263104ad63f0SDan Williams static void cxl_pmem_region_release(struct device *dev) 263204ad63f0SDan Williams { 263304ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); 263404ad63f0SDan Williams int i; 263504ad63f0SDan Williams 263604ad63f0SDan Williams for (i = 0; i < cxlr_pmem->nr_mappings; i++) { 263704ad63f0SDan Williams struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd; 263804ad63f0SDan Williams 263904ad63f0SDan Williams put_device(&cxlmd->dev); 264004ad63f0SDan Williams } 264104ad63f0SDan Williams 264204ad63f0SDan Williams kfree(cxlr_pmem); 264304ad63f0SDan Williams } 264404ad63f0SDan Williams 264504ad63f0SDan Williams static const struct attribute_group *cxl_pmem_region_attribute_groups[] = { 264604ad63f0SDan Williams &cxl_base_attribute_group, 264704ad63f0SDan Williams NULL, 264804ad63f0SDan Williams }; 264904ad63f0SDan Williams 265004ad63f0SDan Williams const struct device_type cxl_pmem_region_type = { 265104ad63f0SDan Williams .name = "cxl_pmem_region", 265204ad63f0SDan Williams .release = cxl_pmem_region_release, 265304ad63f0SDan Williams .groups = cxl_pmem_region_attribute_groups, 265404ad63f0SDan Williams }; 265504ad63f0SDan Williams 265604ad63f0SDan Williams bool is_cxl_pmem_region(struct device *dev) 265704ad63f0SDan Williams { 265804ad63f0SDan Williams return dev->type == &cxl_pmem_region_type; 265904ad63f0SDan Williams } 266004ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, CXL); 266104ad63f0SDan Williams 266204ad63f0SDan Williams struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev) 266304ad63f0SDan Williams { 266404ad63f0SDan Williams if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev), 266504ad63f0SDan Williams "not a cxl_pmem_region device\n")) 266604ad63f0SDan Williams return NULL; 266704ad63f0SDan Williams return container_of(dev, struct cxl_pmem_region, dev); 266804ad63f0SDan Williams } 266904ad63f0SDan Williams EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, CXL); 267004ad63f0SDan Williams 2671f0832a58SAlison Schofield struct cxl_poison_context { 2672f0832a58SAlison Schofield struct cxl_port *port; 2673f0832a58SAlison Schofield enum cxl_decoder_mode mode; 2674f0832a58SAlison Schofield u64 offset; 2675f0832a58SAlison Schofield }; 2676f0832a58SAlison Schofield 2677f0832a58SAlison Schofield static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd, 2678f0832a58SAlison Schofield struct cxl_poison_context *ctx) 2679f0832a58SAlison Schofield { 2680f0832a58SAlison Schofield struct cxl_dev_state *cxlds = cxlmd->cxlds; 2681f0832a58SAlison Schofield u64 offset, length; 2682f0832a58SAlison Schofield int rc = 0; 2683f0832a58SAlison Schofield 2684f0832a58SAlison Schofield /* 2685f0832a58SAlison Schofield * Collect poison for the remaining unmapped resources 2686f0832a58SAlison Schofield * after poison is collected by committed endpoints. 2687f0832a58SAlison Schofield * 2688f0832a58SAlison Schofield * Knowing that PMEM must always follow RAM, get poison 2689f0832a58SAlison Schofield * for unmapped resources based on the last decoder's mode: 2690f0832a58SAlison Schofield * ram: scan remains of ram range, then any pmem range 2691f0832a58SAlison Schofield * pmem: scan remains of pmem range 2692f0832a58SAlison Schofield */ 2693f0832a58SAlison Schofield 2694f0832a58SAlison Schofield if (ctx->mode == CXL_DECODER_RAM) { 2695f0832a58SAlison Schofield offset = ctx->offset; 2696f0832a58SAlison Schofield length = resource_size(&cxlds->ram_res) - offset; 2697f0832a58SAlison Schofield rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 2698f0832a58SAlison Schofield if (rc == -EFAULT) 2699f0832a58SAlison Schofield rc = 0; 2700f0832a58SAlison Schofield if (rc) 2701f0832a58SAlison Schofield return rc; 2702f0832a58SAlison Schofield } 2703f0832a58SAlison Schofield if (ctx->mode == CXL_DECODER_PMEM) { 2704f0832a58SAlison Schofield offset = ctx->offset; 2705f0832a58SAlison Schofield length = resource_size(&cxlds->dpa_res) - offset; 2706f0832a58SAlison Schofield if (!length) 2707f0832a58SAlison Schofield return 0; 2708f0832a58SAlison Schofield } else if (resource_size(&cxlds->pmem_res)) { 2709f0832a58SAlison Schofield offset = cxlds->pmem_res.start; 2710f0832a58SAlison Schofield length = resource_size(&cxlds->pmem_res); 2711f0832a58SAlison Schofield } else { 2712f0832a58SAlison Schofield return 0; 2713f0832a58SAlison Schofield } 2714f0832a58SAlison Schofield 2715f0832a58SAlison Schofield return cxl_mem_get_poison(cxlmd, offset, length, NULL); 2716f0832a58SAlison Schofield } 2717f0832a58SAlison Schofield 2718f0832a58SAlison Schofield static int poison_by_decoder(struct device *dev, void *arg) 2719f0832a58SAlison Schofield { 2720f0832a58SAlison Schofield struct cxl_poison_context *ctx = arg; 2721f0832a58SAlison Schofield struct cxl_endpoint_decoder *cxled; 2722f0832a58SAlison Schofield struct cxl_memdev *cxlmd; 2723f0832a58SAlison Schofield u64 offset, length; 2724f0832a58SAlison Schofield int rc = 0; 2725f0832a58SAlison Schofield 2726f0832a58SAlison Schofield if (!is_endpoint_decoder(dev)) 2727f0832a58SAlison Schofield return rc; 2728f0832a58SAlison Schofield 2729f0832a58SAlison Schofield cxled = to_cxl_endpoint_decoder(dev); 2730f0832a58SAlison Schofield if (!cxled->dpa_res || !resource_size(cxled->dpa_res)) 2731f0832a58SAlison Schofield return rc; 2732f0832a58SAlison Schofield 2733f0832a58SAlison Schofield /* 2734f0832a58SAlison Schofield * Regions are only created with single mode decoders: pmem or ram. 2735f0832a58SAlison Schofield * Linux does not support mixed mode decoders. This means that 2736f0832a58SAlison Schofield * reading poison per endpoint decoder adheres to the requirement 2737f0832a58SAlison Schofield * that poison reads of pmem and ram must be separated. 2738f0832a58SAlison Schofield * CXL 3.0 Spec 8.2.9.8.4.1 2739f0832a58SAlison Schofield */ 2740f0832a58SAlison Schofield if (cxled->mode == CXL_DECODER_MIXED) { 2741f0832a58SAlison Schofield dev_dbg(dev, "poison list read unsupported in mixed mode\n"); 2742f0832a58SAlison Schofield return rc; 2743f0832a58SAlison Schofield } 2744f0832a58SAlison Schofield 2745f0832a58SAlison Schofield cxlmd = cxled_to_memdev(cxled); 2746f0832a58SAlison Schofield if (cxled->skip) { 2747f0832a58SAlison Schofield offset = cxled->dpa_res->start - cxled->skip; 2748f0832a58SAlison Schofield length = cxled->skip; 2749f0832a58SAlison Schofield rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 2750f0832a58SAlison Schofield if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM) 2751f0832a58SAlison Schofield rc = 0; 2752f0832a58SAlison Schofield if (rc) 2753f0832a58SAlison Schofield return rc; 2754f0832a58SAlison Schofield } 2755f0832a58SAlison Schofield 2756f0832a58SAlison Schofield offset = cxled->dpa_res->start; 2757f0832a58SAlison Schofield length = cxled->dpa_res->end - offset + 1; 2758f0832a58SAlison Schofield rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region); 2759f0832a58SAlison Schofield if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM) 2760f0832a58SAlison Schofield rc = 0; 2761f0832a58SAlison Schofield if (rc) 2762f0832a58SAlison Schofield return rc; 2763f0832a58SAlison Schofield 2764f0832a58SAlison Schofield /* Iterate until commit_end is reached */ 2765f0832a58SAlison Schofield if (cxled->cxld.id == ctx->port->commit_end) { 2766f0832a58SAlison Schofield ctx->offset = cxled->dpa_res->end + 1; 2767f0832a58SAlison Schofield ctx->mode = cxled->mode; 2768f0832a58SAlison Schofield return 1; 2769f0832a58SAlison Schofield } 2770f0832a58SAlison Schofield 2771f0832a58SAlison Schofield return 0; 2772f0832a58SAlison Schofield } 2773f0832a58SAlison Schofield 2774f0832a58SAlison Schofield int cxl_get_poison_by_endpoint(struct cxl_port *port) 2775f0832a58SAlison Schofield { 2776f0832a58SAlison Schofield struct cxl_poison_context ctx; 2777f0832a58SAlison Schofield int rc = 0; 2778f0832a58SAlison Schofield 2779f0832a58SAlison Schofield ctx = (struct cxl_poison_context) { 2780f0832a58SAlison Schofield .port = port 2781f0832a58SAlison Schofield }; 2782f0832a58SAlison Schofield 2783f0832a58SAlison Schofield rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder); 2784f0832a58SAlison Schofield if (rc == 1) 27857481653dSDan Williams rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev), 27867481653dSDan Williams &ctx); 2787f0832a58SAlison Schofield 2788f0832a58SAlison Schofield return rc; 2789f0832a58SAlison Schofield } 2790f0832a58SAlison Schofield 2791b98d0426SAlison Schofield struct cxl_dpa_to_region_context { 2792b98d0426SAlison Schofield struct cxl_region *cxlr; 2793b98d0426SAlison Schofield u64 dpa; 2794b98d0426SAlison Schofield }; 2795b98d0426SAlison Schofield 2796b98d0426SAlison Schofield static int __cxl_dpa_to_region(struct device *dev, void *arg) 2797b98d0426SAlison Schofield { 2798b98d0426SAlison Schofield struct cxl_dpa_to_region_context *ctx = arg; 2799b98d0426SAlison Schofield struct cxl_endpoint_decoder *cxled; 2800285f2a08SAlison Schofield struct cxl_region *cxlr; 2801b98d0426SAlison Schofield u64 dpa = ctx->dpa; 2802b98d0426SAlison Schofield 2803b98d0426SAlison Schofield if (!is_endpoint_decoder(dev)) 2804b98d0426SAlison Schofield return 0; 2805b98d0426SAlison Schofield 2806b98d0426SAlison Schofield cxled = to_cxl_endpoint_decoder(dev); 2807285f2a08SAlison Schofield if (!cxled || !cxled->dpa_res || !resource_size(cxled->dpa_res)) 2808b98d0426SAlison Schofield return 0; 2809b98d0426SAlison Schofield 2810b98d0426SAlison Schofield if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start) 2811b98d0426SAlison Schofield return 0; 2812b98d0426SAlison Schofield 2813285f2a08SAlison Schofield /* 2814285f2a08SAlison Schofield * Stop the region search (return 1) when an endpoint mapping is 2815285f2a08SAlison Schofield * found. The region may not be fully constructed so offering 2816285f2a08SAlison Schofield * the cxlr in the context structure is not guaranteed. 2817285f2a08SAlison Schofield */ 2818285f2a08SAlison Schofield cxlr = cxled->cxld.region; 2819285f2a08SAlison Schofield if (cxlr) 2820b98d0426SAlison Schofield dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa, 2821285f2a08SAlison Schofield dev_name(&cxlr->dev)); 2822285f2a08SAlison Schofield else 2823285f2a08SAlison Schofield dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa, 2824285f2a08SAlison Schofield dev_name(dev)); 2825b98d0426SAlison Schofield 2826285f2a08SAlison Schofield ctx->cxlr = cxlr; 2827b98d0426SAlison Schofield 2828b98d0426SAlison Schofield return 1; 2829b98d0426SAlison Schofield } 2830b98d0426SAlison Schofield 2831b98d0426SAlison Schofield struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa) 2832b98d0426SAlison Schofield { 2833b98d0426SAlison Schofield struct cxl_dpa_to_region_context ctx; 2834b98d0426SAlison Schofield struct cxl_port *port; 2835b98d0426SAlison Schofield 2836b98d0426SAlison Schofield ctx = (struct cxl_dpa_to_region_context) { 2837b98d0426SAlison Schofield .dpa = dpa, 2838b98d0426SAlison Schofield }; 2839b98d0426SAlison Schofield port = cxlmd->endpoint; 2840b98d0426SAlison Schofield if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port)) 2841b98d0426SAlison Schofield device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region); 2842b98d0426SAlison Schofield 2843b98d0426SAlison Schofield return ctx.cxlr; 2844b98d0426SAlison Schofield } 2845b98d0426SAlison Schofield 28463b2fedcdSAlison Schofield static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos) 284786954ff5SAlison Schofield { 284886954ff5SAlison Schofield struct cxl_region_params *p = &cxlr->params; 284986954ff5SAlison Schofield int gran = p->interleave_granularity; 285086954ff5SAlison Schofield int ways = p->interleave_ways; 285186954ff5SAlison Schofield u64 offset; 285286954ff5SAlison Schofield 285386954ff5SAlison Schofield /* Is the hpa in an expected chunk for its pos(-ition) */ 285486954ff5SAlison Schofield offset = hpa - p->res->start; 285586954ff5SAlison Schofield offset = do_div(offset, gran * ways); 285686954ff5SAlison Schofield if ((offset >= pos * gran) && (offset < (pos + 1) * gran)) 285786954ff5SAlison Schofield return true; 285886954ff5SAlison Schofield 285986954ff5SAlison Schofield dev_dbg(&cxlr->dev, 286086954ff5SAlison Schofield "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa); 286186954ff5SAlison Schofield 286286954ff5SAlison Schofield return false; 286386954ff5SAlison Schofield } 286486954ff5SAlison Schofield 28659aa5f623SAlison Schofield u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, 28669aa5f623SAlison Schofield u64 dpa) 286786954ff5SAlison Schofield { 28683b2fedcdSAlison Schofield struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 286986954ff5SAlison Schofield u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa; 287086954ff5SAlison Schofield struct cxl_region_params *p = &cxlr->params; 28719aa5f623SAlison Schofield struct cxl_endpoint_decoder *cxled = NULL; 287286954ff5SAlison Schofield u16 eig = 0; 287386954ff5SAlison Schofield u8 eiw = 0; 28749aa5f623SAlison Schofield int pos; 287586954ff5SAlison Schofield 28769aa5f623SAlison Schofield for (int i = 0; i < p->nr_targets; i++) { 28779aa5f623SAlison Schofield cxled = p->targets[i]; 28789aa5f623SAlison Schofield if (cxlmd == cxled_to_memdev(cxled)) 28799aa5f623SAlison Schofield break; 28809aa5f623SAlison Schofield } 28819aa5f623SAlison Schofield if (!cxled || cxlmd != cxled_to_memdev(cxled)) 28829aa5f623SAlison Schofield return ULLONG_MAX; 28839aa5f623SAlison Schofield 28849aa5f623SAlison Schofield pos = cxled->pos; 288586954ff5SAlison Schofield ways_to_eiw(p->interleave_ways, &eiw); 288686954ff5SAlison Schofield granularity_to_eig(p->interleave_granularity, &eig); 288786954ff5SAlison Schofield 288886954ff5SAlison Schofield /* 288986954ff5SAlison Schofield * The device position in the region interleave set was removed 289086954ff5SAlison Schofield * from the offset at HPA->DPA translation. To reconstruct the 289186954ff5SAlison Schofield * HPA, place the 'pos' in the offset. 289286954ff5SAlison Schofield * 289386954ff5SAlison Schofield * The placement of 'pos' in the HPA is determined by interleave 289486954ff5SAlison Schofield * ways and granularity and is defined in the CXL Spec 3.0 Section 289586954ff5SAlison Schofield * 8.2.4.19.13 Implementation Note: Device Decode Logic 289686954ff5SAlison Schofield */ 289786954ff5SAlison Schofield 289886954ff5SAlison Schofield /* Remove the dpa base */ 289986954ff5SAlison Schofield dpa_offset = dpa - cxl_dpa_resource_start(cxled); 290086954ff5SAlison Schofield 290186954ff5SAlison Schofield mask_upper = GENMASK_ULL(51, eig + 8); 290286954ff5SAlison Schofield 290386954ff5SAlison Schofield if (eiw < 8) { 290486954ff5SAlison Schofield hpa_offset = (dpa_offset & mask_upper) << eiw; 290586954ff5SAlison Schofield hpa_offset |= pos << (eig + 8); 290686954ff5SAlison Schofield } else { 290786954ff5SAlison Schofield bits_upper = (dpa_offset & mask_upper) >> (eig + 8); 290886954ff5SAlison Schofield bits_upper = bits_upper * 3; 290986954ff5SAlison Schofield hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8); 291086954ff5SAlison Schofield } 291186954ff5SAlison Schofield 291286954ff5SAlison Schofield /* The lower bits remain unchanged */ 291386954ff5SAlison Schofield hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0); 291486954ff5SAlison Schofield 291586954ff5SAlison Schofield /* Apply the hpa_offset to the region base address */ 291686954ff5SAlison Schofield hpa = hpa_offset + p->res->start; 291786954ff5SAlison Schofield 29183b2fedcdSAlison Schofield /* Root decoder translation overrides typical modulo decode */ 29193b2fedcdSAlison Schofield if (cxlrd->hpa_to_spa) 29203b2fedcdSAlison Schofield hpa = cxlrd->hpa_to_spa(cxlrd, hpa); 29213b2fedcdSAlison Schofield 29223b2fedcdSAlison Schofield if (hpa < p->res->start || hpa > p->res->end) { 29233b2fedcdSAlison Schofield dev_dbg(&cxlr->dev, 29243b2fedcdSAlison Schofield "Addr trans fail: hpa 0x%llx not in region\n", hpa); 29253b2fedcdSAlison Schofield return ULLONG_MAX; 29263b2fedcdSAlison Schofield } 29273b2fedcdSAlison Schofield 29283b2fedcdSAlison Schofield /* Simple chunk check, by pos & gran, only applies to modulo decodes */ 29293b2fedcdSAlison Schofield if (!cxlrd->hpa_to_spa && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos))) 293086954ff5SAlison Schofield return ULLONG_MAX; 293186954ff5SAlison Schofield 293286954ff5SAlison Schofield return hpa; 293386954ff5SAlison Schofield } 293486954ff5SAlison Schofield 293504ad63f0SDan Williams static struct lock_class_key cxl_pmem_region_key; 293604ad63f0SDan Williams 2937d357dd8aSDan Williams static int cxl_pmem_region_alloc(struct cxl_region *cxlr) 293804ad63f0SDan Williams { 293904ad63f0SDan Williams struct cxl_region_params *p = &cxlr->params; 2940f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb; 294104ad63f0SDan Williams struct device *dev; 294204ad63f0SDan Williams int i; 294304ad63f0SDan Williams 2944d357dd8aSDan Williams guard(rwsem_read)(&cxl_region_rwsem); 2945d357dd8aSDan Williams if (p->state != CXL_CONFIG_COMMIT) 2946d357dd8aSDan Williams return -ENXIO; 294704ad63f0SDan Williams 2948d357dd8aSDan Williams struct cxl_pmem_region *cxlr_pmem __free(kfree) = 2949d357dd8aSDan Williams kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL); 2950d357dd8aSDan Williams if (!cxlr_pmem) 2951d357dd8aSDan Williams return -ENOMEM; 295204ad63f0SDan Williams 295304ad63f0SDan Williams cxlr_pmem->hpa_range.start = p->res->start; 295404ad63f0SDan Williams cxlr_pmem->hpa_range.end = p->res->end; 295504ad63f0SDan Williams 295604ad63f0SDan Williams /* Snapshot the region configuration underneath the cxl_region_rwsem */ 295704ad63f0SDan Williams cxlr_pmem->nr_mappings = p->nr_targets; 295804ad63f0SDan Williams for (i = 0; i < p->nr_targets; i++) { 295904ad63f0SDan Williams struct cxl_endpoint_decoder *cxled = p->targets[i]; 296004ad63f0SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 296104ad63f0SDan Williams struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; 296204ad63f0SDan Williams 2963f17b558dSDan Williams /* 2964f17b558dSDan Williams * Regions never span CXL root devices, so by definition the 2965f17b558dSDan Williams * bridge for one device is the same for all. 2966f17b558dSDan Williams */ 2967f17b558dSDan Williams if (i == 0) { 296884ec9859SLi Ming cxl_nvb = cxl_find_nvdimm_bridge(cxlmd->endpoint); 2969d357dd8aSDan Williams if (!cxl_nvb) 2970d357dd8aSDan Williams return -ENODEV; 2971f17b558dSDan Williams cxlr->cxl_nvb = cxl_nvb; 2972f17b558dSDan Williams } 297304ad63f0SDan Williams m->cxlmd = cxlmd; 297404ad63f0SDan Williams get_device(&cxlmd->dev); 297504ad63f0SDan Williams m->start = cxled->dpa_res->start; 297604ad63f0SDan Williams m->size = resource_size(cxled->dpa_res); 297704ad63f0SDan Williams m->position = i; 297804ad63f0SDan Williams } 297904ad63f0SDan Williams 298004ad63f0SDan Williams dev = &cxlr_pmem->dev; 298104ad63f0SDan Williams device_initialize(dev); 298204ad63f0SDan Williams lockdep_set_class(&dev->mutex, &cxl_pmem_region_key); 298304ad63f0SDan Williams device_set_pm_not_required(dev); 298404ad63f0SDan Williams dev->parent = &cxlr->dev; 298504ad63f0SDan Williams dev->bus = &cxl_bus_type; 298604ad63f0SDan Williams dev->type = &cxl_pmem_region_type; 2987d357dd8aSDan Williams cxlr_pmem->cxlr = cxlr; 2988d357dd8aSDan Williams cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem); 298904ad63f0SDan Williams 2990d357dd8aSDan Williams return 0; 299104ad63f0SDan Williams } 299204ad63f0SDan Williams 299309d09e04SDan Williams static void cxl_dax_region_release(struct device *dev) 299409d09e04SDan Williams { 299509d09e04SDan Williams struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev); 299609d09e04SDan Williams 299709d09e04SDan Williams kfree(cxlr_dax); 299809d09e04SDan Williams } 299909d09e04SDan Williams 300009d09e04SDan Williams static const struct attribute_group *cxl_dax_region_attribute_groups[] = { 300109d09e04SDan Williams &cxl_base_attribute_group, 300209d09e04SDan Williams NULL, 300309d09e04SDan Williams }; 300409d09e04SDan Williams 300509d09e04SDan Williams const struct device_type cxl_dax_region_type = { 300609d09e04SDan Williams .name = "cxl_dax_region", 300709d09e04SDan Williams .release = cxl_dax_region_release, 300809d09e04SDan Williams .groups = cxl_dax_region_attribute_groups, 300909d09e04SDan Williams }; 301009d09e04SDan Williams 301109d09e04SDan Williams static bool is_cxl_dax_region(struct device *dev) 301209d09e04SDan Williams { 301309d09e04SDan Williams return dev->type == &cxl_dax_region_type; 301409d09e04SDan Williams } 301509d09e04SDan Williams 301609d09e04SDan Williams struct cxl_dax_region *to_cxl_dax_region(struct device *dev) 301709d09e04SDan Williams { 301809d09e04SDan Williams if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev), 301909d09e04SDan Williams "not a cxl_dax_region device\n")) 302009d09e04SDan Williams return NULL; 302109d09e04SDan Williams return container_of(dev, struct cxl_dax_region, dev); 302209d09e04SDan Williams } 302309d09e04SDan Williams EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, CXL); 302409d09e04SDan Williams 302509d09e04SDan Williams static struct lock_class_key cxl_dax_region_key; 302609d09e04SDan Williams 302709d09e04SDan Williams static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr) 302809d09e04SDan Williams { 302909d09e04SDan Williams struct cxl_region_params *p = &cxlr->params; 303009d09e04SDan Williams struct cxl_dax_region *cxlr_dax; 303109d09e04SDan Williams struct device *dev; 303209d09e04SDan Williams 303309d09e04SDan Williams down_read(&cxl_region_rwsem); 303409d09e04SDan Williams if (p->state != CXL_CONFIG_COMMIT) { 303509d09e04SDan Williams cxlr_dax = ERR_PTR(-ENXIO); 303609d09e04SDan Williams goto out; 303709d09e04SDan Williams } 303809d09e04SDan Williams 303909d09e04SDan Williams cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL); 304009d09e04SDan Williams if (!cxlr_dax) { 304109d09e04SDan Williams cxlr_dax = ERR_PTR(-ENOMEM); 304209d09e04SDan Williams goto out; 304309d09e04SDan Williams } 304409d09e04SDan Williams 304509d09e04SDan Williams cxlr_dax->hpa_range.start = p->res->start; 304609d09e04SDan Williams cxlr_dax->hpa_range.end = p->res->end; 304709d09e04SDan Williams 304809d09e04SDan Williams dev = &cxlr_dax->dev; 304909d09e04SDan Williams cxlr_dax->cxlr = cxlr; 305009d09e04SDan Williams device_initialize(dev); 305109d09e04SDan Williams lockdep_set_class(&dev->mutex, &cxl_dax_region_key); 305209d09e04SDan Williams device_set_pm_not_required(dev); 305309d09e04SDan Williams dev->parent = &cxlr->dev; 305409d09e04SDan Williams dev->bus = &cxl_bus_type; 305509d09e04SDan Williams dev->type = &cxl_dax_region_type; 305609d09e04SDan Williams out: 305709d09e04SDan Williams up_read(&cxl_region_rwsem); 305809d09e04SDan Williams 305909d09e04SDan Williams return cxlr_dax; 306009d09e04SDan Williams } 306109d09e04SDan Williams 3062f17b558dSDan Williams static void cxlr_pmem_unregister(void *_cxlr_pmem) 306304ad63f0SDan Williams { 3064f17b558dSDan Williams struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem; 3065f17b558dSDan Williams struct cxl_region *cxlr = cxlr_pmem->cxlr; 3066f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 3067f17b558dSDan Williams 3068f17b558dSDan Williams /* 3069f17b558dSDan Williams * Either the bridge is in ->remove() context under the device_lock(), 3070f17b558dSDan Williams * or cxlr_release_nvdimm() is cancelling the bridge's release action 3071f17b558dSDan Williams * for @cxlr_pmem and doing it itself (while manually holding the bridge 3072f17b558dSDan Williams * lock). 3073f17b558dSDan Williams */ 3074f17b558dSDan Williams device_lock_assert(&cxl_nvb->dev); 3075f17b558dSDan Williams cxlr->cxlr_pmem = NULL; 3076f17b558dSDan Williams cxlr_pmem->cxlr = NULL; 3077f17b558dSDan Williams device_unregister(&cxlr_pmem->dev); 3078f17b558dSDan Williams } 3079f17b558dSDan Williams 3080f17b558dSDan Williams static void cxlr_release_nvdimm(void *_cxlr) 3081f17b558dSDan Williams { 3082f17b558dSDan Williams struct cxl_region *cxlr = _cxlr; 3083f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 3084f17b558dSDan Williams 30857f569e91SLi Ming scoped_guard(device, &cxl_nvb->dev) { 3086f17b558dSDan Williams if (cxlr->cxlr_pmem) 3087f17b558dSDan Williams devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister, 3088f17b558dSDan Williams cxlr->cxlr_pmem); 30897f569e91SLi Ming } 3090f17b558dSDan Williams cxlr->cxl_nvb = NULL; 3091f17b558dSDan Williams put_device(&cxl_nvb->dev); 309204ad63f0SDan Williams } 309304ad63f0SDan Williams 309404ad63f0SDan Williams /** 309504ad63f0SDan Williams * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge 309604ad63f0SDan Williams * @cxlr: parent CXL region for this pmem region bridge device 309704ad63f0SDan Williams * 309804ad63f0SDan Williams * Return: 0 on success negative error code on failure. 309904ad63f0SDan Williams */ 310004ad63f0SDan Williams static int devm_cxl_add_pmem_region(struct cxl_region *cxlr) 310104ad63f0SDan Williams { 310204ad63f0SDan Williams struct cxl_pmem_region *cxlr_pmem; 3103f17b558dSDan Williams struct cxl_nvdimm_bridge *cxl_nvb; 310404ad63f0SDan Williams struct device *dev; 310504ad63f0SDan Williams int rc; 310604ad63f0SDan Williams 3107d357dd8aSDan Williams rc = cxl_pmem_region_alloc(cxlr); 3108d357dd8aSDan Williams if (rc) 3109d357dd8aSDan Williams return rc; 3110d357dd8aSDan Williams cxlr_pmem = cxlr->cxlr_pmem; 3111f17b558dSDan Williams cxl_nvb = cxlr->cxl_nvb; 311204ad63f0SDan Williams 311304ad63f0SDan Williams dev = &cxlr_pmem->dev; 311404ad63f0SDan Williams rc = dev_set_name(dev, "pmem_region%d", cxlr->id); 311504ad63f0SDan Williams if (rc) 311604ad63f0SDan Williams goto err; 311704ad63f0SDan Williams 311804ad63f0SDan Williams rc = device_add(dev); 311904ad63f0SDan Williams if (rc) 312004ad63f0SDan Williams goto err; 312104ad63f0SDan Williams 312204ad63f0SDan Williams dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 312304ad63f0SDan Williams dev_name(dev)); 312404ad63f0SDan Williams 31257f569e91SLi Ming scoped_guard(device, &cxl_nvb->dev) { 3126f17b558dSDan Williams if (cxl_nvb->dev.driver) 3127f17b558dSDan Williams rc = devm_add_action_or_reset(&cxl_nvb->dev, 31287f569e91SLi Ming cxlr_pmem_unregister, 31297f569e91SLi Ming cxlr_pmem); 3130f17b558dSDan Williams else 3131f17b558dSDan Williams rc = -ENXIO; 31327f569e91SLi Ming } 3133f17b558dSDan Williams 3134f17b558dSDan Williams if (rc) 3135f17b558dSDan Williams goto err_bridge; 3136f17b558dSDan Williams 3137f17b558dSDan Williams /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */ 3138f17b558dSDan Williams return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr); 313904ad63f0SDan Williams 314004ad63f0SDan Williams err: 314104ad63f0SDan Williams put_device(dev); 3142f17b558dSDan Williams err_bridge: 3143f17b558dSDan Williams put_device(&cxl_nvb->dev); 3144f17b558dSDan Williams cxlr->cxl_nvb = NULL; 314504ad63f0SDan Williams return rc; 314604ad63f0SDan Williams } 314704ad63f0SDan Williams 314809d09e04SDan Williams static void cxlr_dax_unregister(void *_cxlr_dax) 314909d09e04SDan Williams { 315009d09e04SDan Williams struct cxl_dax_region *cxlr_dax = _cxlr_dax; 315109d09e04SDan Williams 315209d09e04SDan Williams device_unregister(&cxlr_dax->dev); 315309d09e04SDan Williams } 315409d09e04SDan Williams 315509d09e04SDan Williams static int devm_cxl_add_dax_region(struct cxl_region *cxlr) 315609d09e04SDan Williams { 315709d09e04SDan Williams struct cxl_dax_region *cxlr_dax; 315809d09e04SDan Williams struct device *dev; 315909d09e04SDan Williams int rc; 316009d09e04SDan Williams 316109d09e04SDan Williams cxlr_dax = cxl_dax_region_alloc(cxlr); 316209d09e04SDan Williams if (IS_ERR(cxlr_dax)) 316309d09e04SDan Williams return PTR_ERR(cxlr_dax); 316409d09e04SDan Williams 316509d09e04SDan Williams dev = &cxlr_dax->dev; 316609d09e04SDan Williams rc = dev_set_name(dev, "dax_region%d", cxlr->id); 316709d09e04SDan Williams if (rc) 316809d09e04SDan Williams goto err; 316909d09e04SDan Williams 317009d09e04SDan Williams rc = device_add(dev); 317109d09e04SDan Williams if (rc) 317209d09e04SDan Williams goto err; 317309d09e04SDan Williams 317409d09e04SDan Williams dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 317509d09e04SDan Williams dev_name(dev)); 317609d09e04SDan Williams 317709d09e04SDan Williams return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister, 317809d09e04SDan Williams cxlr_dax); 317909d09e04SDan Williams err: 318009d09e04SDan Williams put_device(dev); 318109d09e04SDan Williams return rc; 318209d09e04SDan Williams } 318309d09e04SDan Williams 318411105814SAlison Schofield static int match_root_decoder_by_range(struct device *dev, void *data) 3185a32320b7SDan Williams { 3186a32320b7SDan Williams struct range *r1, *r2 = data; 3187a32320b7SDan Williams struct cxl_root_decoder *cxlrd; 3188a32320b7SDan Williams 3189a32320b7SDan Williams if (!is_root_decoder(dev)) 3190a32320b7SDan Williams return 0; 3191a32320b7SDan Williams 3192a32320b7SDan Williams cxlrd = to_cxl_root_decoder(dev); 3193a32320b7SDan Williams r1 = &cxlrd->cxlsd.cxld.hpa_range; 3194a32320b7SDan Williams return range_contains(r1, r2); 3195a32320b7SDan Williams } 3196a32320b7SDan Williams 3197a32320b7SDan Williams static int match_region_by_range(struct device *dev, void *data) 3198a32320b7SDan Williams { 3199a32320b7SDan Williams struct cxl_region_params *p; 3200a32320b7SDan Williams struct cxl_region *cxlr; 3201a32320b7SDan Williams struct range *r = data; 3202a32320b7SDan Williams int rc = 0; 3203a32320b7SDan Williams 3204a32320b7SDan Williams if (!is_cxl_region(dev)) 3205a32320b7SDan Williams return 0; 3206a32320b7SDan Williams 3207a32320b7SDan Williams cxlr = to_cxl_region(dev); 3208a32320b7SDan Williams p = &cxlr->params; 3209a32320b7SDan Williams 3210a32320b7SDan Williams down_read(&cxl_region_rwsem); 3211a32320b7SDan Williams if (p->res && p->res->start == r->start && p->res->end == r->end) 3212a32320b7SDan Williams rc = 1; 3213a32320b7SDan Williams up_read(&cxl_region_rwsem); 3214a32320b7SDan Williams 3215a32320b7SDan Williams return rc; 3216a32320b7SDan Williams } 3217a32320b7SDan Williams 3218a32320b7SDan Williams /* Establish an empty region covering the given HPA range */ 3219a32320b7SDan Williams static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, 3220a32320b7SDan Williams struct cxl_endpoint_decoder *cxled) 3221a32320b7SDan Williams { 3222a32320b7SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 3223a32320b7SDan Williams struct cxl_port *port = cxlrd_to_port(cxlrd); 3224a32320b7SDan Williams struct range *hpa = &cxled->cxld.hpa_range; 3225a32320b7SDan Williams struct cxl_region_params *p; 3226a32320b7SDan Williams struct cxl_region *cxlr; 3227a32320b7SDan Williams struct resource *res; 3228a32320b7SDan Williams int rc; 3229a32320b7SDan Williams 3230a32320b7SDan Williams do { 3231a32320b7SDan Williams cxlr = __create_region(cxlrd, cxled->mode, 3232a32320b7SDan Williams atomic_read(&cxlrd->region_id)); 3233a32320b7SDan Williams } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY); 3234a32320b7SDan Williams 3235a32320b7SDan Williams if (IS_ERR(cxlr)) { 3236a32320b7SDan Williams dev_err(cxlmd->dev.parent, 3237a32320b7SDan Williams "%s:%s: %s failed assign region: %ld\n", 3238a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 3239a32320b7SDan Williams __func__, PTR_ERR(cxlr)); 3240a32320b7SDan Williams return cxlr; 3241a32320b7SDan Williams } 3242a32320b7SDan Williams 3243a32320b7SDan Williams down_write(&cxl_region_rwsem); 3244a32320b7SDan Williams p = &cxlr->params; 3245a32320b7SDan Williams if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 3246a32320b7SDan Williams dev_err(cxlmd->dev.parent, 3247a32320b7SDan Williams "%s:%s: %s autodiscovery interrupted\n", 3248a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 3249a32320b7SDan Williams __func__); 3250a32320b7SDan Williams rc = -EBUSY; 3251a32320b7SDan Williams goto err; 3252a32320b7SDan Williams } 3253a32320b7SDan Williams 3254a32320b7SDan Williams set_bit(CXL_REGION_F_AUTO, &cxlr->flags); 3255a32320b7SDan Williams 3256a32320b7SDan Williams res = kmalloc(sizeof(*res), GFP_KERNEL); 3257a32320b7SDan Williams if (!res) { 3258a32320b7SDan Williams rc = -ENOMEM; 3259a32320b7SDan Williams goto err; 3260a32320b7SDan Williams } 3261a32320b7SDan Williams 3262a32320b7SDan Williams *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa), 3263a32320b7SDan Williams dev_name(&cxlr->dev)); 3264a32320b7SDan Williams rc = insert_resource(cxlrd->res, res); 3265a32320b7SDan Williams if (rc) { 3266a32320b7SDan Williams /* 3267a32320b7SDan Williams * Platform-firmware may not have split resources like "System 3268a32320b7SDan Williams * RAM" on CXL window boundaries see cxl_region_iomem_release() 3269a32320b7SDan Williams */ 3270a32320b7SDan Williams dev_warn(cxlmd->dev.parent, 3271a32320b7SDan Williams "%s:%s: %s %s cannot insert resource\n", 3272a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 3273a32320b7SDan Williams __func__, dev_name(&cxlr->dev)); 3274a32320b7SDan Williams } 3275a32320b7SDan Williams 3276a32320b7SDan Williams p->res = res; 3277a32320b7SDan Williams p->interleave_ways = cxled->cxld.interleave_ways; 3278a32320b7SDan Williams p->interleave_granularity = cxled->cxld.interleave_granularity; 3279a32320b7SDan Williams p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 3280a32320b7SDan Williams 3281a32320b7SDan Williams rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 3282a32320b7SDan Williams if (rc) 3283a32320b7SDan Williams goto err; 3284a32320b7SDan Williams 3285a32320b7SDan Williams dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n", 3286a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__, 3287a32320b7SDan Williams dev_name(&cxlr->dev), p->res, p->interleave_ways, 3288a32320b7SDan Williams p->interleave_granularity); 3289a32320b7SDan Williams 3290a32320b7SDan Williams /* ...to match put_device() in cxl_add_to_region() */ 3291a32320b7SDan Williams get_device(&cxlr->dev); 3292a32320b7SDan Williams up_write(&cxl_region_rwsem); 3293a32320b7SDan Williams 3294a32320b7SDan Williams return cxlr; 3295a32320b7SDan Williams 3296a32320b7SDan Williams err: 3297a32320b7SDan Williams up_write(&cxl_region_rwsem); 32987481653dSDan Williams devm_release_action(port->uport_dev, unregister_region, cxlr); 3299a32320b7SDan Williams return ERR_PTR(rc); 3300a32320b7SDan Williams } 3301a32320b7SDan Williams 3302a32320b7SDan Williams int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled) 3303a32320b7SDan Williams { 3304a32320b7SDan Williams struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 3305a32320b7SDan Williams struct range *hpa = &cxled->cxld.hpa_range; 3306a32320b7SDan Williams struct cxl_decoder *cxld = &cxled->cxld; 3307a32320b7SDan Williams struct device *cxlrd_dev, *region_dev; 3308a32320b7SDan Williams struct cxl_root_decoder *cxlrd; 3309a32320b7SDan Williams struct cxl_region_params *p; 3310a32320b7SDan Williams struct cxl_region *cxlr; 3311a32320b7SDan Williams bool attach = false; 3312a32320b7SDan Williams int rc; 3313a32320b7SDan Williams 3314a32320b7SDan Williams cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range, 331511105814SAlison Schofield match_root_decoder_by_range); 3316a32320b7SDan Williams if (!cxlrd_dev) { 3317a32320b7SDan Williams dev_err(cxlmd->dev.parent, 3318a32320b7SDan Williams "%s:%s no CXL window for range %#llx:%#llx\n", 3319a32320b7SDan Williams dev_name(&cxlmd->dev), dev_name(&cxld->dev), 3320a32320b7SDan Williams cxld->hpa_range.start, cxld->hpa_range.end); 3321a32320b7SDan Williams return -ENXIO; 3322a32320b7SDan Williams } 3323a32320b7SDan Williams 3324a32320b7SDan Williams cxlrd = to_cxl_root_decoder(cxlrd_dev); 3325a32320b7SDan Williams 3326a32320b7SDan Williams /* 3327a32320b7SDan Williams * Ensure that if multiple threads race to construct_region() for @hpa 3328a32320b7SDan Williams * one does the construction and the others add to that. 3329a32320b7SDan Williams */ 3330a32320b7SDan Williams mutex_lock(&cxlrd->range_lock); 3331a32320b7SDan Williams region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa, 3332a32320b7SDan Williams match_region_by_range); 3333a32320b7SDan Williams if (!region_dev) { 3334a32320b7SDan Williams cxlr = construct_region(cxlrd, cxled); 3335a32320b7SDan Williams region_dev = &cxlr->dev; 3336a32320b7SDan Williams } else 3337a32320b7SDan Williams cxlr = to_cxl_region(region_dev); 3338a32320b7SDan Williams mutex_unlock(&cxlrd->range_lock); 3339a32320b7SDan Williams 33407abcb0b1SArnd Bergmann rc = PTR_ERR_OR_ZERO(cxlr); 33417abcb0b1SArnd Bergmann if (rc) 3342a32320b7SDan Williams goto out; 3343a32320b7SDan Williams 3344a32320b7SDan Williams attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE); 3345a32320b7SDan Williams 3346a32320b7SDan Williams down_read(&cxl_region_rwsem); 3347a32320b7SDan Williams p = &cxlr->params; 3348a32320b7SDan Williams attach = p->state == CXL_CONFIG_COMMIT; 3349a32320b7SDan Williams up_read(&cxl_region_rwsem); 3350a32320b7SDan Williams 3351a32320b7SDan Williams if (attach) { 3352a32320b7SDan Williams /* 3353a32320b7SDan Williams * If device_attach() fails the range may still be active via 3354a32320b7SDan Williams * the platform-firmware memory map, otherwise the driver for 3355a32320b7SDan Williams * regions is local to this file, so driver matching can't fail. 3356a32320b7SDan Williams */ 3357a32320b7SDan Williams if (device_attach(&cxlr->dev) < 0) 3358a32320b7SDan Williams dev_err(&cxlr->dev, "failed to enable, range: %pr\n", 3359a32320b7SDan Williams p->res); 3360a32320b7SDan Williams } 3361a32320b7SDan Williams 3362a32320b7SDan Williams put_device(region_dev); 3363a32320b7SDan Williams out: 3364a32320b7SDan Williams put_device(cxlrd_dev); 3365a32320b7SDan Williams return rc; 3366a32320b7SDan Williams } 3367a32320b7SDan Williams EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, CXL); 3368a32320b7SDan Williams 3369a32320b7SDan Williams static int is_system_ram(struct resource *res, void *arg) 3370a32320b7SDan Williams { 3371a32320b7SDan Williams struct cxl_region *cxlr = arg; 3372a32320b7SDan Williams struct cxl_region_params *p = &cxlr->params; 3373a32320b7SDan Williams 3374a32320b7SDan Williams dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res); 3375a32320b7SDan Williams return 1; 3376a32320b7SDan Williams } 3377a32320b7SDan Williams 3378d9a476c8SIra Weiny static void shutdown_notifiers(void *_cxlr) 3379d9a476c8SIra Weiny { 3380d9a476c8SIra Weiny struct cxl_region *cxlr = _cxlr; 3381d9a476c8SIra Weiny 3382d9a476c8SIra Weiny unregister_memory_notifier(&cxlr->memory_notifier); 3383d9a476c8SIra Weiny unregister_mt_adistance_algorithm(&cxlr->adist_notifier); 3384d9a476c8SIra Weiny } 3385d9a476c8SIra Weiny 33868d48817dSDan Williams static int cxl_region_probe(struct device *dev) 33878d48817dSDan Williams { 33888d48817dSDan Williams struct cxl_region *cxlr = to_cxl_region(dev); 33898d48817dSDan Williams struct cxl_region_params *p = &cxlr->params; 33908d48817dSDan Williams int rc; 33918d48817dSDan Williams 33928d48817dSDan Williams rc = down_read_interruptible(&cxl_region_rwsem); 33938d48817dSDan Williams if (rc) { 33948d48817dSDan Williams dev_dbg(&cxlr->dev, "probe interrupted\n"); 33958d48817dSDan Williams return rc; 33968d48817dSDan Williams } 33978d48817dSDan Williams 33988d48817dSDan Williams if (p->state < CXL_CONFIG_COMMIT) { 33998d48817dSDan Williams dev_dbg(&cxlr->dev, "config state: %d\n", p->state); 34008d48817dSDan Williams rc = -ENXIO; 3401d18bc74aSDan Williams goto out; 34028d48817dSDan Williams } 34038d48817dSDan Williams 34042ab47045SDan Williams if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) { 34052ab47045SDan Williams dev_err(&cxlr->dev, 34062ab47045SDan Williams "failed to activate, re-commit region and retry\n"); 34072ab47045SDan Williams rc = -ENXIO; 34082ab47045SDan Williams goto out; 34092ab47045SDan Williams } 3410d18bc74aSDan Williams 34118d48817dSDan Williams /* 34128d48817dSDan Williams * From this point on any path that changes the region's state away from 34138d48817dSDan Williams * CXL_CONFIG_COMMIT is also responsible for releasing the driver. 34148d48817dSDan Williams */ 3415d18bc74aSDan Williams out: 34168d48817dSDan Williams up_read(&cxl_region_rwsem); 34178d48817dSDan Williams 3418bf3e5da8SDan Williams if (rc) 3419bf3e5da8SDan Williams return rc; 3420bf3e5da8SDan Williams 3421d9a476c8SIra Weiny cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback; 3422d9a476c8SIra Weiny cxlr->memory_notifier.priority = CXL_CALLBACK_PRI; 3423d9a476c8SIra Weiny register_memory_notifier(&cxlr->memory_notifier); 3424d9a476c8SIra Weiny 3425d9a476c8SIra Weiny cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance; 3426d9a476c8SIra Weiny cxlr->adist_notifier.priority = 100; 3427d9a476c8SIra Weiny register_mt_adistance_algorithm(&cxlr->adist_notifier); 3428d9a476c8SIra Weiny 3429d9a476c8SIra Weiny rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr); 3430d9a476c8SIra Weiny if (rc) 3431d9a476c8SIra Weiny return rc; 3432d9a476c8SIra Weiny 343304ad63f0SDan Williams switch (cxlr->mode) { 343404ad63f0SDan Williams case CXL_DECODER_PMEM: 343504ad63f0SDan Williams return devm_cxl_add_pmem_region(cxlr); 3436a32320b7SDan Williams case CXL_DECODER_RAM: 3437a32320b7SDan Williams /* 3438a32320b7SDan Williams * The region can not be manged by CXL if any portion of 3439a32320b7SDan Williams * it is already online as 'System RAM' 3440a32320b7SDan Williams */ 3441a32320b7SDan Williams if (walk_iomem_res_desc(IORES_DESC_NONE, 3442a32320b7SDan Williams IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY, 3443a32320b7SDan Williams p->res->start, p->res->end, cxlr, 3444a32320b7SDan Williams is_system_ram) > 0) 3445a32320b7SDan Williams return 0; 344609d09e04SDan Williams return devm_cxl_add_dax_region(cxlr); 344704ad63f0SDan Williams default: 344804ad63f0SDan Williams dev_dbg(&cxlr->dev, "unsupported region mode: %d\n", 344904ad63f0SDan Williams cxlr->mode); 345004ad63f0SDan Williams return -ENXIO; 345104ad63f0SDan Williams } 34528d48817dSDan Williams } 34538d48817dSDan Williams 34548d48817dSDan Williams static struct cxl_driver cxl_region_driver = { 34558d48817dSDan Williams .name = "cxl_region", 34568d48817dSDan Williams .probe = cxl_region_probe, 34578d48817dSDan Williams .id = CXL_DEVICE_REGION, 34588d48817dSDan Williams }; 34598d48817dSDan Williams 34608d48817dSDan Williams int cxl_region_init(void) 34618d48817dSDan Williams { 34628d48817dSDan Williams return cxl_driver_register(&cxl_region_driver); 34638d48817dSDan Williams } 34648d48817dSDan Williams 34658d48817dSDan Williams void cxl_region_exit(void) 34668d48817dSDan Williams { 34678d48817dSDan Williams cxl_driver_unregister(&cxl_region_driver); 34688d48817dSDan Williams } 34698d48817dSDan Williams 347023a22cd1SDan Williams MODULE_IMPORT_NS(CXL); 3471d18bc74aSDan Williams MODULE_IMPORT_NS(DEVMEM); 34728d48817dSDan Williams MODULE_ALIAS_CXL(CXL_DEVICE_REGION); 3473