1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3 #include <linux/memregion.h> 4 #include <linux/genalloc.h> 5 #include <linux/device.h> 6 #include <linux/module.h> 7 #include <linux/memory.h> 8 #include <linux/slab.h> 9 #include <linux/uuid.h> 10 #include <linux/sort.h> 11 #include <linux/idr.h> 12 #include <linux/memory-tiers.h> 13 #include <cxlmem.h> 14 #include <cxl.h> 15 #include "core.h" 16 17 /** 18 * DOC: cxl core region 19 * 20 * CXL Regions represent mapped memory capacity in system physical address 21 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL 22 * Memory ranges, Regions represent the active mapped capacity by the HDM 23 * Decoder Capability structures throughout the Host Bridges, Switches, and 24 * Endpoints in the topology. 25 * 26 * Region configuration has ordering constraints. UUID may be set at any time 27 * but is only visible for persistent regions. 28 * 1. Interleave granularity 29 * 2. Interleave size 30 * 3. Decoder targets 31 */ 32 33 static struct cxl_region *to_cxl_region(struct device *dev); 34 35 #define __ACCESS_ATTR_RO(_level, _name) { \ 36 .attr = { .name = __stringify(_name), .mode = 0444 }, \ 37 .show = _name##_access##_level##_show, \ 38 } 39 40 #define ACCESS_DEVICE_ATTR_RO(level, name) \ 41 struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name) 42 43 #define ACCESS_ATTR_RO(level, attrib) \ 44 static ssize_t attrib##_access##level##_show(struct device *dev, \ 45 struct device_attribute *attr, \ 46 char *buf) \ 47 { \ 48 struct cxl_region *cxlr = to_cxl_region(dev); \ 49 \ 50 if (cxlr->coord[level].attrib == 0) \ 51 return -ENOENT; \ 52 \ 53 return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \ 54 } \ 55 static ACCESS_DEVICE_ATTR_RO(level, attrib) 56 57 ACCESS_ATTR_RO(0, read_bandwidth); 58 ACCESS_ATTR_RO(0, read_latency); 59 ACCESS_ATTR_RO(0, write_bandwidth); 60 ACCESS_ATTR_RO(0, write_latency); 61 62 #define ACCESS_ATTR_DECLARE(level, attrib) \ 63 (&dev_attr_access##level##_##attrib.attr) 64 65 static struct attribute *access0_coordinate_attrs[] = { 66 ACCESS_ATTR_DECLARE(0, read_bandwidth), 67 ACCESS_ATTR_DECLARE(0, write_bandwidth), 68 ACCESS_ATTR_DECLARE(0, read_latency), 69 ACCESS_ATTR_DECLARE(0, write_latency), 70 NULL 71 }; 72 73 ACCESS_ATTR_RO(1, read_bandwidth); 74 ACCESS_ATTR_RO(1, read_latency); 75 ACCESS_ATTR_RO(1, write_bandwidth); 76 ACCESS_ATTR_RO(1, write_latency); 77 78 static struct attribute *access1_coordinate_attrs[] = { 79 ACCESS_ATTR_DECLARE(1, read_bandwidth), 80 ACCESS_ATTR_DECLARE(1, write_bandwidth), 81 ACCESS_ATTR_DECLARE(1, read_latency), 82 ACCESS_ATTR_DECLARE(1, write_latency), 83 NULL 84 }; 85 86 #define ACCESS_VISIBLE(level) \ 87 static umode_t cxl_region_access##level##_coordinate_visible( \ 88 struct kobject *kobj, struct attribute *a, int n) \ 89 { \ 90 struct device *dev = kobj_to_dev(kobj); \ 91 struct cxl_region *cxlr = to_cxl_region(dev); \ 92 \ 93 if (a == &dev_attr_access##level##_read_latency.attr && \ 94 cxlr->coord[level].read_latency == 0) \ 95 return 0; \ 96 \ 97 if (a == &dev_attr_access##level##_write_latency.attr && \ 98 cxlr->coord[level].write_latency == 0) \ 99 return 0; \ 100 \ 101 if (a == &dev_attr_access##level##_read_bandwidth.attr && \ 102 cxlr->coord[level].read_bandwidth == 0) \ 103 return 0; \ 104 \ 105 if (a == &dev_attr_access##level##_write_bandwidth.attr && \ 106 cxlr->coord[level].write_bandwidth == 0) \ 107 return 0; \ 108 \ 109 return a->mode; \ 110 } 111 112 ACCESS_VISIBLE(0); 113 ACCESS_VISIBLE(1); 114 115 static const struct attribute_group cxl_region_access0_coordinate_group = { 116 .name = "access0", 117 .attrs = access0_coordinate_attrs, 118 .is_visible = cxl_region_access0_coordinate_visible, 119 }; 120 121 static const struct attribute_group *get_cxl_region_access0_group(void) 122 { 123 return &cxl_region_access0_coordinate_group; 124 } 125 126 static const struct attribute_group cxl_region_access1_coordinate_group = { 127 .name = "access1", 128 .attrs = access1_coordinate_attrs, 129 .is_visible = cxl_region_access1_coordinate_visible, 130 }; 131 132 static const struct attribute_group *get_cxl_region_access1_group(void) 133 { 134 return &cxl_region_access1_coordinate_group; 135 } 136 137 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, 138 char *buf) 139 { 140 struct cxl_region *cxlr = to_cxl_region(dev); 141 struct cxl_region_params *p = &cxlr->params; 142 ssize_t rc; 143 144 rc = down_read_interruptible(&cxl_region_rwsem); 145 if (rc) 146 return rc; 147 if (cxlr->mode != CXL_DECODER_PMEM) 148 rc = sysfs_emit(buf, "\n"); 149 else 150 rc = sysfs_emit(buf, "%pUb\n", &p->uuid); 151 up_read(&cxl_region_rwsem); 152 153 return rc; 154 } 155 156 static int is_dup(struct device *match, void *data) 157 { 158 struct cxl_region_params *p; 159 struct cxl_region *cxlr; 160 uuid_t *uuid = data; 161 162 if (!is_cxl_region(match)) 163 return 0; 164 165 lockdep_assert_held(&cxl_region_rwsem); 166 cxlr = to_cxl_region(match); 167 p = &cxlr->params; 168 169 if (uuid_equal(&p->uuid, uuid)) { 170 dev_dbg(match, "already has uuid: %pUb\n", uuid); 171 return -EBUSY; 172 } 173 174 return 0; 175 } 176 177 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr, 178 const char *buf, size_t len) 179 { 180 struct cxl_region *cxlr = to_cxl_region(dev); 181 struct cxl_region_params *p = &cxlr->params; 182 uuid_t temp; 183 ssize_t rc; 184 185 if (len != UUID_STRING_LEN + 1) 186 return -EINVAL; 187 188 rc = uuid_parse(buf, &temp); 189 if (rc) 190 return rc; 191 192 if (uuid_is_null(&temp)) 193 return -EINVAL; 194 195 rc = down_write_killable(&cxl_region_rwsem); 196 if (rc) 197 return rc; 198 199 if (uuid_equal(&p->uuid, &temp)) 200 goto out; 201 202 rc = -EBUSY; 203 if (p->state >= CXL_CONFIG_ACTIVE) 204 goto out; 205 206 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup); 207 if (rc < 0) 208 goto out; 209 210 uuid_copy(&p->uuid, &temp); 211 out: 212 up_write(&cxl_region_rwsem); 213 214 if (rc) 215 return rc; 216 return len; 217 } 218 static DEVICE_ATTR_RW(uuid); 219 220 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port, 221 struct cxl_region *cxlr) 222 { 223 return xa_load(&port->regions, (unsigned long)cxlr); 224 } 225 226 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) 227 { 228 if (!cpu_cache_has_invalidate_memregion()) { 229 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { 230 dev_info_once( 231 &cxlr->dev, 232 "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); 233 return 0; 234 } else { 235 dev_WARN(&cxlr->dev, 236 "Failed to synchronize CPU cache state\n"); 237 return -ENXIO; 238 } 239 } 240 241 cpu_cache_invalidate_memregion(IORES_DESC_CXL); 242 return 0; 243 } 244 245 static void cxl_region_decode_reset(struct cxl_region *cxlr, int count) 246 { 247 struct cxl_region_params *p = &cxlr->params; 248 int i; 249 250 /* 251 * Before region teardown attempt to flush, evict any data cached for 252 * this region, or scream loudly about missing arch / platform support 253 * for CXL teardown. 254 */ 255 cxl_region_invalidate_memregion(cxlr); 256 257 for (i = count - 1; i >= 0; i--) { 258 struct cxl_endpoint_decoder *cxled = p->targets[i]; 259 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 260 struct cxl_port *iter = cxled_to_port(cxled); 261 struct cxl_dev_state *cxlds = cxlmd->cxlds; 262 struct cxl_ep *ep; 263 264 if (cxlds->rcd) 265 goto endpoint_reset; 266 267 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 268 iter = to_cxl_port(iter->dev.parent); 269 270 for (ep = cxl_ep_load(iter, cxlmd); iter; 271 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 272 struct cxl_region_ref *cxl_rr; 273 struct cxl_decoder *cxld; 274 275 cxl_rr = cxl_rr_load(iter, cxlr); 276 cxld = cxl_rr->decoder; 277 if (cxld->reset) 278 cxld->reset(cxld); 279 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 280 } 281 282 endpoint_reset: 283 cxled->cxld.reset(&cxled->cxld); 284 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 285 } 286 287 /* all decoders associated with this region have been torn down */ 288 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 289 } 290 291 static int commit_decoder(struct cxl_decoder *cxld) 292 { 293 struct cxl_switch_decoder *cxlsd = NULL; 294 295 if (cxld->commit) 296 return cxld->commit(cxld); 297 298 if (is_switch_decoder(&cxld->dev)) 299 cxlsd = to_cxl_switch_decoder(&cxld->dev); 300 301 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1, 302 "->commit() is required\n")) 303 return -ENXIO; 304 return 0; 305 } 306 307 static int cxl_region_decode_commit(struct cxl_region *cxlr) 308 { 309 struct cxl_region_params *p = &cxlr->params; 310 int i, rc = 0; 311 312 for (i = 0; i < p->nr_targets; i++) { 313 struct cxl_endpoint_decoder *cxled = p->targets[i]; 314 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 315 struct cxl_region_ref *cxl_rr; 316 struct cxl_decoder *cxld; 317 struct cxl_port *iter; 318 struct cxl_ep *ep; 319 320 /* commit bottom up */ 321 for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 322 iter = to_cxl_port(iter->dev.parent)) { 323 cxl_rr = cxl_rr_load(iter, cxlr); 324 cxld = cxl_rr->decoder; 325 rc = commit_decoder(cxld); 326 if (rc) 327 break; 328 } 329 330 if (rc) { 331 /* programming @iter failed, teardown */ 332 for (ep = cxl_ep_load(iter, cxlmd); ep && iter; 333 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 334 cxl_rr = cxl_rr_load(iter, cxlr); 335 cxld = cxl_rr->decoder; 336 if (cxld->reset) 337 cxld->reset(cxld); 338 } 339 340 cxled->cxld.reset(&cxled->cxld); 341 goto err; 342 } 343 } 344 345 return 0; 346 347 err: 348 /* undo the targets that were successfully committed */ 349 cxl_region_decode_reset(cxlr, i); 350 return rc; 351 } 352 353 static ssize_t commit_store(struct device *dev, struct device_attribute *attr, 354 const char *buf, size_t len) 355 { 356 struct cxl_region *cxlr = to_cxl_region(dev); 357 struct cxl_region_params *p = &cxlr->params; 358 bool commit; 359 ssize_t rc; 360 361 rc = kstrtobool(buf, &commit); 362 if (rc) 363 return rc; 364 365 rc = down_write_killable(&cxl_region_rwsem); 366 if (rc) 367 return rc; 368 369 /* Already in the requested state? */ 370 if (commit && p->state >= CXL_CONFIG_COMMIT) 371 goto out; 372 if (!commit && p->state < CXL_CONFIG_COMMIT) 373 goto out; 374 375 /* Not ready to commit? */ 376 if (commit && p->state < CXL_CONFIG_ACTIVE) { 377 rc = -ENXIO; 378 goto out; 379 } 380 381 /* 382 * Invalidate caches before region setup to drop any speculative 383 * consumption of this address space 384 */ 385 rc = cxl_region_invalidate_memregion(cxlr); 386 if (rc) 387 goto out; 388 389 if (commit) { 390 rc = cxl_region_decode_commit(cxlr); 391 if (rc == 0) 392 p->state = CXL_CONFIG_COMMIT; 393 } else { 394 p->state = CXL_CONFIG_RESET_PENDING; 395 up_write(&cxl_region_rwsem); 396 device_release_driver(&cxlr->dev); 397 down_write(&cxl_region_rwsem); 398 399 /* 400 * The lock was dropped, so need to revalidate that the reset is 401 * still pending. 402 */ 403 if (p->state == CXL_CONFIG_RESET_PENDING) { 404 cxl_region_decode_reset(cxlr, p->interleave_ways); 405 p->state = CXL_CONFIG_ACTIVE; 406 } 407 } 408 409 out: 410 up_write(&cxl_region_rwsem); 411 412 if (rc) 413 return rc; 414 return len; 415 } 416 417 static ssize_t commit_show(struct device *dev, struct device_attribute *attr, 418 char *buf) 419 { 420 struct cxl_region *cxlr = to_cxl_region(dev); 421 struct cxl_region_params *p = &cxlr->params; 422 ssize_t rc; 423 424 rc = down_read_interruptible(&cxl_region_rwsem); 425 if (rc) 426 return rc; 427 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT); 428 up_read(&cxl_region_rwsem); 429 430 return rc; 431 } 432 static DEVICE_ATTR_RW(commit); 433 434 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a, 435 int n) 436 { 437 struct device *dev = kobj_to_dev(kobj); 438 struct cxl_region *cxlr = to_cxl_region(dev); 439 440 /* 441 * Support tooling that expects to find a 'uuid' attribute for all 442 * regions regardless of mode. 443 */ 444 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_DECODER_PMEM) 445 return 0444; 446 return a->mode; 447 } 448 449 static ssize_t interleave_ways_show(struct device *dev, 450 struct device_attribute *attr, char *buf) 451 { 452 struct cxl_region *cxlr = to_cxl_region(dev); 453 struct cxl_region_params *p = &cxlr->params; 454 ssize_t rc; 455 456 rc = down_read_interruptible(&cxl_region_rwsem); 457 if (rc) 458 return rc; 459 rc = sysfs_emit(buf, "%d\n", p->interleave_ways); 460 up_read(&cxl_region_rwsem); 461 462 return rc; 463 } 464 465 static const struct attribute_group *get_cxl_region_target_group(void); 466 467 static ssize_t interleave_ways_store(struct device *dev, 468 struct device_attribute *attr, 469 const char *buf, size_t len) 470 { 471 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 472 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 473 struct cxl_region *cxlr = to_cxl_region(dev); 474 struct cxl_region_params *p = &cxlr->params; 475 unsigned int val, save; 476 int rc; 477 u8 iw; 478 479 rc = kstrtouint(buf, 0, &val); 480 if (rc) 481 return rc; 482 483 rc = ways_to_eiw(val, &iw); 484 if (rc) 485 return rc; 486 487 /* 488 * Even for x3, x6, and x12 interleaves the region interleave must be a 489 * power of 2 multiple of the host bridge interleave. 490 */ 491 if (!is_power_of_2(val / cxld->interleave_ways) || 492 (val % cxld->interleave_ways)) { 493 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val); 494 return -EINVAL; 495 } 496 497 rc = down_write_killable(&cxl_region_rwsem); 498 if (rc) 499 return rc; 500 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 501 rc = -EBUSY; 502 goto out; 503 } 504 505 save = p->interleave_ways; 506 p->interleave_ways = val; 507 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 508 if (rc) 509 p->interleave_ways = save; 510 out: 511 up_write(&cxl_region_rwsem); 512 if (rc) 513 return rc; 514 return len; 515 } 516 static DEVICE_ATTR_RW(interleave_ways); 517 518 static ssize_t interleave_granularity_show(struct device *dev, 519 struct device_attribute *attr, 520 char *buf) 521 { 522 struct cxl_region *cxlr = to_cxl_region(dev); 523 struct cxl_region_params *p = &cxlr->params; 524 ssize_t rc; 525 526 rc = down_read_interruptible(&cxl_region_rwsem); 527 if (rc) 528 return rc; 529 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity); 530 up_read(&cxl_region_rwsem); 531 532 return rc; 533 } 534 535 static ssize_t interleave_granularity_store(struct device *dev, 536 struct device_attribute *attr, 537 const char *buf, size_t len) 538 { 539 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 540 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 541 struct cxl_region *cxlr = to_cxl_region(dev); 542 struct cxl_region_params *p = &cxlr->params; 543 int rc, val; 544 u16 ig; 545 546 rc = kstrtoint(buf, 0, &val); 547 if (rc) 548 return rc; 549 550 rc = granularity_to_eig(val, &ig); 551 if (rc) 552 return rc; 553 554 /* 555 * When the host-bridge is interleaved, disallow region granularity != 556 * root granularity. Regions with a granularity less than the root 557 * interleave result in needing multiple endpoints to support a single 558 * slot in the interleave (possible to support in the future). Regions 559 * with a granularity greater than the root interleave result in invalid 560 * DPA translations (invalid to support). 561 */ 562 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity) 563 return -EINVAL; 564 565 rc = down_write_killable(&cxl_region_rwsem); 566 if (rc) 567 return rc; 568 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 569 rc = -EBUSY; 570 goto out; 571 } 572 573 p->interleave_granularity = val; 574 out: 575 up_write(&cxl_region_rwsem); 576 if (rc) 577 return rc; 578 return len; 579 } 580 static DEVICE_ATTR_RW(interleave_granularity); 581 582 static ssize_t resource_show(struct device *dev, struct device_attribute *attr, 583 char *buf) 584 { 585 struct cxl_region *cxlr = to_cxl_region(dev); 586 struct cxl_region_params *p = &cxlr->params; 587 u64 resource = -1ULL; 588 ssize_t rc; 589 590 rc = down_read_interruptible(&cxl_region_rwsem); 591 if (rc) 592 return rc; 593 if (p->res) 594 resource = p->res->start; 595 rc = sysfs_emit(buf, "%#llx\n", resource); 596 up_read(&cxl_region_rwsem); 597 598 return rc; 599 } 600 static DEVICE_ATTR_RO(resource); 601 602 static ssize_t mode_show(struct device *dev, struct device_attribute *attr, 603 char *buf) 604 { 605 struct cxl_region *cxlr = to_cxl_region(dev); 606 607 return sysfs_emit(buf, "%s\n", cxl_decoder_mode_name(cxlr->mode)); 608 } 609 static DEVICE_ATTR_RO(mode); 610 611 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) 612 { 613 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 614 struct cxl_region_params *p = &cxlr->params; 615 struct resource *res; 616 u64 remainder = 0; 617 618 lockdep_assert_held_write(&cxl_region_rwsem); 619 620 /* Nothing to do... */ 621 if (p->res && resource_size(p->res) == size) 622 return 0; 623 624 /* To change size the old size must be freed first */ 625 if (p->res) 626 return -EBUSY; 627 628 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) 629 return -EBUSY; 630 631 /* ways, granularity and uuid (if PMEM) need to be set before HPA */ 632 if (!p->interleave_ways || !p->interleave_granularity || 633 (cxlr->mode == CXL_DECODER_PMEM && uuid_is_null(&p->uuid))) 634 return -ENXIO; 635 636 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder); 637 if (remainder) 638 return -EINVAL; 639 640 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, 641 dev_name(&cxlr->dev)); 642 if (IS_ERR(res)) { 643 dev_dbg(&cxlr->dev, 644 "HPA allocation error (%ld) for size:%pap in %s %pr\n", 645 PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res); 646 return PTR_ERR(res); 647 } 648 649 p->res = res; 650 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 651 652 return 0; 653 } 654 655 static void cxl_region_iomem_release(struct cxl_region *cxlr) 656 { 657 struct cxl_region_params *p = &cxlr->params; 658 659 if (device_is_registered(&cxlr->dev)) 660 lockdep_assert_held_write(&cxl_region_rwsem); 661 if (p->res) { 662 /* 663 * Autodiscovered regions may not have been able to insert their 664 * resource. 665 */ 666 if (p->res->parent) 667 remove_resource(p->res); 668 kfree(p->res); 669 p->res = NULL; 670 } 671 } 672 673 static int free_hpa(struct cxl_region *cxlr) 674 { 675 struct cxl_region_params *p = &cxlr->params; 676 677 lockdep_assert_held_write(&cxl_region_rwsem); 678 679 if (!p->res) 680 return 0; 681 682 if (p->state >= CXL_CONFIG_ACTIVE) 683 return -EBUSY; 684 685 cxl_region_iomem_release(cxlr); 686 p->state = CXL_CONFIG_IDLE; 687 return 0; 688 } 689 690 static ssize_t size_store(struct device *dev, struct device_attribute *attr, 691 const char *buf, size_t len) 692 { 693 struct cxl_region *cxlr = to_cxl_region(dev); 694 u64 val; 695 int rc; 696 697 rc = kstrtou64(buf, 0, &val); 698 if (rc) 699 return rc; 700 701 rc = down_write_killable(&cxl_region_rwsem); 702 if (rc) 703 return rc; 704 705 if (val) 706 rc = alloc_hpa(cxlr, val); 707 else 708 rc = free_hpa(cxlr); 709 up_write(&cxl_region_rwsem); 710 711 if (rc) 712 return rc; 713 714 return len; 715 } 716 717 static ssize_t size_show(struct device *dev, struct device_attribute *attr, 718 char *buf) 719 { 720 struct cxl_region *cxlr = to_cxl_region(dev); 721 struct cxl_region_params *p = &cxlr->params; 722 u64 size = 0; 723 ssize_t rc; 724 725 rc = down_read_interruptible(&cxl_region_rwsem); 726 if (rc) 727 return rc; 728 if (p->res) 729 size = resource_size(p->res); 730 rc = sysfs_emit(buf, "%#llx\n", size); 731 up_read(&cxl_region_rwsem); 732 733 return rc; 734 } 735 static DEVICE_ATTR_RW(size); 736 737 static struct attribute *cxl_region_attrs[] = { 738 &dev_attr_uuid.attr, 739 &dev_attr_commit.attr, 740 &dev_attr_interleave_ways.attr, 741 &dev_attr_interleave_granularity.attr, 742 &dev_attr_resource.attr, 743 &dev_attr_size.attr, 744 &dev_attr_mode.attr, 745 NULL, 746 }; 747 748 static const struct attribute_group cxl_region_group = { 749 .attrs = cxl_region_attrs, 750 .is_visible = cxl_region_visible, 751 }; 752 753 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos) 754 { 755 struct cxl_region_params *p = &cxlr->params; 756 struct cxl_endpoint_decoder *cxled; 757 int rc; 758 759 rc = down_read_interruptible(&cxl_region_rwsem); 760 if (rc) 761 return rc; 762 763 if (pos >= p->interleave_ways) { 764 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 765 p->interleave_ways); 766 rc = -ENXIO; 767 goto out; 768 } 769 770 cxled = p->targets[pos]; 771 if (!cxled) 772 rc = sysfs_emit(buf, "\n"); 773 else 774 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev)); 775 out: 776 up_read(&cxl_region_rwsem); 777 778 return rc; 779 } 780 781 static int check_commit_order(struct device *dev, void *data) 782 { 783 struct cxl_decoder *cxld = to_cxl_decoder(dev); 784 785 /* 786 * if port->commit_end is not the only free decoder, then out of 787 * order shutdown has occurred, block further allocations until 788 * that is resolved 789 */ 790 if (((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) 791 return -EBUSY; 792 return 0; 793 } 794 795 static int match_free_decoder(struct device *dev, const void *data) 796 { 797 struct cxl_port *port = to_cxl_port(dev->parent); 798 struct cxl_decoder *cxld; 799 int rc; 800 801 if (!is_switch_decoder(dev)) 802 return 0; 803 804 cxld = to_cxl_decoder(dev); 805 806 if (cxld->id != port->commit_end + 1) 807 return 0; 808 809 if (cxld->region) { 810 dev_dbg(dev->parent, 811 "next decoder to commit (%s) is already reserved (%s)\n", 812 dev_name(dev), dev_name(&cxld->region->dev)); 813 return 0; 814 } 815 816 rc = device_for_each_child_reverse_from(dev->parent, dev, NULL, 817 check_commit_order); 818 if (rc) { 819 dev_dbg(dev->parent, 820 "unable to allocate %s due to out of order shutdown\n", 821 dev_name(dev)); 822 return 0; 823 } 824 return 1; 825 } 826 827 static int match_auto_decoder(struct device *dev, const void *data) 828 { 829 const struct cxl_region_params *p = data; 830 struct cxl_decoder *cxld; 831 struct range *r; 832 833 if (!is_switch_decoder(dev)) 834 return 0; 835 836 cxld = to_cxl_decoder(dev); 837 r = &cxld->hpa_range; 838 839 if (p->res && p->res->start == r->start && p->res->end == r->end) 840 return 1; 841 842 return 0; 843 } 844 845 static struct cxl_decoder * 846 cxl_region_find_decoder(struct cxl_port *port, 847 struct cxl_endpoint_decoder *cxled, 848 struct cxl_region *cxlr) 849 { 850 struct device *dev; 851 852 if (port == cxled_to_port(cxled)) 853 return &cxled->cxld; 854 855 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) 856 dev = device_find_child(&port->dev, &cxlr->params, 857 match_auto_decoder); 858 else 859 dev = device_find_child(&port->dev, NULL, match_free_decoder); 860 if (!dev) 861 return NULL; 862 /* 863 * This decoder is pinned registered as long as the endpoint decoder is 864 * registered, and endpoint decoder unregistration holds the 865 * cxl_region_rwsem over unregister events, so no need to hold on to 866 * this extra reference. 867 */ 868 put_device(dev); 869 return to_cxl_decoder(dev); 870 } 871 872 static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter, 873 struct cxl_decoder *cxld) 874 { 875 struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter); 876 struct cxl_decoder *cxld_iter = rr->decoder; 877 878 /* 879 * Allow the out of order assembly of auto-discovered regions. 880 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders 881 * in HPA order. Confirm that the decoder with the lesser HPA 882 * starting address has the lesser id. 883 */ 884 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n", 885 dev_name(&cxld->dev), cxld->id, 886 dev_name(&cxld_iter->dev), cxld_iter->id); 887 888 if (cxld_iter->id > cxld->id) 889 return true; 890 891 return false; 892 } 893 894 static struct cxl_region_ref * 895 alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr, 896 struct cxl_endpoint_decoder *cxled) 897 { 898 struct cxl_region_params *p = &cxlr->params; 899 struct cxl_region_ref *cxl_rr, *iter; 900 unsigned long index; 901 int rc; 902 903 xa_for_each(&port->regions, index, iter) { 904 struct cxl_region_params *ip = &iter->region->params; 905 906 if (!ip->res || ip->res->start < p->res->start) 907 continue; 908 909 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 910 struct cxl_decoder *cxld; 911 912 cxld = cxl_region_find_decoder(port, cxled, cxlr); 913 if (auto_order_ok(port, iter->region, cxld)) 914 continue; 915 } 916 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n", 917 dev_name(&port->dev), 918 dev_name(&iter->region->dev), ip->res, p->res); 919 920 return ERR_PTR(-EBUSY); 921 } 922 923 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL); 924 if (!cxl_rr) 925 return ERR_PTR(-ENOMEM); 926 cxl_rr->port = port; 927 cxl_rr->region = cxlr; 928 cxl_rr->nr_targets = 1; 929 xa_init(&cxl_rr->endpoints); 930 931 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL); 932 if (rc) { 933 dev_dbg(&cxlr->dev, 934 "%s: failed to track region reference: %d\n", 935 dev_name(&port->dev), rc); 936 kfree(cxl_rr); 937 return ERR_PTR(rc); 938 } 939 940 return cxl_rr; 941 } 942 943 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr) 944 { 945 struct cxl_region *cxlr = cxl_rr->region; 946 struct cxl_decoder *cxld = cxl_rr->decoder; 947 948 if (!cxld) 949 return; 950 951 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n"); 952 if (cxld->region == cxlr) { 953 cxld->region = NULL; 954 put_device(&cxlr->dev); 955 } 956 } 957 958 static void free_region_ref(struct cxl_region_ref *cxl_rr) 959 { 960 struct cxl_port *port = cxl_rr->port; 961 struct cxl_region *cxlr = cxl_rr->region; 962 963 cxl_rr_free_decoder(cxl_rr); 964 xa_erase(&port->regions, (unsigned long)cxlr); 965 xa_destroy(&cxl_rr->endpoints); 966 kfree(cxl_rr); 967 } 968 969 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr, 970 struct cxl_endpoint_decoder *cxled) 971 { 972 int rc; 973 struct cxl_port *port = cxl_rr->port; 974 struct cxl_region *cxlr = cxl_rr->region; 975 struct cxl_decoder *cxld = cxl_rr->decoder; 976 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled)); 977 978 if (ep) { 979 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep, 980 GFP_KERNEL); 981 if (rc) 982 return rc; 983 } 984 cxl_rr->nr_eps++; 985 986 if (!cxld->region) { 987 cxld->region = cxlr; 988 get_device(&cxlr->dev); 989 } 990 991 return 0; 992 } 993 994 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr, 995 struct cxl_endpoint_decoder *cxled, 996 struct cxl_region_ref *cxl_rr) 997 { 998 struct cxl_decoder *cxld; 999 1000 cxld = cxl_region_find_decoder(port, cxled, cxlr); 1001 if (!cxld) { 1002 dev_dbg(&cxlr->dev, "%s: no decoder available\n", 1003 dev_name(&port->dev)); 1004 return -EBUSY; 1005 } 1006 1007 if (cxld->region) { 1008 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", 1009 dev_name(&port->dev), dev_name(&cxld->dev), 1010 dev_name(&cxld->region->dev)); 1011 return -EBUSY; 1012 } 1013 1014 /* 1015 * Endpoints should already match the region type, but backstop that 1016 * assumption with an assertion. Switch-decoders change mapping-type 1017 * based on what is mapped when they are assigned to a region. 1018 */ 1019 dev_WARN_ONCE(&cxlr->dev, 1020 port == cxled_to_port(cxled) && 1021 cxld->target_type != cxlr->type, 1022 "%s:%s mismatch decoder type %d -> %d\n", 1023 dev_name(&cxled_to_memdev(cxled)->dev), 1024 dev_name(&cxld->dev), cxld->target_type, cxlr->type); 1025 cxld->target_type = cxlr->type; 1026 cxl_rr->decoder = cxld; 1027 return 0; 1028 } 1029 1030 /** 1031 * cxl_port_attach_region() - track a region's interest in a port by endpoint 1032 * @port: port to add a new region reference 'struct cxl_region_ref' 1033 * @cxlr: region to attach to @port 1034 * @cxled: endpoint decoder used to create or further pin a region reference 1035 * @pos: interleave position of @cxled in @cxlr 1036 * 1037 * The attach event is an opportunity to validate CXL decode setup 1038 * constraints and record metadata needed for programming HDM decoders, 1039 * in particular decoder target lists. 1040 * 1041 * The steps are: 1042 * 1043 * - validate that there are no other regions with a higher HPA already 1044 * associated with @port 1045 * - establish a region reference if one is not already present 1046 * 1047 * - additionally allocate a decoder instance that will host @cxlr on 1048 * @port 1049 * 1050 * - pin the region reference by the endpoint 1051 * - account for how many entries in @port's target list are needed to 1052 * cover all of the added endpoints. 1053 */ 1054 static int cxl_port_attach_region(struct cxl_port *port, 1055 struct cxl_region *cxlr, 1056 struct cxl_endpoint_decoder *cxled, int pos) 1057 { 1058 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1059 struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 1060 struct cxl_region_ref *cxl_rr; 1061 bool nr_targets_inc = false; 1062 struct cxl_decoder *cxld; 1063 unsigned long index; 1064 int rc = -EBUSY; 1065 1066 lockdep_assert_held_write(&cxl_region_rwsem); 1067 1068 cxl_rr = cxl_rr_load(port, cxlr); 1069 if (cxl_rr) { 1070 struct cxl_ep *ep_iter; 1071 int found = 0; 1072 1073 /* 1074 * Walk the existing endpoints that have been attached to 1075 * @cxlr at @port and see if they share the same 'next' port 1076 * in the downstream direction. I.e. endpoints that share common 1077 * upstream switch. 1078 */ 1079 xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 1080 if (ep_iter == ep) 1081 continue; 1082 if (ep_iter->next == ep->next) { 1083 found++; 1084 break; 1085 } 1086 } 1087 1088 /* 1089 * New target port, or @port is an endpoint port that always 1090 * accounts its own local decode as a target. 1091 */ 1092 if (!found || !ep->next) { 1093 cxl_rr->nr_targets++; 1094 nr_targets_inc = true; 1095 } 1096 } else { 1097 cxl_rr = alloc_region_ref(port, cxlr, cxled); 1098 if (IS_ERR(cxl_rr)) { 1099 dev_dbg(&cxlr->dev, 1100 "%s: failed to allocate region reference\n", 1101 dev_name(&port->dev)); 1102 return PTR_ERR(cxl_rr); 1103 } 1104 nr_targets_inc = true; 1105 1106 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr); 1107 if (rc) 1108 goto out_erase; 1109 } 1110 cxld = cxl_rr->decoder; 1111 1112 /* 1113 * the number of targets should not exceed the target_count 1114 * of the decoder 1115 */ 1116 if (is_switch_decoder(&cxld->dev)) { 1117 struct cxl_switch_decoder *cxlsd; 1118 1119 cxlsd = to_cxl_switch_decoder(&cxld->dev); 1120 if (cxl_rr->nr_targets > cxlsd->nr_targets) { 1121 dev_dbg(&cxlr->dev, 1122 "%s:%s %s add: %s:%s @ %d overflows targets: %d\n", 1123 dev_name(port->uport_dev), dev_name(&port->dev), 1124 dev_name(&cxld->dev), dev_name(&cxlmd->dev), 1125 dev_name(&cxled->cxld.dev), pos, 1126 cxlsd->nr_targets); 1127 rc = -ENXIO; 1128 goto out_erase; 1129 } 1130 } 1131 1132 rc = cxl_rr_ep_add(cxl_rr, cxled); 1133 if (rc) { 1134 dev_dbg(&cxlr->dev, 1135 "%s: failed to track endpoint %s:%s reference\n", 1136 dev_name(&port->dev), dev_name(&cxlmd->dev), 1137 dev_name(&cxld->dev)); 1138 goto out_erase; 1139 } 1140 1141 dev_dbg(&cxlr->dev, 1142 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n", 1143 dev_name(port->uport_dev), dev_name(&port->dev), 1144 dev_name(&cxld->dev), dev_name(&cxlmd->dev), 1145 dev_name(&cxled->cxld.dev), pos, 1146 ep ? ep->next ? dev_name(ep->next->uport_dev) : 1147 dev_name(&cxlmd->dev) : 1148 "none", 1149 cxl_rr->nr_eps, cxl_rr->nr_targets); 1150 1151 return 0; 1152 out_erase: 1153 if (nr_targets_inc) 1154 cxl_rr->nr_targets--; 1155 if (cxl_rr->nr_eps == 0) 1156 free_region_ref(cxl_rr); 1157 return rc; 1158 } 1159 1160 static void cxl_port_detach_region(struct cxl_port *port, 1161 struct cxl_region *cxlr, 1162 struct cxl_endpoint_decoder *cxled) 1163 { 1164 struct cxl_region_ref *cxl_rr; 1165 struct cxl_ep *ep = NULL; 1166 1167 lockdep_assert_held_write(&cxl_region_rwsem); 1168 1169 cxl_rr = cxl_rr_load(port, cxlr); 1170 if (!cxl_rr) 1171 return; 1172 1173 /* 1174 * Endpoint ports do not carry cxl_ep references, and they 1175 * never target more than one endpoint by definition 1176 */ 1177 if (cxl_rr->decoder == &cxled->cxld) 1178 cxl_rr->nr_eps--; 1179 else 1180 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled); 1181 if (ep) { 1182 struct cxl_ep *ep_iter; 1183 unsigned long index; 1184 int found = 0; 1185 1186 cxl_rr->nr_eps--; 1187 xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 1188 if (ep_iter->next == ep->next) { 1189 found++; 1190 break; 1191 } 1192 } 1193 if (!found) 1194 cxl_rr->nr_targets--; 1195 } 1196 1197 if (cxl_rr->nr_eps == 0) 1198 free_region_ref(cxl_rr); 1199 } 1200 1201 static int check_last_peer(struct cxl_endpoint_decoder *cxled, 1202 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr, 1203 int distance) 1204 { 1205 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1206 struct cxl_region *cxlr = cxl_rr->region; 1207 struct cxl_region_params *p = &cxlr->params; 1208 struct cxl_endpoint_decoder *cxled_peer; 1209 struct cxl_port *port = cxl_rr->port; 1210 struct cxl_memdev *cxlmd_peer; 1211 struct cxl_ep *ep_peer; 1212 int pos = cxled->pos; 1213 1214 /* 1215 * If this position wants to share a dport with the last endpoint mapped 1216 * then that endpoint, at index 'position - distance', must also be 1217 * mapped by this dport. 1218 */ 1219 if (pos < distance) { 1220 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", 1221 dev_name(port->uport_dev), dev_name(&port->dev), 1222 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 1223 return -ENXIO; 1224 } 1225 cxled_peer = p->targets[pos - distance]; 1226 cxlmd_peer = cxled_to_memdev(cxled_peer); 1227 ep_peer = cxl_ep_load(port, cxlmd_peer); 1228 if (ep->dport != ep_peer->dport) { 1229 dev_dbg(&cxlr->dev, 1230 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n", 1231 dev_name(port->uport_dev), dev_name(&port->dev), 1232 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, 1233 dev_name(&cxlmd_peer->dev), 1234 dev_name(&cxled_peer->cxld.dev)); 1235 return -ENXIO; 1236 } 1237 1238 return 0; 1239 } 1240 1241 static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig) 1242 { 1243 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 1244 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 1245 unsigned int interleave_mask; 1246 u8 eiw; 1247 u16 eig; 1248 int high_pos, low_pos; 1249 1250 if (!test_bit(iw, &cxlhdm->iw_cap_mask)) 1251 return -ENXIO; 1252 /* 1253 * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection), 1254 * if eiw < 8: 1255 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw] 1256 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0] 1257 * 1258 * when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the 1259 * interleave bits are none. 1260 * 1261 * if eiw >= 8: 1262 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3 1263 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0] 1264 * 1265 * when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the 1266 * interleave bits are none. 1267 */ 1268 ways_to_eiw(iw, &eiw); 1269 if (eiw == 0 || eiw == 8) 1270 return 0; 1271 1272 granularity_to_eig(ig, &eig); 1273 if (eiw > 8) 1274 high_pos = eiw + eig - 1; 1275 else 1276 high_pos = eiw + eig + 7; 1277 low_pos = eig + 8; 1278 interleave_mask = GENMASK(high_pos, low_pos); 1279 if (interleave_mask & ~cxlhdm->interleave_mask) 1280 return -ENXIO; 1281 1282 return 0; 1283 } 1284 1285 static int cxl_port_setup_targets(struct cxl_port *port, 1286 struct cxl_region *cxlr, 1287 struct cxl_endpoint_decoder *cxled) 1288 { 1289 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 1290 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos; 1291 struct cxl_port *parent_port = to_cxl_port(port->dev.parent); 1292 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 1293 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1294 struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 1295 struct cxl_region_params *p = &cxlr->params; 1296 struct cxl_decoder *cxld = cxl_rr->decoder; 1297 struct cxl_switch_decoder *cxlsd; 1298 struct cxl_port *iter = port; 1299 u16 eig, peig; 1300 u8 eiw, peiw; 1301 1302 /* 1303 * While root level decoders support x3, x6, x12, switch level 1304 * decoders only support powers of 2 up to x16. 1305 */ 1306 if (!is_power_of_2(cxl_rr->nr_targets)) { 1307 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n", 1308 dev_name(port->uport_dev), dev_name(&port->dev), 1309 cxl_rr->nr_targets); 1310 return -EINVAL; 1311 } 1312 1313 cxlsd = to_cxl_switch_decoder(&cxld->dev); 1314 if (cxl_rr->nr_targets_set) { 1315 int i, distance = 1; 1316 struct cxl_region_ref *cxl_rr_iter; 1317 1318 /* 1319 * The "distance" between peer downstream ports represents which 1320 * endpoint positions in the region interleave a given port can 1321 * host. 1322 * 1323 * For example, at the root of a hierarchy the distance is 1324 * always 1 as every index targets a different host-bridge. At 1325 * each subsequent switch level those ports map every Nth region 1326 * position where N is the width of the switch == distance. 1327 */ 1328 do { 1329 cxl_rr_iter = cxl_rr_load(iter, cxlr); 1330 distance *= cxl_rr_iter->nr_targets; 1331 iter = to_cxl_port(iter->dev.parent); 1332 } while (!is_cxl_root(iter)); 1333 distance *= cxlrd->cxlsd.cxld.interleave_ways; 1334 1335 for (i = 0; i < cxl_rr->nr_targets_set; i++) 1336 if (ep->dport == cxlsd->target[i]) { 1337 rc = check_last_peer(cxled, ep, cxl_rr, 1338 distance); 1339 if (rc) 1340 return rc; 1341 goto out_target_set; 1342 } 1343 goto add_target; 1344 } 1345 1346 if (is_cxl_root(parent_port)) { 1347 /* 1348 * Root decoder IG is always set to value in CFMWS which 1349 * may be different than this region's IG. We can use the 1350 * region's IG here since interleave_granularity_store() 1351 * does not allow interleaved host-bridges with 1352 * root IG != region IG. 1353 */ 1354 parent_ig = p->interleave_granularity; 1355 parent_iw = cxlrd->cxlsd.cxld.interleave_ways; 1356 /* 1357 * For purposes of address bit routing, use power-of-2 math for 1358 * switch ports. 1359 */ 1360 if (!is_power_of_2(parent_iw)) 1361 parent_iw /= 3; 1362 } else { 1363 struct cxl_region_ref *parent_rr; 1364 struct cxl_decoder *parent_cxld; 1365 1366 parent_rr = cxl_rr_load(parent_port, cxlr); 1367 parent_cxld = parent_rr->decoder; 1368 parent_ig = parent_cxld->interleave_granularity; 1369 parent_iw = parent_cxld->interleave_ways; 1370 } 1371 1372 rc = granularity_to_eig(parent_ig, &peig); 1373 if (rc) { 1374 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n", 1375 dev_name(parent_port->uport_dev), 1376 dev_name(&parent_port->dev), parent_ig); 1377 return rc; 1378 } 1379 1380 rc = ways_to_eiw(parent_iw, &peiw); 1381 if (rc) { 1382 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n", 1383 dev_name(parent_port->uport_dev), 1384 dev_name(&parent_port->dev), parent_iw); 1385 return rc; 1386 } 1387 1388 iw = cxl_rr->nr_targets; 1389 rc = ways_to_eiw(iw, &eiw); 1390 if (rc) { 1391 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n", 1392 dev_name(port->uport_dev), dev_name(&port->dev), iw); 1393 return rc; 1394 } 1395 1396 /* 1397 * Interleave granularity is a multiple of @parent_port granularity. 1398 * Multiplier is the parent port interleave ways. 1399 */ 1400 rc = granularity_to_eig(parent_ig * parent_iw, &eig); 1401 if (rc) { 1402 dev_dbg(&cxlr->dev, 1403 "%s: invalid granularity calculation (%d * %d)\n", 1404 dev_name(&parent_port->dev), parent_ig, parent_iw); 1405 return rc; 1406 } 1407 1408 rc = eig_to_granularity(eig, &ig); 1409 if (rc) { 1410 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n", 1411 dev_name(port->uport_dev), dev_name(&port->dev), 1412 256 << eig); 1413 return rc; 1414 } 1415 1416 if (iw > 8 || iw > cxlsd->nr_targets) { 1417 dev_dbg(&cxlr->dev, 1418 "%s:%s:%s: ways: %d overflows targets: %d\n", 1419 dev_name(port->uport_dev), dev_name(&port->dev), 1420 dev_name(&cxld->dev), iw, cxlsd->nr_targets); 1421 return -ENXIO; 1422 } 1423 1424 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1425 if (cxld->interleave_ways != iw || 1426 cxld->interleave_granularity != ig || 1427 cxld->hpa_range.start != p->res->start || 1428 cxld->hpa_range.end != p->res->end || 1429 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { 1430 dev_err(&cxlr->dev, 1431 "%s:%s %s expected iw: %d ig: %d %pr\n", 1432 dev_name(port->uport_dev), dev_name(&port->dev), 1433 __func__, iw, ig, p->res); 1434 dev_err(&cxlr->dev, 1435 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n", 1436 dev_name(port->uport_dev), dev_name(&port->dev), 1437 __func__, cxld->interleave_ways, 1438 cxld->interleave_granularity, 1439 (cxld->flags & CXL_DECODER_F_ENABLE) ? 1440 "enabled" : 1441 "disabled", 1442 cxld->hpa_range.start, cxld->hpa_range.end); 1443 return -ENXIO; 1444 } 1445 } else { 1446 rc = check_interleave_cap(cxld, iw, ig); 1447 if (rc) { 1448 dev_dbg(&cxlr->dev, 1449 "%s:%s iw: %d ig: %d is not supported\n", 1450 dev_name(port->uport_dev), 1451 dev_name(&port->dev), iw, ig); 1452 return rc; 1453 } 1454 1455 cxld->interleave_ways = iw; 1456 cxld->interleave_granularity = ig; 1457 cxld->hpa_range = (struct range) { 1458 .start = p->res->start, 1459 .end = p->res->end, 1460 }; 1461 } 1462 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev), 1463 dev_name(&port->dev), iw, ig); 1464 add_target: 1465 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) { 1466 dev_dbg(&cxlr->dev, 1467 "%s:%s: targets full trying to add %s:%s at %d\n", 1468 dev_name(port->uport_dev), dev_name(&port->dev), 1469 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 1470 return -ENXIO; 1471 } 1472 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1473 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) { 1474 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n", 1475 dev_name(port->uport_dev), dev_name(&port->dev), 1476 dev_name(&cxlsd->cxld.dev), 1477 dev_name(ep->dport->dport_dev), 1478 cxl_rr->nr_targets_set); 1479 return -ENXIO; 1480 } 1481 } else 1482 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport; 1483 inc = 1; 1484 out_target_set: 1485 cxl_rr->nr_targets_set += inc; 1486 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", 1487 dev_name(port->uport_dev), dev_name(&port->dev), 1488 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev), 1489 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 1490 1491 return 0; 1492 } 1493 1494 static void cxl_port_reset_targets(struct cxl_port *port, 1495 struct cxl_region *cxlr) 1496 { 1497 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 1498 struct cxl_decoder *cxld; 1499 1500 /* 1501 * After the last endpoint has been detached the entire cxl_rr may now 1502 * be gone. 1503 */ 1504 if (!cxl_rr) 1505 return; 1506 cxl_rr->nr_targets_set = 0; 1507 1508 cxld = cxl_rr->decoder; 1509 cxld->hpa_range = (struct range) { 1510 .start = 0, 1511 .end = -1, 1512 }; 1513 } 1514 1515 static void cxl_region_teardown_targets(struct cxl_region *cxlr) 1516 { 1517 struct cxl_region_params *p = &cxlr->params; 1518 struct cxl_endpoint_decoder *cxled; 1519 struct cxl_dev_state *cxlds; 1520 struct cxl_memdev *cxlmd; 1521 struct cxl_port *iter; 1522 struct cxl_ep *ep; 1523 int i; 1524 1525 /* 1526 * In the auto-discovery case skip automatic teardown since the 1527 * address space is already active 1528 */ 1529 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) 1530 return; 1531 1532 for (i = 0; i < p->nr_targets; i++) { 1533 cxled = p->targets[i]; 1534 cxlmd = cxled_to_memdev(cxled); 1535 cxlds = cxlmd->cxlds; 1536 1537 if (cxlds->rcd) 1538 continue; 1539 1540 iter = cxled_to_port(cxled); 1541 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 1542 iter = to_cxl_port(iter->dev.parent); 1543 1544 for (ep = cxl_ep_load(iter, cxlmd); iter; 1545 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) 1546 cxl_port_reset_targets(iter, cxlr); 1547 } 1548 } 1549 1550 static int cxl_region_setup_targets(struct cxl_region *cxlr) 1551 { 1552 struct cxl_region_params *p = &cxlr->params; 1553 struct cxl_endpoint_decoder *cxled; 1554 struct cxl_dev_state *cxlds; 1555 int i, rc, rch = 0, vh = 0; 1556 struct cxl_memdev *cxlmd; 1557 struct cxl_port *iter; 1558 struct cxl_ep *ep; 1559 1560 for (i = 0; i < p->nr_targets; i++) { 1561 cxled = p->targets[i]; 1562 cxlmd = cxled_to_memdev(cxled); 1563 cxlds = cxlmd->cxlds; 1564 1565 /* validate that all targets agree on topology */ 1566 if (!cxlds->rcd) { 1567 vh++; 1568 } else { 1569 rch++; 1570 continue; 1571 } 1572 1573 iter = cxled_to_port(cxled); 1574 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 1575 iter = to_cxl_port(iter->dev.parent); 1576 1577 /* 1578 * Descend the topology tree programming / validating 1579 * targets while looking for conflicts. 1580 */ 1581 for (ep = cxl_ep_load(iter, cxlmd); iter; 1582 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 1583 rc = cxl_port_setup_targets(iter, cxlr, cxled); 1584 if (rc) { 1585 cxl_region_teardown_targets(cxlr); 1586 return rc; 1587 } 1588 } 1589 } 1590 1591 if (rch && vh) { 1592 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n"); 1593 cxl_region_teardown_targets(cxlr); 1594 return -ENXIO; 1595 } 1596 1597 return 0; 1598 } 1599 1600 static int cxl_region_validate_position(struct cxl_region *cxlr, 1601 struct cxl_endpoint_decoder *cxled, 1602 int pos) 1603 { 1604 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1605 struct cxl_region_params *p = &cxlr->params; 1606 int i; 1607 1608 if (pos < 0 || pos >= p->interleave_ways) { 1609 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1610 p->interleave_ways); 1611 return -ENXIO; 1612 } 1613 1614 if (p->targets[pos] == cxled) 1615 return 0; 1616 1617 if (p->targets[pos]) { 1618 struct cxl_endpoint_decoder *cxled_target = p->targets[pos]; 1619 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target); 1620 1621 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n", 1622 pos, dev_name(&cxlmd_target->dev), 1623 dev_name(&cxled_target->cxld.dev)); 1624 return -EBUSY; 1625 } 1626 1627 for (i = 0; i < p->interleave_ways; i++) { 1628 struct cxl_endpoint_decoder *cxled_target; 1629 struct cxl_memdev *cxlmd_target; 1630 1631 cxled_target = p->targets[i]; 1632 if (!cxled_target) 1633 continue; 1634 1635 cxlmd_target = cxled_to_memdev(cxled_target); 1636 if (cxlmd_target == cxlmd) { 1637 dev_dbg(&cxlr->dev, 1638 "%s already specified at position %d via: %s\n", 1639 dev_name(&cxlmd->dev), pos, 1640 dev_name(&cxled_target->cxld.dev)); 1641 return -EBUSY; 1642 } 1643 } 1644 1645 return 0; 1646 } 1647 1648 static int cxl_region_attach_position(struct cxl_region *cxlr, 1649 struct cxl_root_decoder *cxlrd, 1650 struct cxl_endpoint_decoder *cxled, 1651 const struct cxl_dport *dport, int pos) 1652 { 1653 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1654 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd; 1655 struct cxl_decoder *cxld = &cxlsd->cxld; 1656 int iw = cxld->interleave_ways; 1657 struct cxl_port *iter; 1658 int rc; 1659 1660 if (dport != cxlrd->cxlsd.target[pos % iw]) { 1661 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n", 1662 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1663 dev_name(&cxlrd->cxlsd.cxld.dev)); 1664 return -ENXIO; 1665 } 1666 1667 for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 1668 iter = to_cxl_port(iter->dev.parent)) { 1669 rc = cxl_port_attach_region(iter, cxlr, cxled, pos); 1670 if (rc) 1671 goto err; 1672 } 1673 1674 return 0; 1675 1676 err: 1677 for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 1678 iter = to_cxl_port(iter->dev.parent)) 1679 cxl_port_detach_region(iter, cxlr, cxled); 1680 return rc; 1681 } 1682 1683 static int cxl_region_attach_auto(struct cxl_region *cxlr, 1684 struct cxl_endpoint_decoder *cxled, int pos) 1685 { 1686 struct cxl_region_params *p = &cxlr->params; 1687 1688 if (cxled->state != CXL_DECODER_STATE_AUTO) { 1689 dev_err(&cxlr->dev, 1690 "%s: unable to add decoder to autodetected region\n", 1691 dev_name(&cxled->cxld.dev)); 1692 return -EINVAL; 1693 } 1694 1695 if (pos >= 0) { 1696 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n", 1697 dev_name(&cxled->cxld.dev), pos); 1698 return -EINVAL; 1699 } 1700 1701 if (p->nr_targets >= p->interleave_ways) { 1702 dev_err(&cxlr->dev, "%s: no more target slots available\n", 1703 dev_name(&cxled->cxld.dev)); 1704 return -ENXIO; 1705 } 1706 1707 /* 1708 * Temporarily record the endpoint decoder into the target array. Yes, 1709 * this means that userspace can view devices in the wrong position 1710 * before the region activates, and must be careful to understand when 1711 * it might be racing region autodiscovery. 1712 */ 1713 pos = p->nr_targets; 1714 p->targets[pos] = cxled; 1715 cxled->pos = pos; 1716 p->nr_targets++; 1717 1718 return 0; 1719 } 1720 1721 static int cmp_interleave_pos(const void *a, const void *b) 1722 { 1723 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a; 1724 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b; 1725 1726 return cxled_a->pos - cxled_b->pos; 1727 } 1728 1729 static struct cxl_port *next_port(struct cxl_port *port) 1730 { 1731 if (!port->parent_dport) 1732 return NULL; 1733 return port->parent_dport->port; 1734 } 1735 1736 static int match_switch_decoder_by_range(struct device *dev, 1737 const void *data) 1738 { 1739 struct cxl_switch_decoder *cxlsd; 1740 const struct range *r1, *r2 = data; 1741 1742 1743 if (!is_switch_decoder(dev)) 1744 return 0; 1745 1746 cxlsd = to_cxl_switch_decoder(dev); 1747 r1 = &cxlsd->cxld.hpa_range; 1748 1749 if (is_root_decoder(dev)) 1750 return range_contains(r1, r2); 1751 return (r1->start == r2->start && r1->end == r2->end); 1752 } 1753 1754 static int find_pos_and_ways(struct cxl_port *port, struct range *range, 1755 int *pos, int *ways) 1756 { 1757 struct cxl_switch_decoder *cxlsd; 1758 struct cxl_port *parent; 1759 struct device *dev; 1760 int rc = -ENXIO; 1761 1762 parent = next_port(port); 1763 if (!parent) 1764 return rc; 1765 1766 dev = device_find_child(&parent->dev, range, 1767 match_switch_decoder_by_range); 1768 if (!dev) { 1769 dev_err(port->uport_dev, 1770 "failed to find decoder mapping %#llx-%#llx\n", 1771 range->start, range->end); 1772 return rc; 1773 } 1774 cxlsd = to_cxl_switch_decoder(dev); 1775 *ways = cxlsd->cxld.interleave_ways; 1776 1777 for (int i = 0; i < *ways; i++) { 1778 if (cxlsd->target[i] == port->parent_dport) { 1779 *pos = i; 1780 rc = 0; 1781 break; 1782 } 1783 } 1784 put_device(dev); 1785 1786 return rc; 1787 } 1788 1789 /** 1790 * cxl_calc_interleave_pos() - calculate an endpoint position in a region 1791 * @cxled: endpoint decoder member of given region 1792 * 1793 * The endpoint position is calculated by traversing the topology from 1794 * the endpoint to the root decoder and iteratively applying this 1795 * calculation: 1796 * 1797 * position = position * parent_ways + parent_pos; 1798 * 1799 * ...where @position is inferred from switch and root decoder target lists. 1800 * 1801 * Return: position >= 0 on success 1802 * -ENXIO on failure 1803 */ 1804 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled) 1805 { 1806 struct cxl_port *iter, *port = cxled_to_port(cxled); 1807 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1808 struct range *range = &cxled->cxld.hpa_range; 1809 int parent_ways = 0, parent_pos = 0, pos = 0; 1810 int rc; 1811 1812 /* 1813 * Example: the expected interleave order of the 4-way region shown 1814 * below is: mem0, mem2, mem1, mem3 1815 * 1816 * root_port 1817 * / \ 1818 * host_bridge_0 host_bridge_1 1819 * | | | | 1820 * mem0 mem1 mem2 mem3 1821 * 1822 * In the example the calculator will iterate twice. The first iteration 1823 * uses the mem position in the host-bridge and the ways of the host- 1824 * bridge to generate the first, or local, position. The second 1825 * iteration uses the host-bridge position in the root_port and the ways 1826 * of the root_port to refine the position. 1827 * 1828 * A trace of the calculation per endpoint looks like this: 1829 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0 1830 * pos = 0 * 2 + 0 pos = 0 * 2 + 1 1831 * pos: 0 pos: 1 1832 * 1833 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1 1834 * pos = 1 * 2 + 0 pos = 1 * 2 + 1 1835 * pos: 2 pos = 3 1836 * 1837 * Note that while this example is simple, the method applies to more 1838 * complex topologies, including those with switches. 1839 */ 1840 1841 /* Iterate from endpoint to root_port refining the position */ 1842 for (iter = port; iter; iter = next_port(iter)) { 1843 if (is_cxl_root(iter)) 1844 break; 1845 1846 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways); 1847 if (rc) 1848 return rc; 1849 1850 pos = pos * parent_ways + parent_pos; 1851 } 1852 1853 dev_dbg(&cxlmd->dev, 1854 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n", 1855 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent), 1856 dev_name(&port->dev), range->start, range->end, pos); 1857 1858 return pos; 1859 } 1860 1861 static int cxl_region_sort_targets(struct cxl_region *cxlr) 1862 { 1863 struct cxl_region_params *p = &cxlr->params; 1864 int i, rc = 0; 1865 1866 for (i = 0; i < p->nr_targets; i++) { 1867 struct cxl_endpoint_decoder *cxled = p->targets[i]; 1868 1869 cxled->pos = cxl_calc_interleave_pos(cxled); 1870 /* 1871 * Record that sorting failed, but still continue to calc 1872 * cxled->pos so that follow-on code paths can reliably 1873 * do p->targets[cxled->pos] to self-reference their entry. 1874 */ 1875 if (cxled->pos < 0) 1876 rc = -ENXIO; 1877 } 1878 /* Keep the cxlr target list in interleave position order */ 1879 sort(p->targets, p->nr_targets, sizeof(p->targets[0]), 1880 cmp_interleave_pos, NULL); 1881 1882 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful"); 1883 return rc; 1884 } 1885 1886 static int cxl_region_attach(struct cxl_region *cxlr, 1887 struct cxl_endpoint_decoder *cxled, int pos) 1888 { 1889 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 1890 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1891 struct cxl_region_params *p = &cxlr->params; 1892 struct cxl_port *ep_port, *root_port; 1893 struct cxl_dport *dport; 1894 int rc = -ENXIO; 1895 1896 rc = check_interleave_cap(&cxled->cxld, p->interleave_ways, 1897 p->interleave_granularity); 1898 if (rc) { 1899 dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n", 1900 dev_name(&cxled->cxld.dev), p->interleave_ways, 1901 p->interleave_granularity); 1902 return rc; 1903 } 1904 1905 if (cxled->mode != cxlr->mode) { 1906 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch: %d\n", 1907 dev_name(&cxled->cxld.dev), cxlr->mode, cxled->mode); 1908 return -EINVAL; 1909 } 1910 1911 if (cxled->mode == CXL_DECODER_DEAD) { 1912 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev)); 1913 return -ENODEV; 1914 } 1915 1916 /* all full of members, or interleave config not established? */ 1917 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) { 1918 dev_dbg(&cxlr->dev, "region already active\n"); 1919 return -EBUSY; 1920 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) { 1921 dev_dbg(&cxlr->dev, "interleave config missing\n"); 1922 return -ENXIO; 1923 } 1924 1925 if (p->nr_targets >= p->interleave_ways) { 1926 dev_dbg(&cxlr->dev, "region already has %d endpoints\n", 1927 p->nr_targets); 1928 return -EINVAL; 1929 } 1930 1931 ep_port = cxled_to_port(cxled); 1932 root_port = cxlrd_to_port(cxlrd); 1933 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge); 1934 if (!dport) { 1935 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n", 1936 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1937 dev_name(cxlr->dev.parent)); 1938 return -ENXIO; 1939 } 1940 1941 if (cxled->cxld.target_type != cxlr->type) { 1942 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n", 1943 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1944 cxled->cxld.target_type, cxlr->type); 1945 return -ENXIO; 1946 } 1947 1948 if (!cxled->dpa_res) { 1949 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n", 1950 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev)); 1951 return -ENXIO; 1952 } 1953 1954 if (resource_size(cxled->dpa_res) * p->interleave_ways != 1955 resource_size(p->res)) { 1956 dev_dbg(&cxlr->dev, 1957 "%s:%s: decoder-size-%#llx * ways-%d != region-size-%#llx\n", 1958 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1959 (u64)resource_size(cxled->dpa_res), p->interleave_ways, 1960 (u64)resource_size(p->res)); 1961 return -EINVAL; 1962 } 1963 1964 cxl_region_perf_data_calculate(cxlr, cxled); 1965 1966 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1967 int i; 1968 1969 rc = cxl_region_attach_auto(cxlr, cxled, pos); 1970 if (rc) 1971 return rc; 1972 1973 /* await more targets to arrive... */ 1974 if (p->nr_targets < p->interleave_ways) 1975 return 0; 1976 1977 /* 1978 * All targets are here, which implies all PCI enumeration that 1979 * affects this region has been completed. Walk the topology to 1980 * sort the devices into their relative region decode position. 1981 */ 1982 rc = cxl_region_sort_targets(cxlr); 1983 if (rc) 1984 return rc; 1985 1986 for (i = 0; i < p->nr_targets; i++) { 1987 cxled = p->targets[i]; 1988 ep_port = cxled_to_port(cxled); 1989 dport = cxl_find_dport_by_dev(root_port, 1990 ep_port->host_bridge); 1991 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, 1992 dport, i); 1993 if (rc) 1994 return rc; 1995 } 1996 1997 rc = cxl_region_setup_targets(cxlr); 1998 if (rc) 1999 return rc; 2000 2001 /* 2002 * If target setup succeeds in the autodiscovery case 2003 * then the region is already committed. 2004 */ 2005 p->state = CXL_CONFIG_COMMIT; 2006 cxl_region_shared_upstream_bandwidth_update(cxlr); 2007 2008 return 0; 2009 } 2010 2011 rc = cxl_region_validate_position(cxlr, cxled, pos); 2012 if (rc) 2013 return rc; 2014 2015 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos); 2016 if (rc) 2017 return rc; 2018 2019 p->targets[pos] = cxled; 2020 cxled->pos = pos; 2021 p->nr_targets++; 2022 2023 if (p->nr_targets == p->interleave_ways) { 2024 rc = cxl_region_setup_targets(cxlr); 2025 if (rc) 2026 return rc; 2027 p->state = CXL_CONFIG_ACTIVE; 2028 cxl_region_shared_upstream_bandwidth_update(cxlr); 2029 } 2030 2031 cxled->cxld.interleave_ways = p->interleave_ways; 2032 cxled->cxld.interleave_granularity = p->interleave_granularity; 2033 cxled->cxld.hpa_range = (struct range) { 2034 .start = p->res->start, 2035 .end = p->res->end, 2036 }; 2037 2038 if (p->nr_targets != p->interleave_ways) 2039 return 0; 2040 2041 /* 2042 * Test the auto-discovery position calculator function 2043 * against this successfully created user-defined region. 2044 * A fail message here means that this interleave config 2045 * will fail when presented as CXL_REGION_F_AUTO. 2046 */ 2047 for (int i = 0; i < p->nr_targets; i++) { 2048 struct cxl_endpoint_decoder *cxled = p->targets[i]; 2049 int test_pos; 2050 2051 test_pos = cxl_calc_interleave_pos(cxled); 2052 dev_dbg(&cxled->cxld.dev, 2053 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n", 2054 (test_pos == cxled->pos) ? "success" : "fail", 2055 test_pos, cxled->pos); 2056 } 2057 2058 return 0; 2059 } 2060 2061 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled) 2062 { 2063 struct cxl_port *iter, *ep_port = cxled_to_port(cxled); 2064 struct cxl_region *cxlr = cxled->cxld.region; 2065 struct cxl_region_params *p; 2066 int rc = 0; 2067 2068 lockdep_assert_held_write(&cxl_region_rwsem); 2069 2070 if (!cxlr) 2071 return 0; 2072 2073 p = &cxlr->params; 2074 get_device(&cxlr->dev); 2075 2076 if (p->state > CXL_CONFIG_ACTIVE) { 2077 cxl_region_decode_reset(cxlr, p->interleave_ways); 2078 p->state = CXL_CONFIG_ACTIVE; 2079 } 2080 2081 for (iter = ep_port; !is_cxl_root(iter); 2082 iter = to_cxl_port(iter->dev.parent)) 2083 cxl_port_detach_region(iter, cxlr, cxled); 2084 2085 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways || 2086 p->targets[cxled->pos] != cxled) { 2087 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 2088 2089 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n", 2090 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 2091 cxled->pos); 2092 goto out; 2093 } 2094 2095 if (p->state == CXL_CONFIG_ACTIVE) { 2096 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 2097 cxl_region_teardown_targets(cxlr); 2098 } 2099 p->targets[cxled->pos] = NULL; 2100 p->nr_targets--; 2101 cxled->cxld.hpa_range = (struct range) { 2102 .start = 0, 2103 .end = -1, 2104 }; 2105 2106 /* notify the region driver that one of its targets has departed */ 2107 up_write(&cxl_region_rwsem); 2108 device_release_driver(&cxlr->dev); 2109 down_write(&cxl_region_rwsem); 2110 out: 2111 put_device(&cxlr->dev); 2112 return rc; 2113 } 2114 2115 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled) 2116 { 2117 down_write(&cxl_region_rwsem); 2118 cxled->mode = CXL_DECODER_DEAD; 2119 cxl_region_detach(cxled); 2120 up_write(&cxl_region_rwsem); 2121 } 2122 2123 static int attach_target(struct cxl_region *cxlr, 2124 struct cxl_endpoint_decoder *cxled, int pos, 2125 unsigned int state) 2126 { 2127 int rc = 0; 2128 2129 if (state == TASK_INTERRUPTIBLE) 2130 rc = down_write_killable(&cxl_region_rwsem); 2131 else 2132 down_write(&cxl_region_rwsem); 2133 if (rc) 2134 return rc; 2135 2136 down_read(&cxl_dpa_rwsem); 2137 rc = cxl_region_attach(cxlr, cxled, pos); 2138 up_read(&cxl_dpa_rwsem); 2139 up_write(&cxl_region_rwsem); 2140 return rc; 2141 } 2142 2143 static int detach_target(struct cxl_region *cxlr, int pos) 2144 { 2145 struct cxl_region_params *p = &cxlr->params; 2146 int rc; 2147 2148 rc = down_write_killable(&cxl_region_rwsem); 2149 if (rc) 2150 return rc; 2151 2152 if (pos >= p->interleave_ways) { 2153 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 2154 p->interleave_ways); 2155 rc = -ENXIO; 2156 goto out; 2157 } 2158 2159 if (!p->targets[pos]) { 2160 rc = 0; 2161 goto out; 2162 } 2163 2164 rc = cxl_region_detach(p->targets[pos]); 2165 out: 2166 up_write(&cxl_region_rwsem); 2167 return rc; 2168 } 2169 2170 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos, 2171 size_t len) 2172 { 2173 int rc; 2174 2175 if (sysfs_streq(buf, "\n")) 2176 rc = detach_target(cxlr, pos); 2177 else { 2178 struct device *dev; 2179 2180 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf); 2181 if (!dev) 2182 return -ENODEV; 2183 2184 if (!is_endpoint_decoder(dev)) { 2185 rc = -EINVAL; 2186 goto out; 2187 } 2188 2189 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos, 2190 TASK_INTERRUPTIBLE); 2191 out: 2192 put_device(dev); 2193 } 2194 2195 if (rc < 0) 2196 return rc; 2197 return len; 2198 } 2199 2200 #define TARGET_ATTR_RW(n) \ 2201 static ssize_t target##n##_show( \ 2202 struct device *dev, struct device_attribute *attr, char *buf) \ 2203 { \ 2204 return show_targetN(to_cxl_region(dev), buf, (n)); \ 2205 } \ 2206 static ssize_t target##n##_store(struct device *dev, \ 2207 struct device_attribute *attr, \ 2208 const char *buf, size_t len) \ 2209 { \ 2210 return store_targetN(to_cxl_region(dev), buf, (n), len); \ 2211 } \ 2212 static DEVICE_ATTR_RW(target##n) 2213 2214 TARGET_ATTR_RW(0); 2215 TARGET_ATTR_RW(1); 2216 TARGET_ATTR_RW(2); 2217 TARGET_ATTR_RW(3); 2218 TARGET_ATTR_RW(4); 2219 TARGET_ATTR_RW(5); 2220 TARGET_ATTR_RW(6); 2221 TARGET_ATTR_RW(7); 2222 TARGET_ATTR_RW(8); 2223 TARGET_ATTR_RW(9); 2224 TARGET_ATTR_RW(10); 2225 TARGET_ATTR_RW(11); 2226 TARGET_ATTR_RW(12); 2227 TARGET_ATTR_RW(13); 2228 TARGET_ATTR_RW(14); 2229 TARGET_ATTR_RW(15); 2230 2231 static struct attribute *target_attrs[] = { 2232 &dev_attr_target0.attr, 2233 &dev_attr_target1.attr, 2234 &dev_attr_target2.attr, 2235 &dev_attr_target3.attr, 2236 &dev_attr_target4.attr, 2237 &dev_attr_target5.attr, 2238 &dev_attr_target6.attr, 2239 &dev_attr_target7.attr, 2240 &dev_attr_target8.attr, 2241 &dev_attr_target9.attr, 2242 &dev_attr_target10.attr, 2243 &dev_attr_target11.attr, 2244 &dev_attr_target12.attr, 2245 &dev_attr_target13.attr, 2246 &dev_attr_target14.attr, 2247 &dev_attr_target15.attr, 2248 NULL, 2249 }; 2250 2251 static umode_t cxl_region_target_visible(struct kobject *kobj, 2252 struct attribute *a, int n) 2253 { 2254 struct device *dev = kobj_to_dev(kobj); 2255 struct cxl_region *cxlr = to_cxl_region(dev); 2256 struct cxl_region_params *p = &cxlr->params; 2257 2258 if (n < p->interleave_ways) 2259 return a->mode; 2260 return 0; 2261 } 2262 2263 static const struct attribute_group cxl_region_target_group = { 2264 .attrs = target_attrs, 2265 .is_visible = cxl_region_target_visible, 2266 }; 2267 2268 static const struct attribute_group *get_cxl_region_target_group(void) 2269 { 2270 return &cxl_region_target_group; 2271 } 2272 2273 static const struct attribute_group *region_groups[] = { 2274 &cxl_base_attribute_group, 2275 &cxl_region_group, 2276 &cxl_region_target_group, 2277 &cxl_region_access0_coordinate_group, 2278 &cxl_region_access1_coordinate_group, 2279 NULL, 2280 }; 2281 2282 static void cxl_region_release(struct device *dev) 2283 { 2284 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 2285 struct cxl_region *cxlr = to_cxl_region(dev); 2286 int id = atomic_read(&cxlrd->region_id); 2287 2288 /* 2289 * Try to reuse the recently idled id rather than the cached 2290 * next id to prevent the region id space from increasing 2291 * unnecessarily. 2292 */ 2293 if (cxlr->id < id) 2294 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) { 2295 memregion_free(id); 2296 goto out; 2297 } 2298 2299 memregion_free(cxlr->id); 2300 out: 2301 put_device(dev->parent); 2302 kfree(cxlr); 2303 } 2304 2305 const struct device_type cxl_region_type = { 2306 .name = "cxl_region", 2307 .release = cxl_region_release, 2308 .groups = region_groups 2309 }; 2310 2311 bool is_cxl_region(struct device *dev) 2312 { 2313 return dev->type == &cxl_region_type; 2314 } 2315 EXPORT_SYMBOL_NS_GPL(is_cxl_region, "CXL"); 2316 2317 static struct cxl_region *to_cxl_region(struct device *dev) 2318 { 2319 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type, 2320 "not a cxl_region device\n")) 2321 return NULL; 2322 2323 return container_of(dev, struct cxl_region, dev); 2324 } 2325 2326 static void unregister_region(void *_cxlr) 2327 { 2328 struct cxl_region *cxlr = _cxlr; 2329 struct cxl_region_params *p = &cxlr->params; 2330 int i; 2331 2332 device_del(&cxlr->dev); 2333 2334 /* 2335 * Now that region sysfs is shutdown, the parameter block is now 2336 * read-only, so no need to hold the region rwsem to access the 2337 * region parameters. 2338 */ 2339 for (i = 0; i < p->interleave_ways; i++) 2340 detach_target(cxlr, i); 2341 2342 cxl_region_iomem_release(cxlr); 2343 put_device(&cxlr->dev); 2344 } 2345 2346 static struct lock_class_key cxl_region_key; 2347 2348 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id) 2349 { 2350 struct cxl_region *cxlr; 2351 struct device *dev; 2352 2353 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL); 2354 if (!cxlr) { 2355 memregion_free(id); 2356 return ERR_PTR(-ENOMEM); 2357 } 2358 2359 dev = &cxlr->dev; 2360 device_initialize(dev); 2361 lockdep_set_class(&dev->mutex, &cxl_region_key); 2362 dev->parent = &cxlrd->cxlsd.cxld.dev; 2363 /* 2364 * Keep root decoder pinned through cxl_region_release to fixup 2365 * region id allocations 2366 */ 2367 get_device(dev->parent); 2368 device_set_pm_not_required(dev); 2369 dev->bus = &cxl_bus_type; 2370 dev->type = &cxl_region_type; 2371 cxlr->id = id; 2372 2373 return cxlr; 2374 } 2375 2376 static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) 2377 { 2378 int cset = 0; 2379 int rc; 2380 2381 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { 2382 if (cxlr->coord[i].read_bandwidth) { 2383 rc = 0; 2384 if (cxl_need_node_perf_attrs_update(nid)) 2385 node_set_perf_attrs(nid, &cxlr->coord[i], i); 2386 else 2387 rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); 2388 2389 if (rc == 0) 2390 cset++; 2391 } 2392 } 2393 2394 if (!cset) 2395 return false; 2396 2397 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group()); 2398 if (rc) 2399 dev_dbg(&cxlr->dev, "Failed to update access0 group\n"); 2400 2401 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group()); 2402 if (rc) 2403 dev_dbg(&cxlr->dev, "Failed to update access1 group\n"); 2404 2405 return true; 2406 } 2407 2408 static int cxl_region_perf_attrs_callback(struct notifier_block *nb, 2409 unsigned long action, void *arg) 2410 { 2411 struct cxl_region *cxlr = container_of(nb, struct cxl_region, 2412 memory_notifier); 2413 struct memory_notify *mnb = arg; 2414 int nid = mnb->status_change_nid; 2415 int region_nid; 2416 2417 if (nid == NUMA_NO_NODE || action != MEM_ONLINE) 2418 return NOTIFY_DONE; 2419 2420 /* 2421 * No need to hold cxl_region_rwsem; region parameters are stable 2422 * within the cxl_region driver. 2423 */ 2424 region_nid = phys_to_target_node(cxlr->params.res->start); 2425 if (nid != region_nid) 2426 return NOTIFY_DONE; 2427 2428 if (!cxl_region_update_coordinates(cxlr, nid)) 2429 return NOTIFY_DONE; 2430 2431 return NOTIFY_OK; 2432 } 2433 2434 static int cxl_region_calculate_adistance(struct notifier_block *nb, 2435 unsigned long nid, void *data) 2436 { 2437 struct cxl_region *cxlr = container_of(nb, struct cxl_region, 2438 adist_notifier); 2439 struct access_coordinate *perf; 2440 int *adist = data; 2441 int region_nid; 2442 2443 /* 2444 * No need to hold cxl_region_rwsem; region parameters are stable 2445 * within the cxl_region driver. 2446 */ 2447 region_nid = phys_to_target_node(cxlr->params.res->start); 2448 if (nid != region_nid) 2449 return NOTIFY_OK; 2450 2451 perf = &cxlr->coord[ACCESS_COORDINATE_CPU]; 2452 2453 if (mt_perf_to_adistance(perf, adist)) 2454 return NOTIFY_OK; 2455 2456 return NOTIFY_STOP; 2457 } 2458 2459 /** 2460 * devm_cxl_add_region - Adds a region to a decoder 2461 * @cxlrd: root decoder 2462 * @id: memregion id to create, or memregion_free() on failure 2463 * @mode: mode for the endpoint decoders of this region 2464 * @type: select whether this is an expander or accelerator (type-2 or type-3) 2465 * 2466 * This is the second step of region initialization. Regions exist within an 2467 * address space which is mapped by a @cxlrd. 2468 * 2469 * Return: 0 if the region was added to the @cxlrd, else returns negative error 2470 * code. The region will be named "regionZ" where Z is the unique region number. 2471 */ 2472 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, 2473 int id, 2474 enum cxl_decoder_mode mode, 2475 enum cxl_decoder_type type) 2476 { 2477 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent); 2478 struct cxl_region *cxlr; 2479 struct device *dev; 2480 int rc; 2481 2482 cxlr = cxl_region_alloc(cxlrd, id); 2483 if (IS_ERR(cxlr)) 2484 return cxlr; 2485 cxlr->mode = mode; 2486 cxlr->type = type; 2487 2488 dev = &cxlr->dev; 2489 rc = dev_set_name(dev, "region%d", id); 2490 if (rc) 2491 goto err; 2492 2493 rc = device_add(dev); 2494 if (rc) 2495 goto err; 2496 2497 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); 2498 if (rc) 2499 return ERR_PTR(rc); 2500 2501 dev_dbg(port->uport_dev, "%s: created %s\n", 2502 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); 2503 return cxlr; 2504 2505 err: 2506 put_device(dev); 2507 return ERR_PTR(rc); 2508 } 2509 2510 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf) 2511 { 2512 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id)); 2513 } 2514 2515 static ssize_t create_pmem_region_show(struct device *dev, 2516 struct device_attribute *attr, char *buf) 2517 { 2518 return __create_region_show(to_cxl_root_decoder(dev), buf); 2519 } 2520 2521 static ssize_t create_ram_region_show(struct device *dev, 2522 struct device_attribute *attr, char *buf) 2523 { 2524 return __create_region_show(to_cxl_root_decoder(dev), buf); 2525 } 2526 2527 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd, 2528 enum cxl_decoder_mode mode, int id) 2529 { 2530 int rc; 2531 2532 switch (mode) { 2533 case CXL_DECODER_RAM: 2534 case CXL_DECODER_PMEM: 2535 break; 2536 default: 2537 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode); 2538 return ERR_PTR(-EINVAL); 2539 } 2540 2541 rc = memregion_alloc(GFP_KERNEL); 2542 if (rc < 0) 2543 return ERR_PTR(rc); 2544 2545 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) { 2546 memregion_free(rc); 2547 return ERR_PTR(-EBUSY); 2548 } 2549 2550 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM); 2551 } 2552 2553 static ssize_t create_region_store(struct device *dev, const char *buf, 2554 size_t len, enum cxl_decoder_mode mode) 2555 { 2556 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2557 struct cxl_region *cxlr; 2558 int rc, id; 2559 2560 rc = sscanf(buf, "region%d\n", &id); 2561 if (rc != 1) 2562 return -EINVAL; 2563 2564 cxlr = __create_region(cxlrd, mode, id); 2565 if (IS_ERR(cxlr)) 2566 return PTR_ERR(cxlr); 2567 2568 return len; 2569 } 2570 2571 static ssize_t create_pmem_region_store(struct device *dev, 2572 struct device_attribute *attr, 2573 const char *buf, size_t len) 2574 { 2575 return create_region_store(dev, buf, len, CXL_DECODER_PMEM); 2576 } 2577 DEVICE_ATTR_RW(create_pmem_region); 2578 2579 static ssize_t create_ram_region_store(struct device *dev, 2580 struct device_attribute *attr, 2581 const char *buf, size_t len) 2582 { 2583 return create_region_store(dev, buf, len, CXL_DECODER_RAM); 2584 } 2585 DEVICE_ATTR_RW(create_ram_region); 2586 2587 static ssize_t region_show(struct device *dev, struct device_attribute *attr, 2588 char *buf) 2589 { 2590 struct cxl_decoder *cxld = to_cxl_decoder(dev); 2591 ssize_t rc; 2592 2593 rc = down_read_interruptible(&cxl_region_rwsem); 2594 if (rc) 2595 return rc; 2596 2597 if (cxld->region) 2598 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev)); 2599 else 2600 rc = sysfs_emit(buf, "\n"); 2601 up_read(&cxl_region_rwsem); 2602 2603 return rc; 2604 } 2605 DEVICE_ATTR_RO(region); 2606 2607 static struct cxl_region * 2608 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name) 2609 { 2610 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 2611 struct device *region_dev; 2612 2613 region_dev = device_find_child_by_name(&cxld->dev, name); 2614 if (!region_dev) 2615 return ERR_PTR(-ENODEV); 2616 2617 return to_cxl_region(region_dev); 2618 } 2619 2620 static ssize_t delete_region_store(struct device *dev, 2621 struct device_attribute *attr, 2622 const char *buf, size_t len) 2623 { 2624 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2625 struct cxl_port *port = to_cxl_port(dev->parent); 2626 struct cxl_region *cxlr; 2627 2628 cxlr = cxl_find_region_by_name(cxlrd, buf); 2629 if (IS_ERR(cxlr)) 2630 return PTR_ERR(cxlr); 2631 2632 devm_release_action(port->uport_dev, unregister_region, cxlr); 2633 put_device(&cxlr->dev); 2634 2635 return len; 2636 } 2637 DEVICE_ATTR_WO(delete_region); 2638 2639 static void cxl_pmem_region_release(struct device *dev) 2640 { 2641 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); 2642 int i; 2643 2644 for (i = 0; i < cxlr_pmem->nr_mappings; i++) { 2645 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd; 2646 2647 put_device(&cxlmd->dev); 2648 } 2649 2650 kfree(cxlr_pmem); 2651 } 2652 2653 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = { 2654 &cxl_base_attribute_group, 2655 NULL, 2656 }; 2657 2658 const struct device_type cxl_pmem_region_type = { 2659 .name = "cxl_pmem_region", 2660 .release = cxl_pmem_region_release, 2661 .groups = cxl_pmem_region_attribute_groups, 2662 }; 2663 2664 bool is_cxl_pmem_region(struct device *dev) 2665 { 2666 return dev->type == &cxl_pmem_region_type; 2667 } 2668 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, "CXL"); 2669 2670 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev) 2671 { 2672 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev), 2673 "not a cxl_pmem_region device\n")) 2674 return NULL; 2675 return container_of(dev, struct cxl_pmem_region, dev); 2676 } 2677 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, "CXL"); 2678 2679 struct cxl_poison_context { 2680 struct cxl_port *port; 2681 enum cxl_decoder_mode mode; 2682 u64 offset; 2683 }; 2684 2685 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd, 2686 struct cxl_poison_context *ctx) 2687 { 2688 struct cxl_dev_state *cxlds = cxlmd->cxlds; 2689 u64 offset, length; 2690 int rc = 0; 2691 2692 /* 2693 * Collect poison for the remaining unmapped resources 2694 * after poison is collected by committed endpoints. 2695 * 2696 * Knowing that PMEM must always follow RAM, get poison 2697 * for unmapped resources based on the last decoder's mode: 2698 * ram: scan remains of ram range, then any pmem range 2699 * pmem: scan remains of pmem range 2700 */ 2701 2702 if (ctx->mode == CXL_DECODER_RAM) { 2703 offset = ctx->offset; 2704 length = resource_size(&cxlds->ram_res) - offset; 2705 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 2706 if (rc == -EFAULT) 2707 rc = 0; 2708 if (rc) 2709 return rc; 2710 } 2711 if (ctx->mode == CXL_DECODER_PMEM) { 2712 offset = ctx->offset; 2713 length = resource_size(&cxlds->dpa_res) - offset; 2714 if (!length) 2715 return 0; 2716 } else if (resource_size(&cxlds->pmem_res)) { 2717 offset = cxlds->pmem_res.start; 2718 length = resource_size(&cxlds->pmem_res); 2719 } else { 2720 return 0; 2721 } 2722 2723 return cxl_mem_get_poison(cxlmd, offset, length, NULL); 2724 } 2725 2726 static int poison_by_decoder(struct device *dev, void *arg) 2727 { 2728 struct cxl_poison_context *ctx = arg; 2729 struct cxl_endpoint_decoder *cxled; 2730 struct cxl_memdev *cxlmd; 2731 u64 offset, length; 2732 int rc = 0; 2733 2734 if (!is_endpoint_decoder(dev)) 2735 return rc; 2736 2737 cxled = to_cxl_endpoint_decoder(dev); 2738 if (!cxled->dpa_res || !resource_size(cxled->dpa_res)) 2739 return rc; 2740 2741 /* 2742 * Regions are only created with single mode decoders: pmem or ram. 2743 * Linux does not support mixed mode decoders. This means that 2744 * reading poison per endpoint decoder adheres to the requirement 2745 * that poison reads of pmem and ram must be separated. 2746 * CXL 3.0 Spec 8.2.9.8.4.1 2747 */ 2748 if (cxled->mode == CXL_DECODER_MIXED) { 2749 dev_dbg(dev, "poison list read unsupported in mixed mode\n"); 2750 return rc; 2751 } 2752 2753 cxlmd = cxled_to_memdev(cxled); 2754 if (cxled->skip) { 2755 offset = cxled->dpa_res->start - cxled->skip; 2756 length = cxled->skip; 2757 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 2758 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM) 2759 rc = 0; 2760 if (rc) 2761 return rc; 2762 } 2763 2764 offset = cxled->dpa_res->start; 2765 length = cxled->dpa_res->end - offset + 1; 2766 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region); 2767 if (rc == -EFAULT && cxled->mode == CXL_DECODER_RAM) 2768 rc = 0; 2769 if (rc) 2770 return rc; 2771 2772 /* Iterate until commit_end is reached */ 2773 if (cxled->cxld.id == ctx->port->commit_end) { 2774 ctx->offset = cxled->dpa_res->end + 1; 2775 ctx->mode = cxled->mode; 2776 return 1; 2777 } 2778 2779 return 0; 2780 } 2781 2782 int cxl_get_poison_by_endpoint(struct cxl_port *port) 2783 { 2784 struct cxl_poison_context ctx; 2785 int rc = 0; 2786 2787 ctx = (struct cxl_poison_context) { 2788 .port = port 2789 }; 2790 2791 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder); 2792 if (rc == 1) 2793 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev), 2794 &ctx); 2795 2796 return rc; 2797 } 2798 2799 struct cxl_dpa_to_region_context { 2800 struct cxl_region *cxlr; 2801 u64 dpa; 2802 }; 2803 2804 static int __cxl_dpa_to_region(struct device *dev, void *arg) 2805 { 2806 struct cxl_dpa_to_region_context *ctx = arg; 2807 struct cxl_endpoint_decoder *cxled; 2808 struct cxl_region *cxlr; 2809 u64 dpa = ctx->dpa; 2810 2811 if (!is_endpoint_decoder(dev)) 2812 return 0; 2813 2814 cxled = to_cxl_endpoint_decoder(dev); 2815 if (!cxled || !cxled->dpa_res || !resource_size(cxled->dpa_res)) 2816 return 0; 2817 2818 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start) 2819 return 0; 2820 2821 /* 2822 * Stop the region search (return 1) when an endpoint mapping is 2823 * found. The region may not be fully constructed so offering 2824 * the cxlr in the context structure is not guaranteed. 2825 */ 2826 cxlr = cxled->cxld.region; 2827 if (cxlr) 2828 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa, 2829 dev_name(&cxlr->dev)); 2830 else 2831 dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa, 2832 dev_name(dev)); 2833 2834 ctx->cxlr = cxlr; 2835 2836 return 1; 2837 } 2838 2839 struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa) 2840 { 2841 struct cxl_dpa_to_region_context ctx; 2842 struct cxl_port *port; 2843 2844 ctx = (struct cxl_dpa_to_region_context) { 2845 .dpa = dpa, 2846 }; 2847 port = cxlmd->endpoint; 2848 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port)) 2849 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region); 2850 2851 return ctx.cxlr; 2852 } 2853 2854 static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos) 2855 { 2856 struct cxl_region_params *p = &cxlr->params; 2857 int gran = p->interleave_granularity; 2858 int ways = p->interleave_ways; 2859 u64 offset; 2860 2861 /* Is the hpa in an expected chunk for its pos(-ition) */ 2862 offset = hpa - p->res->start; 2863 offset = do_div(offset, gran * ways); 2864 if ((offset >= pos * gran) && (offset < (pos + 1) * gran)) 2865 return true; 2866 2867 dev_dbg(&cxlr->dev, 2868 "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa); 2869 2870 return false; 2871 } 2872 2873 u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, 2874 u64 dpa) 2875 { 2876 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 2877 u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa; 2878 struct cxl_region_params *p = &cxlr->params; 2879 struct cxl_endpoint_decoder *cxled = NULL; 2880 u16 eig = 0; 2881 u8 eiw = 0; 2882 int pos; 2883 2884 for (int i = 0; i < p->nr_targets; i++) { 2885 cxled = p->targets[i]; 2886 if (cxlmd == cxled_to_memdev(cxled)) 2887 break; 2888 } 2889 if (!cxled || cxlmd != cxled_to_memdev(cxled)) 2890 return ULLONG_MAX; 2891 2892 pos = cxled->pos; 2893 ways_to_eiw(p->interleave_ways, &eiw); 2894 granularity_to_eig(p->interleave_granularity, &eig); 2895 2896 /* 2897 * The device position in the region interleave set was removed 2898 * from the offset at HPA->DPA translation. To reconstruct the 2899 * HPA, place the 'pos' in the offset. 2900 * 2901 * The placement of 'pos' in the HPA is determined by interleave 2902 * ways and granularity and is defined in the CXL Spec 3.0 Section 2903 * 8.2.4.19.13 Implementation Note: Device Decode Logic 2904 */ 2905 2906 /* Remove the dpa base */ 2907 dpa_offset = dpa - cxl_dpa_resource_start(cxled); 2908 2909 mask_upper = GENMASK_ULL(51, eig + 8); 2910 2911 if (eiw < 8) { 2912 hpa_offset = (dpa_offset & mask_upper) << eiw; 2913 hpa_offset |= pos << (eig + 8); 2914 } else { 2915 bits_upper = (dpa_offset & mask_upper) >> (eig + 8); 2916 bits_upper = bits_upper * 3; 2917 hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8); 2918 } 2919 2920 /* The lower bits remain unchanged */ 2921 hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0); 2922 2923 /* Apply the hpa_offset to the region base address */ 2924 hpa = hpa_offset + p->res->start; 2925 2926 /* Root decoder translation overrides typical modulo decode */ 2927 if (cxlrd->hpa_to_spa) 2928 hpa = cxlrd->hpa_to_spa(cxlrd, hpa); 2929 2930 if (hpa < p->res->start || hpa > p->res->end) { 2931 dev_dbg(&cxlr->dev, 2932 "Addr trans fail: hpa 0x%llx not in region\n", hpa); 2933 return ULLONG_MAX; 2934 } 2935 2936 /* Simple chunk check, by pos & gran, only applies to modulo decodes */ 2937 if (!cxlrd->hpa_to_spa && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos))) 2938 return ULLONG_MAX; 2939 2940 return hpa; 2941 } 2942 2943 static struct lock_class_key cxl_pmem_region_key; 2944 2945 static int cxl_pmem_region_alloc(struct cxl_region *cxlr) 2946 { 2947 struct cxl_region_params *p = &cxlr->params; 2948 struct cxl_nvdimm_bridge *cxl_nvb; 2949 struct device *dev; 2950 int i; 2951 2952 guard(rwsem_read)(&cxl_region_rwsem); 2953 if (p->state != CXL_CONFIG_COMMIT) 2954 return -ENXIO; 2955 2956 struct cxl_pmem_region *cxlr_pmem __free(kfree) = 2957 kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL); 2958 if (!cxlr_pmem) 2959 return -ENOMEM; 2960 2961 cxlr_pmem->hpa_range.start = p->res->start; 2962 cxlr_pmem->hpa_range.end = p->res->end; 2963 2964 /* Snapshot the region configuration underneath the cxl_region_rwsem */ 2965 cxlr_pmem->nr_mappings = p->nr_targets; 2966 for (i = 0; i < p->nr_targets; i++) { 2967 struct cxl_endpoint_decoder *cxled = p->targets[i]; 2968 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 2969 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; 2970 2971 /* 2972 * Regions never span CXL root devices, so by definition the 2973 * bridge for one device is the same for all. 2974 */ 2975 if (i == 0) { 2976 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd->endpoint); 2977 if (!cxl_nvb) 2978 return -ENODEV; 2979 cxlr->cxl_nvb = cxl_nvb; 2980 } 2981 m->cxlmd = cxlmd; 2982 get_device(&cxlmd->dev); 2983 m->start = cxled->dpa_res->start; 2984 m->size = resource_size(cxled->dpa_res); 2985 m->position = i; 2986 } 2987 2988 dev = &cxlr_pmem->dev; 2989 device_initialize(dev); 2990 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key); 2991 device_set_pm_not_required(dev); 2992 dev->parent = &cxlr->dev; 2993 dev->bus = &cxl_bus_type; 2994 dev->type = &cxl_pmem_region_type; 2995 cxlr_pmem->cxlr = cxlr; 2996 cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem); 2997 2998 return 0; 2999 } 3000 3001 static void cxl_dax_region_release(struct device *dev) 3002 { 3003 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev); 3004 3005 kfree(cxlr_dax); 3006 } 3007 3008 static const struct attribute_group *cxl_dax_region_attribute_groups[] = { 3009 &cxl_base_attribute_group, 3010 NULL, 3011 }; 3012 3013 const struct device_type cxl_dax_region_type = { 3014 .name = "cxl_dax_region", 3015 .release = cxl_dax_region_release, 3016 .groups = cxl_dax_region_attribute_groups, 3017 }; 3018 3019 static bool is_cxl_dax_region(struct device *dev) 3020 { 3021 return dev->type == &cxl_dax_region_type; 3022 } 3023 3024 struct cxl_dax_region *to_cxl_dax_region(struct device *dev) 3025 { 3026 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev), 3027 "not a cxl_dax_region device\n")) 3028 return NULL; 3029 return container_of(dev, struct cxl_dax_region, dev); 3030 } 3031 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, "CXL"); 3032 3033 static struct lock_class_key cxl_dax_region_key; 3034 3035 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr) 3036 { 3037 struct cxl_region_params *p = &cxlr->params; 3038 struct cxl_dax_region *cxlr_dax; 3039 struct device *dev; 3040 3041 down_read(&cxl_region_rwsem); 3042 if (p->state != CXL_CONFIG_COMMIT) { 3043 cxlr_dax = ERR_PTR(-ENXIO); 3044 goto out; 3045 } 3046 3047 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL); 3048 if (!cxlr_dax) { 3049 cxlr_dax = ERR_PTR(-ENOMEM); 3050 goto out; 3051 } 3052 3053 cxlr_dax->hpa_range.start = p->res->start; 3054 cxlr_dax->hpa_range.end = p->res->end; 3055 3056 dev = &cxlr_dax->dev; 3057 cxlr_dax->cxlr = cxlr; 3058 device_initialize(dev); 3059 lockdep_set_class(&dev->mutex, &cxl_dax_region_key); 3060 device_set_pm_not_required(dev); 3061 dev->parent = &cxlr->dev; 3062 dev->bus = &cxl_bus_type; 3063 dev->type = &cxl_dax_region_type; 3064 out: 3065 up_read(&cxl_region_rwsem); 3066 3067 return cxlr_dax; 3068 } 3069 3070 static void cxlr_pmem_unregister(void *_cxlr_pmem) 3071 { 3072 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem; 3073 struct cxl_region *cxlr = cxlr_pmem->cxlr; 3074 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 3075 3076 /* 3077 * Either the bridge is in ->remove() context under the device_lock(), 3078 * or cxlr_release_nvdimm() is cancelling the bridge's release action 3079 * for @cxlr_pmem and doing it itself (while manually holding the bridge 3080 * lock). 3081 */ 3082 device_lock_assert(&cxl_nvb->dev); 3083 cxlr->cxlr_pmem = NULL; 3084 cxlr_pmem->cxlr = NULL; 3085 device_unregister(&cxlr_pmem->dev); 3086 } 3087 3088 static void cxlr_release_nvdimm(void *_cxlr) 3089 { 3090 struct cxl_region *cxlr = _cxlr; 3091 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 3092 3093 scoped_guard(device, &cxl_nvb->dev) { 3094 if (cxlr->cxlr_pmem) 3095 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister, 3096 cxlr->cxlr_pmem); 3097 } 3098 cxlr->cxl_nvb = NULL; 3099 put_device(&cxl_nvb->dev); 3100 } 3101 3102 /** 3103 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge 3104 * @cxlr: parent CXL region for this pmem region bridge device 3105 * 3106 * Return: 0 on success negative error code on failure. 3107 */ 3108 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr) 3109 { 3110 struct cxl_pmem_region *cxlr_pmem; 3111 struct cxl_nvdimm_bridge *cxl_nvb; 3112 struct device *dev; 3113 int rc; 3114 3115 rc = cxl_pmem_region_alloc(cxlr); 3116 if (rc) 3117 return rc; 3118 cxlr_pmem = cxlr->cxlr_pmem; 3119 cxl_nvb = cxlr->cxl_nvb; 3120 3121 dev = &cxlr_pmem->dev; 3122 rc = dev_set_name(dev, "pmem_region%d", cxlr->id); 3123 if (rc) 3124 goto err; 3125 3126 rc = device_add(dev); 3127 if (rc) 3128 goto err; 3129 3130 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 3131 dev_name(dev)); 3132 3133 scoped_guard(device, &cxl_nvb->dev) { 3134 if (cxl_nvb->dev.driver) 3135 rc = devm_add_action_or_reset(&cxl_nvb->dev, 3136 cxlr_pmem_unregister, 3137 cxlr_pmem); 3138 else 3139 rc = -ENXIO; 3140 } 3141 3142 if (rc) 3143 goto err_bridge; 3144 3145 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */ 3146 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr); 3147 3148 err: 3149 put_device(dev); 3150 err_bridge: 3151 put_device(&cxl_nvb->dev); 3152 cxlr->cxl_nvb = NULL; 3153 return rc; 3154 } 3155 3156 static void cxlr_dax_unregister(void *_cxlr_dax) 3157 { 3158 struct cxl_dax_region *cxlr_dax = _cxlr_dax; 3159 3160 device_unregister(&cxlr_dax->dev); 3161 } 3162 3163 static int devm_cxl_add_dax_region(struct cxl_region *cxlr) 3164 { 3165 struct cxl_dax_region *cxlr_dax; 3166 struct device *dev; 3167 int rc; 3168 3169 cxlr_dax = cxl_dax_region_alloc(cxlr); 3170 if (IS_ERR(cxlr_dax)) 3171 return PTR_ERR(cxlr_dax); 3172 3173 dev = &cxlr_dax->dev; 3174 rc = dev_set_name(dev, "dax_region%d", cxlr->id); 3175 if (rc) 3176 goto err; 3177 3178 rc = device_add(dev); 3179 if (rc) 3180 goto err; 3181 3182 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 3183 dev_name(dev)); 3184 3185 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister, 3186 cxlr_dax); 3187 err: 3188 put_device(dev); 3189 return rc; 3190 } 3191 3192 static int match_root_decoder_by_range(struct device *dev, 3193 const void *data) 3194 { 3195 const struct range *r1, *r2 = data; 3196 struct cxl_root_decoder *cxlrd; 3197 3198 if (!is_root_decoder(dev)) 3199 return 0; 3200 3201 cxlrd = to_cxl_root_decoder(dev); 3202 r1 = &cxlrd->cxlsd.cxld.hpa_range; 3203 return range_contains(r1, r2); 3204 } 3205 3206 static int match_region_by_range(struct device *dev, const void *data) 3207 { 3208 struct cxl_region_params *p; 3209 struct cxl_region *cxlr; 3210 const struct range *r = data; 3211 int rc = 0; 3212 3213 if (!is_cxl_region(dev)) 3214 return 0; 3215 3216 cxlr = to_cxl_region(dev); 3217 p = &cxlr->params; 3218 3219 down_read(&cxl_region_rwsem); 3220 if (p->res && p->res->start == r->start && p->res->end == r->end) 3221 rc = 1; 3222 up_read(&cxl_region_rwsem); 3223 3224 return rc; 3225 } 3226 3227 /* Establish an empty region covering the given HPA range */ 3228 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, 3229 struct cxl_endpoint_decoder *cxled) 3230 { 3231 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 3232 struct cxl_port *port = cxlrd_to_port(cxlrd); 3233 struct range *hpa = &cxled->cxld.hpa_range; 3234 struct cxl_region_params *p; 3235 struct cxl_region *cxlr; 3236 struct resource *res; 3237 int rc; 3238 3239 do { 3240 cxlr = __create_region(cxlrd, cxled->mode, 3241 atomic_read(&cxlrd->region_id)); 3242 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY); 3243 3244 if (IS_ERR(cxlr)) { 3245 dev_err(cxlmd->dev.parent, 3246 "%s:%s: %s failed assign region: %ld\n", 3247 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 3248 __func__, PTR_ERR(cxlr)); 3249 return cxlr; 3250 } 3251 3252 down_write(&cxl_region_rwsem); 3253 p = &cxlr->params; 3254 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 3255 dev_err(cxlmd->dev.parent, 3256 "%s:%s: %s autodiscovery interrupted\n", 3257 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 3258 __func__); 3259 rc = -EBUSY; 3260 goto err; 3261 } 3262 3263 set_bit(CXL_REGION_F_AUTO, &cxlr->flags); 3264 3265 res = kmalloc(sizeof(*res), GFP_KERNEL); 3266 if (!res) { 3267 rc = -ENOMEM; 3268 goto err; 3269 } 3270 3271 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa), 3272 dev_name(&cxlr->dev)); 3273 rc = insert_resource(cxlrd->res, res); 3274 if (rc) { 3275 /* 3276 * Platform-firmware may not have split resources like "System 3277 * RAM" on CXL window boundaries see cxl_region_iomem_release() 3278 */ 3279 dev_warn(cxlmd->dev.parent, 3280 "%s:%s: %s %s cannot insert resource\n", 3281 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 3282 __func__, dev_name(&cxlr->dev)); 3283 } 3284 3285 p->res = res; 3286 p->interleave_ways = cxled->cxld.interleave_ways; 3287 p->interleave_granularity = cxled->cxld.interleave_granularity; 3288 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 3289 3290 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 3291 if (rc) 3292 goto err; 3293 3294 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n", 3295 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__, 3296 dev_name(&cxlr->dev), p->res, p->interleave_ways, 3297 p->interleave_granularity); 3298 3299 /* ...to match put_device() in cxl_add_to_region() */ 3300 get_device(&cxlr->dev); 3301 up_write(&cxl_region_rwsem); 3302 3303 return cxlr; 3304 3305 err: 3306 up_write(&cxl_region_rwsem); 3307 devm_release_action(port->uport_dev, unregister_region, cxlr); 3308 return ERR_PTR(rc); 3309 } 3310 3311 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled) 3312 { 3313 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 3314 struct range *hpa = &cxled->cxld.hpa_range; 3315 struct cxl_decoder *cxld = &cxled->cxld; 3316 struct device *cxlrd_dev, *region_dev; 3317 struct cxl_root_decoder *cxlrd; 3318 struct cxl_region_params *p; 3319 struct cxl_region *cxlr; 3320 bool attach = false; 3321 int rc; 3322 3323 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range, 3324 match_root_decoder_by_range); 3325 if (!cxlrd_dev) { 3326 dev_err(cxlmd->dev.parent, 3327 "%s:%s no CXL window for range %#llx:%#llx\n", 3328 dev_name(&cxlmd->dev), dev_name(&cxld->dev), 3329 cxld->hpa_range.start, cxld->hpa_range.end); 3330 return -ENXIO; 3331 } 3332 3333 cxlrd = to_cxl_root_decoder(cxlrd_dev); 3334 3335 /* 3336 * Ensure that if multiple threads race to construct_region() for @hpa 3337 * one does the construction and the others add to that. 3338 */ 3339 mutex_lock(&cxlrd->range_lock); 3340 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa, 3341 match_region_by_range); 3342 if (!region_dev) { 3343 cxlr = construct_region(cxlrd, cxled); 3344 region_dev = &cxlr->dev; 3345 } else 3346 cxlr = to_cxl_region(region_dev); 3347 mutex_unlock(&cxlrd->range_lock); 3348 3349 rc = PTR_ERR_OR_ZERO(cxlr); 3350 if (rc) 3351 goto out; 3352 3353 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE); 3354 3355 down_read(&cxl_region_rwsem); 3356 p = &cxlr->params; 3357 attach = p->state == CXL_CONFIG_COMMIT; 3358 up_read(&cxl_region_rwsem); 3359 3360 if (attach) { 3361 /* 3362 * If device_attach() fails the range may still be active via 3363 * the platform-firmware memory map, otherwise the driver for 3364 * regions is local to this file, so driver matching can't fail. 3365 */ 3366 if (device_attach(&cxlr->dev) < 0) 3367 dev_err(&cxlr->dev, "failed to enable, range: %pr\n", 3368 p->res); 3369 } 3370 3371 put_device(region_dev); 3372 out: 3373 put_device(cxlrd_dev); 3374 return rc; 3375 } 3376 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL"); 3377 3378 static int is_system_ram(struct resource *res, void *arg) 3379 { 3380 struct cxl_region *cxlr = arg; 3381 struct cxl_region_params *p = &cxlr->params; 3382 3383 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res); 3384 return 1; 3385 } 3386 3387 static void shutdown_notifiers(void *_cxlr) 3388 { 3389 struct cxl_region *cxlr = _cxlr; 3390 3391 unregister_memory_notifier(&cxlr->memory_notifier); 3392 unregister_mt_adistance_algorithm(&cxlr->adist_notifier); 3393 } 3394 3395 static int cxl_region_probe(struct device *dev) 3396 { 3397 struct cxl_region *cxlr = to_cxl_region(dev); 3398 struct cxl_region_params *p = &cxlr->params; 3399 int rc; 3400 3401 rc = down_read_interruptible(&cxl_region_rwsem); 3402 if (rc) { 3403 dev_dbg(&cxlr->dev, "probe interrupted\n"); 3404 return rc; 3405 } 3406 3407 if (p->state < CXL_CONFIG_COMMIT) { 3408 dev_dbg(&cxlr->dev, "config state: %d\n", p->state); 3409 rc = -ENXIO; 3410 goto out; 3411 } 3412 3413 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) { 3414 dev_err(&cxlr->dev, 3415 "failed to activate, re-commit region and retry\n"); 3416 rc = -ENXIO; 3417 goto out; 3418 } 3419 3420 /* 3421 * From this point on any path that changes the region's state away from 3422 * CXL_CONFIG_COMMIT is also responsible for releasing the driver. 3423 */ 3424 out: 3425 up_read(&cxl_region_rwsem); 3426 3427 if (rc) 3428 return rc; 3429 3430 cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback; 3431 cxlr->memory_notifier.priority = CXL_CALLBACK_PRI; 3432 register_memory_notifier(&cxlr->memory_notifier); 3433 3434 cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance; 3435 cxlr->adist_notifier.priority = 100; 3436 register_mt_adistance_algorithm(&cxlr->adist_notifier); 3437 3438 rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr); 3439 if (rc) 3440 return rc; 3441 3442 switch (cxlr->mode) { 3443 case CXL_DECODER_PMEM: 3444 return devm_cxl_add_pmem_region(cxlr); 3445 case CXL_DECODER_RAM: 3446 /* 3447 * The region can not be manged by CXL if any portion of 3448 * it is already online as 'System RAM' 3449 */ 3450 if (walk_iomem_res_desc(IORES_DESC_NONE, 3451 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY, 3452 p->res->start, p->res->end, cxlr, 3453 is_system_ram) > 0) 3454 return 0; 3455 return devm_cxl_add_dax_region(cxlr); 3456 default: 3457 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n", 3458 cxlr->mode); 3459 return -ENXIO; 3460 } 3461 } 3462 3463 static struct cxl_driver cxl_region_driver = { 3464 .name = "cxl_region", 3465 .probe = cxl_region_probe, 3466 .id = CXL_DEVICE_REGION, 3467 }; 3468 3469 int cxl_region_init(void) 3470 { 3471 return cxl_driver_register(&cxl_region_driver); 3472 } 3473 3474 void cxl_region_exit(void) 3475 { 3476 cxl_driver_unregister(&cxl_region_driver); 3477 } 3478 3479 MODULE_IMPORT_NS("CXL"); 3480 MODULE_IMPORT_NS("DEVMEM"); 3481 MODULE_ALIAS_CXL(CXL_DEVICE_REGION); 3482