1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3 #include <linux/memregion.h> 4 #include <linux/genalloc.h> 5 #include <linux/device.h> 6 #include <linux/module.h> 7 #include <linux/memory.h> 8 #include <linux/slab.h> 9 #include <linux/uuid.h> 10 #include <linux/sort.h> 11 #include <linux/idr.h> 12 #include <linux/memory-tiers.h> 13 #include <cxlmem.h> 14 #include <cxl.h> 15 #include "core.h" 16 17 /** 18 * DOC: cxl core region 19 * 20 * CXL Regions represent mapped memory capacity in system physical address 21 * space. Whereas the CXL Root Decoders identify the bounds of potential CXL 22 * Memory ranges, Regions represent the active mapped capacity by the HDM 23 * Decoder Capability structures throughout the Host Bridges, Switches, and 24 * Endpoints in the topology. 25 * 26 * Region configuration has ordering constraints. UUID may be set at any time 27 * but is only visible for persistent regions. 28 * 1. Interleave granularity 29 * 2. Interleave size 30 * 3. Decoder targets 31 */ 32 33 static struct cxl_region *to_cxl_region(struct device *dev); 34 35 #define __ACCESS_ATTR_RO(_level, _name) { \ 36 .attr = { .name = __stringify(_name), .mode = 0444 }, \ 37 .show = _name##_access##_level##_show, \ 38 } 39 40 #define ACCESS_DEVICE_ATTR_RO(level, name) \ 41 struct device_attribute dev_attr_access##level##_##name = __ACCESS_ATTR_RO(level, name) 42 43 #define ACCESS_ATTR_RO(level, attrib) \ 44 static ssize_t attrib##_access##level##_show(struct device *dev, \ 45 struct device_attribute *attr, \ 46 char *buf) \ 47 { \ 48 struct cxl_region *cxlr = to_cxl_region(dev); \ 49 \ 50 if (cxlr->coord[level].attrib == 0) \ 51 return -ENOENT; \ 52 \ 53 return sysfs_emit(buf, "%u\n", cxlr->coord[level].attrib); \ 54 } \ 55 static ACCESS_DEVICE_ATTR_RO(level, attrib) 56 57 ACCESS_ATTR_RO(0, read_bandwidth); 58 ACCESS_ATTR_RO(0, read_latency); 59 ACCESS_ATTR_RO(0, write_bandwidth); 60 ACCESS_ATTR_RO(0, write_latency); 61 62 #define ACCESS_ATTR_DECLARE(level, attrib) \ 63 (&dev_attr_access##level##_##attrib.attr) 64 65 static struct attribute *access0_coordinate_attrs[] = { 66 ACCESS_ATTR_DECLARE(0, read_bandwidth), 67 ACCESS_ATTR_DECLARE(0, write_bandwidth), 68 ACCESS_ATTR_DECLARE(0, read_latency), 69 ACCESS_ATTR_DECLARE(0, write_latency), 70 NULL 71 }; 72 73 ACCESS_ATTR_RO(1, read_bandwidth); 74 ACCESS_ATTR_RO(1, read_latency); 75 ACCESS_ATTR_RO(1, write_bandwidth); 76 ACCESS_ATTR_RO(1, write_latency); 77 78 static struct attribute *access1_coordinate_attrs[] = { 79 ACCESS_ATTR_DECLARE(1, read_bandwidth), 80 ACCESS_ATTR_DECLARE(1, write_bandwidth), 81 ACCESS_ATTR_DECLARE(1, read_latency), 82 ACCESS_ATTR_DECLARE(1, write_latency), 83 NULL 84 }; 85 86 #define ACCESS_VISIBLE(level) \ 87 static umode_t cxl_region_access##level##_coordinate_visible( \ 88 struct kobject *kobj, struct attribute *a, int n) \ 89 { \ 90 struct device *dev = kobj_to_dev(kobj); \ 91 struct cxl_region *cxlr = to_cxl_region(dev); \ 92 \ 93 if (a == &dev_attr_access##level##_read_latency.attr && \ 94 cxlr->coord[level].read_latency == 0) \ 95 return 0; \ 96 \ 97 if (a == &dev_attr_access##level##_write_latency.attr && \ 98 cxlr->coord[level].write_latency == 0) \ 99 return 0; \ 100 \ 101 if (a == &dev_attr_access##level##_read_bandwidth.attr && \ 102 cxlr->coord[level].read_bandwidth == 0) \ 103 return 0; \ 104 \ 105 if (a == &dev_attr_access##level##_write_bandwidth.attr && \ 106 cxlr->coord[level].write_bandwidth == 0) \ 107 return 0; \ 108 \ 109 return a->mode; \ 110 } 111 112 ACCESS_VISIBLE(0); 113 ACCESS_VISIBLE(1); 114 115 static const struct attribute_group cxl_region_access0_coordinate_group = { 116 .name = "access0", 117 .attrs = access0_coordinate_attrs, 118 .is_visible = cxl_region_access0_coordinate_visible, 119 }; 120 121 static const struct attribute_group *get_cxl_region_access0_group(void) 122 { 123 return &cxl_region_access0_coordinate_group; 124 } 125 126 static const struct attribute_group cxl_region_access1_coordinate_group = { 127 .name = "access1", 128 .attrs = access1_coordinate_attrs, 129 .is_visible = cxl_region_access1_coordinate_visible, 130 }; 131 132 static const struct attribute_group *get_cxl_region_access1_group(void) 133 { 134 return &cxl_region_access1_coordinate_group; 135 } 136 137 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr, 138 char *buf) 139 { 140 struct cxl_region *cxlr = to_cxl_region(dev); 141 struct cxl_region_params *p = &cxlr->params; 142 ssize_t rc; 143 144 rc = down_read_interruptible(&cxl_region_rwsem); 145 if (rc) 146 return rc; 147 if (cxlr->mode != CXL_PARTMODE_PMEM) 148 rc = sysfs_emit(buf, "\n"); 149 else 150 rc = sysfs_emit(buf, "%pUb\n", &p->uuid); 151 up_read(&cxl_region_rwsem); 152 153 return rc; 154 } 155 156 static int is_dup(struct device *match, void *data) 157 { 158 struct cxl_region_params *p; 159 struct cxl_region *cxlr; 160 uuid_t *uuid = data; 161 162 if (!is_cxl_region(match)) 163 return 0; 164 165 lockdep_assert_held(&cxl_region_rwsem); 166 cxlr = to_cxl_region(match); 167 p = &cxlr->params; 168 169 if (uuid_equal(&p->uuid, uuid)) { 170 dev_dbg(match, "already has uuid: %pUb\n", uuid); 171 return -EBUSY; 172 } 173 174 return 0; 175 } 176 177 static ssize_t uuid_store(struct device *dev, struct device_attribute *attr, 178 const char *buf, size_t len) 179 { 180 struct cxl_region *cxlr = to_cxl_region(dev); 181 struct cxl_region_params *p = &cxlr->params; 182 uuid_t temp; 183 ssize_t rc; 184 185 if (len != UUID_STRING_LEN + 1) 186 return -EINVAL; 187 188 rc = uuid_parse(buf, &temp); 189 if (rc) 190 return rc; 191 192 if (uuid_is_null(&temp)) 193 return -EINVAL; 194 195 rc = down_write_killable(&cxl_region_rwsem); 196 if (rc) 197 return rc; 198 199 if (uuid_equal(&p->uuid, &temp)) 200 goto out; 201 202 rc = -EBUSY; 203 if (p->state >= CXL_CONFIG_ACTIVE) 204 goto out; 205 206 rc = bus_for_each_dev(&cxl_bus_type, NULL, &temp, is_dup); 207 if (rc < 0) 208 goto out; 209 210 uuid_copy(&p->uuid, &temp); 211 out: 212 up_write(&cxl_region_rwsem); 213 214 if (rc) 215 return rc; 216 return len; 217 } 218 static DEVICE_ATTR_RW(uuid); 219 220 static struct cxl_region_ref *cxl_rr_load(struct cxl_port *port, 221 struct cxl_region *cxlr) 222 { 223 return xa_load(&port->regions, (unsigned long)cxlr); 224 } 225 226 static int cxl_region_invalidate_memregion(struct cxl_region *cxlr) 227 { 228 if (!cpu_cache_has_invalidate_memregion()) { 229 if (IS_ENABLED(CONFIG_CXL_REGION_INVALIDATION_TEST)) { 230 dev_info_once( 231 &cxlr->dev, 232 "Bypassing cpu_cache_invalidate_memregion() for testing!\n"); 233 return 0; 234 } else { 235 dev_WARN(&cxlr->dev, 236 "Failed to synchronize CPU cache state\n"); 237 return -ENXIO; 238 } 239 } 240 241 cpu_cache_invalidate_memregion(IORES_DESC_CXL); 242 return 0; 243 } 244 245 static void cxl_region_decode_reset(struct cxl_region *cxlr, int count) 246 { 247 struct cxl_region_params *p = &cxlr->params; 248 int i; 249 250 /* 251 * Before region teardown attempt to flush, evict any data cached for 252 * this region, or scream loudly about missing arch / platform support 253 * for CXL teardown. 254 */ 255 cxl_region_invalidate_memregion(cxlr); 256 257 for (i = count - 1; i >= 0; i--) { 258 struct cxl_endpoint_decoder *cxled = p->targets[i]; 259 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 260 struct cxl_port *iter = cxled_to_port(cxled); 261 struct cxl_dev_state *cxlds = cxlmd->cxlds; 262 struct cxl_ep *ep; 263 264 if (cxlds->rcd) 265 goto endpoint_reset; 266 267 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 268 iter = to_cxl_port(iter->dev.parent); 269 270 for (ep = cxl_ep_load(iter, cxlmd); iter; 271 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 272 struct cxl_region_ref *cxl_rr; 273 struct cxl_decoder *cxld; 274 275 cxl_rr = cxl_rr_load(iter, cxlr); 276 cxld = cxl_rr->decoder; 277 if (cxld->reset) 278 cxld->reset(cxld); 279 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 280 } 281 282 endpoint_reset: 283 cxled->cxld.reset(&cxled->cxld); 284 set_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 285 } 286 287 /* all decoders associated with this region have been torn down */ 288 clear_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags); 289 } 290 291 static int commit_decoder(struct cxl_decoder *cxld) 292 { 293 struct cxl_switch_decoder *cxlsd = NULL; 294 295 if (cxld->commit) 296 return cxld->commit(cxld); 297 298 if (is_switch_decoder(&cxld->dev)) 299 cxlsd = to_cxl_switch_decoder(&cxld->dev); 300 301 if (dev_WARN_ONCE(&cxld->dev, !cxlsd || cxlsd->nr_targets > 1, 302 "->commit() is required\n")) 303 return -ENXIO; 304 return 0; 305 } 306 307 static int cxl_region_decode_commit(struct cxl_region *cxlr) 308 { 309 struct cxl_region_params *p = &cxlr->params; 310 int i, rc = 0; 311 312 for (i = 0; i < p->nr_targets; i++) { 313 struct cxl_endpoint_decoder *cxled = p->targets[i]; 314 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 315 struct cxl_region_ref *cxl_rr; 316 struct cxl_decoder *cxld; 317 struct cxl_port *iter; 318 struct cxl_ep *ep; 319 320 /* commit bottom up */ 321 for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 322 iter = to_cxl_port(iter->dev.parent)) { 323 cxl_rr = cxl_rr_load(iter, cxlr); 324 cxld = cxl_rr->decoder; 325 rc = commit_decoder(cxld); 326 if (rc) 327 break; 328 } 329 330 if (rc) { 331 /* programming @iter failed, teardown */ 332 for (ep = cxl_ep_load(iter, cxlmd); ep && iter; 333 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 334 cxl_rr = cxl_rr_load(iter, cxlr); 335 cxld = cxl_rr->decoder; 336 if (cxld->reset) 337 cxld->reset(cxld); 338 } 339 340 cxled->cxld.reset(&cxled->cxld); 341 goto err; 342 } 343 } 344 345 return 0; 346 347 err: 348 /* undo the targets that were successfully committed */ 349 cxl_region_decode_reset(cxlr, i); 350 return rc; 351 } 352 353 static ssize_t commit_store(struct device *dev, struct device_attribute *attr, 354 const char *buf, size_t len) 355 { 356 struct cxl_region *cxlr = to_cxl_region(dev); 357 struct cxl_region_params *p = &cxlr->params; 358 bool commit; 359 ssize_t rc; 360 361 rc = kstrtobool(buf, &commit); 362 if (rc) 363 return rc; 364 365 rc = down_write_killable(&cxl_region_rwsem); 366 if (rc) 367 return rc; 368 369 /* Already in the requested state? */ 370 if (commit && p->state >= CXL_CONFIG_COMMIT) 371 goto out; 372 if (!commit && p->state < CXL_CONFIG_COMMIT) 373 goto out; 374 375 /* Not ready to commit? */ 376 if (commit && p->state < CXL_CONFIG_ACTIVE) { 377 rc = -ENXIO; 378 goto out; 379 } 380 381 /* 382 * Invalidate caches before region setup to drop any speculative 383 * consumption of this address space 384 */ 385 rc = cxl_region_invalidate_memregion(cxlr); 386 if (rc) 387 goto out; 388 389 if (commit) { 390 rc = cxl_region_decode_commit(cxlr); 391 if (rc == 0) 392 p->state = CXL_CONFIG_COMMIT; 393 } else { 394 p->state = CXL_CONFIG_RESET_PENDING; 395 up_write(&cxl_region_rwsem); 396 device_release_driver(&cxlr->dev); 397 down_write(&cxl_region_rwsem); 398 399 /* 400 * The lock was dropped, so need to revalidate that the reset is 401 * still pending. 402 */ 403 if (p->state == CXL_CONFIG_RESET_PENDING) { 404 cxl_region_decode_reset(cxlr, p->interleave_ways); 405 p->state = CXL_CONFIG_ACTIVE; 406 } 407 } 408 409 out: 410 up_write(&cxl_region_rwsem); 411 412 if (rc) 413 return rc; 414 return len; 415 } 416 417 static ssize_t commit_show(struct device *dev, struct device_attribute *attr, 418 char *buf) 419 { 420 struct cxl_region *cxlr = to_cxl_region(dev); 421 struct cxl_region_params *p = &cxlr->params; 422 ssize_t rc; 423 424 rc = down_read_interruptible(&cxl_region_rwsem); 425 if (rc) 426 return rc; 427 rc = sysfs_emit(buf, "%d\n", p->state >= CXL_CONFIG_COMMIT); 428 up_read(&cxl_region_rwsem); 429 430 return rc; 431 } 432 static DEVICE_ATTR_RW(commit); 433 434 static umode_t cxl_region_visible(struct kobject *kobj, struct attribute *a, 435 int n) 436 { 437 struct device *dev = kobj_to_dev(kobj); 438 struct cxl_region *cxlr = to_cxl_region(dev); 439 440 /* 441 * Support tooling that expects to find a 'uuid' attribute for all 442 * regions regardless of mode. 443 */ 444 if (a == &dev_attr_uuid.attr && cxlr->mode != CXL_PARTMODE_PMEM) 445 return 0444; 446 return a->mode; 447 } 448 449 static ssize_t interleave_ways_show(struct device *dev, 450 struct device_attribute *attr, char *buf) 451 { 452 struct cxl_region *cxlr = to_cxl_region(dev); 453 struct cxl_region_params *p = &cxlr->params; 454 ssize_t rc; 455 456 rc = down_read_interruptible(&cxl_region_rwsem); 457 if (rc) 458 return rc; 459 rc = sysfs_emit(buf, "%d\n", p->interleave_ways); 460 up_read(&cxl_region_rwsem); 461 462 return rc; 463 } 464 465 static const struct attribute_group *get_cxl_region_target_group(void); 466 467 static ssize_t interleave_ways_store(struct device *dev, 468 struct device_attribute *attr, 469 const char *buf, size_t len) 470 { 471 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 472 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 473 struct cxl_region *cxlr = to_cxl_region(dev); 474 struct cxl_region_params *p = &cxlr->params; 475 unsigned int val, save; 476 int rc; 477 u8 iw; 478 479 rc = kstrtouint(buf, 0, &val); 480 if (rc) 481 return rc; 482 483 rc = ways_to_eiw(val, &iw); 484 if (rc) 485 return rc; 486 487 /* 488 * Even for x3, x6, and x12 interleaves the region interleave must be a 489 * power of 2 multiple of the host bridge interleave. 490 */ 491 if (!is_power_of_2(val / cxld->interleave_ways) || 492 (val % cxld->interleave_ways)) { 493 dev_dbg(&cxlr->dev, "invalid interleave: %d\n", val); 494 return -EINVAL; 495 } 496 497 rc = down_write_killable(&cxl_region_rwsem); 498 if (rc) 499 return rc; 500 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 501 rc = -EBUSY; 502 goto out; 503 } 504 505 save = p->interleave_ways; 506 p->interleave_ways = val; 507 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 508 if (rc) 509 p->interleave_ways = save; 510 out: 511 up_write(&cxl_region_rwsem); 512 if (rc) 513 return rc; 514 return len; 515 } 516 static DEVICE_ATTR_RW(interleave_ways); 517 518 static ssize_t interleave_granularity_show(struct device *dev, 519 struct device_attribute *attr, 520 char *buf) 521 { 522 struct cxl_region *cxlr = to_cxl_region(dev); 523 struct cxl_region_params *p = &cxlr->params; 524 ssize_t rc; 525 526 rc = down_read_interruptible(&cxl_region_rwsem); 527 if (rc) 528 return rc; 529 rc = sysfs_emit(buf, "%d\n", p->interleave_granularity); 530 up_read(&cxl_region_rwsem); 531 532 return rc; 533 } 534 535 static ssize_t interleave_granularity_store(struct device *dev, 536 struct device_attribute *attr, 537 const char *buf, size_t len) 538 { 539 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 540 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 541 struct cxl_region *cxlr = to_cxl_region(dev); 542 struct cxl_region_params *p = &cxlr->params; 543 int rc, val; 544 u16 ig; 545 546 rc = kstrtoint(buf, 0, &val); 547 if (rc) 548 return rc; 549 550 rc = granularity_to_eig(val, &ig); 551 if (rc) 552 return rc; 553 554 /* 555 * When the host-bridge is interleaved, disallow region granularity != 556 * root granularity. Regions with a granularity less than the root 557 * interleave result in needing multiple endpoints to support a single 558 * slot in the interleave (possible to support in the future). Regions 559 * with a granularity greater than the root interleave result in invalid 560 * DPA translations (invalid to support). 561 */ 562 if (cxld->interleave_ways > 1 && val != cxld->interleave_granularity) 563 return -EINVAL; 564 565 rc = down_write_killable(&cxl_region_rwsem); 566 if (rc) 567 return rc; 568 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 569 rc = -EBUSY; 570 goto out; 571 } 572 573 p->interleave_granularity = val; 574 out: 575 up_write(&cxl_region_rwsem); 576 if (rc) 577 return rc; 578 return len; 579 } 580 static DEVICE_ATTR_RW(interleave_granularity); 581 582 static ssize_t resource_show(struct device *dev, struct device_attribute *attr, 583 char *buf) 584 { 585 struct cxl_region *cxlr = to_cxl_region(dev); 586 struct cxl_region_params *p = &cxlr->params; 587 u64 resource = -1ULL; 588 ssize_t rc; 589 590 rc = down_read_interruptible(&cxl_region_rwsem); 591 if (rc) 592 return rc; 593 if (p->res) 594 resource = p->res->start; 595 rc = sysfs_emit(buf, "%#llx\n", resource); 596 up_read(&cxl_region_rwsem); 597 598 return rc; 599 } 600 static DEVICE_ATTR_RO(resource); 601 602 static ssize_t mode_show(struct device *dev, struct device_attribute *attr, 603 char *buf) 604 { 605 struct cxl_region *cxlr = to_cxl_region(dev); 606 const char *desc; 607 608 if (cxlr->mode == CXL_PARTMODE_RAM) 609 desc = "ram"; 610 else if (cxlr->mode == CXL_PARTMODE_PMEM) 611 desc = "pmem"; 612 else 613 desc = ""; 614 615 return sysfs_emit(buf, "%s\n", desc); 616 } 617 static DEVICE_ATTR_RO(mode); 618 619 static int alloc_hpa(struct cxl_region *cxlr, resource_size_t size) 620 { 621 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 622 struct cxl_region_params *p = &cxlr->params; 623 struct resource *res; 624 u64 remainder = 0; 625 626 lockdep_assert_held_write(&cxl_region_rwsem); 627 628 /* Nothing to do... */ 629 if (p->res && resource_size(p->res) == size) 630 return 0; 631 632 /* To change size the old size must be freed first */ 633 if (p->res) 634 return -EBUSY; 635 636 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) 637 return -EBUSY; 638 639 /* ways, granularity and uuid (if PMEM) need to be set before HPA */ 640 if (!p->interleave_ways || !p->interleave_granularity || 641 (cxlr->mode == CXL_PARTMODE_PMEM && uuid_is_null(&p->uuid))) 642 return -ENXIO; 643 644 div64_u64_rem(size, (u64)SZ_256M * p->interleave_ways, &remainder); 645 if (remainder) 646 return -EINVAL; 647 648 res = alloc_free_mem_region(cxlrd->res, size, SZ_256M, 649 dev_name(&cxlr->dev)); 650 if (IS_ERR(res)) { 651 dev_dbg(&cxlr->dev, 652 "HPA allocation error (%ld) for size:%pap in %s %pr\n", 653 PTR_ERR(res), &size, cxlrd->res->name, cxlrd->res); 654 return PTR_ERR(res); 655 } 656 657 p->res = res; 658 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 659 660 return 0; 661 } 662 663 static void cxl_region_iomem_release(struct cxl_region *cxlr) 664 { 665 struct cxl_region_params *p = &cxlr->params; 666 667 if (device_is_registered(&cxlr->dev)) 668 lockdep_assert_held_write(&cxl_region_rwsem); 669 if (p->res) { 670 /* 671 * Autodiscovered regions may not have been able to insert their 672 * resource. 673 */ 674 if (p->res->parent) 675 remove_resource(p->res); 676 kfree(p->res); 677 p->res = NULL; 678 } 679 } 680 681 static int free_hpa(struct cxl_region *cxlr) 682 { 683 struct cxl_region_params *p = &cxlr->params; 684 685 lockdep_assert_held_write(&cxl_region_rwsem); 686 687 if (!p->res) 688 return 0; 689 690 if (p->state >= CXL_CONFIG_ACTIVE) 691 return -EBUSY; 692 693 cxl_region_iomem_release(cxlr); 694 p->state = CXL_CONFIG_IDLE; 695 return 0; 696 } 697 698 static ssize_t size_store(struct device *dev, struct device_attribute *attr, 699 const char *buf, size_t len) 700 { 701 struct cxl_region *cxlr = to_cxl_region(dev); 702 u64 val; 703 int rc; 704 705 rc = kstrtou64(buf, 0, &val); 706 if (rc) 707 return rc; 708 709 rc = down_write_killable(&cxl_region_rwsem); 710 if (rc) 711 return rc; 712 713 if (val) 714 rc = alloc_hpa(cxlr, val); 715 else 716 rc = free_hpa(cxlr); 717 up_write(&cxl_region_rwsem); 718 719 if (rc) 720 return rc; 721 722 return len; 723 } 724 725 static ssize_t size_show(struct device *dev, struct device_attribute *attr, 726 char *buf) 727 { 728 struct cxl_region *cxlr = to_cxl_region(dev); 729 struct cxl_region_params *p = &cxlr->params; 730 u64 size = 0; 731 ssize_t rc; 732 733 rc = down_read_interruptible(&cxl_region_rwsem); 734 if (rc) 735 return rc; 736 if (p->res) 737 size = resource_size(p->res); 738 rc = sysfs_emit(buf, "%#llx\n", size); 739 up_read(&cxl_region_rwsem); 740 741 return rc; 742 } 743 static DEVICE_ATTR_RW(size); 744 745 static struct attribute *cxl_region_attrs[] = { 746 &dev_attr_uuid.attr, 747 &dev_attr_commit.attr, 748 &dev_attr_interleave_ways.attr, 749 &dev_attr_interleave_granularity.attr, 750 &dev_attr_resource.attr, 751 &dev_attr_size.attr, 752 &dev_attr_mode.attr, 753 NULL, 754 }; 755 756 static const struct attribute_group cxl_region_group = { 757 .attrs = cxl_region_attrs, 758 .is_visible = cxl_region_visible, 759 }; 760 761 static size_t show_targetN(struct cxl_region *cxlr, char *buf, int pos) 762 { 763 struct cxl_region_params *p = &cxlr->params; 764 struct cxl_endpoint_decoder *cxled; 765 int rc; 766 767 rc = down_read_interruptible(&cxl_region_rwsem); 768 if (rc) 769 return rc; 770 771 if (pos >= p->interleave_ways) { 772 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 773 p->interleave_ways); 774 rc = -ENXIO; 775 goto out; 776 } 777 778 cxled = p->targets[pos]; 779 if (!cxled) 780 rc = sysfs_emit(buf, "\n"); 781 else 782 rc = sysfs_emit(buf, "%s\n", dev_name(&cxled->cxld.dev)); 783 out: 784 up_read(&cxl_region_rwsem); 785 786 return rc; 787 } 788 789 static int check_commit_order(struct device *dev, void *data) 790 { 791 struct cxl_decoder *cxld = to_cxl_decoder(dev); 792 793 /* 794 * if port->commit_end is not the only free decoder, then out of 795 * order shutdown has occurred, block further allocations until 796 * that is resolved 797 */ 798 if (((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) 799 return -EBUSY; 800 return 0; 801 } 802 803 static int match_free_decoder(struct device *dev, const void *data) 804 { 805 struct cxl_port *port = to_cxl_port(dev->parent); 806 struct cxl_decoder *cxld; 807 int rc; 808 809 if (!is_switch_decoder(dev)) 810 return 0; 811 812 cxld = to_cxl_decoder(dev); 813 814 if (cxld->id != port->commit_end + 1) 815 return 0; 816 817 if (cxld->region) { 818 dev_dbg(dev->parent, 819 "next decoder to commit (%s) is already reserved (%s)\n", 820 dev_name(dev), dev_name(&cxld->region->dev)); 821 return 0; 822 } 823 824 rc = device_for_each_child_reverse_from(dev->parent, dev, NULL, 825 check_commit_order); 826 if (rc) { 827 dev_dbg(dev->parent, 828 "unable to allocate %s due to out of order shutdown\n", 829 dev_name(dev)); 830 return 0; 831 } 832 return 1; 833 } 834 835 static bool region_res_match_cxl_range(const struct cxl_region_params *p, 836 struct range *range) 837 { 838 if (!p->res) 839 return false; 840 841 /* 842 * If an extended linear cache region then the CXL range is assumed 843 * to be fronted by the DRAM range in current known implementation. 844 * This assumption will be made until a variant implementation exists. 845 */ 846 return p->res->start + p->cache_size == range->start && 847 p->res->end == range->end; 848 } 849 850 static int match_auto_decoder(struct device *dev, const void *data) 851 { 852 const struct cxl_region_params *p = data; 853 struct cxl_decoder *cxld; 854 struct range *r; 855 856 if (!is_switch_decoder(dev)) 857 return 0; 858 859 cxld = to_cxl_decoder(dev); 860 r = &cxld->hpa_range; 861 862 if (region_res_match_cxl_range(p, r)) 863 return 1; 864 865 return 0; 866 } 867 868 static struct cxl_decoder * 869 cxl_region_find_decoder(struct cxl_port *port, 870 struct cxl_endpoint_decoder *cxled, 871 struct cxl_region *cxlr) 872 { 873 struct device *dev; 874 875 if (port == cxled_to_port(cxled)) 876 return &cxled->cxld; 877 878 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) 879 dev = device_find_child(&port->dev, &cxlr->params, 880 match_auto_decoder); 881 else 882 dev = device_find_child(&port->dev, NULL, match_free_decoder); 883 if (!dev) 884 return NULL; 885 /* 886 * This decoder is pinned registered as long as the endpoint decoder is 887 * registered, and endpoint decoder unregistration holds the 888 * cxl_region_rwsem over unregister events, so no need to hold on to 889 * this extra reference. 890 */ 891 put_device(dev); 892 return to_cxl_decoder(dev); 893 } 894 895 static bool auto_order_ok(struct cxl_port *port, struct cxl_region *cxlr_iter, 896 struct cxl_decoder *cxld) 897 { 898 struct cxl_region_ref *rr = cxl_rr_load(port, cxlr_iter); 899 struct cxl_decoder *cxld_iter = rr->decoder; 900 901 /* 902 * Allow the out of order assembly of auto-discovered regions. 903 * Per CXL Spec 3.1 8.2.4.20.12 software must commit decoders 904 * in HPA order. Confirm that the decoder with the lesser HPA 905 * starting address has the lesser id. 906 */ 907 dev_dbg(&cxld->dev, "check for HPA violation %s:%d < %s:%d\n", 908 dev_name(&cxld->dev), cxld->id, 909 dev_name(&cxld_iter->dev), cxld_iter->id); 910 911 if (cxld_iter->id > cxld->id) 912 return true; 913 914 return false; 915 } 916 917 static struct cxl_region_ref * 918 alloc_region_ref(struct cxl_port *port, struct cxl_region *cxlr, 919 struct cxl_endpoint_decoder *cxled) 920 { 921 struct cxl_region_params *p = &cxlr->params; 922 struct cxl_region_ref *cxl_rr, *iter; 923 unsigned long index; 924 int rc; 925 926 xa_for_each(&port->regions, index, iter) { 927 struct cxl_region_params *ip = &iter->region->params; 928 929 if (!ip->res || ip->res->start < p->res->start) 930 continue; 931 932 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 933 struct cxl_decoder *cxld; 934 935 cxld = cxl_region_find_decoder(port, cxled, cxlr); 936 if (auto_order_ok(port, iter->region, cxld)) 937 continue; 938 } 939 dev_dbg(&cxlr->dev, "%s: HPA order violation %s:%pr vs %pr\n", 940 dev_name(&port->dev), 941 dev_name(&iter->region->dev), ip->res, p->res); 942 943 return ERR_PTR(-EBUSY); 944 } 945 946 cxl_rr = kzalloc(sizeof(*cxl_rr), GFP_KERNEL); 947 if (!cxl_rr) 948 return ERR_PTR(-ENOMEM); 949 cxl_rr->port = port; 950 cxl_rr->region = cxlr; 951 cxl_rr->nr_targets = 1; 952 xa_init(&cxl_rr->endpoints); 953 954 rc = xa_insert(&port->regions, (unsigned long)cxlr, cxl_rr, GFP_KERNEL); 955 if (rc) { 956 dev_dbg(&cxlr->dev, 957 "%s: failed to track region reference: %d\n", 958 dev_name(&port->dev), rc); 959 kfree(cxl_rr); 960 return ERR_PTR(rc); 961 } 962 963 return cxl_rr; 964 } 965 966 static void cxl_rr_free_decoder(struct cxl_region_ref *cxl_rr) 967 { 968 struct cxl_region *cxlr = cxl_rr->region; 969 struct cxl_decoder *cxld = cxl_rr->decoder; 970 971 if (!cxld) 972 return; 973 974 dev_WARN_ONCE(&cxlr->dev, cxld->region != cxlr, "region mismatch\n"); 975 if (cxld->region == cxlr) { 976 cxld->region = NULL; 977 put_device(&cxlr->dev); 978 } 979 } 980 981 static void free_region_ref(struct cxl_region_ref *cxl_rr) 982 { 983 struct cxl_port *port = cxl_rr->port; 984 struct cxl_region *cxlr = cxl_rr->region; 985 986 cxl_rr_free_decoder(cxl_rr); 987 xa_erase(&port->regions, (unsigned long)cxlr); 988 xa_destroy(&cxl_rr->endpoints); 989 kfree(cxl_rr); 990 } 991 992 static int cxl_rr_ep_add(struct cxl_region_ref *cxl_rr, 993 struct cxl_endpoint_decoder *cxled) 994 { 995 int rc; 996 struct cxl_port *port = cxl_rr->port; 997 struct cxl_region *cxlr = cxl_rr->region; 998 struct cxl_decoder *cxld = cxl_rr->decoder; 999 struct cxl_ep *ep = cxl_ep_load(port, cxled_to_memdev(cxled)); 1000 1001 if (ep) { 1002 rc = xa_insert(&cxl_rr->endpoints, (unsigned long)cxled, ep, 1003 GFP_KERNEL); 1004 if (rc) 1005 return rc; 1006 } 1007 cxl_rr->nr_eps++; 1008 1009 if (!cxld->region) { 1010 cxld->region = cxlr; 1011 get_device(&cxlr->dev); 1012 } 1013 1014 return 0; 1015 } 1016 1017 static int cxl_rr_alloc_decoder(struct cxl_port *port, struct cxl_region *cxlr, 1018 struct cxl_endpoint_decoder *cxled, 1019 struct cxl_region_ref *cxl_rr) 1020 { 1021 struct cxl_decoder *cxld; 1022 1023 cxld = cxl_region_find_decoder(port, cxled, cxlr); 1024 if (!cxld) { 1025 dev_dbg(&cxlr->dev, "%s: no decoder available\n", 1026 dev_name(&port->dev)); 1027 return -EBUSY; 1028 } 1029 1030 if (cxld->region) { 1031 dev_dbg(&cxlr->dev, "%s: %s already attached to %s\n", 1032 dev_name(&port->dev), dev_name(&cxld->dev), 1033 dev_name(&cxld->region->dev)); 1034 return -EBUSY; 1035 } 1036 1037 /* 1038 * Endpoints should already match the region type, but backstop that 1039 * assumption with an assertion. Switch-decoders change mapping-type 1040 * based on what is mapped when they are assigned to a region. 1041 */ 1042 dev_WARN_ONCE(&cxlr->dev, 1043 port == cxled_to_port(cxled) && 1044 cxld->target_type != cxlr->type, 1045 "%s:%s mismatch decoder type %d -> %d\n", 1046 dev_name(&cxled_to_memdev(cxled)->dev), 1047 dev_name(&cxld->dev), cxld->target_type, cxlr->type); 1048 cxld->target_type = cxlr->type; 1049 cxl_rr->decoder = cxld; 1050 return 0; 1051 } 1052 1053 /** 1054 * cxl_port_attach_region() - track a region's interest in a port by endpoint 1055 * @port: port to add a new region reference 'struct cxl_region_ref' 1056 * @cxlr: region to attach to @port 1057 * @cxled: endpoint decoder used to create or further pin a region reference 1058 * @pos: interleave position of @cxled in @cxlr 1059 * 1060 * The attach event is an opportunity to validate CXL decode setup 1061 * constraints and record metadata needed for programming HDM decoders, 1062 * in particular decoder target lists. 1063 * 1064 * The steps are: 1065 * 1066 * - validate that there are no other regions with a higher HPA already 1067 * associated with @port 1068 * - establish a region reference if one is not already present 1069 * 1070 * - additionally allocate a decoder instance that will host @cxlr on 1071 * @port 1072 * 1073 * - pin the region reference by the endpoint 1074 * - account for how many entries in @port's target list are needed to 1075 * cover all of the added endpoints. 1076 */ 1077 static int cxl_port_attach_region(struct cxl_port *port, 1078 struct cxl_region *cxlr, 1079 struct cxl_endpoint_decoder *cxled, int pos) 1080 { 1081 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1082 struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 1083 struct cxl_region_ref *cxl_rr; 1084 bool nr_targets_inc = false; 1085 struct cxl_decoder *cxld; 1086 unsigned long index; 1087 int rc = -EBUSY; 1088 1089 lockdep_assert_held_write(&cxl_region_rwsem); 1090 1091 cxl_rr = cxl_rr_load(port, cxlr); 1092 if (cxl_rr) { 1093 struct cxl_ep *ep_iter; 1094 int found = 0; 1095 1096 /* 1097 * Walk the existing endpoints that have been attached to 1098 * @cxlr at @port and see if they share the same 'next' port 1099 * in the downstream direction. I.e. endpoints that share common 1100 * upstream switch. 1101 */ 1102 xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 1103 if (ep_iter == ep) 1104 continue; 1105 if (ep_iter->next == ep->next) { 1106 found++; 1107 break; 1108 } 1109 } 1110 1111 /* 1112 * New target port, or @port is an endpoint port that always 1113 * accounts its own local decode as a target. 1114 */ 1115 if (!found || !ep->next) { 1116 cxl_rr->nr_targets++; 1117 nr_targets_inc = true; 1118 } 1119 } else { 1120 cxl_rr = alloc_region_ref(port, cxlr, cxled); 1121 if (IS_ERR(cxl_rr)) { 1122 dev_dbg(&cxlr->dev, 1123 "%s: failed to allocate region reference\n", 1124 dev_name(&port->dev)); 1125 return PTR_ERR(cxl_rr); 1126 } 1127 nr_targets_inc = true; 1128 1129 rc = cxl_rr_alloc_decoder(port, cxlr, cxled, cxl_rr); 1130 if (rc) 1131 goto out_erase; 1132 } 1133 cxld = cxl_rr->decoder; 1134 1135 /* 1136 * the number of targets should not exceed the target_count 1137 * of the decoder 1138 */ 1139 if (is_switch_decoder(&cxld->dev)) { 1140 struct cxl_switch_decoder *cxlsd; 1141 1142 cxlsd = to_cxl_switch_decoder(&cxld->dev); 1143 if (cxl_rr->nr_targets > cxlsd->nr_targets) { 1144 dev_dbg(&cxlr->dev, 1145 "%s:%s %s add: %s:%s @ %d overflows targets: %d\n", 1146 dev_name(port->uport_dev), dev_name(&port->dev), 1147 dev_name(&cxld->dev), dev_name(&cxlmd->dev), 1148 dev_name(&cxled->cxld.dev), pos, 1149 cxlsd->nr_targets); 1150 rc = -ENXIO; 1151 goto out_erase; 1152 } 1153 } 1154 1155 rc = cxl_rr_ep_add(cxl_rr, cxled); 1156 if (rc) { 1157 dev_dbg(&cxlr->dev, 1158 "%s: failed to track endpoint %s:%s reference\n", 1159 dev_name(&port->dev), dev_name(&cxlmd->dev), 1160 dev_name(&cxld->dev)); 1161 goto out_erase; 1162 } 1163 1164 dev_dbg(&cxlr->dev, 1165 "%s:%s %s add: %s:%s @ %d next: %s nr_eps: %d nr_targets: %d\n", 1166 dev_name(port->uport_dev), dev_name(&port->dev), 1167 dev_name(&cxld->dev), dev_name(&cxlmd->dev), 1168 dev_name(&cxled->cxld.dev), pos, 1169 ep ? ep->next ? dev_name(ep->next->uport_dev) : 1170 dev_name(&cxlmd->dev) : 1171 "none", 1172 cxl_rr->nr_eps, cxl_rr->nr_targets); 1173 1174 return 0; 1175 out_erase: 1176 if (nr_targets_inc) 1177 cxl_rr->nr_targets--; 1178 if (cxl_rr->nr_eps == 0) 1179 free_region_ref(cxl_rr); 1180 return rc; 1181 } 1182 1183 static void cxl_port_detach_region(struct cxl_port *port, 1184 struct cxl_region *cxlr, 1185 struct cxl_endpoint_decoder *cxled) 1186 { 1187 struct cxl_region_ref *cxl_rr; 1188 struct cxl_ep *ep = NULL; 1189 1190 lockdep_assert_held_write(&cxl_region_rwsem); 1191 1192 cxl_rr = cxl_rr_load(port, cxlr); 1193 if (!cxl_rr) 1194 return; 1195 1196 /* 1197 * Endpoint ports do not carry cxl_ep references, and they 1198 * never target more than one endpoint by definition 1199 */ 1200 if (cxl_rr->decoder == &cxled->cxld) 1201 cxl_rr->nr_eps--; 1202 else 1203 ep = xa_erase(&cxl_rr->endpoints, (unsigned long)cxled); 1204 if (ep) { 1205 struct cxl_ep *ep_iter; 1206 unsigned long index; 1207 int found = 0; 1208 1209 cxl_rr->nr_eps--; 1210 xa_for_each(&cxl_rr->endpoints, index, ep_iter) { 1211 if (ep_iter->next == ep->next) { 1212 found++; 1213 break; 1214 } 1215 } 1216 if (!found) 1217 cxl_rr->nr_targets--; 1218 } 1219 1220 if (cxl_rr->nr_eps == 0) 1221 free_region_ref(cxl_rr); 1222 } 1223 1224 static int check_last_peer(struct cxl_endpoint_decoder *cxled, 1225 struct cxl_ep *ep, struct cxl_region_ref *cxl_rr, 1226 int distance) 1227 { 1228 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1229 struct cxl_region *cxlr = cxl_rr->region; 1230 struct cxl_region_params *p = &cxlr->params; 1231 struct cxl_endpoint_decoder *cxled_peer; 1232 struct cxl_port *port = cxl_rr->port; 1233 struct cxl_memdev *cxlmd_peer; 1234 struct cxl_ep *ep_peer; 1235 int pos = cxled->pos; 1236 1237 /* 1238 * If this position wants to share a dport with the last endpoint mapped 1239 * then that endpoint, at index 'position - distance', must also be 1240 * mapped by this dport. 1241 */ 1242 if (pos < distance) { 1243 dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", 1244 dev_name(port->uport_dev), dev_name(&port->dev), 1245 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 1246 return -ENXIO; 1247 } 1248 cxled_peer = p->targets[pos - distance]; 1249 cxlmd_peer = cxled_to_memdev(cxled_peer); 1250 ep_peer = cxl_ep_load(port, cxlmd_peer); 1251 if (ep->dport != ep_peer->dport) { 1252 dev_dbg(&cxlr->dev, 1253 "%s:%s: %s:%s pos %d mismatched peer %s:%s\n", 1254 dev_name(port->uport_dev), dev_name(&port->dev), 1255 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos, 1256 dev_name(&cxlmd_peer->dev), 1257 dev_name(&cxled_peer->cxld.dev)); 1258 return -ENXIO; 1259 } 1260 1261 return 0; 1262 } 1263 1264 static int check_interleave_cap(struct cxl_decoder *cxld, int iw, int ig) 1265 { 1266 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 1267 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 1268 unsigned int interleave_mask; 1269 u8 eiw; 1270 u16 eig; 1271 int high_pos, low_pos; 1272 1273 if (!test_bit(iw, &cxlhdm->iw_cap_mask)) 1274 return -ENXIO; 1275 /* 1276 * Per CXL specification r3.1(8.2.4.20.13 Decoder Protection), 1277 * if eiw < 8: 1278 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + 8 + eiw] 1279 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0] 1280 * 1281 * when the eiw is 0, all the bits of HPAOFFSET[51: 0] are used, the 1282 * interleave bits are none. 1283 * 1284 * if eiw >= 8: 1285 * DPAOFFSET[51: eig + 8] = HPAOFFSET[51: eig + eiw] / 3 1286 * DPAOFFSET[eig + 7: 0] = HPAOFFSET[eig + 7: 0] 1287 * 1288 * when the eiw is 8, all the bits of HPAOFFSET[51: 0] are used, the 1289 * interleave bits are none. 1290 */ 1291 ways_to_eiw(iw, &eiw); 1292 if (eiw == 0 || eiw == 8) 1293 return 0; 1294 1295 granularity_to_eig(ig, &eig); 1296 if (eiw > 8) 1297 high_pos = eiw + eig - 1; 1298 else 1299 high_pos = eiw + eig + 7; 1300 low_pos = eig + 8; 1301 interleave_mask = GENMASK(high_pos, low_pos); 1302 if (interleave_mask & ~cxlhdm->interleave_mask) 1303 return -ENXIO; 1304 1305 return 0; 1306 } 1307 1308 static int cxl_port_setup_targets(struct cxl_port *port, 1309 struct cxl_region *cxlr, 1310 struct cxl_endpoint_decoder *cxled) 1311 { 1312 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 1313 int parent_iw, parent_ig, ig, iw, rc, inc = 0, pos = cxled->pos; 1314 struct cxl_port *parent_port = to_cxl_port(port->dev.parent); 1315 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 1316 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1317 struct cxl_ep *ep = cxl_ep_load(port, cxlmd); 1318 struct cxl_region_params *p = &cxlr->params; 1319 struct cxl_decoder *cxld = cxl_rr->decoder; 1320 struct cxl_switch_decoder *cxlsd; 1321 struct cxl_port *iter = port; 1322 u16 eig, peig; 1323 u8 eiw, peiw; 1324 1325 /* 1326 * While root level decoders support x3, x6, x12, switch level 1327 * decoders only support powers of 2 up to x16. 1328 */ 1329 if (!is_power_of_2(cxl_rr->nr_targets)) { 1330 dev_dbg(&cxlr->dev, "%s:%s: invalid target count %d\n", 1331 dev_name(port->uport_dev), dev_name(&port->dev), 1332 cxl_rr->nr_targets); 1333 return -EINVAL; 1334 } 1335 1336 cxlsd = to_cxl_switch_decoder(&cxld->dev); 1337 if (cxl_rr->nr_targets_set) { 1338 int i, distance = 1; 1339 struct cxl_region_ref *cxl_rr_iter; 1340 1341 /* 1342 * The "distance" between peer downstream ports represents which 1343 * endpoint positions in the region interleave a given port can 1344 * host. 1345 * 1346 * For example, at the root of a hierarchy the distance is 1347 * always 1 as every index targets a different host-bridge. At 1348 * each subsequent switch level those ports map every Nth region 1349 * position where N is the width of the switch == distance. 1350 */ 1351 do { 1352 cxl_rr_iter = cxl_rr_load(iter, cxlr); 1353 distance *= cxl_rr_iter->nr_targets; 1354 iter = to_cxl_port(iter->dev.parent); 1355 } while (!is_cxl_root(iter)); 1356 distance *= cxlrd->cxlsd.cxld.interleave_ways; 1357 1358 for (i = 0; i < cxl_rr->nr_targets_set; i++) 1359 if (ep->dport == cxlsd->target[i]) { 1360 rc = check_last_peer(cxled, ep, cxl_rr, 1361 distance); 1362 if (rc) 1363 return rc; 1364 goto out_target_set; 1365 } 1366 goto add_target; 1367 } 1368 1369 if (is_cxl_root(parent_port)) { 1370 /* 1371 * Root decoder IG is always set to value in CFMWS which 1372 * may be different than this region's IG. We can use the 1373 * region's IG here since interleave_granularity_store() 1374 * does not allow interleaved host-bridges with 1375 * root IG != region IG. 1376 */ 1377 parent_ig = p->interleave_granularity; 1378 parent_iw = cxlrd->cxlsd.cxld.interleave_ways; 1379 /* 1380 * For purposes of address bit routing, use power-of-2 math for 1381 * switch ports. 1382 */ 1383 if (!is_power_of_2(parent_iw)) 1384 parent_iw /= 3; 1385 } else { 1386 struct cxl_region_ref *parent_rr; 1387 struct cxl_decoder *parent_cxld; 1388 1389 parent_rr = cxl_rr_load(parent_port, cxlr); 1390 parent_cxld = parent_rr->decoder; 1391 parent_ig = parent_cxld->interleave_granularity; 1392 parent_iw = parent_cxld->interleave_ways; 1393 } 1394 1395 rc = granularity_to_eig(parent_ig, &peig); 1396 if (rc) { 1397 dev_dbg(&cxlr->dev, "%s:%s: invalid parent granularity: %d\n", 1398 dev_name(parent_port->uport_dev), 1399 dev_name(&parent_port->dev), parent_ig); 1400 return rc; 1401 } 1402 1403 rc = ways_to_eiw(parent_iw, &peiw); 1404 if (rc) { 1405 dev_dbg(&cxlr->dev, "%s:%s: invalid parent interleave: %d\n", 1406 dev_name(parent_port->uport_dev), 1407 dev_name(&parent_port->dev), parent_iw); 1408 return rc; 1409 } 1410 1411 iw = cxl_rr->nr_targets; 1412 rc = ways_to_eiw(iw, &eiw); 1413 if (rc) { 1414 dev_dbg(&cxlr->dev, "%s:%s: invalid port interleave: %d\n", 1415 dev_name(port->uport_dev), dev_name(&port->dev), iw); 1416 return rc; 1417 } 1418 1419 /* 1420 * Interleave granularity is a multiple of @parent_port granularity. 1421 * Multiplier is the parent port interleave ways. 1422 */ 1423 rc = granularity_to_eig(parent_ig * parent_iw, &eig); 1424 if (rc) { 1425 dev_dbg(&cxlr->dev, 1426 "%s: invalid granularity calculation (%d * %d)\n", 1427 dev_name(&parent_port->dev), parent_ig, parent_iw); 1428 return rc; 1429 } 1430 1431 rc = eig_to_granularity(eig, &ig); 1432 if (rc) { 1433 dev_dbg(&cxlr->dev, "%s:%s: invalid interleave: %d\n", 1434 dev_name(port->uport_dev), dev_name(&port->dev), 1435 256 << eig); 1436 return rc; 1437 } 1438 1439 if (iw > 8 || iw > cxlsd->nr_targets) { 1440 dev_dbg(&cxlr->dev, 1441 "%s:%s:%s: ways: %d overflows targets: %d\n", 1442 dev_name(port->uport_dev), dev_name(&port->dev), 1443 dev_name(&cxld->dev), iw, cxlsd->nr_targets); 1444 return -ENXIO; 1445 } 1446 1447 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1448 if (cxld->interleave_ways != iw || 1449 cxld->interleave_granularity != ig || 1450 !region_res_match_cxl_range(p, &cxld->hpa_range) || 1451 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { 1452 dev_err(&cxlr->dev, 1453 "%s:%s %s expected iw: %d ig: %d %pr\n", 1454 dev_name(port->uport_dev), dev_name(&port->dev), 1455 __func__, iw, ig, p->res); 1456 dev_err(&cxlr->dev, 1457 "%s:%s %s got iw: %d ig: %d state: %s %#llx:%#llx\n", 1458 dev_name(port->uport_dev), dev_name(&port->dev), 1459 __func__, cxld->interleave_ways, 1460 cxld->interleave_granularity, 1461 (cxld->flags & CXL_DECODER_F_ENABLE) ? 1462 "enabled" : 1463 "disabled", 1464 cxld->hpa_range.start, cxld->hpa_range.end); 1465 return -ENXIO; 1466 } 1467 } else { 1468 rc = check_interleave_cap(cxld, iw, ig); 1469 if (rc) { 1470 dev_dbg(&cxlr->dev, 1471 "%s:%s iw: %d ig: %d is not supported\n", 1472 dev_name(port->uport_dev), 1473 dev_name(&port->dev), iw, ig); 1474 return rc; 1475 } 1476 1477 cxld->interleave_ways = iw; 1478 cxld->interleave_granularity = ig; 1479 cxld->hpa_range = (struct range) { 1480 .start = p->res->start, 1481 .end = p->res->end, 1482 }; 1483 } 1484 dev_dbg(&cxlr->dev, "%s:%s iw: %d ig: %d\n", dev_name(port->uport_dev), 1485 dev_name(&port->dev), iw, ig); 1486 add_target: 1487 if (cxl_rr->nr_targets_set == cxl_rr->nr_targets) { 1488 dev_dbg(&cxlr->dev, 1489 "%s:%s: targets full trying to add %s:%s at %d\n", 1490 dev_name(port->uport_dev), dev_name(&port->dev), 1491 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 1492 return -ENXIO; 1493 } 1494 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1495 if (cxlsd->target[cxl_rr->nr_targets_set] != ep->dport) { 1496 dev_dbg(&cxlr->dev, "%s:%s: %s expected %s at %d\n", 1497 dev_name(port->uport_dev), dev_name(&port->dev), 1498 dev_name(&cxlsd->cxld.dev), 1499 dev_name(ep->dport->dport_dev), 1500 cxl_rr->nr_targets_set); 1501 return -ENXIO; 1502 } 1503 } else 1504 cxlsd->target[cxl_rr->nr_targets_set] = ep->dport; 1505 inc = 1; 1506 out_target_set: 1507 cxl_rr->nr_targets_set += inc; 1508 dev_dbg(&cxlr->dev, "%s:%s target[%d] = %s for %s:%s @ %d\n", 1509 dev_name(port->uport_dev), dev_name(&port->dev), 1510 cxl_rr->nr_targets_set - 1, dev_name(ep->dport->dport_dev), 1511 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), pos); 1512 1513 return 0; 1514 } 1515 1516 static void cxl_port_reset_targets(struct cxl_port *port, 1517 struct cxl_region *cxlr) 1518 { 1519 struct cxl_region_ref *cxl_rr = cxl_rr_load(port, cxlr); 1520 struct cxl_decoder *cxld; 1521 1522 /* 1523 * After the last endpoint has been detached the entire cxl_rr may now 1524 * be gone. 1525 */ 1526 if (!cxl_rr) 1527 return; 1528 cxl_rr->nr_targets_set = 0; 1529 1530 cxld = cxl_rr->decoder; 1531 cxld->hpa_range = (struct range) { 1532 .start = 0, 1533 .end = -1, 1534 }; 1535 } 1536 1537 static void cxl_region_teardown_targets(struct cxl_region *cxlr) 1538 { 1539 struct cxl_region_params *p = &cxlr->params; 1540 struct cxl_endpoint_decoder *cxled; 1541 struct cxl_dev_state *cxlds; 1542 struct cxl_memdev *cxlmd; 1543 struct cxl_port *iter; 1544 struct cxl_ep *ep; 1545 int i; 1546 1547 /* 1548 * In the auto-discovery case skip automatic teardown since the 1549 * address space is already active 1550 */ 1551 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) 1552 return; 1553 1554 for (i = 0; i < p->nr_targets; i++) { 1555 cxled = p->targets[i]; 1556 cxlmd = cxled_to_memdev(cxled); 1557 cxlds = cxlmd->cxlds; 1558 1559 if (cxlds->rcd) 1560 continue; 1561 1562 iter = cxled_to_port(cxled); 1563 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 1564 iter = to_cxl_port(iter->dev.parent); 1565 1566 for (ep = cxl_ep_load(iter, cxlmd); iter; 1567 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) 1568 cxl_port_reset_targets(iter, cxlr); 1569 } 1570 } 1571 1572 static int cxl_region_setup_targets(struct cxl_region *cxlr) 1573 { 1574 struct cxl_region_params *p = &cxlr->params; 1575 struct cxl_endpoint_decoder *cxled; 1576 struct cxl_dev_state *cxlds; 1577 int i, rc, rch = 0, vh = 0; 1578 struct cxl_memdev *cxlmd; 1579 struct cxl_port *iter; 1580 struct cxl_ep *ep; 1581 1582 for (i = 0; i < p->nr_targets; i++) { 1583 cxled = p->targets[i]; 1584 cxlmd = cxled_to_memdev(cxled); 1585 cxlds = cxlmd->cxlds; 1586 1587 /* validate that all targets agree on topology */ 1588 if (!cxlds->rcd) { 1589 vh++; 1590 } else { 1591 rch++; 1592 continue; 1593 } 1594 1595 iter = cxled_to_port(cxled); 1596 while (!is_cxl_root(to_cxl_port(iter->dev.parent))) 1597 iter = to_cxl_port(iter->dev.parent); 1598 1599 /* 1600 * Descend the topology tree programming / validating 1601 * targets while looking for conflicts. 1602 */ 1603 for (ep = cxl_ep_load(iter, cxlmd); iter; 1604 iter = ep->next, ep = cxl_ep_load(iter, cxlmd)) { 1605 rc = cxl_port_setup_targets(iter, cxlr, cxled); 1606 if (rc) { 1607 cxl_region_teardown_targets(cxlr); 1608 return rc; 1609 } 1610 } 1611 } 1612 1613 if (rch && vh) { 1614 dev_err(&cxlr->dev, "mismatched CXL topologies detected\n"); 1615 cxl_region_teardown_targets(cxlr); 1616 return -ENXIO; 1617 } 1618 1619 return 0; 1620 } 1621 1622 static int cxl_region_validate_position(struct cxl_region *cxlr, 1623 struct cxl_endpoint_decoder *cxled, 1624 int pos) 1625 { 1626 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1627 struct cxl_region_params *p = &cxlr->params; 1628 int i; 1629 1630 if (pos < 0 || pos >= p->interleave_ways) { 1631 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 1632 p->interleave_ways); 1633 return -ENXIO; 1634 } 1635 1636 if (p->targets[pos] == cxled) 1637 return 0; 1638 1639 if (p->targets[pos]) { 1640 struct cxl_endpoint_decoder *cxled_target = p->targets[pos]; 1641 struct cxl_memdev *cxlmd_target = cxled_to_memdev(cxled_target); 1642 1643 dev_dbg(&cxlr->dev, "position %d already assigned to %s:%s\n", 1644 pos, dev_name(&cxlmd_target->dev), 1645 dev_name(&cxled_target->cxld.dev)); 1646 return -EBUSY; 1647 } 1648 1649 for (i = 0; i < p->interleave_ways; i++) { 1650 struct cxl_endpoint_decoder *cxled_target; 1651 struct cxl_memdev *cxlmd_target; 1652 1653 cxled_target = p->targets[i]; 1654 if (!cxled_target) 1655 continue; 1656 1657 cxlmd_target = cxled_to_memdev(cxled_target); 1658 if (cxlmd_target == cxlmd) { 1659 dev_dbg(&cxlr->dev, 1660 "%s already specified at position %d via: %s\n", 1661 dev_name(&cxlmd->dev), pos, 1662 dev_name(&cxled_target->cxld.dev)); 1663 return -EBUSY; 1664 } 1665 } 1666 1667 return 0; 1668 } 1669 1670 static int cxl_region_attach_position(struct cxl_region *cxlr, 1671 struct cxl_root_decoder *cxlrd, 1672 struct cxl_endpoint_decoder *cxled, 1673 const struct cxl_dport *dport, int pos) 1674 { 1675 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1676 struct cxl_switch_decoder *cxlsd = &cxlrd->cxlsd; 1677 struct cxl_decoder *cxld = &cxlsd->cxld; 1678 int iw = cxld->interleave_ways; 1679 struct cxl_port *iter; 1680 int rc; 1681 1682 if (dport != cxlrd->cxlsd.target[pos % iw]) { 1683 dev_dbg(&cxlr->dev, "%s:%s invalid target position for %s\n", 1684 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1685 dev_name(&cxlrd->cxlsd.cxld.dev)); 1686 return -ENXIO; 1687 } 1688 1689 for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 1690 iter = to_cxl_port(iter->dev.parent)) { 1691 rc = cxl_port_attach_region(iter, cxlr, cxled, pos); 1692 if (rc) 1693 goto err; 1694 } 1695 1696 return 0; 1697 1698 err: 1699 for (iter = cxled_to_port(cxled); !is_cxl_root(iter); 1700 iter = to_cxl_port(iter->dev.parent)) 1701 cxl_port_detach_region(iter, cxlr, cxled); 1702 return rc; 1703 } 1704 1705 static int cxl_region_attach_auto(struct cxl_region *cxlr, 1706 struct cxl_endpoint_decoder *cxled, int pos) 1707 { 1708 struct cxl_region_params *p = &cxlr->params; 1709 1710 if (cxled->state != CXL_DECODER_STATE_AUTO) { 1711 dev_err(&cxlr->dev, 1712 "%s: unable to add decoder to autodetected region\n", 1713 dev_name(&cxled->cxld.dev)); 1714 return -EINVAL; 1715 } 1716 1717 if (pos >= 0) { 1718 dev_dbg(&cxlr->dev, "%s: expected auto position, not %d\n", 1719 dev_name(&cxled->cxld.dev), pos); 1720 return -EINVAL; 1721 } 1722 1723 if (p->nr_targets >= p->interleave_ways) { 1724 dev_err(&cxlr->dev, "%s: no more target slots available\n", 1725 dev_name(&cxled->cxld.dev)); 1726 return -ENXIO; 1727 } 1728 1729 /* 1730 * Temporarily record the endpoint decoder into the target array. Yes, 1731 * this means that userspace can view devices in the wrong position 1732 * before the region activates, and must be careful to understand when 1733 * it might be racing region autodiscovery. 1734 */ 1735 pos = p->nr_targets; 1736 p->targets[pos] = cxled; 1737 cxled->pos = pos; 1738 p->nr_targets++; 1739 1740 return 0; 1741 } 1742 1743 static int cmp_interleave_pos(const void *a, const void *b) 1744 { 1745 struct cxl_endpoint_decoder *cxled_a = *(typeof(cxled_a) *)a; 1746 struct cxl_endpoint_decoder *cxled_b = *(typeof(cxled_b) *)b; 1747 1748 return cxled_a->pos - cxled_b->pos; 1749 } 1750 1751 static struct cxl_port *next_port(struct cxl_port *port) 1752 { 1753 if (!port->parent_dport) 1754 return NULL; 1755 return port->parent_dport->port; 1756 } 1757 1758 static int match_switch_decoder_by_range(struct device *dev, 1759 const void *data) 1760 { 1761 struct cxl_switch_decoder *cxlsd; 1762 const struct range *r1, *r2 = data; 1763 1764 1765 if (!is_switch_decoder(dev)) 1766 return 0; 1767 1768 cxlsd = to_cxl_switch_decoder(dev); 1769 r1 = &cxlsd->cxld.hpa_range; 1770 1771 if (is_root_decoder(dev)) 1772 return range_contains(r1, r2); 1773 return (r1->start == r2->start && r1->end == r2->end); 1774 } 1775 1776 static int find_pos_and_ways(struct cxl_port *port, struct range *range, 1777 int *pos, int *ways) 1778 { 1779 struct cxl_switch_decoder *cxlsd; 1780 struct cxl_port *parent; 1781 struct device *dev; 1782 int rc = -ENXIO; 1783 1784 parent = next_port(port); 1785 if (!parent) 1786 return rc; 1787 1788 dev = device_find_child(&parent->dev, range, 1789 match_switch_decoder_by_range); 1790 if (!dev) { 1791 dev_err(port->uport_dev, 1792 "failed to find decoder mapping %#llx-%#llx\n", 1793 range->start, range->end); 1794 return rc; 1795 } 1796 cxlsd = to_cxl_switch_decoder(dev); 1797 *ways = cxlsd->cxld.interleave_ways; 1798 1799 for (int i = 0; i < *ways; i++) { 1800 if (cxlsd->target[i] == port->parent_dport) { 1801 *pos = i; 1802 rc = 0; 1803 break; 1804 } 1805 } 1806 put_device(dev); 1807 1808 return rc; 1809 } 1810 1811 /** 1812 * cxl_calc_interleave_pos() - calculate an endpoint position in a region 1813 * @cxled: endpoint decoder member of given region 1814 * 1815 * The endpoint position is calculated by traversing the topology from 1816 * the endpoint to the root decoder and iteratively applying this 1817 * calculation: 1818 * 1819 * position = position * parent_ways + parent_pos; 1820 * 1821 * ...where @position is inferred from switch and root decoder target lists. 1822 * 1823 * Return: position >= 0 on success 1824 * -ENXIO on failure 1825 */ 1826 static int cxl_calc_interleave_pos(struct cxl_endpoint_decoder *cxled) 1827 { 1828 struct cxl_port *iter, *port = cxled_to_port(cxled); 1829 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1830 struct range *range = &cxled->cxld.hpa_range; 1831 int parent_ways = 0, parent_pos = 0, pos = 0; 1832 int rc; 1833 1834 /* 1835 * Example: the expected interleave order of the 4-way region shown 1836 * below is: mem0, mem2, mem1, mem3 1837 * 1838 * root_port 1839 * / \ 1840 * host_bridge_0 host_bridge_1 1841 * | | | | 1842 * mem0 mem1 mem2 mem3 1843 * 1844 * In the example the calculator will iterate twice. The first iteration 1845 * uses the mem position in the host-bridge and the ways of the host- 1846 * bridge to generate the first, or local, position. The second 1847 * iteration uses the host-bridge position in the root_port and the ways 1848 * of the root_port to refine the position. 1849 * 1850 * A trace of the calculation per endpoint looks like this: 1851 * mem0: pos = 0 * 2 + 0 mem2: pos = 0 * 2 + 0 1852 * pos = 0 * 2 + 0 pos = 0 * 2 + 1 1853 * pos: 0 pos: 1 1854 * 1855 * mem1: pos = 0 * 2 + 1 mem3: pos = 0 * 2 + 1 1856 * pos = 1 * 2 + 0 pos = 1 * 2 + 1 1857 * pos: 2 pos = 3 1858 * 1859 * Note that while this example is simple, the method applies to more 1860 * complex topologies, including those with switches. 1861 */ 1862 1863 /* Iterate from endpoint to root_port refining the position */ 1864 for (iter = port; iter; iter = next_port(iter)) { 1865 if (is_cxl_root(iter)) 1866 break; 1867 1868 rc = find_pos_and_ways(iter, range, &parent_pos, &parent_ways); 1869 if (rc) 1870 return rc; 1871 1872 pos = pos * parent_ways + parent_pos; 1873 } 1874 1875 dev_dbg(&cxlmd->dev, 1876 "decoder:%s parent:%s port:%s range:%#llx-%#llx pos:%d\n", 1877 dev_name(&cxled->cxld.dev), dev_name(cxlmd->dev.parent), 1878 dev_name(&port->dev), range->start, range->end, pos); 1879 1880 return pos; 1881 } 1882 1883 static int cxl_region_sort_targets(struct cxl_region *cxlr) 1884 { 1885 struct cxl_region_params *p = &cxlr->params; 1886 int i, rc = 0; 1887 1888 for (i = 0; i < p->nr_targets; i++) { 1889 struct cxl_endpoint_decoder *cxled = p->targets[i]; 1890 1891 cxled->pos = cxl_calc_interleave_pos(cxled); 1892 /* 1893 * Record that sorting failed, but still continue to calc 1894 * cxled->pos so that follow-on code paths can reliably 1895 * do p->targets[cxled->pos] to self-reference their entry. 1896 */ 1897 if (cxled->pos < 0) 1898 rc = -ENXIO; 1899 } 1900 /* Keep the cxlr target list in interleave position order */ 1901 sort(p->targets, p->nr_targets, sizeof(p->targets[0]), 1902 cmp_interleave_pos, NULL); 1903 1904 dev_dbg(&cxlr->dev, "region sort %s\n", rc ? "failed" : "successful"); 1905 return rc; 1906 } 1907 1908 static int cxl_region_attach(struct cxl_region *cxlr, 1909 struct cxl_endpoint_decoder *cxled, int pos) 1910 { 1911 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 1912 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1913 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1914 struct cxl_region_params *p = &cxlr->params; 1915 struct cxl_port *ep_port, *root_port; 1916 struct cxl_dport *dport; 1917 int rc = -ENXIO; 1918 1919 rc = check_interleave_cap(&cxled->cxld, p->interleave_ways, 1920 p->interleave_granularity); 1921 if (rc) { 1922 dev_dbg(&cxlr->dev, "%s iw: %d ig: %d is not supported\n", 1923 dev_name(&cxled->cxld.dev), p->interleave_ways, 1924 p->interleave_granularity); 1925 return rc; 1926 } 1927 1928 if (cxled->part < 0) { 1929 dev_dbg(&cxlr->dev, "%s dead\n", dev_name(&cxled->cxld.dev)); 1930 return -ENODEV; 1931 } 1932 1933 if (cxlds->part[cxled->part].mode != cxlr->mode) { 1934 dev_dbg(&cxlr->dev, "%s region mode: %d mismatch\n", 1935 dev_name(&cxled->cxld.dev), cxlr->mode); 1936 return -EINVAL; 1937 } 1938 1939 /* all full of members, or interleave config not established? */ 1940 if (p->state > CXL_CONFIG_INTERLEAVE_ACTIVE) { 1941 dev_dbg(&cxlr->dev, "region already active\n"); 1942 return -EBUSY; 1943 } else if (p->state < CXL_CONFIG_INTERLEAVE_ACTIVE) { 1944 dev_dbg(&cxlr->dev, "interleave config missing\n"); 1945 return -ENXIO; 1946 } 1947 1948 if (p->nr_targets >= p->interleave_ways) { 1949 dev_dbg(&cxlr->dev, "region already has %d endpoints\n", 1950 p->nr_targets); 1951 return -EINVAL; 1952 } 1953 1954 ep_port = cxled_to_port(cxled); 1955 root_port = cxlrd_to_port(cxlrd); 1956 dport = cxl_find_dport_by_dev(root_port, ep_port->host_bridge); 1957 if (!dport) { 1958 dev_dbg(&cxlr->dev, "%s:%s invalid target for %s\n", 1959 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1960 dev_name(cxlr->dev.parent)); 1961 return -ENXIO; 1962 } 1963 1964 if (cxled->cxld.target_type != cxlr->type) { 1965 dev_dbg(&cxlr->dev, "%s:%s type mismatch: %d vs %d\n", 1966 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1967 cxled->cxld.target_type, cxlr->type); 1968 return -ENXIO; 1969 } 1970 1971 if (!cxled->dpa_res) { 1972 dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n", 1973 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev)); 1974 return -ENXIO; 1975 } 1976 1977 if (resource_size(cxled->dpa_res) * p->interleave_ways + p->cache_size != 1978 resource_size(p->res)) { 1979 dev_dbg(&cxlr->dev, 1980 "%s:%s-size-%#llx * ways-%d + cache-%#llx != region-size-%#llx\n", 1981 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 1982 (u64)resource_size(cxled->dpa_res), p->interleave_ways, 1983 (u64)p->cache_size, (u64)resource_size(p->res)); 1984 return -EINVAL; 1985 } 1986 1987 cxl_region_perf_data_calculate(cxlr, cxled); 1988 1989 if (test_bit(CXL_REGION_F_AUTO, &cxlr->flags)) { 1990 int i; 1991 1992 rc = cxl_region_attach_auto(cxlr, cxled, pos); 1993 if (rc) 1994 return rc; 1995 1996 /* await more targets to arrive... */ 1997 if (p->nr_targets < p->interleave_ways) 1998 return 0; 1999 2000 /* 2001 * All targets are here, which implies all PCI enumeration that 2002 * affects this region has been completed. Walk the topology to 2003 * sort the devices into their relative region decode position. 2004 */ 2005 rc = cxl_region_sort_targets(cxlr); 2006 if (rc) 2007 return rc; 2008 2009 for (i = 0; i < p->nr_targets; i++) { 2010 cxled = p->targets[i]; 2011 ep_port = cxled_to_port(cxled); 2012 dport = cxl_find_dport_by_dev(root_port, 2013 ep_port->host_bridge); 2014 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, 2015 dport, i); 2016 if (rc) 2017 return rc; 2018 } 2019 2020 rc = cxl_region_setup_targets(cxlr); 2021 if (rc) 2022 return rc; 2023 2024 /* 2025 * If target setup succeeds in the autodiscovery case 2026 * then the region is already committed. 2027 */ 2028 p->state = CXL_CONFIG_COMMIT; 2029 cxl_region_shared_upstream_bandwidth_update(cxlr); 2030 2031 return 0; 2032 } 2033 2034 rc = cxl_region_validate_position(cxlr, cxled, pos); 2035 if (rc) 2036 return rc; 2037 2038 rc = cxl_region_attach_position(cxlr, cxlrd, cxled, dport, pos); 2039 if (rc) 2040 return rc; 2041 2042 p->targets[pos] = cxled; 2043 cxled->pos = pos; 2044 p->nr_targets++; 2045 2046 if (p->nr_targets == p->interleave_ways) { 2047 rc = cxl_region_setup_targets(cxlr); 2048 if (rc) 2049 return rc; 2050 p->state = CXL_CONFIG_ACTIVE; 2051 cxl_region_shared_upstream_bandwidth_update(cxlr); 2052 } 2053 2054 cxled->cxld.interleave_ways = p->interleave_ways; 2055 cxled->cxld.interleave_granularity = p->interleave_granularity; 2056 cxled->cxld.hpa_range = (struct range) { 2057 .start = p->res->start, 2058 .end = p->res->end, 2059 }; 2060 2061 if (p->nr_targets != p->interleave_ways) 2062 return 0; 2063 2064 /* 2065 * Test the auto-discovery position calculator function 2066 * against this successfully created user-defined region. 2067 * A fail message here means that this interleave config 2068 * will fail when presented as CXL_REGION_F_AUTO. 2069 */ 2070 for (int i = 0; i < p->nr_targets; i++) { 2071 struct cxl_endpoint_decoder *cxled = p->targets[i]; 2072 int test_pos; 2073 2074 test_pos = cxl_calc_interleave_pos(cxled); 2075 dev_dbg(&cxled->cxld.dev, 2076 "Test cxl_calc_interleave_pos(): %s test_pos:%d cxled->pos:%d\n", 2077 (test_pos == cxled->pos) ? "success" : "fail", 2078 test_pos, cxled->pos); 2079 } 2080 2081 return 0; 2082 } 2083 2084 static int cxl_region_detach(struct cxl_endpoint_decoder *cxled) 2085 { 2086 struct cxl_port *iter, *ep_port = cxled_to_port(cxled); 2087 struct cxl_region *cxlr = cxled->cxld.region; 2088 struct cxl_region_params *p; 2089 int rc = 0; 2090 2091 lockdep_assert_held_write(&cxl_region_rwsem); 2092 2093 if (!cxlr) 2094 return 0; 2095 2096 p = &cxlr->params; 2097 get_device(&cxlr->dev); 2098 2099 if (p->state > CXL_CONFIG_ACTIVE) { 2100 cxl_region_decode_reset(cxlr, p->interleave_ways); 2101 p->state = CXL_CONFIG_ACTIVE; 2102 } 2103 2104 for (iter = ep_port; !is_cxl_root(iter); 2105 iter = to_cxl_port(iter->dev.parent)) 2106 cxl_port_detach_region(iter, cxlr, cxled); 2107 2108 if (cxled->pos < 0 || cxled->pos >= p->interleave_ways || 2109 p->targets[cxled->pos] != cxled) { 2110 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 2111 2112 dev_WARN_ONCE(&cxlr->dev, 1, "expected %s:%s at position %d\n", 2113 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 2114 cxled->pos); 2115 goto out; 2116 } 2117 2118 if (p->state == CXL_CONFIG_ACTIVE) { 2119 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 2120 cxl_region_teardown_targets(cxlr); 2121 } 2122 p->targets[cxled->pos] = NULL; 2123 p->nr_targets--; 2124 cxled->cxld.hpa_range = (struct range) { 2125 .start = 0, 2126 .end = -1, 2127 }; 2128 2129 /* notify the region driver that one of its targets has departed */ 2130 up_write(&cxl_region_rwsem); 2131 device_release_driver(&cxlr->dev); 2132 down_write(&cxl_region_rwsem); 2133 out: 2134 put_device(&cxlr->dev); 2135 return rc; 2136 } 2137 2138 void cxl_decoder_kill_region(struct cxl_endpoint_decoder *cxled) 2139 { 2140 down_write(&cxl_region_rwsem); 2141 cxled->part = -1; 2142 cxl_region_detach(cxled); 2143 up_write(&cxl_region_rwsem); 2144 } 2145 2146 static int attach_target(struct cxl_region *cxlr, 2147 struct cxl_endpoint_decoder *cxled, int pos, 2148 unsigned int state) 2149 { 2150 int rc = 0; 2151 2152 if (state == TASK_INTERRUPTIBLE) 2153 rc = down_write_killable(&cxl_region_rwsem); 2154 else 2155 down_write(&cxl_region_rwsem); 2156 if (rc) 2157 return rc; 2158 2159 down_read(&cxl_dpa_rwsem); 2160 rc = cxl_region_attach(cxlr, cxled, pos); 2161 up_read(&cxl_dpa_rwsem); 2162 up_write(&cxl_region_rwsem); 2163 return rc; 2164 } 2165 2166 static int detach_target(struct cxl_region *cxlr, int pos) 2167 { 2168 struct cxl_region_params *p = &cxlr->params; 2169 int rc; 2170 2171 rc = down_write_killable(&cxl_region_rwsem); 2172 if (rc) 2173 return rc; 2174 2175 if (pos >= p->interleave_ways) { 2176 dev_dbg(&cxlr->dev, "position %d out of range %d\n", pos, 2177 p->interleave_ways); 2178 rc = -ENXIO; 2179 goto out; 2180 } 2181 2182 if (!p->targets[pos]) { 2183 rc = 0; 2184 goto out; 2185 } 2186 2187 rc = cxl_region_detach(p->targets[pos]); 2188 out: 2189 up_write(&cxl_region_rwsem); 2190 return rc; 2191 } 2192 2193 static size_t store_targetN(struct cxl_region *cxlr, const char *buf, int pos, 2194 size_t len) 2195 { 2196 int rc; 2197 2198 if (sysfs_streq(buf, "\n")) 2199 rc = detach_target(cxlr, pos); 2200 else { 2201 struct device *dev; 2202 2203 dev = bus_find_device_by_name(&cxl_bus_type, NULL, buf); 2204 if (!dev) 2205 return -ENODEV; 2206 2207 if (!is_endpoint_decoder(dev)) { 2208 rc = -EINVAL; 2209 goto out; 2210 } 2211 2212 rc = attach_target(cxlr, to_cxl_endpoint_decoder(dev), pos, 2213 TASK_INTERRUPTIBLE); 2214 out: 2215 put_device(dev); 2216 } 2217 2218 if (rc < 0) 2219 return rc; 2220 return len; 2221 } 2222 2223 #define TARGET_ATTR_RW(n) \ 2224 static ssize_t target##n##_show( \ 2225 struct device *dev, struct device_attribute *attr, char *buf) \ 2226 { \ 2227 return show_targetN(to_cxl_region(dev), buf, (n)); \ 2228 } \ 2229 static ssize_t target##n##_store(struct device *dev, \ 2230 struct device_attribute *attr, \ 2231 const char *buf, size_t len) \ 2232 { \ 2233 return store_targetN(to_cxl_region(dev), buf, (n), len); \ 2234 } \ 2235 static DEVICE_ATTR_RW(target##n) 2236 2237 TARGET_ATTR_RW(0); 2238 TARGET_ATTR_RW(1); 2239 TARGET_ATTR_RW(2); 2240 TARGET_ATTR_RW(3); 2241 TARGET_ATTR_RW(4); 2242 TARGET_ATTR_RW(5); 2243 TARGET_ATTR_RW(6); 2244 TARGET_ATTR_RW(7); 2245 TARGET_ATTR_RW(8); 2246 TARGET_ATTR_RW(9); 2247 TARGET_ATTR_RW(10); 2248 TARGET_ATTR_RW(11); 2249 TARGET_ATTR_RW(12); 2250 TARGET_ATTR_RW(13); 2251 TARGET_ATTR_RW(14); 2252 TARGET_ATTR_RW(15); 2253 2254 static struct attribute *target_attrs[] = { 2255 &dev_attr_target0.attr, 2256 &dev_attr_target1.attr, 2257 &dev_attr_target2.attr, 2258 &dev_attr_target3.attr, 2259 &dev_attr_target4.attr, 2260 &dev_attr_target5.attr, 2261 &dev_attr_target6.attr, 2262 &dev_attr_target7.attr, 2263 &dev_attr_target8.attr, 2264 &dev_attr_target9.attr, 2265 &dev_attr_target10.attr, 2266 &dev_attr_target11.attr, 2267 &dev_attr_target12.attr, 2268 &dev_attr_target13.attr, 2269 &dev_attr_target14.attr, 2270 &dev_attr_target15.attr, 2271 NULL, 2272 }; 2273 2274 static umode_t cxl_region_target_visible(struct kobject *kobj, 2275 struct attribute *a, int n) 2276 { 2277 struct device *dev = kobj_to_dev(kobj); 2278 struct cxl_region *cxlr = to_cxl_region(dev); 2279 struct cxl_region_params *p = &cxlr->params; 2280 2281 if (n < p->interleave_ways) 2282 return a->mode; 2283 return 0; 2284 } 2285 2286 static const struct attribute_group cxl_region_target_group = { 2287 .attrs = target_attrs, 2288 .is_visible = cxl_region_target_visible, 2289 }; 2290 2291 static const struct attribute_group *get_cxl_region_target_group(void) 2292 { 2293 return &cxl_region_target_group; 2294 } 2295 2296 static const struct attribute_group *region_groups[] = { 2297 &cxl_base_attribute_group, 2298 &cxl_region_group, 2299 &cxl_region_target_group, 2300 &cxl_region_access0_coordinate_group, 2301 &cxl_region_access1_coordinate_group, 2302 NULL, 2303 }; 2304 2305 static void cxl_region_release(struct device *dev) 2306 { 2307 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev->parent); 2308 struct cxl_region *cxlr = to_cxl_region(dev); 2309 int id = atomic_read(&cxlrd->region_id); 2310 2311 /* 2312 * Try to reuse the recently idled id rather than the cached 2313 * next id to prevent the region id space from increasing 2314 * unnecessarily. 2315 */ 2316 if (cxlr->id < id) 2317 if (atomic_try_cmpxchg(&cxlrd->region_id, &id, cxlr->id)) { 2318 memregion_free(id); 2319 goto out; 2320 } 2321 2322 memregion_free(cxlr->id); 2323 out: 2324 put_device(dev->parent); 2325 kfree(cxlr); 2326 } 2327 2328 const struct device_type cxl_region_type = { 2329 .name = "cxl_region", 2330 .release = cxl_region_release, 2331 .groups = region_groups 2332 }; 2333 2334 bool is_cxl_region(struct device *dev) 2335 { 2336 return dev->type == &cxl_region_type; 2337 } 2338 EXPORT_SYMBOL_NS_GPL(is_cxl_region, "CXL"); 2339 2340 static struct cxl_region *to_cxl_region(struct device *dev) 2341 { 2342 if (dev_WARN_ONCE(dev, dev->type != &cxl_region_type, 2343 "not a cxl_region device\n")) 2344 return NULL; 2345 2346 return container_of(dev, struct cxl_region, dev); 2347 } 2348 2349 static void unregister_region(void *_cxlr) 2350 { 2351 struct cxl_region *cxlr = _cxlr; 2352 struct cxl_region_params *p = &cxlr->params; 2353 int i; 2354 2355 device_del(&cxlr->dev); 2356 2357 /* 2358 * Now that region sysfs is shutdown, the parameter block is now 2359 * read-only, so no need to hold the region rwsem to access the 2360 * region parameters. 2361 */ 2362 for (i = 0; i < p->interleave_ways; i++) 2363 detach_target(cxlr, i); 2364 2365 cxl_region_iomem_release(cxlr); 2366 put_device(&cxlr->dev); 2367 } 2368 2369 static struct lock_class_key cxl_region_key; 2370 2371 static struct cxl_region *cxl_region_alloc(struct cxl_root_decoder *cxlrd, int id) 2372 { 2373 struct cxl_region *cxlr; 2374 struct device *dev; 2375 2376 cxlr = kzalloc(sizeof(*cxlr), GFP_KERNEL); 2377 if (!cxlr) { 2378 memregion_free(id); 2379 return ERR_PTR(-ENOMEM); 2380 } 2381 2382 dev = &cxlr->dev; 2383 device_initialize(dev); 2384 lockdep_set_class(&dev->mutex, &cxl_region_key); 2385 dev->parent = &cxlrd->cxlsd.cxld.dev; 2386 /* 2387 * Keep root decoder pinned through cxl_region_release to fixup 2388 * region id allocations 2389 */ 2390 get_device(dev->parent); 2391 device_set_pm_not_required(dev); 2392 dev->bus = &cxl_bus_type; 2393 dev->type = &cxl_region_type; 2394 cxlr->id = id; 2395 2396 return cxlr; 2397 } 2398 2399 static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) 2400 { 2401 int cset = 0; 2402 int rc; 2403 2404 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { 2405 if (cxlr->coord[i].read_bandwidth) { 2406 rc = 0; 2407 if (cxl_need_node_perf_attrs_update(nid)) 2408 node_set_perf_attrs(nid, &cxlr->coord[i], i); 2409 else 2410 rc = cxl_update_hmat_access_coordinates(nid, cxlr, i); 2411 2412 if (rc == 0) 2413 cset++; 2414 } 2415 } 2416 2417 if (!cset) 2418 return false; 2419 2420 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access0_group()); 2421 if (rc) 2422 dev_dbg(&cxlr->dev, "Failed to update access0 group\n"); 2423 2424 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_access1_group()); 2425 if (rc) 2426 dev_dbg(&cxlr->dev, "Failed to update access1 group\n"); 2427 2428 return true; 2429 } 2430 2431 static int cxl_region_perf_attrs_callback(struct notifier_block *nb, 2432 unsigned long action, void *arg) 2433 { 2434 struct cxl_region *cxlr = container_of(nb, struct cxl_region, 2435 memory_notifier); 2436 struct memory_notify *mnb = arg; 2437 int nid = mnb->status_change_nid; 2438 int region_nid; 2439 2440 if (nid == NUMA_NO_NODE || action != MEM_ONLINE) 2441 return NOTIFY_DONE; 2442 2443 /* 2444 * No need to hold cxl_region_rwsem; region parameters are stable 2445 * within the cxl_region driver. 2446 */ 2447 region_nid = phys_to_target_node(cxlr->params.res->start); 2448 if (nid != region_nid) 2449 return NOTIFY_DONE; 2450 2451 if (!cxl_region_update_coordinates(cxlr, nid)) 2452 return NOTIFY_DONE; 2453 2454 return NOTIFY_OK; 2455 } 2456 2457 static int cxl_region_calculate_adistance(struct notifier_block *nb, 2458 unsigned long nid, void *data) 2459 { 2460 struct cxl_region *cxlr = container_of(nb, struct cxl_region, 2461 adist_notifier); 2462 struct access_coordinate *perf; 2463 int *adist = data; 2464 int region_nid; 2465 2466 /* 2467 * No need to hold cxl_region_rwsem; region parameters are stable 2468 * within the cxl_region driver. 2469 */ 2470 region_nid = phys_to_target_node(cxlr->params.res->start); 2471 if (nid != region_nid) 2472 return NOTIFY_OK; 2473 2474 perf = &cxlr->coord[ACCESS_COORDINATE_CPU]; 2475 2476 if (mt_perf_to_adistance(perf, adist)) 2477 return NOTIFY_OK; 2478 2479 return NOTIFY_STOP; 2480 } 2481 2482 /** 2483 * devm_cxl_add_region - Adds a region to a decoder 2484 * @cxlrd: root decoder 2485 * @id: memregion id to create, or memregion_free() on failure 2486 * @mode: mode for the endpoint decoders of this region 2487 * @type: select whether this is an expander or accelerator (type-2 or type-3) 2488 * 2489 * This is the second step of region initialization. Regions exist within an 2490 * address space which is mapped by a @cxlrd. 2491 * 2492 * Return: 0 if the region was added to the @cxlrd, else returns negative error 2493 * code. The region will be named "regionZ" where Z is the unique region number. 2494 */ 2495 static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, 2496 int id, 2497 enum cxl_partition_mode mode, 2498 enum cxl_decoder_type type) 2499 { 2500 struct cxl_port *port = to_cxl_port(cxlrd->cxlsd.cxld.dev.parent); 2501 struct cxl_region *cxlr; 2502 struct device *dev; 2503 int rc; 2504 2505 cxlr = cxl_region_alloc(cxlrd, id); 2506 if (IS_ERR(cxlr)) 2507 return cxlr; 2508 cxlr->mode = mode; 2509 cxlr->type = type; 2510 2511 dev = &cxlr->dev; 2512 rc = dev_set_name(dev, "region%d", id); 2513 if (rc) 2514 goto err; 2515 2516 rc = device_add(dev); 2517 if (rc) 2518 goto err; 2519 2520 rc = devm_add_action_or_reset(port->uport_dev, unregister_region, cxlr); 2521 if (rc) 2522 return ERR_PTR(rc); 2523 2524 dev_dbg(port->uport_dev, "%s: created %s\n", 2525 dev_name(&cxlrd->cxlsd.cxld.dev), dev_name(dev)); 2526 return cxlr; 2527 2528 err: 2529 put_device(dev); 2530 return ERR_PTR(rc); 2531 } 2532 2533 static ssize_t __create_region_show(struct cxl_root_decoder *cxlrd, char *buf) 2534 { 2535 return sysfs_emit(buf, "region%u\n", atomic_read(&cxlrd->region_id)); 2536 } 2537 2538 static ssize_t create_pmem_region_show(struct device *dev, 2539 struct device_attribute *attr, char *buf) 2540 { 2541 return __create_region_show(to_cxl_root_decoder(dev), buf); 2542 } 2543 2544 static ssize_t create_ram_region_show(struct device *dev, 2545 struct device_attribute *attr, char *buf) 2546 { 2547 return __create_region_show(to_cxl_root_decoder(dev), buf); 2548 } 2549 2550 static struct cxl_region *__create_region(struct cxl_root_decoder *cxlrd, 2551 enum cxl_partition_mode mode, int id) 2552 { 2553 int rc; 2554 2555 switch (mode) { 2556 case CXL_PARTMODE_RAM: 2557 case CXL_PARTMODE_PMEM: 2558 break; 2559 default: 2560 dev_err(&cxlrd->cxlsd.cxld.dev, "unsupported mode %d\n", mode); 2561 return ERR_PTR(-EINVAL); 2562 } 2563 2564 rc = memregion_alloc(GFP_KERNEL); 2565 if (rc < 0) 2566 return ERR_PTR(rc); 2567 2568 if (atomic_cmpxchg(&cxlrd->region_id, id, rc) != id) { 2569 memregion_free(rc); 2570 return ERR_PTR(-EBUSY); 2571 } 2572 2573 return devm_cxl_add_region(cxlrd, id, mode, CXL_DECODER_HOSTONLYMEM); 2574 } 2575 2576 static ssize_t create_region_store(struct device *dev, const char *buf, 2577 size_t len, enum cxl_partition_mode mode) 2578 { 2579 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2580 struct cxl_region *cxlr; 2581 int rc, id; 2582 2583 rc = sscanf(buf, "region%d\n", &id); 2584 if (rc != 1) 2585 return -EINVAL; 2586 2587 cxlr = __create_region(cxlrd, mode, id); 2588 if (IS_ERR(cxlr)) 2589 return PTR_ERR(cxlr); 2590 2591 return len; 2592 } 2593 2594 static ssize_t create_pmem_region_store(struct device *dev, 2595 struct device_attribute *attr, 2596 const char *buf, size_t len) 2597 { 2598 return create_region_store(dev, buf, len, CXL_PARTMODE_PMEM); 2599 } 2600 DEVICE_ATTR_RW(create_pmem_region); 2601 2602 static ssize_t create_ram_region_store(struct device *dev, 2603 struct device_attribute *attr, 2604 const char *buf, size_t len) 2605 { 2606 return create_region_store(dev, buf, len, CXL_PARTMODE_RAM); 2607 } 2608 DEVICE_ATTR_RW(create_ram_region); 2609 2610 static ssize_t region_show(struct device *dev, struct device_attribute *attr, 2611 char *buf) 2612 { 2613 struct cxl_decoder *cxld = to_cxl_decoder(dev); 2614 ssize_t rc; 2615 2616 rc = down_read_interruptible(&cxl_region_rwsem); 2617 if (rc) 2618 return rc; 2619 2620 if (cxld->region) 2621 rc = sysfs_emit(buf, "%s\n", dev_name(&cxld->region->dev)); 2622 else 2623 rc = sysfs_emit(buf, "\n"); 2624 up_read(&cxl_region_rwsem); 2625 2626 return rc; 2627 } 2628 DEVICE_ATTR_RO(region); 2629 2630 static struct cxl_region * 2631 cxl_find_region_by_name(struct cxl_root_decoder *cxlrd, const char *name) 2632 { 2633 struct cxl_decoder *cxld = &cxlrd->cxlsd.cxld; 2634 struct device *region_dev; 2635 2636 region_dev = device_find_child_by_name(&cxld->dev, name); 2637 if (!region_dev) 2638 return ERR_PTR(-ENODEV); 2639 2640 return to_cxl_region(region_dev); 2641 } 2642 2643 static ssize_t delete_region_store(struct device *dev, 2644 struct device_attribute *attr, 2645 const char *buf, size_t len) 2646 { 2647 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(dev); 2648 struct cxl_port *port = to_cxl_port(dev->parent); 2649 struct cxl_region *cxlr; 2650 2651 cxlr = cxl_find_region_by_name(cxlrd, buf); 2652 if (IS_ERR(cxlr)) 2653 return PTR_ERR(cxlr); 2654 2655 devm_release_action(port->uport_dev, unregister_region, cxlr); 2656 put_device(&cxlr->dev); 2657 2658 return len; 2659 } 2660 DEVICE_ATTR_WO(delete_region); 2661 2662 static void cxl_pmem_region_release(struct device *dev) 2663 { 2664 struct cxl_pmem_region *cxlr_pmem = to_cxl_pmem_region(dev); 2665 int i; 2666 2667 for (i = 0; i < cxlr_pmem->nr_mappings; i++) { 2668 struct cxl_memdev *cxlmd = cxlr_pmem->mapping[i].cxlmd; 2669 2670 put_device(&cxlmd->dev); 2671 } 2672 2673 kfree(cxlr_pmem); 2674 } 2675 2676 static const struct attribute_group *cxl_pmem_region_attribute_groups[] = { 2677 &cxl_base_attribute_group, 2678 NULL, 2679 }; 2680 2681 const struct device_type cxl_pmem_region_type = { 2682 .name = "cxl_pmem_region", 2683 .release = cxl_pmem_region_release, 2684 .groups = cxl_pmem_region_attribute_groups, 2685 }; 2686 2687 bool is_cxl_pmem_region(struct device *dev) 2688 { 2689 return dev->type == &cxl_pmem_region_type; 2690 } 2691 EXPORT_SYMBOL_NS_GPL(is_cxl_pmem_region, "CXL"); 2692 2693 struct cxl_pmem_region *to_cxl_pmem_region(struct device *dev) 2694 { 2695 if (dev_WARN_ONCE(dev, !is_cxl_pmem_region(dev), 2696 "not a cxl_pmem_region device\n")) 2697 return NULL; 2698 return container_of(dev, struct cxl_pmem_region, dev); 2699 } 2700 EXPORT_SYMBOL_NS_GPL(to_cxl_pmem_region, "CXL"); 2701 2702 struct cxl_poison_context { 2703 struct cxl_port *port; 2704 int part; 2705 u64 offset; 2706 }; 2707 2708 static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd, 2709 struct cxl_poison_context *ctx) 2710 { 2711 struct cxl_dev_state *cxlds = cxlmd->cxlds; 2712 const struct resource *res; 2713 struct resource *p, *last; 2714 u64 offset, length; 2715 int rc = 0; 2716 2717 if (ctx->part < 0) 2718 return 0; 2719 2720 /* 2721 * Collect poison for the remaining unmapped resources after 2722 * poison is collected by committed endpoints decoders. 2723 */ 2724 for (int i = ctx->part; i < cxlds->nr_partitions; i++) { 2725 res = &cxlds->part[i].res; 2726 for (p = res->child, last = NULL; p; p = p->sibling) 2727 last = p; 2728 if (last) 2729 offset = last->end + 1; 2730 else 2731 offset = res->start; 2732 length = res->end - offset + 1; 2733 if (!length) 2734 break; 2735 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 2736 if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM) 2737 continue; 2738 if (rc) 2739 break; 2740 } 2741 2742 return rc; 2743 } 2744 2745 static int poison_by_decoder(struct device *dev, void *arg) 2746 { 2747 struct cxl_poison_context *ctx = arg; 2748 struct cxl_endpoint_decoder *cxled; 2749 enum cxl_partition_mode mode; 2750 struct cxl_dev_state *cxlds; 2751 struct cxl_memdev *cxlmd; 2752 u64 offset, length; 2753 int rc = 0; 2754 2755 if (!is_endpoint_decoder(dev)) 2756 return rc; 2757 2758 cxled = to_cxl_endpoint_decoder(dev); 2759 if (!cxled->dpa_res) 2760 return rc; 2761 2762 cxlmd = cxled_to_memdev(cxled); 2763 cxlds = cxlmd->cxlds; 2764 mode = cxlds->part[cxled->part].mode; 2765 2766 if (cxled->skip) { 2767 offset = cxled->dpa_res->start - cxled->skip; 2768 length = cxled->skip; 2769 rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); 2770 if (rc == -EFAULT && mode == CXL_PARTMODE_RAM) 2771 rc = 0; 2772 if (rc) 2773 return rc; 2774 } 2775 2776 offset = cxled->dpa_res->start; 2777 length = cxled->dpa_res->end - offset + 1; 2778 rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region); 2779 if (rc == -EFAULT && mode == CXL_PARTMODE_RAM) 2780 rc = 0; 2781 if (rc) 2782 return rc; 2783 2784 /* Iterate until commit_end is reached */ 2785 if (cxled->cxld.id == ctx->port->commit_end) { 2786 ctx->offset = cxled->dpa_res->end + 1; 2787 ctx->part = cxled->part; 2788 return 1; 2789 } 2790 2791 return 0; 2792 } 2793 2794 int cxl_get_poison_by_endpoint(struct cxl_port *port) 2795 { 2796 struct cxl_poison_context ctx; 2797 int rc = 0; 2798 2799 ctx = (struct cxl_poison_context) { 2800 .port = port, 2801 .part = -1, 2802 }; 2803 2804 rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder); 2805 if (rc == 1) 2806 rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev), 2807 &ctx); 2808 2809 return rc; 2810 } 2811 2812 struct cxl_dpa_to_region_context { 2813 struct cxl_region *cxlr; 2814 u64 dpa; 2815 }; 2816 2817 static int __cxl_dpa_to_region(struct device *dev, void *arg) 2818 { 2819 struct cxl_dpa_to_region_context *ctx = arg; 2820 struct cxl_endpoint_decoder *cxled; 2821 struct cxl_region *cxlr; 2822 u64 dpa = ctx->dpa; 2823 2824 if (!is_endpoint_decoder(dev)) 2825 return 0; 2826 2827 cxled = to_cxl_endpoint_decoder(dev); 2828 if (!cxled || !cxled->dpa_res || !resource_size(cxled->dpa_res)) 2829 return 0; 2830 2831 if (dpa > cxled->dpa_res->end || dpa < cxled->dpa_res->start) 2832 return 0; 2833 2834 /* 2835 * Stop the region search (return 1) when an endpoint mapping is 2836 * found. The region may not be fully constructed so offering 2837 * the cxlr in the context structure is not guaranteed. 2838 */ 2839 cxlr = cxled->cxld.region; 2840 if (cxlr) 2841 dev_dbg(dev, "dpa:0x%llx mapped in region:%s\n", dpa, 2842 dev_name(&cxlr->dev)); 2843 else 2844 dev_dbg(dev, "dpa:0x%llx mapped in endpoint:%s\n", dpa, 2845 dev_name(dev)); 2846 2847 ctx->cxlr = cxlr; 2848 2849 return 1; 2850 } 2851 2852 struct cxl_region *cxl_dpa_to_region(const struct cxl_memdev *cxlmd, u64 dpa) 2853 { 2854 struct cxl_dpa_to_region_context ctx; 2855 struct cxl_port *port; 2856 2857 ctx = (struct cxl_dpa_to_region_context) { 2858 .dpa = dpa, 2859 }; 2860 port = cxlmd->endpoint; 2861 if (port && is_cxl_endpoint(port) && cxl_num_decoders_committed(port)) 2862 device_for_each_child(&port->dev, &ctx, __cxl_dpa_to_region); 2863 2864 return ctx.cxlr; 2865 } 2866 2867 static bool cxl_is_hpa_in_chunk(u64 hpa, struct cxl_region *cxlr, int pos) 2868 { 2869 struct cxl_region_params *p = &cxlr->params; 2870 int gran = p->interleave_granularity; 2871 int ways = p->interleave_ways; 2872 u64 offset; 2873 2874 /* Is the hpa in an expected chunk for its pos(-ition) */ 2875 offset = hpa - p->res->start; 2876 offset = do_div(offset, gran * ways); 2877 if ((offset >= pos * gran) && (offset < (pos + 1) * gran)) 2878 return true; 2879 2880 dev_dbg(&cxlr->dev, 2881 "Addr trans fail: hpa 0x%llx not in expected chunk\n", hpa); 2882 2883 return false; 2884 } 2885 2886 u64 cxl_dpa_to_hpa(struct cxl_region *cxlr, const struct cxl_memdev *cxlmd, 2887 u64 dpa) 2888 { 2889 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 2890 u64 dpa_offset, hpa_offset, bits_upper, mask_upper, hpa; 2891 struct cxl_region_params *p = &cxlr->params; 2892 struct cxl_endpoint_decoder *cxled = NULL; 2893 u16 eig = 0; 2894 u8 eiw = 0; 2895 int pos; 2896 2897 for (int i = 0; i < p->nr_targets; i++) { 2898 cxled = p->targets[i]; 2899 if (cxlmd == cxled_to_memdev(cxled)) 2900 break; 2901 } 2902 if (!cxled || cxlmd != cxled_to_memdev(cxled)) 2903 return ULLONG_MAX; 2904 2905 pos = cxled->pos; 2906 ways_to_eiw(p->interleave_ways, &eiw); 2907 granularity_to_eig(p->interleave_granularity, &eig); 2908 2909 /* 2910 * The device position in the region interleave set was removed 2911 * from the offset at HPA->DPA translation. To reconstruct the 2912 * HPA, place the 'pos' in the offset. 2913 * 2914 * The placement of 'pos' in the HPA is determined by interleave 2915 * ways and granularity and is defined in the CXL Spec 3.0 Section 2916 * 8.2.4.19.13 Implementation Note: Device Decode Logic 2917 */ 2918 2919 /* Remove the dpa base */ 2920 dpa_offset = dpa - cxl_dpa_resource_start(cxled); 2921 2922 mask_upper = GENMASK_ULL(51, eig + 8); 2923 2924 if (eiw < 8) { 2925 hpa_offset = (dpa_offset & mask_upper) << eiw; 2926 hpa_offset |= pos << (eig + 8); 2927 } else { 2928 bits_upper = (dpa_offset & mask_upper) >> (eig + 8); 2929 bits_upper = bits_upper * 3; 2930 hpa_offset = ((bits_upper << (eiw - 8)) + pos) << (eig + 8); 2931 } 2932 2933 /* The lower bits remain unchanged */ 2934 hpa_offset |= dpa_offset & GENMASK_ULL(eig + 7, 0); 2935 2936 /* Apply the hpa_offset to the region base address */ 2937 hpa = hpa_offset + p->res->start + p->cache_size; 2938 2939 /* Root decoder translation overrides typical modulo decode */ 2940 if (cxlrd->hpa_to_spa) 2941 hpa = cxlrd->hpa_to_spa(cxlrd, hpa); 2942 2943 if (hpa < p->res->start || hpa > p->res->end) { 2944 dev_dbg(&cxlr->dev, 2945 "Addr trans fail: hpa 0x%llx not in region\n", hpa); 2946 return ULLONG_MAX; 2947 } 2948 2949 /* Simple chunk check, by pos & gran, only applies to modulo decodes */ 2950 if (!cxlrd->hpa_to_spa && (!cxl_is_hpa_in_chunk(hpa, cxlr, pos))) 2951 return ULLONG_MAX; 2952 2953 return hpa; 2954 } 2955 2956 static struct lock_class_key cxl_pmem_region_key; 2957 2958 static int cxl_pmem_region_alloc(struct cxl_region *cxlr) 2959 { 2960 struct cxl_region_params *p = &cxlr->params; 2961 struct cxl_nvdimm_bridge *cxl_nvb; 2962 struct device *dev; 2963 int i; 2964 2965 guard(rwsem_read)(&cxl_region_rwsem); 2966 if (p->state != CXL_CONFIG_COMMIT) 2967 return -ENXIO; 2968 2969 struct cxl_pmem_region *cxlr_pmem __free(kfree) = 2970 kzalloc(struct_size(cxlr_pmem, mapping, p->nr_targets), GFP_KERNEL); 2971 if (!cxlr_pmem) 2972 return -ENOMEM; 2973 2974 cxlr_pmem->hpa_range.start = p->res->start; 2975 cxlr_pmem->hpa_range.end = p->res->end; 2976 2977 /* Snapshot the region configuration underneath the cxl_region_rwsem */ 2978 cxlr_pmem->nr_mappings = p->nr_targets; 2979 for (i = 0; i < p->nr_targets; i++) { 2980 struct cxl_endpoint_decoder *cxled = p->targets[i]; 2981 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 2982 struct cxl_pmem_region_mapping *m = &cxlr_pmem->mapping[i]; 2983 2984 /* 2985 * Regions never span CXL root devices, so by definition the 2986 * bridge for one device is the same for all. 2987 */ 2988 if (i == 0) { 2989 cxl_nvb = cxl_find_nvdimm_bridge(cxlmd->endpoint); 2990 if (!cxl_nvb) 2991 return -ENODEV; 2992 cxlr->cxl_nvb = cxl_nvb; 2993 } 2994 m->cxlmd = cxlmd; 2995 get_device(&cxlmd->dev); 2996 m->start = cxled->dpa_res->start; 2997 m->size = resource_size(cxled->dpa_res); 2998 m->position = i; 2999 } 3000 3001 dev = &cxlr_pmem->dev; 3002 device_initialize(dev); 3003 lockdep_set_class(&dev->mutex, &cxl_pmem_region_key); 3004 device_set_pm_not_required(dev); 3005 dev->parent = &cxlr->dev; 3006 dev->bus = &cxl_bus_type; 3007 dev->type = &cxl_pmem_region_type; 3008 cxlr_pmem->cxlr = cxlr; 3009 cxlr->cxlr_pmem = no_free_ptr(cxlr_pmem); 3010 3011 return 0; 3012 } 3013 3014 static void cxl_dax_region_release(struct device *dev) 3015 { 3016 struct cxl_dax_region *cxlr_dax = to_cxl_dax_region(dev); 3017 3018 kfree(cxlr_dax); 3019 } 3020 3021 static const struct attribute_group *cxl_dax_region_attribute_groups[] = { 3022 &cxl_base_attribute_group, 3023 NULL, 3024 }; 3025 3026 const struct device_type cxl_dax_region_type = { 3027 .name = "cxl_dax_region", 3028 .release = cxl_dax_region_release, 3029 .groups = cxl_dax_region_attribute_groups, 3030 }; 3031 3032 static bool is_cxl_dax_region(struct device *dev) 3033 { 3034 return dev->type == &cxl_dax_region_type; 3035 } 3036 3037 struct cxl_dax_region *to_cxl_dax_region(struct device *dev) 3038 { 3039 if (dev_WARN_ONCE(dev, !is_cxl_dax_region(dev), 3040 "not a cxl_dax_region device\n")) 3041 return NULL; 3042 return container_of(dev, struct cxl_dax_region, dev); 3043 } 3044 EXPORT_SYMBOL_NS_GPL(to_cxl_dax_region, "CXL"); 3045 3046 static struct lock_class_key cxl_dax_region_key; 3047 3048 static struct cxl_dax_region *cxl_dax_region_alloc(struct cxl_region *cxlr) 3049 { 3050 struct cxl_region_params *p = &cxlr->params; 3051 struct cxl_dax_region *cxlr_dax; 3052 struct device *dev; 3053 3054 guard(rwsem_read)(&cxl_region_rwsem); 3055 if (p->state != CXL_CONFIG_COMMIT) 3056 return ERR_PTR(-ENXIO); 3057 3058 cxlr_dax = kzalloc(sizeof(*cxlr_dax), GFP_KERNEL); 3059 if (!cxlr_dax) 3060 return ERR_PTR(-ENOMEM); 3061 3062 cxlr_dax->hpa_range.start = p->res->start; 3063 cxlr_dax->hpa_range.end = p->res->end; 3064 3065 dev = &cxlr_dax->dev; 3066 cxlr_dax->cxlr = cxlr; 3067 device_initialize(dev); 3068 lockdep_set_class(&dev->mutex, &cxl_dax_region_key); 3069 device_set_pm_not_required(dev); 3070 dev->parent = &cxlr->dev; 3071 dev->bus = &cxl_bus_type; 3072 dev->type = &cxl_dax_region_type; 3073 3074 return cxlr_dax; 3075 } 3076 3077 static void cxlr_pmem_unregister(void *_cxlr_pmem) 3078 { 3079 struct cxl_pmem_region *cxlr_pmem = _cxlr_pmem; 3080 struct cxl_region *cxlr = cxlr_pmem->cxlr; 3081 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 3082 3083 /* 3084 * Either the bridge is in ->remove() context under the device_lock(), 3085 * or cxlr_release_nvdimm() is cancelling the bridge's release action 3086 * for @cxlr_pmem and doing it itself (while manually holding the bridge 3087 * lock). 3088 */ 3089 device_lock_assert(&cxl_nvb->dev); 3090 cxlr->cxlr_pmem = NULL; 3091 cxlr_pmem->cxlr = NULL; 3092 device_unregister(&cxlr_pmem->dev); 3093 } 3094 3095 static void cxlr_release_nvdimm(void *_cxlr) 3096 { 3097 struct cxl_region *cxlr = _cxlr; 3098 struct cxl_nvdimm_bridge *cxl_nvb = cxlr->cxl_nvb; 3099 3100 scoped_guard(device, &cxl_nvb->dev) { 3101 if (cxlr->cxlr_pmem) 3102 devm_release_action(&cxl_nvb->dev, cxlr_pmem_unregister, 3103 cxlr->cxlr_pmem); 3104 } 3105 cxlr->cxl_nvb = NULL; 3106 put_device(&cxl_nvb->dev); 3107 } 3108 3109 /** 3110 * devm_cxl_add_pmem_region() - add a cxl_region-to-nd_region bridge 3111 * @cxlr: parent CXL region for this pmem region bridge device 3112 * 3113 * Return: 0 on success negative error code on failure. 3114 */ 3115 static int devm_cxl_add_pmem_region(struct cxl_region *cxlr) 3116 { 3117 struct cxl_pmem_region *cxlr_pmem; 3118 struct cxl_nvdimm_bridge *cxl_nvb; 3119 struct device *dev; 3120 int rc; 3121 3122 rc = cxl_pmem_region_alloc(cxlr); 3123 if (rc) 3124 return rc; 3125 cxlr_pmem = cxlr->cxlr_pmem; 3126 cxl_nvb = cxlr->cxl_nvb; 3127 3128 dev = &cxlr_pmem->dev; 3129 rc = dev_set_name(dev, "pmem_region%d", cxlr->id); 3130 if (rc) 3131 goto err; 3132 3133 rc = device_add(dev); 3134 if (rc) 3135 goto err; 3136 3137 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 3138 dev_name(dev)); 3139 3140 scoped_guard(device, &cxl_nvb->dev) { 3141 if (cxl_nvb->dev.driver) 3142 rc = devm_add_action_or_reset(&cxl_nvb->dev, 3143 cxlr_pmem_unregister, 3144 cxlr_pmem); 3145 else 3146 rc = -ENXIO; 3147 } 3148 3149 if (rc) 3150 goto err_bridge; 3151 3152 /* @cxlr carries a reference on @cxl_nvb until cxlr_release_nvdimm */ 3153 return devm_add_action_or_reset(&cxlr->dev, cxlr_release_nvdimm, cxlr); 3154 3155 err: 3156 put_device(dev); 3157 err_bridge: 3158 put_device(&cxl_nvb->dev); 3159 cxlr->cxl_nvb = NULL; 3160 return rc; 3161 } 3162 3163 static void cxlr_dax_unregister(void *_cxlr_dax) 3164 { 3165 struct cxl_dax_region *cxlr_dax = _cxlr_dax; 3166 3167 device_unregister(&cxlr_dax->dev); 3168 } 3169 3170 static int devm_cxl_add_dax_region(struct cxl_region *cxlr) 3171 { 3172 struct cxl_dax_region *cxlr_dax; 3173 struct device *dev; 3174 int rc; 3175 3176 cxlr_dax = cxl_dax_region_alloc(cxlr); 3177 if (IS_ERR(cxlr_dax)) 3178 return PTR_ERR(cxlr_dax); 3179 3180 dev = &cxlr_dax->dev; 3181 rc = dev_set_name(dev, "dax_region%d", cxlr->id); 3182 if (rc) 3183 goto err; 3184 3185 rc = device_add(dev); 3186 if (rc) 3187 goto err; 3188 3189 dev_dbg(&cxlr->dev, "%s: register %s\n", dev_name(dev->parent), 3190 dev_name(dev)); 3191 3192 return devm_add_action_or_reset(&cxlr->dev, cxlr_dax_unregister, 3193 cxlr_dax); 3194 err: 3195 put_device(dev); 3196 return rc; 3197 } 3198 3199 static int match_root_decoder_by_range(struct device *dev, 3200 const void *data) 3201 { 3202 const struct range *r1, *r2 = data; 3203 struct cxl_root_decoder *cxlrd; 3204 3205 if (!is_root_decoder(dev)) 3206 return 0; 3207 3208 cxlrd = to_cxl_root_decoder(dev); 3209 r1 = &cxlrd->cxlsd.cxld.hpa_range; 3210 return range_contains(r1, r2); 3211 } 3212 3213 static int match_region_by_range(struct device *dev, const void *data) 3214 { 3215 struct cxl_region_params *p; 3216 struct cxl_region *cxlr; 3217 const struct range *r = data; 3218 3219 if (!is_cxl_region(dev)) 3220 return 0; 3221 3222 cxlr = to_cxl_region(dev); 3223 p = &cxlr->params; 3224 3225 guard(rwsem_read)(&cxl_region_rwsem); 3226 if (p->res && p->res->start == r->start && p->res->end == r->end) 3227 return 1; 3228 3229 return 0; 3230 } 3231 3232 static int cxl_extended_linear_cache_resize(struct cxl_region *cxlr, 3233 struct resource *res) 3234 { 3235 struct cxl_root_decoder *cxlrd = to_cxl_root_decoder(cxlr->dev.parent); 3236 struct cxl_region_params *p = &cxlr->params; 3237 int nid = phys_to_target_node(res->start); 3238 resource_size_t size = resource_size(res); 3239 resource_size_t cache_size, start; 3240 int rc; 3241 3242 rc = cxl_acpi_get_extended_linear_cache_size(res, nid, &cache_size); 3243 if (rc) 3244 return rc; 3245 3246 if (!cache_size) 3247 return 0; 3248 3249 if (size != cache_size) { 3250 dev_warn(&cxlr->dev, 3251 "Extended Linear Cache size %pa != CXL size %pa. No Support!", 3252 &cache_size, &size); 3253 return -ENXIO; 3254 } 3255 3256 /* 3257 * Move the start of the range to where the cache range starts. The 3258 * implementation assumes that the cache range is in front of the 3259 * CXL range. This is not dictated by the HMAT spec but is how the 3260 * current known implementation is configured. 3261 * 3262 * The cache range is expected to be within the CFMWS. The adjusted 3263 * res->start should not be less than cxlrd->res->start. 3264 */ 3265 start = res->start - cache_size; 3266 if (start < cxlrd->res->start) 3267 return -ENXIO; 3268 3269 res->start = start; 3270 p->cache_size = cache_size; 3271 3272 return 0; 3273 } 3274 3275 static int __construct_region(struct cxl_region *cxlr, 3276 struct cxl_root_decoder *cxlrd, 3277 struct cxl_endpoint_decoder *cxled) 3278 { 3279 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 3280 struct range *hpa = &cxled->cxld.hpa_range; 3281 struct cxl_region_params *p; 3282 struct resource *res; 3283 int rc; 3284 3285 guard(rwsem_write)(&cxl_region_rwsem); 3286 p = &cxlr->params; 3287 if (p->state >= CXL_CONFIG_INTERLEAVE_ACTIVE) { 3288 dev_err(cxlmd->dev.parent, 3289 "%s:%s: %s autodiscovery interrupted\n", 3290 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 3291 __func__); 3292 return -EBUSY; 3293 } 3294 3295 set_bit(CXL_REGION_F_AUTO, &cxlr->flags); 3296 3297 res = kmalloc(sizeof(*res), GFP_KERNEL); 3298 if (!res) 3299 return -ENOMEM; 3300 3301 *res = DEFINE_RES_MEM_NAMED(hpa->start, range_len(hpa), 3302 dev_name(&cxlr->dev)); 3303 3304 rc = cxl_extended_linear_cache_resize(cxlr, res); 3305 if (rc && rc != -EOPNOTSUPP) { 3306 /* 3307 * Failing to support extended linear cache region resize does not 3308 * prevent the region from functioning. Only causes cxl list showing 3309 * incorrect region size. 3310 */ 3311 dev_warn(cxlmd->dev.parent, 3312 "Extended linear cache calculation failed rc:%d\n", rc); 3313 } 3314 3315 rc = insert_resource(cxlrd->res, res); 3316 if (rc) { 3317 /* 3318 * Platform-firmware may not have split resources like "System 3319 * RAM" on CXL window boundaries see cxl_region_iomem_release() 3320 */ 3321 dev_warn(cxlmd->dev.parent, 3322 "%s:%s: %s %s cannot insert resource\n", 3323 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 3324 __func__, dev_name(&cxlr->dev)); 3325 } 3326 3327 p->res = res; 3328 p->interleave_ways = cxled->cxld.interleave_ways; 3329 p->interleave_granularity = cxled->cxld.interleave_granularity; 3330 p->state = CXL_CONFIG_INTERLEAVE_ACTIVE; 3331 3332 rc = sysfs_update_group(&cxlr->dev.kobj, get_cxl_region_target_group()); 3333 if (rc) 3334 return rc; 3335 3336 dev_dbg(cxlmd->dev.parent, "%s:%s: %s %s res: %pr iw: %d ig: %d\n", 3337 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), __func__, 3338 dev_name(&cxlr->dev), p->res, p->interleave_ways, 3339 p->interleave_granularity); 3340 3341 /* ...to match put_device() in cxl_add_to_region() */ 3342 get_device(&cxlr->dev); 3343 3344 return 0; 3345 } 3346 3347 /* Establish an empty region covering the given HPA range */ 3348 static struct cxl_region *construct_region(struct cxl_root_decoder *cxlrd, 3349 struct cxl_endpoint_decoder *cxled) 3350 { 3351 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 3352 struct cxl_port *port = cxlrd_to_port(cxlrd); 3353 struct cxl_dev_state *cxlds = cxlmd->cxlds; 3354 int rc, part = READ_ONCE(cxled->part); 3355 struct cxl_region *cxlr; 3356 3357 do { 3358 cxlr = __create_region(cxlrd, cxlds->part[part].mode, 3359 atomic_read(&cxlrd->region_id)); 3360 } while (IS_ERR(cxlr) && PTR_ERR(cxlr) == -EBUSY); 3361 3362 if (IS_ERR(cxlr)) { 3363 dev_err(cxlmd->dev.parent, 3364 "%s:%s: %s failed assign region: %ld\n", 3365 dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev), 3366 __func__, PTR_ERR(cxlr)); 3367 return cxlr; 3368 } 3369 3370 rc = __construct_region(cxlr, cxlrd, cxled); 3371 if (rc) { 3372 devm_release_action(port->uport_dev, unregister_region, cxlr); 3373 return ERR_PTR(rc); 3374 } 3375 3376 return cxlr; 3377 } 3378 3379 int cxl_add_to_region(struct cxl_port *root, struct cxl_endpoint_decoder *cxled) 3380 { 3381 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 3382 struct range *hpa = &cxled->cxld.hpa_range; 3383 struct cxl_decoder *cxld = &cxled->cxld; 3384 struct device *cxlrd_dev, *region_dev; 3385 struct cxl_root_decoder *cxlrd; 3386 struct cxl_region_params *p; 3387 struct cxl_region *cxlr; 3388 bool attach = false; 3389 int rc; 3390 3391 cxlrd_dev = device_find_child(&root->dev, &cxld->hpa_range, 3392 match_root_decoder_by_range); 3393 if (!cxlrd_dev) { 3394 dev_err(cxlmd->dev.parent, 3395 "%s:%s no CXL window for range %#llx:%#llx\n", 3396 dev_name(&cxlmd->dev), dev_name(&cxld->dev), 3397 cxld->hpa_range.start, cxld->hpa_range.end); 3398 return -ENXIO; 3399 } 3400 3401 cxlrd = to_cxl_root_decoder(cxlrd_dev); 3402 3403 /* 3404 * Ensure that if multiple threads race to construct_region() for @hpa 3405 * one does the construction and the others add to that. 3406 */ 3407 mutex_lock(&cxlrd->range_lock); 3408 region_dev = device_find_child(&cxlrd->cxlsd.cxld.dev, hpa, 3409 match_region_by_range); 3410 if (!region_dev) { 3411 cxlr = construct_region(cxlrd, cxled); 3412 region_dev = &cxlr->dev; 3413 } else 3414 cxlr = to_cxl_region(region_dev); 3415 mutex_unlock(&cxlrd->range_lock); 3416 3417 rc = PTR_ERR_OR_ZERO(cxlr); 3418 if (rc) 3419 goto out; 3420 3421 attach_target(cxlr, cxled, -1, TASK_UNINTERRUPTIBLE); 3422 3423 down_read(&cxl_region_rwsem); 3424 p = &cxlr->params; 3425 attach = p->state == CXL_CONFIG_COMMIT; 3426 up_read(&cxl_region_rwsem); 3427 3428 if (attach) { 3429 /* 3430 * If device_attach() fails the range may still be active via 3431 * the platform-firmware memory map, otherwise the driver for 3432 * regions is local to this file, so driver matching can't fail. 3433 */ 3434 if (device_attach(&cxlr->dev) < 0) 3435 dev_err(&cxlr->dev, "failed to enable, range: %pr\n", 3436 p->res); 3437 } 3438 3439 put_device(region_dev); 3440 out: 3441 put_device(cxlrd_dev); 3442 return rc; 3443 } 3444 EXPORT_SYMBOL_NS_GPL(cxl_add_to_region, "CXL"); 3445 3446 u64 cxl_port_get_spa_cache_alias(struct cxl_port *endpoint, u64 spa) 3447 { 3448 struct cxl_region_ref *iter; 3449 unsigned long index; 3450 3451 if (!endpoint) 3452 return ~0ULL; 3453 3454 guard(rwsem_write)(&cxl_region_rwsem); 3455 3456 xa_for_each(&endpoint->regions, index, iter) { 3457 struct cxl_region_params *p = &iter->region->params; 3458 3459 if (p->res->start <= spa && spa <= p->res->end) { 3460 if (!p->cache_size) 3461 return ~0ULL; 3462 3463 if (spa >= p->res->start + p->cache_size) 3464 return spa - p->cache_size; 3465 3466 return spa + p->cache_size; 3467 } 3468 } 3469 3470 return ~0ULL; 3471 } 3472 EXPORT_SYMBOL_NS_GPL(cxl_port_get_spa_cache_alias, "CXL"); 3473 3474 static int is_system_ram(struct resource *res, void *arg) 3475 { 3476 struct cxl_region *cxlr = arg; 3477 struct cxl_region_params *p = &cxlr->params; 3478 3479 dev_dbg(&cxlr->dev, "%pr has System RAM: %pr\n", p->res, res); 3480 return 1; 3481 } 3482 3483 static void shutdown_notifiers(void *_cxlr) 3484 { 3485 struct cxl_region *cxlr = _cxlr; 3486 3487 unregister_memory_notifier(&cxlr->memory_notifier); 3488 unregister_mt_adistance_algorithm(&cxlr->adist_notifier); 3489 } 3490 3491 static int cxl_region_probe(struct device *dev) 3492 { 3493 struct cxl_region *cxlr = to_cxl_region(dev); 3494 struct cxl_region_params *p = &cxlr->params; 3495 int rc; 3496 3497 rc = down_read_interruptible(&cxl_region_rwsem); 3498 if (rc) { 3499 dev_dbg(&cxlr->dev, "probe interrupted\n"); 3500 return rc; 3501 } 3502 3503 if (p->state < CXL_CONFIG_COMMIT) { 3504 dev_dbg(&cxlr->dev, "config state: %d\n", p->state); 3505 rc = -ENXIO; 3506 goto out; 3507 } 3508 3509 if (test_bit(CXL_REGION_F_NEEDS_RESET, &cxlr->flags)) { 3510 dev_err(&cxlr->dev, 3511 "failed to activate, re-commit region and retry\n"); 3512 rc = -ENXIO; 3513 goto out; 3514 } 3515 3516 /* 3517 * From this point on any path that changes the region's state away from 3518 * CXL_CONFIG_COMMIT is also responsible for releasing the driver. 3519 */ 3520 out: 3521 up_read(&cxl_region_rwsem); 3522 3523 if (rc) 3524 return rc; 3525 3526 cxlr->memory_notifier.notifier_call = cxl_region_perf_attrs_callback; 3527 cxlr->memory_notifier.priority = CXL_CALLBACK_PRI; 3528 register_memory_notifier(&cxlr->memory_notifier); 3529 3530 cxlr->adist_notifier.notifier_call = cxl_region_calculate_adistance; 3531 cxlr->adist_notifier.priority = 100; 3532 register_mt_adistance_algorithm(&cxlr->adist_notifier); 3533 3534 rc = devm_add_action_or_reset(&cxlr->dev, shutdown_notifiers, cxlr); 3535 if (rc) 3536 return rc; 3537 3538 switch (cxlr->mode) { 3539 case CXL_PARTMODE_PMEM: 3540 return devm_cxl_add_pmem_region(cxlr); 3541 case CXL_PARTMODE_RAM: 3542 /* 3543 * The region can not be manged by CXL if any portion of 3544 * it is already online as 'System RAM' 3545 */ 3546 if (walk_iomem_res_desc(IORES_DESC_NONE, 3547 IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY, 3548 p->res->start, p->res->end, cxlr, 3549 is_system_ram) > 0) 3550 return 0; 3551 return devm_cxl_add_dax_region(cxlr); 3552 default: 3553 dev_dbg(&cxlr->dev, "unsupported region mode: %d\n", 3554 cxlr->mode); 3555 return -ENXIO; 3556 } 3557 } 3558 3559 static struct cxl_driver cxl_region_driver = { 3560 .name = "cxl_region", 3561 .probe = cxl_region_probe, 3562 .id = CXL_DEVICE_REGION, 3563 }; 3564 3565 int cxl_region_init(void) 3566 { 3567 return cxl_driver_register(&cxl_region_driver); 3568 } 3569 3570 void cxl_region_exit(void) 3571 { 3572 cxl_driver_unregister(&cxl_region_driver); 3573 } 3574 3575 MODULE_IMPORT_NS("CXL"); 3576 MODULE_IMPORT_NS("DEVMEM"); 3577 MODULE_ALIAS_CXL(CXL_DEVICE_REGION); 3578