1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ 3 #include <linux/seq_file.h> 4 #include <linux/device.h> 5 #include <linux/delay.h> 6 7 #include "cxlmem.h" 8 #include "core.h" 9 10 /** 11 * DOC: cxl core hdm 12 * 13 * Compute Express Link Host Managed Device Memory, starting with the 14 * CXL 2.0 specification, is managed by an array of HDM Decoder register 15 * instances per CXL port and per CXL endpoint. Define common helpers 16 * for enumerating these registers and capabilities. 17 */ 18 19 struct cxl_rwsem cxl_rwsem = { 20 .region = __RWSEM_INITIALIZER(cxl_rwsem.region), 21 .dpa = __RWSEM_INITIALIZER(cxl_rwsem.dpa), 22 }; 23 24 static int add_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 25 int *target_map) 26 { 27 int rc; 28 29 rc = cxl_decoder_add_locked(cxld, target_map); 30 if (rc) { 31 put_device(&cxld->dev); 32 dev_err(&port->dev, "Failed to add decoder\n"); 33 return rc; 34 } 35 36 rc = cxl_decoder_autoremove(&port->dev, cxld); 37 if (rc) 38 return rc; 39 40 dev_dbg(port->uport_dev, "%s added to %s\n", 41 dev_name(&cxld->dev), dev_name(&port->dev)); 42 43 return 0; 44 } 45 46 /* 47 * Per the CXL specification (8.2.5.12 CXL HDM Decoder Capability Structure) 48 * single ported host-bridges need not publish a decoder capability when a 49 * passthrough decode can be assumed, i.e. all transactions that the uport sees 50 * are claimed and passed to the single dport. Disable the range until the first 51 * CXL region is enumerated / activated. 52 */ 53 int devm_cxl_add_passthrough_decoder(struct cxl_port *port) 54 { 55 struct cxl_switch_decoder *cxlsd; 56 struct cxl_dport *dport = NULL; 57 int single_port_map[1]; 58 unsigned long index; 59 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 60 61 /* 62 * Capability checks are moot for passthrough decoders, support 63 * any and all possibilities. 64 */ 65 cxlhdm->interleave_mask = ~0U; 66 cxlhdm->iw_cap_mask = ~0UL; 67 68 cxlsd = cxl_switch_decoder_alloc(port, 1); 69 if (IS_ERR(cxlsd)) 70 return PTR_ERR(cxlsd); 71 72 device_lock_assert(&port->dev); 73 74 xa_for_each(&port->dports, index, dport) 75 break; 76 single_port_map[0] = dport->port_id; 77 78 return add_hdm_decoder(port, &cxlsd->cxld, single_port_map); 79 } 80 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_passthrough_decoder, "CXL"); 81 82 static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm) 83 { 84 u32 hdm_cap; 85 86 hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET); 87 cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap); 88 cxlhdm->target_count = 89 FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap); 90 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap)) 91 cxlhdm->interleave_mask |= GENMASK(11, 8); 92 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_14_12, hdm_cap)) 93 cxlhdm->interleave_mask |= GENMASK(14, 12); 94 cxlhdm->iw_cap_mask = BIT(1) | BIT(2) | BIT(4) | BIT(8); 95 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_3_6_12_WAY, hdm_cap)) 96 cxlhdm->iw_cap_mask |= BIT(3) | BIT(6) | BIT(12); 97 if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_16_WAY, hdm_cap)) 98 cxlhdm->iw_cap_mask |= BIT(16); 99 } 100 101 static bool should_emulate_decoders(struct cxl_endpoint_dvsec_info *info) 102 { 103 struct cxl_hdm *cxlhdm; 104 void __iomem *hdm; 105 u32 ctrl; 106 int i; 107 108 if (!info) 109 return false; 110 111 cxlhdm = dev_get_drvdata(&info->port->dev); 112 hdm = cxlhdm->regs.hdm_decoder; 113 114 if (!hdm) 115 return true; 116 117 /* 118 * If HDM decoders are present and the driver is in control of 119 * Mem_Enable skip DVSEC based emulation 120 */ 121 if (!info->mem_enabled) 122 return false; 123 124 /* 125 * If any decoders are committed already, there should not be any 126 * emulated DVSEC decoders. 127 */ 128 for (i = 0; i < cxlhdm->decoder_count; i++) { 129 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); 130 dev_dbg(&info->port->dev, 131 "decoder%d.%d: committed: %ld base: %#x_%.8x size: %#x_%.8x\n", 132 info->port->id, i, 133 FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl), 134 readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i)), 135 readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(i)), 136 readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i)), 137 readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i))); 138 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) 139 return false; 140 } 141 142 return true; 143 } 144 145 /** 146 * devm_cxl_setup_hdm - map HDM decoder component registers 147 * @port: cxl_port to map 148 * @info: cached DVSEC range register info 149 */ 150 struct cxl_hdm *devm_cxl_setup_hdm(struct cxl_port *port, 151 struct cxl_endpoint_dvsec_info *info) 152 { 153 struct cxl_register_map *reg_map = &port->reg_map; 154 struct device *dev = &port->dev; 155 struct cxl_hdm *cxlhdm; 156 int rc; 157 158 cxlhdm = devm_kzalloc(dev, sizeof(*cxlhdm), GFP_KERNEL); 159 if (!cxlhdm) 160 return ERR_PTR(-ENOMEM); 161 cxlhdm->port = port; 162 dev_set_drvdata(dev, cxlhdm); 163 164 /* Memory devices can configure device HDM using DVSEC range regs. */ 165 if (reg_map->resource == CXL_RESOURCE_NONE) { 166 if (!info || !info->mem_enabled) { 167 dev_err(dev, "No component registers mapped\n"); 168 return ERR_PTR(-ENXIO); 169 } 170 171 cxlhdm->decoder_count = info->ranges; 172 return cxlhdm; 173 } 174 175 if (!reg_map->component_map.hdm_decoder.valid) { 176 dev_dbg(&port->dev, "HDM decoder registers not implemented\n"); 177 /* unique error code to indicate no HDM decoder capability */ 178 return ERR_PTR(-ENODEV); 179 } 180 181 rc = cxl_map_component_regs(reg_map, &cxlhdm->regs, 182 BIT(CXL_CM_CAP_CAP_ID_HDM)); 183 if (rc) { 184 dev_err(dev, "Failed to map HDM capability.\n"); 185 return ERR_PTR(rc); 186 } 187 188 parse_hdm_decoder_caps(cxlhdm); 189 if (cxlhdm->decoder_count == 0) { 190 dev_err(dev, "Spec violation. Caps invalid\n"); 191 return ERR_PTR(-ENXIO); 192 } 193 194 /* 195 * Now that the hdm capability is parsed, decide if range 196 * register emulation is needed and fixup cxlhdm accordingly. 197 */ 198 if (should_emulate_decoders(info)) { 199 dev_dbg(dev, "Fallback map %d range register%s\n", info->ranges, 200 info->ranges > 1 ? "s" : ""); 201 cxlhdm->decoder_count = info->ranges; 202 } 203 204 return cxlhdm; 205 } 206 EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_hdm, "CXL"); 207 208 static void __cxl_dpa_debug(struct seq_file *file, struct resource *r, int depth) 209 { 210 unsigned long long start = r->start, end = r->end; 211 212 seq_printf(file, "%*s%08llx-%08llx : %s\n", depth * 2, "", start, end, 213 r->name); 214 } 215 216 void cxl_dpa_debug(struct seq_file *file, struct cxl_dev_state *cxlds) 217 { 218 struct resource *p1, *p2; 219 220 guard(rwsem_read)(&cxl_rwsem.dpa); 221 for (p1 = cxlds->dpa_res.child; p1; p1 = p1->sibling) { 222 __cxl_dpa_debug(file, p1, 0); 223 for (p2 = p1->child; p2; p2 = p2->sibling) 224 __cxl_dpa_debug(file, p2, 1); 225 } 226 } 227 EXPORT_SYMBOL_NS_GPL(cxl_dpa_debug, "CXL"); 228 229 /* See request_skip() kernel-doc */ 230 static resource_size_t __adjust_skip(struct cxl_dev_state *cxlds, 231 const resource_size_t skip_base, 232 const resource_size_t skip_len, 233 const char *requester) 234 { 235 const resource_size_t skip_end = skip_base + skip_len - 1; 236 237 for (int i = 0; i < cxlds->nr_partitions; i++) { 238 const struct resource *part_res = &cxlds->part[i].res; 239 resource_size_t adjust_start, adjust_end, size; 240 241 adjust_start = max(skip_base, part_res->start); 242 adjust_end = min(skip_end, part_res->end); 243 244 if (adjust_end < adjust_start) 245 continue; 246 247 size = adjust_end - adjust_start + 1; 248 249 if (!requester) 250 __release_region(&cxlds->dpa_res, adjust_start, size); 251 else if (!__request_region(&cxlds->dpa_res, adjust_start, size, 252 requester, 0)) 253 return adjust_start - skip_base; 254 } 255 256 return skip_len; 257 } 258 #define release_skip(c, b, l) __adjust_skip((c), (b), (l), NULL) 259 260 /* 261 * Must be called in a context that synchronizes against this decoder's 262 * port ->remove() callback (like an endpoint decoder sysfs attribute) 263 */ 264 static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 265 { 266 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 267 struct cxl_port *port = cxled_to_port(cxled); 268 struct cxl_dev_state *cxlds = cxlmd->cxlds; 269 struct resource *res = cxled->dpa_res; 270 resource_size_t skip_start; 271 272 lockdep_assert_held_write(&cxl_rwsem.dpa); 273 274 /* save @skip_start, before @res is released */ 275 skip_start = res->start - cxled->skip; 276 __release_region(&cxlds->dpa_res, res->start, resource_size(res)); 277 if (cxled->skip) 278 release_skip(cxlds, skip_start, cxled->skip); 279 cxled->skip = 0; 280 cxled->dpa_res = NULL; 281 put_device(&cxled->cxld.dev); 282 port->hdm_end--; 283 } 284 285 static void cxl_dpa_release(void *cxled) 286 { 287 guard(rwsem_write)(&cxl_rwsem.dpa); 288 __cxl_dpa_release(cxled); 289 } 290 291 /* 292 * Must be called from context that will not race port device 293 * unregistration, like decoder sysfs attribute methods 294 */ 295 static void devm_cxl_dpa_release(struct cxl_endpoint_decoder *cxled) 296 { 297 struct cxl_port *port = cxled_to_port(cxled); 298 299 lockdep_assert_held_write(&cxl_rwsem.dpa); 300 devm_remove_action(&port->dev, cxl_dpa_release, cxled); 301 __cxl_dpa_release(cxled); 302 } 303 304 /** 305 * request_skip() - Track DPA 'skip' in @cxlds->dpa_res resource tree 306 * @cxlds: CXL.mem device context that parents @cxled 307 * @cxled: Endpoint decoder establishing new allocation that skips lower DPA 308 * @skip_base: DPA < start of new DPA allocation (DPAnew) 309 * @skip_len: @skip_base + @skip_len == DPAnew 310 * 311 * DPA 'skip' arises from out-of-sequence DPA allocation events relative 312 * to free capacity across multiple partitions. It is a wasteful event 313 * as usable DPA gets thrown away, but if a deployment has, for example, 314 * a dual RAM+PMEM device, wants to use PMEM, and has unallocated RAM 315 * DPA, the free RAM DPA must be sacrificed to start allocating PMEM. 316 * See third "Implementation Note" in CXL 3.1 8.2.4.19.13 "Decoder 317 * Protection" for more details. 318 * 319 * A 'skip' always covers the last allocated DPA in a previous partition 320 * to the start of the current partition to allocate. Allocations never 321 * start in the middle of a partition, and allocations are always 322 * de-allocated in reverse order (see cxl_dpa_free(), or natural devm 323 * unwind order from forced in-order allocation). 324 * 325 * If @cxlds->nr_partitions was guaranteed to be <= 2 then the 'skip' 326 * would always be contained to a single partition. Given 327 * @cxlds->nr_partitions may be > 2 it results in cases where the 'skip' 328 * might span "tail capacity of partition[0], all of partition[1], ..., 329 * all of partition[N-1]" to support allocating from partition[N]. That 330 * in turn interacts with the partition 'struct resource' boundaries 331 * within @cxlds->dpa_res whereby 'skip' requests need to be divided by 332 * partition. I.e. this is a quirk of using a 'struct resource' tree to 333 * detect range conflicts while also tracking partition boundaries in 334 * @cxlds->dpa_res. 335 */ 336 static int request_skip(struct cxl_dev_state *cxlds, 337 struct cxl_endpoint_decoder *cxled, 338 const resource_size_t skip_base, 339 const resource_size_t skip_len) 340 { 341 resource_size_t skipped = __adjust_skip(cxlds, skip_base, skip_len, 342 dev_name(&cxled->cxld.dev)); 343 344 if (skipped == skip_len) 345 return 0; 346 347 dev_dbg(cxlds->dev, 348 "%s: failed to reserve skipped space (%pa %pa %pa)\n", 349 dev_name(&cxled->cxld.dev), &skip_base, &skip_len, &skipped); 350 351 release_skip(cxlds, skip_base, skipped); 352 353 return -EBUSY; 354 } 355 356 static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 357 resource_size_t base, resource_size_t len, 358 resource_size_t skipped) 359 { 360 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 361 struct cxl_port *port = cxled_to_port(cxled); 362 struct cxl_dev_state *cxlds = cxlmd->cxlds; 363 struct device *dev = &port->dev; 364 struct resource *res; 365 int rc; 366 367 lockdep_assert_held_write(&cxl_rwsem.dpa); 368 369 if (!len) { 370 dev_warn(dev, "decoder%d.%d: empty reservation attempted\n", 371 port->id, cxled->cxld.id); 372 return -EINVAL; 373 } 374 375 if (cxled->dpa_res) { 376 dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n", 377 port->id, cxled->cxld.id, cxled->dpa_res); 378 return -EBUSY; 379 } 380 381 if (port->hdm_end + 1 != cxled->cxld.id) { 382 /* 383 * Assumes alloc and commit order is always in hardware instance 384 * order per expectations from 8.2.5.12.20 Committing Decoder 385 * Programming that enforce decoder[m] committed before 386 * decoder[m+1] commit start. 387 */ 388 dev_dbg(dev, "decoder%d.%d: expected decoder%d.%d\n", port->id, 389 cxled->cxld.id, port->id, port->hdm_end + 1); 390 return -EBUSY; 391 } 392 393 if (skipped) { 394 rc = request_skip(cxlds, cxled, base - skipped, skipped); 395 if (rc) 396 return rc; 397 } 398 res = __request_region(&cxlds->dpa_res, base, len, 399 dev_name(&cxled->cxld.dev), 0); 400 if (!res) { 401 dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n", 402 port->id, cxled->cxld.id); 403 if (skipped) 404 release_skip(cxlds, base - skipped, skipped); 405 return -EBUSY; 406 } 407 cxled->dpa_res = res; 408 cxled->skip = skipped; 409 410 /* 411 * When allocating new capacity, ->part is already set, when 412 * discovering decoder settings at initial enumeration, ->part 413 * is not set. 414 */ 415 if (cxled->part < 0) 416 for (int i = 0; cxlds->nr_partitions; i++) 417 if (resource_contains(&cxlds->part[i].res, res)) { 418 cxled->part = i; 419 break; 420 } 421 422 if (cxled->part < 0) 423 dev_warn(dev, "decoder%d.%d: %pr does not map any partition\n", 424 port->id, cxled->cxld.id, res); 425 426 port->hdm_end++; 427 get_device(&cxled->cxld.dev); 428 return 0; 429 } 430 431 static int add_dpa_res(struct device *dev, struct resource *parent, 432 struct resource *res, resource_size_t start, 433 resource_size_t size, const char *type) 434 { 435 int rc; 436 437 *res = (struct resource) { 438 .name = type, 439 .start = start, 440 .end = start + size - 1, 441 .flags = IORESOURCE_MEM, 442 }; 443 if (resource_size(res) == 0) { 444 dev_dbg(dev, "DPA(%s): no capacity\n", res->name); 445 return 0; 446 } 447 rc = request_resource(parent, res); 448 if (rc) { 449 dev_err(dev, "DPA(%s): failed to track %pr (%d)\n", res->name, 450 res, rc); 451 return rc; 452 } 453 454 dev_dbg(dev, "DPA(%s): %pr\n", res->name, res); 455 456 return 0; 457 } 458 459 static const char *cxl_mode_name(enum cxl_partition_mode mode) 460 { 461 switch (mode) { 462 case CXL_PARTMODE_RAM: 463 return "ram"; 464 case CXL_PARTMODE_PMEM: 465 return "pmem"; 466 default: 467 return ""; 468 }; 469 } 470 471 /* if this fails the caller must destroy @cxlds, there is no recovery */ 472 int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info) 473 { 474 struct device *dev = cxlds->dev; 475 476 guard(rwsem_write)(&cxl_rwsem.dpa); 477 478 if (cxlds->nr_partitions) 479 return -EBUSY; 480 481 if (!info->size || !info->nr_partitions) { 482 cxlds->dpa_res = DEFINE_RES_MEM(0, 0); 483 cxlds->nr_partitions = 0; 484 return 0; 485 } 486 487 cxlds->dpa_res = DEFINE_RES_MEM(0, info->size); 488 489 for (int i = 0; i < info->nr_partitions; i++) { 490 const struct cxl_dpa_part_info *part = &info->part[i]; 491 int rc; 492 493 cxlds->part[i].perf.qos_class = CXL_QOS_CLASS_INVALID; 494 cxlds->part[i].mode = part->mode; 495 496 /* Require ordered + contiguous partitions */ 497 if (i) { 498 const struct cxl_dpa_part_info *prev = &info->part[i - 1]; 499 500 if (prev->range.end + 1 != part->range.start) 501 return -EINVAL; 502 } 503 rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->part[i].res, 504 part->range.start, range_len(&part->range), 505 cxl_mode_name(part->mode)); 506 if (rc) 507 return rc; 508 cxlds->nr_partitions++; 509 } 510 511 return 0; 512 } 513 EXPORT_SYMBOL_GPL(cxl_dpa_setup); 514 515 int devm_cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, 516 resource_size_t base, resource_size_t len, 517 resource_size_t skipped) 518 { 519 struct cxl_port *port = cxled_to_port(cxled); 520 int rc; 521 522 scoped_guard(rwsem_write, &cxl_rwsem.dpa) 523 rc = __cxl_dpa_reserve(cxled, base, len, skipped); 524 525 if (rc) 526 return rc; 527 528 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); 529 } 530 EXPORT_SYMBOL_NS_GPL(devm_cxl_dpa_reserve, "CXL"); 531 532 resource_size_t cxl_dpa_size(struct cxl_endpoint_decoder *cxled) 533 { 534 guard(rwsem_read)(&cxl_rwsem.dpa); 535 if (cxled->dpa_res) 536 return resource_size(cxled->dpa_res); 537 538 return 0; 539 } 540 541 resource_size_t cxl_dpa_resource_start(struct cxl_endpoint_decoder *cxled) 542 { 543 resource_size_t base = -1; 544 545 lockdep_assert_held(&cxl_rwsem.dpa); 546 if (cxled->dpa_res) 547 base = cxled->dpa_res->start; 548 549 return base; 550 } 551 552 bool cxl_resource_contains_addr(const struct resource *res, const resource_size_t addr) 553 { 554 struct resource _addr = DEFINE_RES_MEM(addr, 1); 555 556 return resource_contains(res, &_addr); 557 } 558 559 int cxl_dpa_free(struct cxl_endpoint_decoder *cxled) 560 { 561 struct cxl_port *port = cxled_to_port(cxled); 562 struct device *dev = &cxled->cxld.dev; 563 564 guard(rwsem_write)(&cxl_rwsem.dpa); 565 if (!cxled->dpa_res) 566 return 0; 567 if (cxled->cxld.region) { 568 dev_dbg(dev, "decoder assigned to: %s\n", 569 dev_name(&cxled->cxld.region->dev)); 570 return -EBUSY; 571 } 572 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 573 dev_dbg(dev, "decoder enabled\n"); 574 return -EBUSY; 575 } 576 if (cxled->cxld.id != port->hdm_end) { 577 dev_dbg(dev, "expected decoder%d.%d\n", port->id, 578 port->hdm_end); 579 return -EBUSY; 580 } 581 582 devm_cxl_dpa_release(cxled); 583 return 0; 584 } 585 586 int cxl_dpa_set_part(struct cxl_endpoint_decoder *cxled, 587 enum cxl_partition_mode mode) 588 { 589 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 590 struct cxl_dev_state *cxlds = cxlmd->cxlds; 591 struct device *dev = &cxled->cxld.dev; 592 int part; 593 594 guard(rwsem_write)(&cxl_rwsem.dpa); 595 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) 596 return -EBUSY; 597 598 for (part = 0; part < cxlds->nr_partitions; part++) 599 if (cxlds->part[part].mode == mode) 600 break; 601 602 if (part >= cxlds->nr_partitions) { 603 dev_dbg(dev, "unsupported mode: %d\n", mode); 604 return -EINVAL; 605 } 606 607 if (!resource_size(&cxlds->part[part].res)) { 608 dev_dbg(dev, "no available capacity for mode: %d\n", mode); 609 return -ENXIO; 610 } 611 612 cxled->part = part; 613 return 0; 614 } 615 616 static int __cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size) 617 { 618 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 619 struct cxl_dev_state *cxlds = cxlmd->cxlds; 620 struct device *dev = &cxled->cxld.dev; 621 struct resource *res, *prev = NULL; 622 resource_size_t start, avail, skip, skip_start; 623 struct resource *p, *last; 624 int part; 625 626 guard(rwsem_write)(&cxl_rwsem.dpa); 627 if (cxled->cxld.region) { 628 dev_dbg(dev, "decoder attached to %s\n", 629 dev_name(&cxled->cxld.region->dev)); 630 return -EBUSY; 631 } 632 633 if (cxled->cxld.flags & CXL_DECODER_F_ENABLE) { 634 dev_dbg(dev, "decoder enabled\n"); 635 return -EBUSY; 636 } 637 638 part = cxled->part; 639 if (part < 0) { 640 dev_dbg(dev, "partition not set\n"); 641 return -EBUSY; 642 } 643 644 res = &cxlds->part[part].res; 645 for (p = res->child, last = NULL; p; p = p->sibling) 646 last = p; 647 if (last) 648 start = last->end + 1; 649 else 650 start = res->start; 651 652 /* 653 * To allocate at partition N, a skip needs to be calculated for all 654 * unallocated space at lower partitions indices. 655 * 656 * If a partition has any allocations, the search can end because a 657 * previous cxl_dpa_alloc() invocation is assumed to have accounted for 658 * all previous partitions. 659 */ 660 skip_start = CXL_RESOURCE_NONE; 661 for (int i = part; i; i--) { 662 prev = &cxlds->part[i - 1].res; 663 for (p = prev->child, last = NULL; p; p = p->sibling) 664 last = p; 665 if (last) { 666 skip_start = last->end + 1; 667 break; 668 } 669 skip_start = prev->start; 670 } 671 672 avail = res->end - start + 1; 673 if (skip_start == CXL_RESOURCE_NONE) 674 skip = 0; 675 else 676 skip = res->start - skip_start; 677 678 if (size > avail) { 679 dev_dbg(dev, "%llu exceeds available %s capacity: %llu\n", size, 680 res->name, (u64)avail); 681 return -ENOSPC; 682 } 683 684 return __cxl_dpa_reserve(cxled, start, size, skip); 685 } 686 687 int cxl_dpa_alloc(struct cxl_endpoint_decoder *cxled, u64 size) 688 { 689 struct cxl_port *port = cxled_to_port(cxled); 690 int rc; 691 692 rc = __cxl_dpa_alloc(cxled, size); 693 if (rc) 694 return rc; 695 696 return devm_add_action_or_reset(&port->dev, cxl_dpa_release, cxled); 697 } 698 699 static void cxld_set_interleave(struct cxl_decoder *cxld, u32 *ctrl) 700 { 701 u16 eig; 702 u8 eiw; 703 704 /* 705 * Input validation ensures these warns never fire, but otherwise 706 * suppress unititalized variable usage warnings. 707 */ 708 if (WARN_ONCE(ways_to_eiw(cxld->interleave_ways, &eiw), 709 "invalid interleave_ways: %d\n", cxld->interleave_ways)) 710 return; 711 if (WARN_ONCE(granularity_to_eig(cxld->interleave_granularity, &eig), 712 "invalid interleave_granularity: %d\n", 713 cxld->interleave_granularity)) 714 return; 715 716 u32p_replace_bits(ctrl, eig, CXL_HDM_DECODER0_CTRL_IG_MASK); 717 u32p_replace_bits(ctrl, eiw, CXL_HDM_DECODER0_CTRL_IW_MASK); 718 *ctrl |= CXL_HDM_DECODER0_CTRL_COMMIT; 719 } 720 721 static void cxld_set_type(struct cxl_decoder *cxld, u32 *ctrl) 722 { 723 u32p_replace_bits(ctrl, 724 !!(cxld->target_type == CXL_DECODER_HOSTONLYMEM), 725 CXL_HDM_DECODER0_CTRL_HOSTONLY); 726 } 727 728 static void cxlsd_set_targets(struct cxl_switch_decoder *cxlsd, u64 *tgt) 729 { 730 struct cxl_dport **t = &cxlsd->target[0]; 731 int ways = cxlsd->cxld.interleave_ways; 732 733 *tgt = FIELD_PREP(GENMASK(7, 0), t[0]->port_id); 734 if (ways > 1) 735 *tgt |= FIELD_PREP(GENMASK(15, 8), t[1]->port_id); 736 if (ways > 2) 737 *tgt |= FIELD_PREP(GENMASK(23, 16), t[2]->port_id); 738 if (ways > 3) 739 *tgt |= FIELD_PREP(GENMASK(31, 24), t[3]->port_id); 740 if (ways > 4) 741 *tgt |= FIELD_PREP(GENMASK_ULL(39, 32), t[4]->port_id); 742 if (ways > 5) 743 *tgt |= FIELD_PREP(GENMASK_ULL(47, 40), t[5]->port_id); 744 if (ways > 6) 745 *tgt |= FIELD_PREP(GENMASK_ULL(55, 48), t[6]->port_id); 746 if (ways > 7) 747 *tgt |= FIELD_PREP(GENMASK_ULL(63, 56), t[7]->port_id); 748 } 749 750 /* 751 * Per CXL 2.0 8.2.5.12.20 Committing Decoder Programming, hardware must set 752 * committed or error within 10ms, but just be generous with 20ms to account for 753 * clock skew and other marginal behavior 754 */ 755 #define COMMIT_TIMEOUT_MS 20 756 static int cxld_await_commit(void __iomem *hdm, int id) 757 { 758 u32 ctrl; 759 int i; 760 761 for (i = 0; i < COMMIT_TIMEOUT_MS; i++) { 762 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 763 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMIT_ERROR, ctrl)) { 764 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; 765 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 766 return -EIO; 767 } 768 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl)) 769 return 0; 770 fsleep(1000); 771 } 772 773 return -ETIMEDOUT; 774 } 775 776 static void setup_hw_decoder(struct cxl_decoder *cxld, void __iomem *hdm) 777 { 778 int id = cxld->id; 779 u64 base, size; 780 u32 ctrl; 781 782 /* common decoder settings */ 783 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id)); 784 cxld_set_interleave(cxld, &ctrl); 785 cxld_set_type(cxld, &ctrl); 786 base = cxld->hpa_range.start; 787 size = range_len(&cxld->hpa_range); 788 789 writel(upper_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); 790 writel(lower_32_bits(base), hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); 791 writel(upper_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); 792 writel(lower_32_bits(size), hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); 793 794 if (is_switch_decoder(&cxld->dev)) { 795 struct cxl_switch_decoder *cxlsd = 796 to_cxl_switch_decoder(&cxld->dev); 797 void __iomem *tl_hi = hdm + CXL_HDM_DECODER0_TL_HIGH(id); 798 void __iomem *tl_lo = hdm + CXL_HDM_DECODER0_TL_LOW(id); 799 u64 targets; 800 801 cxlsd_set_targets(cxlsd, &targets); 802 writel(upper_32_bits(targets), tl_hi); 803 writel(lower_32_bits(targets), tl_lo); 804 } else { 805 struct cxl_endpoint_decoder *cxled = 806 to_cxl_endpoint_decoder(&cxld->dev); 807 void __iomem *sk_hi = hdm + CXL_HDM_DECODER0_SKIP_HIGH(id); 808 void __iomem *sk_lo = hdm + CXL_HDM_DECODER0_SKIP_LOW(id); 809 810 writel(upper_32_bits(cxled->skip), sk_hi); 811 writel(lower_32_bits(cxled->skip), sk_lo); 812 } 813 814 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 815 } 816 817 static int cxl_decoder_commit(struct cxl_decoder *cxld) 818 { 819 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 820 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 821 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 822 int id = cxld->id, rc; 823 824 if (cxld->flags & CXL_DECODER_F_ENABLE) 825 return 0; 826 827 if (cxl_num_decoders_committed(port) != id) { 828 dev_dbg(&port->dev, 829 "%s: out of order commit, expected decoder%d.%d\n", 830 dev_name(&cxld->dev), port->id, 831 cxl_num_decoders_committed(port)); 832 return -EBUSY; 833 } 834 835 /* 836 * For endpoint decoders hosted on CXL memory devices that 837 * support the sanitize operation, make sure sanitize is not in-flight. 838 */ 839 if (is_endpoint_decoder(&cxld->dev)) { 840 struct cxl_endpoint_decoder *cxled = 841 to_cxl_endpoint_decoder(&cxld->dev); 842 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 843 struct cxl_memdev_state *mds = 844 to_cxl_memdev_state(cxlmd->cxlds); 845 846 if (mds && mds->security.sanitize_active) { 847 dev_dbg(&cxlmd->dev, 848 "attempted to commit %s during sanitize\n", 849 dev_name(&cxld->dev)); 850 return -EBUSY; 851 } 852 } 853 854 scoped_guard(rwsem_read, &cxl_rwsem.dpa) 855 setup_hw_decoder(cxld, hdm); 856 857 port->commit_end++; 858 rc = cxld_await_commit(hdm, cxld->id); 859 if (rc) { 860 dev_dbg(&port->dev, "%s: error %d committing decoder\n", 861 dev_name(&cxld->dev), rc); 862 cxld->reset(cxld); 863 return rc; 864 } 865 cxld->flags |= CXL_DECODER_F_ENABLE; 866 867 return 0; 868 } 869 870 static int commit_reap(struct device *dev, void *data) 871 { 872 struct cxl_port *port = to_cxl_port(dev->parent); 873 struct cxl_decoder *cxld; 874 875 if (!is_switch_decoder(dev) && !is_endpoint_decoder(dev)) 876 return 0; 877 878 cxld = to_cxl_decoder(dev); 879 if (port->commit_end == cxld->id && 880 ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)) { 881 port->commit_end--; 882 dev_dbg(&port->dev, "reap: %s commit_end: %d\n", 883 dev_name(&cxld->dev), port->commit_end); 884 } 885 886 return 0; 887 } 888 889 void cxl_port_commit_reap(struct cxl_decoder *cxld) 890 { 891 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 892 893 lockdep_assert_held_write(&cxl_rwsem.region); 894 895 /* 896 * Once the highest committed decoder is disabled, free any other 897 * decoders that were pinned allocated by out-of-order release. 898 */ 899 port->commit_end--; 900 dev_dbg(&port->dev, "reap: %s commit_end: %d\n", dev_name(&cxld->dev), 901 port->commit_end); 902 device_for_each_child_reverse_from(&port->dev, &cxld->dev, NULL, 903 commit_reap); 904 } 905 EXPORT_SYMBOL_NS_GPL(cxl_port_commit_reap, "CXL"); 906 907 static void cxl_decoder_reset(struct cxl_decoder *cxld) 908 { 909 struct cxl_port *port = to_cxl_port(cxld->dev.parent); 910 struct cxl_hdm *cxlhdm = dev_get_drvdata(&port->dev); 911 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 912 int id = cxld->id; 913 u32 ctrl; 914 915 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0) 916 return; 917 918 if (port->commit_end == id) 919 cxl_port_commit_reap(cxld); 920 else 921 dev_dbg(&port->dev, 922 "%s: out of order reset, expected decoder%d.%d\n", 923 dev_name(&cxld->dev), port->id, port->commit_end); 924 925 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 926 ctrl &= ~CXL_HDM_DECODER0_CTRL_COMMIT; 927 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(id)); 928 929 writel(0, hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(id)); 930 writel(0, hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(id)); 931 writel(0, hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(id)); 932 writel(0, hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(id)); 933 934 cxld->flags &= ~CXL_DECODER_F_ENABLE; 935 936 /* Userspace is now responsible for reconfiguring this decoder */ 937 if (is_endpoint_decoder(&cxld->dev)) { 938 struct cxl_endpoint_decoder *cxled; 939 940 cxled = to_cxl_endpoint_decoder(&cxld->dev); 941 cxled->state = CXL_DECODER_STATE_MANUAL; 942 } 943 } 944 945 static int cxl_setup_hdm_decoder_from_dvsec( 946 struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base, 947 int which, struct cxl_endpoint_dvsec_info *info) 948 { 949 struct cxl_endpoint_decoder *cxled; 950 u64 len; 951 int rc; 952 953 if (!is_cxl_endpoint(port)) 954 return -EOPNOTSUPP; 955 956 cxled = to_cxl_endpoint_decoder(&cxld->dev); 957 len = range_len(&info->dvsec_range[which]); 958 if (!len) 959 return -ENOENT; 960 961 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 962 cxld->commit = NULL; 963 cxld->reset = NULL; 964 cxld->hpa_range = info->dvsec_range[which]; 965 966 /* 967 * Set the emulated decoder as locked pending additional support to 968 * change the range registers at run time. 969 */ 970 cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK; 971 port->commit_end = cxld->id; 972 973 rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0); 974 if (rc) { 975 dev_err(&port->dev, 976 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", 977 port->id, cxld->id, *dpa_base, *dpa_base + len - 1, rc); 978 return rc; 979 } 980 *dpa_base += len; 981 cxled->state = CXL_DECODER_STATE_AUTO; 982 983 return 0; 984 } 985 986 static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, 987 int *target_map, void __iomem *hdm, int which, 988 u64 *dpa_base, struct cxl_endpoint_dvsec_info *info) 989 { 990 struct cxl_endpoint_decoder *cxled = NULL; 991 u64 size, base, skip, dpa_size, lo, hi; 992 bool committed; 993 u32 remainder; 994 int i, rc; 995 u32 ctrl; 996 union { 997 u64 value; 998 unsigned char target_id[8]; 999 } target_list; 1000 1001 if (should_emulate_decoders(info)) 1002 return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base, 1003 which, info); 1004 1005 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); 1006 lo = readl(hdm + CXL_HDM_DECODER0_BASE_LOW_OFFSET(which)); 1007 hi = readl(hdm + CXL_HDM_DECODER0_BASE_HIGH_OFFSET(which)); 1008 base = (hi << 32) + lo; 1009 lo = readl(hdm + CXL_HDM_DECODER0_SIZE_LOW_OFFSET(which)); 1010 hi = readl(hdm + CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(which)); 1011 size = (hi << 32) + lo; 1012 committed = !!(ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED); 1013 cxld->commit = cxl_decoder_commit; 1014 cxld->reset = cxl_decoder_reset; 1015 1016 if (!committed) 1017 size = 0; 1018 if (base == U64_MAX || size == U64_MAX) { 1019 dev_warn(&port->dev, "decoder%d.%d: Invalid resource range\n", 1020 port->id, cxld->id); 1021 return -ENXIO; 1022 } 1023 1024 if (info) 1025 cxled = to_cxl_endpoint_decoder(&cxld->dev); 1026 cxld->hpa_range = (struct range) { 1027 .start = base, 1028 .end = base + size - 1, 1029 }; 1030 1031 /* decoders are enabled if committed */ 1032 if (committed) { 1033 cxld->flags |= CXL_DECODER_F_ENABLE; 1034 if (ctrl & CXL_HDM_DECODER0_CTRL_LOCK) 1035 cxld->flags |= CXL_DECODER_F_LOCK; 1036 if (FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl)) 1037 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 1038 else 1039 cxld->target_type = CXL_DECODER_DEVMEM; 1040 1041 guard(rwsem_write)(&cxl_rwsem.region); 1042 if (cxld->id != cxl_num_decoders_committed(port)) { 1043 dev_warn(&port->dev, 1044 "decoder%d.%d: Committed out of order\n", 1045 port->id, cxld->id); 1046 return -ENXIO; 1047 } 1048 1049 if (size == 0) { 1050 dev_warn(&port->dev, 1051 "decoder%d.%d: Committed with zero size\n", 1052 port->id, cxld->id); 1053 return -ENXIO; 1054 } 1055 port->commit_end = cxld->id; 1056 } else { 1057 if (cxled) { 1058 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 1059 struct cxl_dev_state *cxlds = cxlmd->cxlds; 1060 1061 /* 1062 * Default by devtype until a device arrives that needs 1063 * more precision. 1064 */ 1065 if (cxlds->type == CXL_DEVTYPE_CLASSMEM) 1066 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 1067 else 1068 cxld->target_type = CXL_DECODER_DEVMEM; 1069 } else { 1070 /* To be overridden by region type at commit time */ 1071 cxld->target_type = CXL_DECODER_HOSTONLYMEM; 1072 } 1073 1074 if (!FIELD_GET(CXL_HDM_DECODER0_CTRL_HOSTONLY, ctrl) && 1075 cxld->target_type == CXL_DECODER_HOSTONLYMEM) { 1076 ctrl |= CXL_HDM_DECODER0_CTRL_HOSTONLY; 1077 writel(ctrl, hdm + CXL_HDM_DECODER0_CTRL_OFFSET(which)); 1078 } 1079 } 1080 rc = eiw_to_ways(FIELD_GET(CXL_HDM_DECODER0_CTRL_IW_MASK, ctrl), 1081 &cxld->interleave_ways); 1082 if (rc) { 1083 dev_warn(&port->dev, 1084 "decoder%d.%d: Invalid interleave ways (ctrl: %#x)\n", 1085 port->id, cxld->id, ctrl); 1086 return rc; 1087 } 1088 rc = eig_to_granularity(FIELD_GET(CXL_HDM_DECODER0_CTRL_IG_MASK, ctrl), 1089 &cxld->interleave_granularity); 1090 if (rc) { 1091 dev_warn(&port->dev, 1092 "decoder%d.%d: Invalid interleave granularity (ctrl: %#x)\n", 1093 port->id, cxld->id, ctrl); 1094 return rc; 1095 } 1096 1097 dev_dbg(&port->dev, "decoder%d.%d: range: %#llx-%#llx iw: %d ig: %d\n", 1098 port->id, cxld->id, cxld->hpa_range.start, cxld->hpa_range.end, 1099 cxld->interleave_ways, cxld->interleave_granularity); 1100 1101 if (!cxled) { 1102 lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which)); 1103 hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which)); 1104 target_list.value = (hi << 32) + lo; 1105 for (i = 0; i < cxld->interleave_ways; i++) 1106 target_map[i] = target_list.target_id[i]; 1107 1108 return 0; 1109 } 1110 1111 if (!committed) 1112 return 0; 1113 1114 dpa_size = div_u64_rem(size, cxld->interleave_ways, &remainder); 1115 if (remainder) { 1116 dev_err(&port->dev, 1117 "decoder%d.%d: invalid committed configuration size: %#llx ways: %d\n", 1118 port->id, cxld->id, size, cxld->interleave_ways); 1119 return -ENXIO; 1120 } 1121 lo = readl(hdm + CXL_HDM_DECODER0_SKIP_LOW(which)); 1122 hi = readl(hdm + CXL_HDM_DECODER0_SKIP_HIGH(which)); 1123 skip = (hi << 32) + lo; 1124 rc = devm_cxl_dpa_reserve(cxled, *dpa_base + skip, dpa_size, skip); 1125 if (rc) { 1126 dev_err(&port->dev, 1127 "decoder%d.%d: Failed to reserve DPA range %#llx - %#llx\n (%d)", 1128 port->id, cxld->id, *dpa_base, 1129 *dpa_base + dpa_size + skip - 1, rc); 1130 return rc; 1131 } 1132 *dpa_base += dpa_size + skip; 1133 1134 cxled->state = CXL_DECODER_STATE_AUTO; 1135 1136 return 0; 1137 } 1138 1139 static void cxl_settle_decoders(struct cxl_hdm *cxlhdm) 1140 { 1141 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 1142 int committed, i; 1143 u32 ctrl; 1144 1145 if (!hdm) 1146 return; 1147 1148 /* 1149 * Since the register resource was recently claimed via request_region() 1150 * be careful about trusting the "not-committed" status until the commit 1151 * timeout has elapsed. The commit timeout is 10ms (CXL 2.0 1152 * 8.2.5.12.20), but double it to be tolerant of any clock skew between 1153 * host and target. 1154 */ 1155 for (i = 0, committed = 0; i < cxlhdm->decoder_count; i++) { 1156 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(i)); 1157 if (ctrl & CXL_HDM_DECODER0_CTRL_COMMITTED) 1158 committed++; 1159 } 1160 1161 /* ensure that future checks of committed can be trusted */ 1162 if (committed != cxlhdm->decoder_count) 1163 msleep(20); 1164 } 1165 1166 /** 1167 * devm_cxl_enumerate_decoders - add decoder objects per HDM register set 1168 * @cxlhdm: Structure to populate with HDM capabilities 1169 * @info: cached DVSEC range register info 1170 */ 1171 int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm, 1172 struct cxl_endpoint_dvsec_info *info) 1173 { 1174 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 1175 struct cxl_port *port = cxlhdm->port; 1176 int i; 1177 u64 dpa_base = 0; 1178 1179 cxl_settle_decoders(cxlhdm); 1180 1181 for (i = 0; i < cxlhdm->decoder_count; i++) { 1182 int target_map[CXL_DECODER_MAX_INTERLEAVE] = { 0 }; 1183 int rc, target_count = cxlhdm->target_count; 1184 struct cxl_decoder *cxld; 1185 1186 if (is_cxl_endpoint(port)) { 1187 struct cxl_endpoint_decoder *cxled; 1188 1189 cxled = cxl_endpoint_decoder_alloc(port); 1190 if (IS_ERR(cxled)) { 1191 dev_warn(&port->dev, 1192 "Failed to allocate decoder%d.%d\n", 1193 port->id, i); 1194 return PTR_ERR(cxled); 1195 } 1196 cxld = &cxled->cxld; 1197 } else { 1198 struct cxl_switch_decoder *cxlsd; 1199 1200 cxlsd = cxl_switch_decoder_alloc(port, target_count); 1201 if (IS_ERR(cxlsd)) { 1202 dev_warn(&port->dev, 1203 "Failed to allocate decoder%d.%d\n", 1204 port->id, i); 1205 return PTR_ERR(cxlsd); 1206 } 1207 cxld = &cxlsd->cxld; 1208 } 1209 1210 rc = init_hdm_decoder(port, cxld, target_map, hdm, i, 1211 &dpa_base, info); 1212 if (rc) { 1213 dev_warn(&port->dev, 1214 "Failed to initialize decoder%d.%d\n", 1215 port->id, i); 1216 put_device(&cxld->dev); 1217 return rc; 1218 } 1219 rc = add_hdm_decoder(port, cxld, target_map); 1220 if (rc) { 1221 dev_warn(&port->dev, 1222 "Failed to add decoder%d.%d\n", port->id, i); 1223 return rc; 1224 } 1225 } 1226 1227 return 0; 1228 } 1229 EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_decoders, "CXL"); 1230