1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 #include <linux/io-64-nonatomic-lo-hi.h> 4 #include <linux/device.h> 5 #include <linux/slab.h> 6 #include <linux/pci.h> 7 #include <cxl/pci.h> 8 #include <cxlmem.h> 9 #include <cxlpci.h> 10 #include <pmu.h> 11 12 #include "core.h" 13 14 /** 15 * DOC: cxl registers 16 * 17 * CXL device capabilities are enumerated by PCI DVSEC (Designated 18 * Vendor-specific) and / or descriptors provided by platform firmware. 19 * They can be defined as a set like the device and component registers 20 * mandated by CXL Section 8.1.12.2 Memory Device PCIe Capabilities and 21 * Extended Capabilities, or they can be individual capabilities 22 * appended to bridged and endpoint devices. 23 * 24 * Provide common infrastructure for enumerating and mapping these 25 * discrete capabilities. 26 */ 27 28 /** 29 * cxl_probe_component_regs() - Detect CXL Component register blocks 30 * @dev: Host device of the @base mapping 31 * @base: Mapping containing the HDM Decoder Capability Header 32 * @map: Map object describing the register block information found 33 * 34 * See CXL 2.0 8.2.4 Component Register Layout and Definition 35 * See CXL 2.0 8.2.5.5 CXL Device Register Interface 36 * 37 * Probe for component register information and return it in map object. 38 */ 39 void cxl_probe_component_regs(struct device *dev, void __iomem *base, 40 struct cxl_component_reg_map *map) 41 { 42 int cap, cap_count; 43 u32 cap_array; 44 45 *map = (struct cxl_component_reg_map) { 0 }; 46 47 /* 48 * CXL.cache and CXL.mem registers are at offset 0x1000 as defined in 49 * CXL 2.0 8.2.4 Table 141. 50 */ 51 base += CXL_CM_OFFSET; 52 53 cap_array = readl(base + CXL_CM_CAP_HDR_OFFSET); 54 55 if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) { 56 dev_dbg(dev, 57 "Couldn't locate the CXL.cache and CXL.mem capability array header.\n"); 58 return; 59 } 60 61 /* It's assumed that future versions will be backward compatible */ 62 cap_count = FIELD_GET(CXL_CM_CAP_HDR_ARRAY_SIZE_MASK, cap_array); 63 64 for (cap = 1; cap <= cap_count; cap++) { 65 void __iomem *register_block; 66 struct cxl_reg_map *rmap; 67 u16 cap_id, offset; 68 u32 length, hdr; 69 70 hdr = readl(base + cap * 0x4); 71 72 cap_id = FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, hdr); 73 offset = FIELD_GET(CXL_CM_CAP_PTR_MASK, hdr); 74 register_block = base + offset; 75 hdr = readl(register_block); 76 77 rmap = NULL; 78 switch (cap_id) { 79 case CXL_CM_CAP_CAP_ID_HDM: { 80 int decoder_cnt; 81 82 dev_dbg(dev, "found HDM decoder capability (0x%x)\n", 83 offset); 84 85 decoder_cnt = cxl_hdm_decoder_count(hdr); 86 length = 0x20 * decoder_cnt + 0x10; 87 rmap = &map->hdm_decoder; 88 break; 89 } 90 case CXL_CM_CAP_CAP_ID_RAS: 91 dev_dbg(dev, "found RAS capability (0x%x)\n", 92 offset); 93 length = CXL_RAS_CAPABILITY_LENGTH; 94 rmap = &map->ras; 95 break; 96 default: 97 dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id, 98 offset); 99 break; 100 } 101 102 if (!rmap) 103 continue; 104 rmap->valid = true; 105 rmap->id = cap_id; 106 rmap->offset = CXL_CM_OFFSET + offset; 107 rmap->size = length; 108 } 109 } 110 EXPORT_SYMBOL_NS_GPL(cxl_probe_component_regs, "CXL"); 111 112 /** 113 * cxl_probe_device_regs() - Detect CXL Device register blocks 114 * @dev: Host device of the @base mapping 115 * @base: Mapping of CXL 2.0 8.2.8 CXL Device Register Interface 116 * @map: Map object describing the register block information found 117 * 118 * Probe for device register information and return it in map object. 119 */ 120 void cxl_probe_device_regs(struct device *dev, void __iomem *base, 121 struct cxl_device_reg_map *map) 122 { 123 int cap, cap_count; 124 u64 cap_array; 125 126 *map = (struct cxl_device_reg_map){ 0 }; 127 128 cap_array = readq(base + CXLDEV_CAP_ARRAY_OFFSET); 129 if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != 130 CXLDEV_CAP_ARRAY_CAP_ID) 131 return; 132 133 cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); 134 135 for (cap = 1; cap <= cap_count; cap++) { 136 struct cxl_reg_map *rmap; 137 u32 offset, length; 138 u16 cap_id; 139 140 cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, 141 readl(base + cap * 0x10)); 142 offset = readl(base + cap * 0x10 + 0x4); 143 length = readl(base + cap * 0x10 + 0x8); 144 145 rmap = NULL; 146 switch (cap_id) { 147 case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: 148 dev_dbg(dev, "found Status capability (0x%x)\n", offset); 149 rmap = &map->status; 150 break; 151 case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: 152 dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); 153 rmap = &map->mbox; 154 break; 155 case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: 156 dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); 157 break; 158 case CXLDEV_CAP_CAP_ID_MEMDEV: 159 dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); 160 rmap = &map->memdev; 161 break; 162 default: 163 if (cap_id >= 0x8000) 164 dev_dbg(dev, "Vendor cap ID: %#x offset: %#x\n", cap_id, offset); 165 else 166 dev_dbg(dev, "Unknown cap ID: %#x offset: %#x\n", cap_id, offset); 167 break; 168 } 169 170 if (!rmap) 171 continue; 172 rmap->valid = true; 173 rmap->id = cap_id; 174 rmap->offset = offset; 175 rmap->size = length; 176 } 177 } 178 EXPORT_SYMBOL_NS_GPL(cxl_probe_device_regs, "CXL"); 179 180 void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr, 181 resource_size_t length) 182 { 183 void __iomem *ret_val; 184 struct resource *res; 185 186 if (WARN_ON_ONCE(addr == CXL_RESOURCE_NONE)) 187 return NULL; 188 189 res = devm_request_mem_region(dev, addr, length, dev_name(dev)); 190 if (!res) { 191 resource_size_t end = addr + length - 1; 192 193 dev_err(dev, "Failed to request region %pa-%pa\n", &addr, &end); 194 return NULL; 195 } 196 197 ret_val = devm_ioremap(dev, addr, length); 198 if (!ret_val) 199 dev_err(dev, "Failed to map region %pr\n", res); 200 201 return ret_val; 202 } 203 204 int cxl_map_component_regs(const struct cxl_register_map *map, 205 struct cxl_component_regs *regs, 206 unsigned long map_mask) 207 { 208 struct device *host = map->host; 209 struct mapinfo { 210 const struct cxl_reg_map *rmap; 211 void __iomem **addr; 212 } mapinfo[] = { 213 { &map->component_map.hdm_decoder, ®s->hdm_decoder }, 214 { &map->component_map.ras, ®s->ras }, 215 }; 216 int i; 217 218 for (i = 0; i < ARRAY_SIZE(mapinfo); i++) { 219 struct mapinfo *mi = &mapinfo[i]; 220 resource_size_t addr; 221 resource_size_t length; 222 223 if (!mi->rmap->valid) 224 continue; 225 if (!test_bit(mi->rmap->id, &map_mask)) 226 continue; 227 addr = map->resource + mi->rmap->offset; 228 length = mi->rmap->size; 229 *(mi->addr) = devm_cxl_iomap_block(host, addr, length); 230 if (!*(mi->addr)) 231 return -ENOMEM; 232 } 233 234 return 0; 235 } 236 EXPORT_SYMBOL_NS_GPL(cxl_map_component_regs, "CXL"); 237 238 int cxl_map_device_regs(const struct cxl_register_map *map, 239 struct cxl_device_regs *regs) 240 { 241 struct device *host = map->host; 242 resource_size_t phys_addr = map->resource; 243 struct mapinfo { 244 const struct cxl_reg_map *rmap; 245 void __iomem **addr; 246 } mapinfo[] = { 247 { &map->device_map.status, ®s->status, }, 248 { &map->device_map.mbox, ®s->mbox, }, 249 { &map->device_map.memdev, ®s->memdev, }, 250 }; 251 int i; 252 253 for (i = 0; i < ARRAY_SIZE(mapinfo); i++) { 254 struct mapinfo *mi = &mapinfo[i]; 255 resource_size_t length; 256 resource_size_t addr; 257 258 if (!mi->rmap->valid) 259 continue; 260 261 addr = phys_addr + mi->rmap->offset; 262 length = mi->rmap->size; 263 *(mi->addr) = devm_cxl_iomap_block(host, addr, length); 264 if (!*(mi->addr)) 265 return -ENOMEM; 266 } 267 268 return 0; 269 } 270 EXPORT_SYMBOL_NS_GPL(cxl_map_device_regs, "CXL"); 271 272 static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi, 273 struct cxl_register_map *map) 274 { 275 u8 reg_type = FIELD_GET(PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_ID, reg_lo); 276 int bar = FIELD_GET(PCI_DVSEC_CXL_REG_LOCATOR_BIR, reg_lo); 277 u64 offset = ((u64)reg_hi << 32) | 278 (reg_lo & PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW); 279 280 if (offset > pci_resource_len(pdev, bar)) { 281 dev_warn(&pdev->dev, 282 "BAR%d: %pr: too small (offset: %pa, type: %d)\n", bar, 283 &pdev->resource[bar], &offset, reg_type); 284 return false; 285 } 286 287 map->reg_type = reg_type; 288 map->resource = pci_resource_start(pdev, bar) + offset; 289 map->max_size = pci_resource_len(pdev, bar) - offset; 290 return true; 291 } 292 293 /* 294 * __cxl_find_regblock_instance() - Locate a register block or count instances by type / index 295 * Use CXL_INSTANCES_COUNT for @index if counting instances. 296 * 297 * __cxl_find_regblock_instance() may return: 298 * 0 - if register block enumerated. 299 * >= 0 - if counting instances. 300 * < 0 - error code otherwise. 301 */ 302 static int __cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type, 303 struct cxl_register_map *map, int index) 304 { 305 u32 regloc_size, regblocks; 306 int instance = 0; 307 int regloc, i; 308 309 *map = (struct cxl_register_map) { 310 .host = &pdev->dev, 311 .resource = CXL_RESOURCE_NONE, 312 }; 313 314 regloc = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, 315 PCI_DVSEC_CXL_REG_LOCATOR); 316 if (!regloc) 317 return -ENXIO; 318 319 pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); 320 regloc_size = PCI_DVSEC_HEADER1_LEN(regloc_size); 321 322 regloc += PCI_DVSEC_CXL_REG_LOCATOR_BLOCK1; 323 regblocks = (regloc_size - PCI_DVSEC_CXL_REG_LOCATOR_BLOCK1) / 8; 324 325 for (i = 0; i < regblocks; i++, regloc += 8) { 326 u32 reg_lo, reg_hi; 327 328 pci_read_config_dword(pdev, regloc, ®_lo); 329 pci_read_config_dword(pdev, regloc + 4, ®_hi); 330 331 if (!cxl_decode_regblock(pdev, reg_lo, reg_hi, map)) 332 continue; 333 334 if (map->reg_type == type) { 335 if (index == instance) 336 return 0; 337 instance++; 338 } 339 } 340 341 map->resource = CXL_RESOURCE_NONE; 342 if (index == CXL_INSTANCES_COUNT) 343 return instance; 344 345 return -ENODEV; 346 } 347 348 /** 349 * cxl_find_regblock_instance() - Locate a register block by type / index 350 * @pdev: The CXL PCI device to enumerate. 351 * @type: Register Block Indicator id 352 * @map: Enumeration output, clobbered on error 353 * @index: Index into which particular instance of a regblock wanted in the 354 * order found in register locator DVSEC. 355 * 356 * Return: 0 if register block enumerated, negative error code otherwise 357 * 358 * A CXL DVSEC may point to one or more register blocks, search for them 359 * by @type and @index. 360 */ 361 int cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_type type, 362 struct cxl_register_map *map, unsigned int index) 363 { 364 return __cxl_find_regblock_instance(pdev, type, map, index); 365 } 366 EXPORT_SYMBOL_NS_GPL(cxl_find_regblock_instance, "CXL"); 367 368 /** 369 * cxl_find_regblock() - Locate register blocks by type 370 * @pdev: The CXL PCI device to enumerate. 371 * @type: Register Block Indicator id 372 * @map: Enumeration output, clobbered on error 373 * 374 * Return: 0 if register block enumerated, negative error code otherwise 375 * 376 * A CXL DVSEC may point to one or more register blocks, search for them 377 * by @type. 378 */ 379 int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, 380 struct cxl_register_map *map) 381 { 382 return __cxl_find_regblock_instance(pdev, type, map, 0); 383 } 384 EXPORT_SYMBOL_NS_GPL(cxl_find_regblock, "CXL"); 385 386 /** 387 * cxl_count_regblock() - Count instances of a given regblock type. 388 * @pdev: The CXL PCI device to enumerate. 389 * @type: Register Block Indicator id 390 * 391 * Some regblocks may be repeated. Count how many instances. 392 * 393 * Return: non-negative count of matching regblocks, negative error code otherwise. 394 */ 395 int cxl_count_regblock(struct pci_dev *pdev, enum cxl_regloc_type type) 396 { 397 struct cxl_register_map map; 398 399 return __cxl_find_regblock_instance(pdev, type, &map, CXL_INSTANCES_COUNT); 400 } 401 EXPORT_SYMBOL_NS_GPL(cxl_count_regblock, "CXL"); 402 403 int cxl_map_pmu_regs(struct cxl_register_map *map, struct cxl_pmu_regs *regs) 404 { 405 struct device *dev = map->host; 406 resource_size_t phys_addr; 407 408 phys_addr = map->resource; 409 regs->pmu = devm_cxl_iomap_block(dev, phys_addr, CXL_PMU_REGMAP_SIZE); 410 if (!regs->pmu) 411 return -ENOMEM; 412 413 return 0; 414 } 415 EXPORT_SYMBOL_NS_GPL(cxl_map_pmu_regs, "CXL"); 416 417 static int cxl_map_regblock(struct cxl_register_map *map) 418 { 419 struct device *host = map->host; 420 421 map->base = ioremap(map->resource, map->max_size); 422 if (!map->base) { 423 dev_err(host, "failed to map registers\n"); 424 return -ENOMEM; 425 } 426 427 dev_dbg(host, "Mapped CXL Memory Device resource %pa\n", &map->resource); 428 return 0; 429 } 430 431 static void cxl_unmap_regblock(struct cxl_register_map *map) 432 { 433 iounmap(map->base); 434 map->base = NULL; 435 } 436 437 static int cxl_probe_regs(struct cxl_register_map *map) 438 { 439 struct cxl_component_reg_map *comp_map; 440 struct cxl_device_reg_map *dev_map; 441 struct device *host = map->host; 442 void __iomem *base = map->base; 443 444 switch (map->reg_type) { 445 case CXL_REGLOC_RBI_COMPONENT: 446 comp_map = &map->component_map; 447 cxl_probe_component_regs(host, base, comp_map); 448 dev_dbg(host, "Set up component registers\n"); 449 break; 450 case CXL_REGLOC_RBI_MEMDEV: 451 dev_map = &map->device_map; 452 cxl_probe_device_regs(host, base, dev_map); 453 if (!dev_map->status.valid || !dev_map->mbox.valid || 454 !dev_map->memdev.valid) { 455 dev_err(host, "registers not found: %s%s%s\n", 456 !dev_map->status.valid ? "status " : "", 457 !dev_map->mbox.valid ? "mbox " : "", 458 !dev_map->memdev.valid ? "memdev " : ""); 459 return -ENXIO; 460 } 461 462 dev_dbg(host, "Probing device registers...\n"); 463 break; 464 default: 465 break; 466 } 467 468 return 0; 469 } 470 471 int cxl_setup_regs(struct cxl_register_map *map) 472 { 473 int rc; 474 475 rc = cxl_map_regblock(map); 476 if (rc) 477 return rc; 478 479 rc = cxl_probe_regs(map); 480 cxl_unmap_regblock(map); 481 482 return rc; 483 } 484 EXPORT_SYMBOL_NS_GPL(cxl_setup_regs, "CXL"); 485 486 u16 cxl_rcrb_to_aer(struct device *dev, resource_size_t rcrb) 487 { 488 void __iomem *addr; 489 u16 offset = 0; 490 u32 cap_hdr; 491 492 if (WARN_ON_ONCE(rcrb == CXL_RESOURCE_NONE)) 493 return 0; 494 495 if (!request_mem_region(rcrb, SZ_4K, dev_name(dev))) 496 return 0; 497 498 addr = ioremap(rcrb, SZ_4K); 499 if (!addr) 500 goto out; 501 502 cap_hdr = readl(addr + offset); 503 while (PCI_EXT_CAP_ID(cap_hdr) != PCI_EXT_CAP_ID_ERR) { 504 offset = PCI_EXT_CAP_NEXT(cap_hdr); 505 506 /* Offset 0 terminates capability list. */ 507 if (!offset) 508 break; 509 cap_hdr = readl(addr + offset); 510 } 511 512 if (offset) 513 dev_dbg(dev, "found AER extended capability (0x%x)\n", offset); 514 515 iounmap(addr); 516 out: 517 release_mem_region(rcrb, SZ_4K); 518 519 return offset; 520 } 521 522 static resource_size_t cxl_rcrb_to_linkcap(struct device *dev, struct cxl_dport *dport) 523 { 524 resource_size_t rcrb = dport->rcrb.base; 525 void __iomem *addr; 526 u32 cap_hdr; 527 u16 offset; 528 529 if (!request_mem_region(rcrb, SZ_4K, "CXL RCRB")) 530 return CXL_RESOURCE_NONE; 531 532 addr = ioremap(rcrb, SZ_4K); 533 if (!addr) { 534 dev_err(dev, "Failed to map region %pr\n", addr); 535 release_mem_region(rcrb, SZ_4K); 536 return CXL_RESOURCE_NONE; 537 } 538 539 offset = FIELD_GET(PCI_RCRB_CAP_LIST_ID_MASK, readw(addr + PCI_CAPABILITY_LIST)); 540 cap_hdr = readl(addr + offset); 541 while ((FIELD_GET(PCI_RCRB_CAP_HDR_ID_MASK, cap_hdr)) != PCI_CAP_ID_EXP) { 542 offset = FIELD_GET(PCI_RCRB_CAP_HDR_NEXT_MASK, cap_hdr); 543 if (offset == 0 || offset > SZ_4K) { 544 offset = 0; 545 break; 546 } 547 cap_hdr = readl(addr + offset); 548 } 549 550 iounmap(addr); 551 release_mem_region(rcrb, SZ_4K); 552 if (!offset) 553 return CXL_RESOURCE_NONE; 554 555 return offset; 556 } 557 558 int cxl_dport_map_rcd_linkcap(struct pci_dev *pdev, struct cxl_dport *dport) 559 { 560 void __iomem *dport_pcie_cap = NULL; 561 resource_size_t pos; 562 struct cxl_rcrb_info *ri; 563 564 ri = &dport->rcrb; 565 pos = cxl_rcrb_to_linkcap(&pdev->dev, dport); 566 if (pos == CXL_RESOURCE_NONE) 567 return -ENXIO; 568 569 dport_pcie_cap = devm_cxl_iomap_block(&pdev->dev, 570 ri->base + pos, 571 PCI_CAP_EXP_SIZEOF); 572 dport->regs.rcd_pcie_cap = dport_pcie_cap; 573 574 return 0; 575 } 576 EXPORT_SYMBOL_NS_GPL(cxl_dport_map_rcd_linkcap, "CXL"); 577 578 resource_size_t __rcrb_to_component(struct device *dev, struct cxl_rcrb_info *ri, 579 enum cxl_rcrb which) 580 { 581 resource_size_t component_reg_phys; 582 resource_size_t rcrb = ri->base; 583 void __iomem *addr; 584 u32 bar0, bar1; 585 u32 id; 586 587 if (which == CXL_RCRB_UPSTREAM) 588 rcrb += SZ_4K; 589 590 /* 591 * RCRB's BAR[0..1] point to component block containing CXL 592 * subsystem component registers. MEMBAR extraction follows 593 * the PCI Base spec here, esp. 64 bit extraction and memory 594 * ranges alignment (6.0, 7.5.1.2.1). 595 */ 596 if (!request_mem_region(rcrb, SZ_4K, "CXL RCRB")) 597 return CXL_RESOURCE_NONE; 598 addr = ioremap(rcrb, SZ_4K); 599 if (!addr) { 600 dev_err(dev, "Failed to map region %pr\n", addr); 601 release_mem_region(rcrb, SZ_4K); 602 return CXL_RESOURCE_NONE; 603 } 604 605 id = readl(addr + PCI_VENDOR_ID); 606 bar0 = readl(addr + PCI_BASE_ADDRESS_0); 607 bar1 = readl(addr + PCI_BASE_ADDRESS_1); 608 iounmap(addr); 609 release_mem_region(rcrb, SZ_4K); 610 611 /* 612 * Sanity check, see CXL 3.0 Figure 9-8 CXL Device that Does Not 613 * Remap Upstream Port and Component Registers 614 */ 615 if (id == U32_MAX) { 616 if (which == CXL_RCRB_DOWNSTREAM) 617 dev_err(dev, "Failed to access Downstream Port RCRB\n"); 618 return CXL_RESOURCE_NONE; 619 } 620 /* The RCRB is a Memory Window, and the MEM_TYPE_1M bit is obsolete */ 621 if (bar0 & (PCI_BASE_ADDRESS_MEM_TYPE_1M | PCI_BASE_ADDRESS_SPACE_IO)) 622 return CXL_RESOURCE_NONE; 623 624 component_reg_phys = bar0 & PCI_BASE_ADDRESS_MEM_MASK; 625 if (bar0 & PCI_BASE_ADDRESS_MEM_TYPE_64) 626 component_reg_phys |= ((u64)bar1) << 32; 627 628 if (!component_reg_phys) 629 return CXL_RESOURCE_NONE; 630 631 /* MEMBAR is block size (64k) aligned. */ 632 if (!IS_ALIGNED(component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE)) 633 return CXL_RESOURCE_NONE; 634 635 return component_reg_phys; 636 } 637 638 resource_size_t cxl_rcd_component_reg_phys(struct device *dev, 639 struct cxl_dport *dport) 640 { 641 if (!dport->rch) 642 return CXL_RESOURCE_NONE; 643 return __rcrb_to_component(dev, &dport->rcrb, CXL_RCRB_UPSTREAM); 644 } 645