1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ 3 #include <linux/units.h> 4 #include <linux/io-64-nonatomic-lo-hi.h> 5 #include <linux/device.h> 6 #include <linux/delay.h> 7 #include <linux/pci.h> 8 #include <linux/pci-doe.h> 9 #include <cxl/pci.h> 10 #include <linux/aer.h> 11 #include <cxlpci.h> 12 #include <cxlmem.h> 13 #include <cxl.h> 14 #include "core.h" 15 #include "trace.h" 16 17 /** 18 * DOC: cxl core pci 19 * 20 * Compute Express Link protocols are layered on top of PCIe. CXL core provides 21 * a set of helpers for CXL interactions which occur via PCIe. 22 */ 23 24 static unsigned short media_ready_timeout = 60; 25 module_param(media_ready_timeout, ushort, 0644); 26 MODULE_PARM_DESC(media_ready_timeout, "seconds to wait for media ready"); 27 28 static int pci_get_port_num(struct pci_dev *pdev) 29 { 30 u32 lnkcap; 31 int type; 32 33 type = pci_pcie_type(pdev); 34 if (type != PCI_EXP_TYPE_DOWNSTREAM && type != PCI_EXP_TYPE_ROOT_PORT) 35 return -EINVAL; 36 37 if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP, 38 &lnkcap)) 39 return -ENXIO; 40 41 return FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap); 42 } 43 44 /** 45 * devm_cxl_add_dport_by_dev - allocate a dport by dport device 46 * @port: cxl_port that hosts the dport 47 * @dport_dev: 'struct device' of the dport 48 * 49 * Returns the allocated dport on success or ERR_PTR() of -errno on error 50 */ 51 struct cxl_dport *devm_cxl_add_dport_by_dev(struct cxl_port *port, 52 struct device *dport_dev) 53 { 54 struct cxl_register_map map; 55 struct pci_dev *pdev; 56 int port_num, rc; 57 58 if (!dev_is_pci(dport_dev)) 59 return ERR_PTR(-EINVAL); 60 61 pdev = to_pci_dev(dport_dev); 62 port_num = pci_get_port_num(pdev); 63 if (port_num < 0) 64 return ERR_PTR(port_num); 65 66 rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map); 67 if (rc) 68 return ERR_PTR(rc); 69 70 device_lock_assert(&port->dev); 71 return devm_cxl_add_dport(port, dport_dev, port_num, map.resource); 72 } 73 EXPORT_SYMBOL_NS_GPL(devm_cxl_add_dport_by_dev, "CXL"); 74 75 static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id) 76 { 77 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 78 int d = cxlds->cxl_dvsec; 79 bool valid = false; 80 int rc, i; 81 u32 temp; 82 83 if (id > CXL_DVSEC_RANGE_MAX) 84 return -EINVAL; 85 86 /* Check MEM INFO VALID bit first, give up after 1s */ 87 i = 1; 88 do { 89 rc = pci_read_config_dword(pdev, 90 d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(id), 91 &temp); 92 if (rc) 93 return pcibios_err_to_errno(rc); 94 95 valid = FIELD_GET(PCI_DVSEC_CXL_MEM_INFO_VALID, temp); 96 if (valid) 97 break; 98 msleep(1000); 99 } while (i--); 100 101 if (!valid) { 102 dev_err(&pdev->dev, 103 "Timeout awaiting memory range %d valid after 1s.\n", 104 id); 105 return -ETIMEDOUT; 106 } 107 108 return 0; 109 } 110 111 static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id) 112 { 113 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 114 int d = cxlds->cxl_dvsec; 115 bool active = false; 116 int rc, i; 117 u32 temp; 118 119 if (id > CXL_DVSEC_RANGE_MAX) 120 return -EINVAL; 121 122 /* Check MEM ACTIVE bit, up to 60s timeout by default */ 123 for (i = media_ready_timeout; i; i--) { 124 rc = pci_read_config_dword( 125 pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(id), &temp); 126 if (rc) 127 return pcibios_err_to_errno(rc); 128 129 active = FIELD_GET(PCI_DVSEC_CXL_MEM_ACTIVE, temp); 130 if (active) 131 break; 132 msleep(1000); 133 } 134 135 if (!active) { 136 dev_err(&pdev->dev, 137 "timeout awaiting memory active after %d seconds\n", 138 media_ready_timeout); 139 return -ETIMEDOUT; 140 } 141 142 return 0; 143 } 144 145 /* 146 * Wait up to @media_ready_timeout for the device to report memory 147 * active. 148 */ 149 int cxl_await_media_ready(struct cxl_dev_state *cxlds) 150 { 151 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 152 int d = cxlds->cxl_dvsec; 153 int rc, i, hdm_count; 154 u64 md_status; 155 u16 cap; 156 157 rc = pci_read_config_word(pdev, 158 d + PCI_DVSEC_CXL_CAP, &cap); 159 if (rc) 160 return pcibios_err_to_errno(rc); 161 162 hdm_count = FIELD_GET(PCI_DVSEC_CXL_HDM_COUNT, cap); 163 for (i = 0; i < hdm_count; i++) { 164 rc = cxl_dvsec_mem_range_valid(cxlds, i); 165 if (rc) 166 return rc; 167 } 168 169 for (i = 0; i < hdm_count; i++) { 170 rc = cxl_dvsec_mem_range_active(cxlds, i); 171 if (rc) 172 return rc; 173 } 174 175 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 176 if (!CXLMDEV_READY(md_status)) 177 return -EIO; 178 179 return 0; 180 } 181 EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, "CXL"); 182 183 static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val) 184 { 185 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 186 int d = cxlds->cxl_dvsec; 187 u16 ctrl; 188 int rc; 189 190 rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, &ctrl); 191 if (rc) 192 return pcibios_err_to_errno(rc); 193 194 if ((ctrl & PCI_DVSEC_CXL_MEM_ENABLE) == val) 195 return 1; 196 ctrl &= ~PCI_DVSEC_CXL_MEM_ENABLE; 197 ctrl |= val; 198 199 rc = pci_write_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, ctrl); 200 if (rc) 201 return pcibios_err_to_errno(rc); 202 203 return 0; 204 } 205 206 static void clear_mem_enable(void *cxlds) 207 { 208 cxl_set_mem_enable(cxlds, 0); 209 } 210 211 static int devm_cxl_enable_mem(struct device *host, struct cxl_dev_state *cxlds) 212 { 213 int rc; 214 215 rc = cxl_set_mem_enable(cxlds, PCI_DVSEC_CXL_MEM_ENABLE); 216 if (rc < 0) 217 return rc; 218 if (rc > 0) 219 return 0; 220 return devm_add_action_or_reset(host, clear_mem_enable, cxlds); 221 } 222 223 /* require dvsec ranges to be covered by a locked platform window */ 224 static int dvsec_range_allowed(struct device *dev, const void *arg) 225 { 226 const struct range *dev_range = arg; 227 struct cxl_decoder *cxld; 228 229 if (!is_root_decoder(dev)) 230 return 0; 231 232 cxld = to_cxl_decoder(dev); 233 234 if (!(cxld->flags & CXL_DECODER_F_RAM)) 235 return 0; 236 237 return range_contains(&cxld->hpa_range, dev_range); 238 } 239 240 static void disable_hdm(void *_cxlhdm) 241 { 242 u32 global_ctrl; 243 struct cxl_hdm *cxlhdm = _cxlhdm; 244 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 245 246 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 247 writel(global_ctrl & ~CXL_HDM_DECODER_ENABLE, 248 hdm + CXL_HDM_DECODER_CTRL_OFFSET); 249 } 250 251 static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm) 252 { 253 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 254 u32 global_ctrl; 255 256 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 257 writel(global_ctrl | CXL_HDM_DECODER_ENABLE, 258 hdm + CXL_HDM_DECODER_CTRL_OFFSET); 259 260 return devm_add_action_or_reset(host, disable_hdm, cxlhdm); 261 } 262 263 int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds, 264 struct cxl_endpoint_dvsec_info *info) 265 { 266 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 267 struct device *dev = cxlds->dev; 268 int hdm_count, rc, i, ranges = 0; 269 int d = cxlds->cxl_dvsec; 270 u16 cap, ctrl; 271 272 if (!d) { 273 dev_dbg(dev, "No DVSEC Capability\n"); 274 return -ENXIO; 275 } 276 277 rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CAP, &cap); 278 if (rc) 279 return pcibios_err_to_errno(rc); 280 281 if (!(cap & PCI_DVSEC_CXL_MEM_CAPABLE)) { 282 dev_dbg(dev, "Not MEM Capable\n"); 283 return -ENXIO; 284 } 285 286 /* 287 * It is not allowed by spec for MEM.capable to be set and have 0 legacy 288 * HDM decoders (values > 2 are also undefined as of CXL 2.0). As this 289 * driver is for a spec defined class code which must be CXL.mem 290 * capable, there is no point in continuing to enable CXL.mem. 291 */ 292 hdm_count = FIELD_GET(PCI_DVSEC_CXL_HDM_COUNT, cap); 293 if (!hdm_count || hdm_count > 2) 294 return -EINVAL; 295 296 /* 297 * The current DVSEC values are moot if the memory capability is 298 * disabled, and they will remain moot after the HDM Decoder 299 * capability is enabled. 300 */ 301 rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, &ctrl); 302 if (rc) 303 return pcibios_err_to_errno(rc); 304 305 info->mem_enabled = FIELD_GET(PCI_DVSEC_CXL_MEM_ENABLE, ctrl); 306 if (!info->mem_enabled) 307 return 0; 308 309 for (i = 0; i < hdm_count; i++) { 310 u64 base, size; 311 u32 temp; 312 313 rc = cxl_dvsec_mem_range_valid(cxlds, i); 314 if (rc) 315 return rc; 316 317 rc = pci_read_config_dword( 318 pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_HIGH(i), &temp); 319 if (rc) 320 return pcibios_err_to_errno(rc); 321 322 size = (u64)temp << 32; 323 324 rc = pci_read_config_dword( 325 pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(i), &temp); 326 if (rc) 327 return pcibios_err_to_errno(rc); 328 329 size |= temp & PCI_DVSEC_CXL_MEM_SIZE_LOW; 330 if (!size) { 331 continue; 332 } 333 334 rc = pci_read_config_dword( 335 pdev, d + PCI_DVSEC_CXL_RANGE_BASE_HIGH(i), &temp); 336 if (rc) 337 return pcibios_err_to_errno(rc); 338 339 base = (u64)temp << 32; 340 341 rc = pci_read_config_dword( 342 pdev, d + PCI_DVSEC_CXL_RANGE_BASE_LOW(i), &temp); 343 if (rc) 344 return pcibios_err_to_errno(rc); 345 346 base |= temp & PCI_DVSEC_CXL_MEM_BASE_LOW; 347 348 info->dvsec_range[ranges++] = (struct range) { 349 .start = base, 350 .end = base + size - 1 351 }; 352 } 353 354 info->ranges = ranges; 355 356 return 0; 357 } 358 EXPORT_SYMBOL_NS_GPL(cxl_dvsec_rr_decode, "CXL"); 359 360 /** 361 * cxl_hdm_decode_init() - Setup HDM decoding for the endpoint 362 * @cxlds: Device state 363 * @cxlhdm: Mapped HDM decoder Capability 364 * @info: Cached DVSEC range registers info 365 * 366 * Try to enable the endpoint's HDM Decoder Capability 367 */ 368 int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, 369 struct cxl_endpoint_dvsec_info *info) 370 { 371 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 372 struct cxl_port *port = cxlhdm->port; 373 struct device *dev = cxlds->dev; 374 struct cxl_port *root; 375 int i, rc, allowed; 376 u32 global_ctrl = 0; 377 378 if (hdm) 379 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 380 381 /* 382 * If the HDM Decoder Capability is already enabled then assume 383 * that some other agent like platform firmware set it up. 384 */ 385 if (global_ctrl & CXL_HDM_DECODER_ENABLE || (!hdm && info->mem_enabled)) 386 return devm_cxl_enable_mem(&port->dev, cxlds); 387 388 /* 389 * If the HDM Decoder Capability does not exist and DVSEC was 390 * not setup, the DVSEC based emulation cannot be used. 391 */ 392 if (!hdm) 393 return -ENODEV; 394 395 /* The HDM Decoder Capability exists but is globally disabled. */ 396 397 /* 398 * If the DVSEC CXL Range registers are not enabled, just 399 * enable and use the HDM Decoder Capability registers. 400 */ 401 if (!info->mem_enabled) { 402 rc = devm_cxl_enable_hdm(&port->dev, cxlhdm); 403 if (rc) 404 return rc; 405 406 return devm_cxl_enable_mem(&port->dev, cxlds); 407 } 408 409 /* 410 * Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base 411 * [High,Low] when HDM operation is enabled the range register values 412 * are ignored by the device, but the spec also recommends matching the 413 * DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges 414 * are expected even though Linux does not require or maintain that 415 * match. Check if at least one DVSEC range is enabled and allowed by 416 * the platform. That is, the DVSEC range must be covered by a locked 417 * platform window (CFMWS). Fail otherwise as the endpoint's decoders 418 * cannot be used. 419 */ 420 421 root = to_cxl_port(port->dev.parent); 422 while (!is_cxl_root(root) && is_cxl_port(root->dev.parent)) 423 root = to_cxl_port(root->dev.parent); 424 if (!is_cxl_root(root)) { 425 dev_err(dev, "Failed to acquire root port for HDM enable\n"); 426 return -ENODEV; 427 } 428 429 for (i = 0, allowed = 0; i < info->ranges; i++) { 430 struct device *cxld_dev; 431 432 cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i], 433 dvsec_range_allowed); 434 if (!cxld_dev) { 435 dev_dbg(dev, "DVSEC Range%d denied by platform\n", i); 436 continue; 437 } 438 dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i); 439 put_device(cxld_dev); 440 allowed++; 441 } 442 443 if (!allowed) { 444 dev_err(dev, "Range register decodes outside platform defined CXL ranges.\n"); 445 return -ENXIO; 446 } 447 448 return 0; 449 } 450 EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, "CXL"); 451 452 #define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff 453 #define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0 454 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00 455 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA 0 456 #define CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE 0xffff0000 457 #define CXL_DOE_TABLE_ACCESS_LAST_ENTRY 0xffff 458 #define CXL_DOE_PROTOCOL_TABLE_ACCESS 2 459 460 #define CDAT_DOE_REQ(entry_handle) cpu_to_le32 \ 461 (FIELD_PREP(CXL_DOE_TABLE_ACCESS_REQ_CODE, \ 462 CXL_DOE_TABLE_ACCESS_REQ_CODE_READ) | \ 463 FIELD_PREP(CXL_DOE_TABLE_ACCESS_TABLE_TYPE, \ 464 CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) | \ 465 FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle))) 466 467 static int cxl_cdat_get_length(struct device *dev, 468 struct pci_doe_mb *doe_mb, 469 size_t *length) 470 { 471 __le32 request = CDAT_DOE_REQ(0); 472 __le32 response[2]; 473 int rc; 474 475 rc = pci_doe(doe_mb, PCI_VENDOR_ID_CXL, 476 CXL_DOE_PROTOCOL_TABLE_ACCESS, 477 &request, sizeof(request), 478 &response, sizeof(response)); 479 if (rc < 0) { 480 dev_err(dev, "DOE failed: %d", rc); 481 return rc; 482 } 483 if (rc < sizeof(response)) 484 return -EIO; 485 486 *length = le32_to_cpu(response[1]); 487 dev_dbg(dev, "CDAT length %zu\n", *length); 488 489 return 0; 490 } 491 492 static int cxl_cdat_read_table(struct device *dev, 493 struct pci_doe_mb *doe_mb, 494 struct cdat_doe_rsp *rsp, size_t *length) 495 { 496 size_t received, remaining = *length; 497 unsigned int entry_handle = 0; 498 union cdat_data *data; 499 __le32 saved_dw = 0; 500 501 do { 502 __le32 request = CDAT_DOE_REQ(entry_handle); 503 int rc; 504 505 rc = pci_doe(doe_mb, PCI_VENDOR_ID_CXL, 506 CXL_DOE_PROTOCOL_TABLE_ACCESS, 507 &request, sizeof(request), 508 rsp, sizeof(*rsp) + remaining); 509 if (rc < 0) { 510 dev_err(dev, "DOE failed: %d", rc); 511 return rc; 512 } 513 514 if (rc < sizeof(*rsp)) 515 return -EIO; 516 517 data = (union cdat_data *)rsp->data; 518 received = rc - sizeof(*rsp); 519 520 if (entry_handle == 0) { 521 if (received != sizeof(data->header)) 522 return -EIO; 523 } else { 524 if (received < sizeof(data->entry) || 525 received != le16_to_cpu(data->entry.length)) 526 return -EIO; 527 } 528 529 /* Get the CXL table access header entry handle */ 530 entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, 531 le32_to_cpu(rsp->doe_header)); 532 533 /* 534 * Table Access Response Header overwrote the last DW of 535 * previous entry, so restore that DW 536 */ 537 rsp->doe_header = saved_dw; 538 remaining -= received; 539 rsp = (void *)rsp + received; 540 saved_dw = rsp->doe_header; 541 } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY); 542 543 /* Length in CDAT header may exceed concatenation of CDAT entries */ 544 *length -= remaining; 545 546 return 0; 547 } 548 549 static unsigned char cdat_checksum(void *buf, size_t size) 550 { 551 unsigned char sum, *data = buf; 552 size_t i; 553 554 for (sum = 0, i = 0; i < size; i++) 555 sum += data[i]; 556 return sum; 557 } 558 559 /** 560 * read_cdat_data - Read the CDAT data on this port 561 * @port: Port to read data from 562 * 563 * This call will sleep waiting for responses from the DOE mailbox. 564 */ 565 void read_cdat_data(struct cxl_port *port) 566 { 567 struct device *uport = port->uport_dev; 568 struct device *dev = &port->dev; 569 struct pci_doe_mb *doe_mb; 570 struct pci_dev *pdev = NULL; 571 struct cxl_memdev *cxlmd; 572 struct cdat_doe_rsp *buf; 573 size_t table_length, length; 574 int rc; 575 576 if (is_cxl_memdev(uport)) { 577 struct device *host; 578 579 cxlmd = to_cxl_memdev(uport); 580 host = cxlmd->dev.parent; 581 if (dev_is_pci(host)) 582 pdev = to_pci_dev(host); 583 } else if (dev_is_pci(uport)) { 584 pdev = to_pci_dev(uport); 585 } 586 587 if (!pdev) 588 return; 589 590 doe_mb = pci_find_doe_mailbox(pdev, PCI_VENDOR_ID_CXL, 591 CXL_DOE_PROTOCOL_TABLE_ACCESS); 592 if (!doe_mb) { 593 dev_dbg(dev, "No CDAT mailbox\n"); 594 return; 595 } 596 597 port->cdat_available = true; 598 599 if (cxl_cdat_get_length(dev, doe_mb, &length)) { 600 dev_dbg(dev, "No CDAT length\n"); 601 return; 602 } 603 604 /* 605 * The begin of the CDAT buffer needs space for additional 4 606 * bytes for the DOE header. Table data starts afterwards. 607 */ 608 buf = devm_kzalloc(dev, sizeof(*buf) + length, GFP_KERNEL); 609 if (!buf) 610 goto err; 611 612 table_length = length; 613 614 rc = cxl_cdat_read_table(dev, doe_mb, buf, &length); 615 if (rc) 616 goto err; 617 618 if (table_length != length) 619 dev_warn(dev, "Malformed CDAT table length (%zu:%zu), discarding trailing data\n", 620 table_length, length); 621 622 if (cdat_checksum(buf->data, length)) 623 goto err; 624 625 port->cdat.table = buf->data; 626 port->cdat.length = length; 627 628 return; 629 err: 630 /* Don't leave table data allocated on error */ 631 devm_kfree(dev, buf); 632 dev_err(dev, "Failed to read/validate CDAT.\n"); 633 } 634 EXPORT_SYMBOL_NS_GPL(read_cdat_data, "CXL"); 635 636 static int cxl_flit_size(struct pci_dev *pdev) 637 { 638 if (cxl_pci_flit_256(pdev)) 639 return 256; 640 641 return 68; 642 } 643 644 /** 645 * cxl_pci_get_latency - calculate the link latency for the PCIe link 646 * @pdev: PCI device 647 * 648 * return: calculated latency or 0 for no latency 649 * 650 * CXL Memory Device SW Guide v1.0 2.11.4 Link latency calculation 651 * Link latency = LinkPropagationLatency + FlitLatency + RetimerLatency 652 * LinkProgationLatency is negligible, so 0 will be used 653 * RetimerLatency is assumed to be negligible and 0 will be used 654 * FlitLatency = FlitSize / LinkBandwidth 655 * FlitSize is defined by spec. CXL rev3.0 4.2.1. 656 * 68B flit is used up to 32GT/s. >32GT/s, 256B flit size is used. 657 * The FlitLatency is converted to picoseconds. 658 */ 659 long cxl_pci_get_latency(struct pci_dev *pdev) 660 { 661 long bw; 662 663 bw = pcie_link_speed_mbps(pdev); 664 if (bw < 0) 665 return 0; 666 bw /= BITS_PER_BYTE; 667 668 return cxl_flit_size(pdev) * MEGA / bw; 669 } 670 671 static int __cxl_endpoint_decoder_reset_detected(struct device *dev, void *data) 672 { 673 struct cxl_port *port = data; 674 struct cxl_decoder *cxld; 675 struct cxl_hdm *cxlhdm; 676 void __iomem *hdm; 677 u32 ctrl; 678 679 if (!is_endpoint_decoder(dev)) 680 return 0; 681 682 cxld = to_cxl_decoder(dev); 683 if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0) 684 return 0; 685 686 cxlhdm = dev_get_drvdata(&port->dev); 687 hdm = cxlhdm->regs.hdm_decoder; 688 ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id)); 689 690 return !FIELD_GET(CXL_HDM_DECODER0_CTRL_COMMITTED, ctrl); 691 } 692 693 bool cxl_endpoint_decoder_reset_detected(struct cxl_port *port) 694 { 695 return device_for_each_child(&port->dev, port, 696 __cxl_endpoint_decoder_reset_detected); 697 } 698 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_decoder_reset_detected, "CXL"); 699 700 static int cxl_rcrb_get_comp_regs(struct pci_dev *pdev, 701 struct cxl_register_map *map, 702 struct cxl_dport *dport) 703 { 704 resource_size_t component_reg_phys; 705 706 *map = (struct cxl_register_map) { 707 .host = &pdev->dev, 708 .resource = CXL_RESOURCE_NONE, 709 }; 710 711 component_reg_phys = cxl_rcd_component_reg_phys(&pdev->dev, dport); 712 if (component_reg_phys == CXL_RESOURCE_NONE) 713 return -ENXIO; 714 715 map->resource = component_reg_phys; 716 map->reg_type = CXL_REGLOC_RBI_COMPONENT; 717 map->max_size = CXL_COMPONENT_REG_BLOCK_SIZE; 718 719 return 0; 720 } 721 722 int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, 723 struct cxl_register_map *map) 724 { 725 int rc; 726 727 rc = cxl_find_regblock(pdev, type, map); 728 729 /* 730 * If the Register Locator DVSEC does not exist, check if it 731 * is an RCH and try to extract the Component Registers from 732 * an RCRB. 733 */ 734 if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev)) { 735 struct cxl_dport *dport; 736 struct cxl_port *port __free(put_cxl_port) = 737 cxl_pci_find_port(pdev, &dport); 738 if (!port) 739 return -EPROBE_DEFER; 740 741 rc = cxl_rcrb_get_comp_regs(pdev, map, dport); 742 if (rc) 743 return rc; 744 745 rc = cxl_dport_map_rcd_linkcap(pdev, dport); 746 if (rc) 747 return rc; 748 749 } else if (rc) { 750 return rc; 751 } 752 753 return cxl_setup_regs(map); 754 } 755 EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, "CXL"); 756 757 int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c) 758 { 759 int speed, bw; 760 u16 lnksta; 761 u32 width; 762 763 speed = pcie_link_speed_mbps(pdev); 764 if (speed < 0) 765 return speed; 766 speed /= BITS_PER_BYTE; 767 768 pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); 769 width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); 770 bw = speed * width; 771 772 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { 773 c[i].read_bandwidth = bw; 774 c[i].write_bandwidth = bw; 775 } 776 777 return 0; 778 } 779 780 /* 781 * Set max timeout such that platforms will optimize GPF flow to avoid 782 * the implied worst-case scenario delays. On a sane platform, all 783 * devices should always complete GPF within the energy budget of 784 * the GPF flow. The kernel does not have enough information to pick 785 * anything better than "maximize timeouts and hope it works". 786 * 787 * A misbehaving device could block forward progress of GPF for all 788 * the other devices, exhausting the energy budget of the platform. 789 * However, the spec seems to assume that moving on from slow to respond 790 * devices is a virtue. It is not possible to know that, in actuality, 791 * the slow to respond device is *the* most critical device in the 792 * system to wait. 793 */ 794 #define GPF_TIMEOUT_BASE_MAX 2 795 #define GPF_TIMEOUT_SCALE_MAX 7 /* 10 seconds */ 796 797 u16 cxl_gpf_get_dvsec(struct device *dev) 798 { 799 struct pci_dev *pdev; 800 bool is_port = true; 801 u16 dvsec; 802 803 if (!dev_is_pci(dev)) 804 return 0; 805 806 pdev = to_pci_dev(dev); 807 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ENDPOINT) 808 is_port = false; 809 810 dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, 811 is_port ? PCI_DVSEC_CXL_PORT_GPF : PCI_DVSEC_CXL_DEVICE_GPF); 812 if (!dvsec) 813 dev_warn(dev, "%s GPF DVSEC not present\n", 814 is_port ? "Port" : "Device"); 815 return dvsec; 816 } 817 EXPORT_SYMBOL_NS_GPL(cxl_gpf_get_dvsec, "CXL"); 818 819 static int update_gpf_port_dvsec(struct pci_dev *pdev, int dvsec, int phase) 820 { 821 u64 base, scale; 822 int rc, offset; 823 u16 ctrl; 824 825 switch (phase) { 826 case 1: 827 offset = PCI_DVSEC_CXL_PORT_GPF_PHASE_1_CONTROL; 828 base = PCI_DVSEC_CXL_PORT_GPF_PHASE_1_TMO_BASE; 829 scale = PCI_DVSEC_CXL_PORT_GPF_PHASE_1_TMO_SCALE; 830 break; 831 case 2: 832 offset = PCI_DVSEC_CXL_PORT_GPF_PHASE_2_CONTROL; 833 base = PCI_DVSEC_CXL_PORT_GPF_PHASE_2_TMO_BASE; 834 scale = PCI_DVSEC_CXL_PORT_GPF_PHASE_2_TMO_SCALE; 835 break; 836 default: 837 return -EINVAL; 838 } 839 840 rc = pci_read_config_word(pdev, dvsec + offset, &ctrl); 841 if (rc) 842 return rc; 843 844 if (FIELD_GET(base, ctrl) == GPF_TIMEOUT_BASE_MAX && 845 FIELD_GET(scale, ctrl) == GPF_TIMEOUT_SCALE_MAX) 846 return 0; 847 848 ctrl = FIELD_PREP(base, GPF_TIMEOUT_BASE_MAX); 849 ctrl |= FIELD_PREP(scale, GPF_TIMEOUT_SCALE_MAX); 850 851 rc = pci_write_config_word(pdev, dvsec + offset, ctrl); 852 if (!rc) 853 pci_dbg(pdev, "Port GPF phase %d timeout: %d0 secs\n", 854 phase, GPF_TIMEOUT_BASE_MAX); 855 856 return rc; 857 } 858 859 int cxl_gpf_port_setup(struct cxl_dport *dport) 860 { 861 if (!dport) 862 return -EINVAL; 863 864 if (!dport->gpf_dvsec) { 865 struct pci_dev *pdev; 866 int dvsec; 867 868 dvsec = cxl_gpf_get_dvsec(dport->dport_dev); 869 if (!dvsec) 870 return -EINVAL; 871 872 dport->gpf_dvsec = dvsec; 873 pdev = to_pci_dev(dport->dport_dev); 874 update_gpf_port_dvsec(pdev, dport->gpf_dvsec, 1); 875 update_gpf_port_dvsec(pdev, dport->gpf_dvsec, 2); 876 } 877 878 return 0; 879 } 880 881 struct cxl_walk_context { 882 struct pci_bus *bus; 883 struct cxl_port *port; 884 int type; 885 int error; 886 int count; 887 }; 888 889 static int count_dports(struct pci_dev *pdev, void *data) 890 { 891 struct cxl_walk_context *ctx = data; 892 int type = pci_pcie_type(pdev); 893 894 if (pdev->bus != ctx->bus) 895 return 0; 896 if (!pci_is_pcie(pdev)) 897 return 0; 898 if (type != ctx->type) 899 return 0; 900 901 ctx->count++; 902 return 0; 903 } 904 905 int cxl_port_get_possible_dports(struct cxl_port *port) 906 { 907 struct pci_bus *bus = cxl_port_to_pci_bus(port); 908 struct cxl_walk_context ctx; 909 int type; 910 911 if (!bus) { 912 dev_err(&port->dev, "No PCI bus found for port %s\n", 913 dev_name(&port->dev)); 914 return -ENXIO; 915 } 916 917 if (pci_is_root_bus(bus)) 918 type = PCI_EXP_TYPE_ROOT_PORT; 919 else 920 type = PCI_EXP_TYPE_DOWNSTREAM; 921 922 ctx = (struct cxl_walk_context) { 923 .bus = bus, 924 .type = type, 925 }; 926 pci_walk_bus(bus, count_dports, &ctx); 927 928 return ctx.count; 929 } 930