1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ 3 #include <linux/io-64-nonatomic-lo-hi.h> 4 #include <linux/device.h> 5 #include <linux/delay.h> 6 #include <linux/pci.h> 7 #include <linux/pci-doe.h> 8 #include <linux/aer.h> 9 #include <cxlpci.h> 10 #include <cxlmem.h> 11 #include <cxl.h> 12 #include "core.h" 13 #include "trace.h" 14 15 /** 16 * DOC: cxl core pci 17 * 18 * Compute Express Link protocols are layered on top of PCIe. CXL core provides 19 * a set of helpers for CXL interactions which occur via PCIe. 20 */ 21 22 static unsigned short media_ready_timeout = 60; 23 module_param(media_ready_timeout, ushort, 0644); 24 MODULE_PARM_DESC(media_ready_timeout, "seconds to wait for media ready"); 25 26 struct cxl_walk_context { 27 struct pci_bus *bus; 28 struct cxl_port *port; 29 int type; 30 int error; 31 int count; 32 }; 33 34 static int match_add_dports(struct pci_dev *pdev, void *data) 35 { 36 struct cxl_walk_context *ctx = data; 37 struct cxl_port *port = ctx->port; 38 int type = pci_pcie_type(pdev); 39 struct cxl_register_map map; 40 struct cxl_dport *dport; 41 u32 lnkcap, port_num; 42 int rc; 43 44 if (pdev->bus != ctx->bus) 45 return 0; 46 if (!pci_is_pcie(pdev)) 47 return 0; 48 if (type != ctx->type) 49 return 0; 50 if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP, 51 &lnkcap)) 52 return 0; 53 54 rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map); 55 if (rc) 56 dev_dbg(&port->dev, "failed to find component registers\n"); 57 58 port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap); 59 dport = devm_cxl_add_dport(port, &pdev->dev, port_num, map.resource); 60 if (IS_ERR(dport)) { 61 ctx->error = PTR_ERR(dport); 62 return PTR_ERR(dport); 63 } 64 ctx->count++; 65 66 return 0; 67 } 68 69 /** 70 * devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port 71 * @port: cxl_port whose ->uport_dev is the upstream of dports to be enumerated 72 * 73 * Returns a positive number of dports enumerated or a negative error 74 * code. 75 */ 76 int devm_cxl_port_enumerate_dports(struct cxl_port *port) 77 { 78 struct pci_bus *bus = cxl_port_to_pci_bus(port); 79 struct cxl_walk_context ctx; 80 int type; 81 82 if (!bus) 83 return -ENXIO; 84 85 if (pci_is_root_bus(bus)) 86 type = PCI_EXP_TYPE_ROOT_PORT; 87 else 88 type = PCI_EXP_TYPE_DOWNSTREAM; 89 90 ctx = (struct cxl_walk_context) { 91 .port = port, 92 .bus = bus, 93 .type = type, 94 }; 95 pci_walk_bus(bus, match_add_dports, &ctx); 96 97 if (ctx.count == 0) 98 return -ENODEV; 99 if (ctx.error) 100 return ctx.error; 101 return ctx.count; 102 } 103 EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL); 104 105 static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id) 106 { 107 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 108 int d = cxlds->cxl_dvsec; 109 bool valid = false; 110 int rc, i; 111 u32 temp; 112 113 if (id > CXL_DVSEC_RANGE_MAX) 114 return -EINVAL; 115 116 /* Check MEM INFO VALID bit first, give up after 1s */ 117 i = 1; 118 do { 119 rc = pci_read_config_dword(pdev, 120 d + CXL_DVSEC_RANGE_SIZE_LOW(id), 121 &temp); 122 if (rc) 123 return rc; 124 125 valid = FIELD_GET(CXL_DVSEC_MEM_INFO_VALID, temp); 126 if (valid) 127 break; 128 msleep(1000); 129 } while (i--); 130 131 if (!valid) { 132 dev_err(&pdev->dev, 133 "Timeout awaiting memory range %d valid after 1s.\n", 134 id); 135 return -ETIMEDOUT; 136 } 137 138 return 0; 139 } 140 141 static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id) 142 { 143 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 144 int d = cxlds->cxl_dvsec; 145 bool active = false; 146 int rc, i; 147 u32 temp; 148 149 if (id > CXL_DVSEC_RANGE_MAX) 150 return -EINVAL; 151 152 /* Check MEM ACTIVE bit, up to 60s timeout by default */ 153 for (i = media_ready_timeout; i; i--) { 154 rc = pci_read_config_dword( 155 pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(id), &temp); 156 if (rc) 157 return rc; 158 159 active = FIELD_GET(CXL_DVSEC_MEM_ACTIVE, temp); 160 if (active) 161 break; 162 msleep(1000); 163 } 164 165 if (!active) { 166 dev_err(&pdev->dev, 167 "timeout awaiting memory active after %d seconds\n", 168 media_ready_timeout); 169 return -ETIMEDOUT; 170 } 171 172 return 0; 173 } 174 175 /* 176 * Wait up to @media_ready_timeout for the device to report memory 177 * active. 178 */ 179 int cxl_await_media_ready(struct cxl_dev_state *cxlds) 180 { 181 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 182 int d = cxlds->cxl_dvsec; 183 int rc, i, hdm_count; 184 u64 md_status; 185 u16 cap; 186 187 rc = pci_read_config_word(pdev, 188 d + CXL_DVSEC_CAP_OFFSET, &cap); 189 if (rc) 190 return rc; 191 192 hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap); 193 for (i = 0; i < hdm_count; i++) { 194 rc = cxl_dvsec_mem_range_valid(cxlds, i); 195 if (rc) 196 return rc; 197 } 198 199 for (i = 0; i < hdm_count; i++) { 200 rc = cxl_dvsec_mem_range_active(cxlds, i); 201 if (rc) 202 return rc; 203 } 204 205 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 206 if (!CXLMDEV_READY(md_status)) 207 return -EIO; 208 209 return 0; 210 } 211 EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, CXL); 212 213 static int wait_for_valid(struct pci_dev *pdev, int d) 214 { 215 u32 val; 216 int rc; 217 218 /* 219 * Memory_Info_Valid: When set, indicates that the CXL Range 1 Size high 220 * and Size Low registers are valid. Must be set within 1 second of 221 * deassertion of reset to CXL device. Likely it is already set by the 222 * time this runs, but otherwise give a 1.5 second timeout in case of 223 * clock skew. 224 */ 225 rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val); 226 if (rc) 227 return rc; 228 229 if (val & CXL_DVSEC_MEM_INFO_VALID) 230 return 0; 231 232 msleep(1500); 233 234 rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val); 235 if (rc) 236 return rc; 237 238 if (val & CXL_DVSEC_MEM_INFO_VALID) 239 return 0; 240 241 return -ETIMEDOUT; 242 } 243 244 static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val) 245 { 246 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 247 int d = cxlds->cxl_dvsec; 248 u16 ctrl; 249 int rc; 250 251 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl); 252 if (rc < 0) 253 return rc; 254 255 if ((ctrl & CXL_DVSEC_MEM_ENABLE) == val) 256 return 1; 257 ctrl &= ~CXL_DVSEC_MEM_ENABLE; 258 ctrl |= val; 259 260 rc = pci_write_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, ctrl); 261 if (rc < 0) 262 return rc; 263 264 return 0; 265 } 266 267 static void clear_mem_enable(void *cxlds) 268 { 269 cxl_set_mem_enable(cxlds, 0); 270 } 271 272 static int devm_cxl_enable_mem(struct device *host, struct cxl_dev_state *cxlds) 273 { 274 int rc; 275 276 rc = cxl_set_mem_enable(cxlds, CXL_DVSEC_MEM_ENABLE); 277 if (rc < 0) 278 return rc; 279 if (rc > 0) 280 return 0; 281 return devm_add_action_or_reset(host, clear_mem_enable, cxlds); 282 } 283 284 /* require dvsec ranges to be covered by a locked platform window */ 285 static int dvsec_range_allowed(struct device *dev, void *arg) 286 { 287 struct range *dev_range = arg; 288 struct cxl_decoder *cxld; 289 290 if (!is_root_decoder(dev)) 291 return 0; 292 293 cxld = to_cxl_decoder(dev); 294 295 if (!(cxld->flags & CXL_DECODER_F_RAM)) 296 return 0; 297 298 return range_contains(&cxld->hpa_range, dev_range); 299 } 300 301 static void disable_hdm(void *_cxlhdm) 302 { 303 u32 global_ctrl; 304 struct cxl_hdm *cxlhdm = _cxlhdm; 305 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 306 307 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 308 writel(global_ctrl & ~CXL_HDM_DECODER_ENABLE, 309 hdm + CXL_HDM_DECODER_CTRL_OFFSET); 310 } 311 312 static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm) 313 { 314 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 315 u32 global_ctrl; 316 317 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 318 writel(global_ctrl | CXL_HDM_DECODER_ENABLE, 319 hdm + CXL_HDM_DECODER_CTRL_OFFSET); 320 321 return devm_add_action_or_reset(host, disable_hdm, cxlhdm); 322 } 323 324 int cxl_dvsec_rr_decode(struct device *dev, int d, 325 struct cxl_endpoint_dvsec_info *info) 326 { 327 struct pci_dev *pdev = to_pci_dev(dev); 328 int hdm_count, rc, i, ranges = 0; 329 u16 cap, ctrl; 330 331 if (!d) { 332 dev_dbg(dev, "No DVSEC Capability\n"); 333 return -ENXIO; 334 } 335 336 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CAP_OFFSET, &cap); 337 if (rc) 338 return rc; 339 340 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl); 341 if (rc) 342 return rc; 343 344 if (!(cap & CXL_DVSEC_MEM_CAPABLE)) { 345 dev_dbg(dev, "Not MEM Capable\n"); 346 return -ENXIO; 347 } 348 349 /* 350 * It is not allowed by spec for MEM.capable to be set and have 0 legacy 351 * HDM decoders (values > 2 are also undefined as of CXL 2.0). As this 352 * driver is for a spec defined class code which must be CXL.mem 353 * capable, there is no point in continuing to enable CXL.mem. 354 */ 355 hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap); 356 if (!hdm_count || hdm_count > 2) 357 return -EINVAL; 358 359 rc = wait_for_valid(pdev, d); 360 if (rc) { 361 dev_dbg(dev, "Failure awaiting MEM_INFO_VALID (%d)\n", rc); 362 return rc; 363 } 364 365 /* 366 * The current DVSEC values are moot if the memory capability is 367 * disabled, and they will remain moot after the HDM Decoder 368 * capability is enabled. 369 */ 370 info->mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl); 371 if (!info->mem_enabled) 372 return 0; 373 374 for (i = 0; i < hdm_count; i++) { 375 u64 base, size; 376 u32 temp; 377 378 rc = pci_read_config_dword( 379 pdev, d + CXL_DVSEC_RANGE_SIZE_HIGH(i), &temp); 380 if (rc) 381 return rc; 382 383 size = (u64)temp << 32; 384 385 rc = pci_read_config_dword( 386 pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(i), &temp); 387 if (rc) 388 return rc; 389 390 size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK; 391 if (!size) { 392 info->dvsec_range[i] = (struct range) { 393 .start = 0, 394 .end = CXL_RESOURCE_NONE, 395 }; 396 continue; 397 } 398 399 rc = pci_read_config_dword( 400 pdev, d + CXL_DVSEC_RANGE_BASE_HIGH(i), &temp); 401 if (rc) 402 return rc; 403 404 base = (u64)temp << 32; 405 406 rc = pci_read_config_dword( 407 pdev, d + CXL_DVSEC_RANGE_BASE_LOW(i), &temp); 408 if (rc) 409 return rc; 410 411 base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK; 412 413 info->dvsec_range[i] = (struct range) { 414 .start = base, 415 .end = base + size - 1 416 }; 417 418 ranges++; 419 } 420 421 info->ranges = ranges; 422 423 return 0; 424 } 425 EXPORT_SYMBOL_NS_GPL(cxl_dvsec_rr_decode, CXL); 426 427 /** 428 * cxl_hdm_decode_init() - Setup HDM decoding for the endpoint 429 * @cxlds: Device state 430 * @cxlhdm: Mapped HDM decoder Capability 431 * @info: Cached DVSEC range registers info 432 * 433 * Try to enable the endpoint's HDM Decoder Capability 434 */ 435 int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm, 436 struct cxl_endpoint_dvsec_info *info) 437 { 438 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 439 struct cxl_port *port = cxlhdm->port; 440 struct device *dev = cxlds->dev; 441 struct cxl_port *root; 442 int i, rc, allowed; 443 u32 global_ctrl = 0; 444 445 if (hdm) 446 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 447 448 /* 449 * If the HDM Decoder Capability is already enabled then assume 450 * that some other agent like platform firmware set it up. 451 */ 452 if (global_ctrl & CXL_HDM_DECODER_ENABLE || (!hdm && info->mem_enabled)) 453 return devm_cxl_enable_mem(&port->dev, cxlds); 454 else if (!hdm) 455 return -ENODEV; 456 457 root = to_cxl_port(port->dev.parent); 458 while (!is_cxl_root(root) && is_cxl_port(root->dev.parent)) 459 root = to_cxl_port(root->dev.parent); 460 if (!is_cxl_root(root)) { 461 dev_err(dev, "Failed to acquire root port for HDM enable\n"); 462 return -ENODEV; 463 } 464 465 for (i = 0, allowed = 0; info->mem_enabled && i < info->ranges; i++) { 466 struct device *cxld_dev; 467 468 cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i], 469 dvsec_range_allowed); 470 if (!cxld_dev) { 471 dev_dbg(dev, "DVSEC Range%d denied by platform\n", i); 472 continue; 473 } 474 dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i); 475 put_device(cxld_dev); 476 allowed++; 477 } 478 479 if (!allowed) { 480 cxl_set_mem_enable(cxlds, 0); 481 info->mem_enabled = 0; 482 } 483 484 /* 485 * Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base 486 * [High,Low] when HDM operation is enabled the range register values 487 * are ignored by the device, but the spec also recommends matching the 488 * DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges 489 * are expected even though Linux does not require or maintain that 490 * match. If at least one DVSEC range is enabled and allowed, skip HDM 491 * Decoder Capability Enable. 492 */ 493 if (info->mem_enabled) 494 return 0; 495 496 rc = devm_cxl_enable_hdm(&port->dev, cxlhdm); 497 if (rc) 498 return rc; 499 500 return devm_cxl_enable_mem(&port->dev, cxlds); 501 } 502 EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL); 503 504 #define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff 505 #define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0 506 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00 507 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA 0 508 #define CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE 0xffff0000 509 #define CXL_DOE_TABLE_ACCESS_LAST_ENTRY 0xffff 510 #define CXL_DOE_PROTOCOL_TABLE_ACCESS 2 511 512 #define CDAT_DOE_REQ(entry_handle) cpu_to_le32 \ 513 (FIELD_PREP(CXL_DOE_TABLE_ACCESS_REQ_CODE, \ 514 CXL_DOE_TABLE_ACCESS_REQ_CODE_READ) | \ 515 FIELD_PREP(CXL_DOE_TABLE_ACCESS_TABLE_TYPE, \ 516 CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) | \ 517 FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle))) 518 519 static int cxl_cdat_get_length(struct device *dev, 520 struct pci_doe_mb *cdat_doe, 521 size_t *length) 522 { 523 __le32 request = CDAT_DOE_REQ(0); 524 __le32 response[2]; 525 int rc; 526 527 rc = pci_doe(cdat_doe, PCI_DVSEC_VENDOR_ID_CXL, 528 CXL_DOE_PROTOCOL_TABLE_ACCESS, 529 &request, sizeof(request), 530 &response, sizeof(response)); 531 if (rc < 0) { 532 dev_err(dev, "DOE failed: %d", rc); 533 return rc; 534 } 535 if (rc < sizeof(response)) 536 return -EIO; 537 538 *length = le32_to_cpu(response[1]); 539 dev_dbg(dev, "CDAT length %zu\n", *length); 540 541 return 0; 542 } 543 544 static int cxl_cdat_read_table(struct device *dev, 545 struct pci_doe_mb *cdat_doe, 546 void *cdat_table, size_t *cdat_length) 547 { 548 size_t length = *cdat_length + sizeof(__le32); 549 __le32 *data = cdat_table; 550 int entry_handle = 0; 551 __le32 saved_dw = 0; 552 553 do { 554 __le32 request = CDAT_DOE_REQ(entry_handle); 555 struct cdat_entry_header *entry; 556 size_t entry_dw; 557 int rc; 558 559 rc = pci_doe(cdat_doe, PCI_DVSEC_VENDOR_ID_CXL, 560 CXL_DOE_PROTOCOL_TABLE_ACCESS, 561 &request, sizeof(request), 562 data, length); 563 if (rc < 0) { 564 dev_err(dev, "DOE failed: %d", rc); 565 return rc; 566 } 567 568 /* 1 DW Table Access Response Header + CDAT entry */ 569 entry = (struct cdat_entry_header *)(data + 1); 570 if ((entry_handle == 0 && 571 rc != sizeof(__le32) + sizeof(struct cdat_header)) || 572 (entry_handle > 0 && 573 (rc < sizeof(__le32) + sizeof(*entry) || 574 rc != sizeof(__le32) + le16_to_cpu(entry->length)))) 575 return -EIO; 576 577 /* Get the CXL table access header entry handle */ 578 entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, 579 le32_to_cpu(data[0])); 580 entry_dw = rc / sizeof(__le32); 581 /* Skip Header */ 582 entry_dw -= 1; 583 /* 584 * Table Access Response Header overwrote the last DW of 585 * previous entry, so restore that DW 586 */ 587 *data = saved_dw; 588 length -= entry_dw * sizeof(__le32); 589 data += entry_dw; 590 saved_dw = *data; 591 } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY); 592 593 /* Length in CDAT header may exceed concatenation of CDAT entries */ 594 *cdat_length -= length - sizeof(__le32); 595 596 return 0; 597 } 598 599 static unsigned char cdat_checksum(void *buf, size_t size) 600 { 601 unsigned char sum, *data = buf; 602 size_t i; 603 604 for (sum = 0, i = 0; i < size; i++) 605 sum += data[i]; 606 return sum; 607 } 608 609 /** 610 * read_cdat_data - Read the CDAT data on this port 611 * @port: Port to read data from 612 * 613 * This call will sleep waiting for responses from the DOE mailbox. 614 */ 615 void read_cdat_data(struct cxl_port *port) 616 { 617 struct device *uport = port->uport_dev; 618 struct device *dev = &port->dev; 619 struct pci_doe_mb *cdat_doe; 620 struct pci_dev *pdev = NULL; 621 struct cxl_memdev *cxlmd; 622 size_t cdat_length; 623 void *cdat_table; 624 int rc; 625 626 if (is_cxl_memdev(uport)) { 627 struct device *host; 628 629 cxlmd = to_cxl_memdev(uport); 630 host = cxlmd->dev.parent; 631 if (dev_is_pci(host)) 632 pdev = to_pci_dev(host); 633 } else if (dev_is_pci(uport)) { 634 pdev = to_pci_dev(uport); 635 } 636 637 if (!pdev) 638 return; 639 640 cdat_doe = pci_find_doe_mailbox(pdev, PCI_DVSEC_VENDOR_ID_CXL, 641 CXL_DOE_PROTOCOL_TABLE_ACCESS); 642 if (!cdat_doe) { 643 dev_dbg(dev, "No CDAT mailbox\n"); 644 return; 645 } 646 647 port->cdat_available = true; 648 649 if (cxl_cdat_get_length(dev, cdat_doe, &cdat_length)) { 650 dev_dbg(dev, "No CDAT length\n"); 651 return; 652 } 653 654 cdat_table = devm_kzalloc(dev, cdat_length + sizeof(__le32), 655 GFP_KERNEL); 656 if (!cdat_table) 657 return; 658 659 rc = cxl_cdat_read_table(dev, cdat_doe, cdat_table, &cdat_length); 660 if (rc) 661 goto err; 662 663 cdat_table = cdat_table + sizeof(__le32); 664 if (cdat_checksum(cdat_table, cdat_length)) 665 goto err; 666 667 port->cdat.table = cdat_table; 668 port->cdat.length = cdat_length; 669 return; 670 671 err: 672 /* Don't leave table data allocated on error */ 673 devm_kfree(dev, cdat_table); 674 dev_err(dev, "Failed to read/validate CDAT.\n"); 675 } 676 EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL); 677 678 static void __cxl_handle_cor_ras(struct cxl_dev_state *cxlds, 679 void __iomem *ras_base) 680 { 681 void __iomem *addr; 682 u32 status; 683 684 if (!ras_base) 685 return; 686 687 addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET; 688 status = readl(addr); 689 if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) { 690 writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr); 691 trace_cxl_aer_correctable_error(cxlds->cxlmd, status); 692 } 693 } 694 695 static void cxl_handle_endpoint_cor_ras(struct cxl_dev_state *cxlds) 696 { 697 return __cxl_handle_cor_ras(cxlds, cxlds->regs.ras); 698 } 699 700 /* CXL spec rev3.0 8.2.4.16.1 */ 701 static void header_log_copy(void __iomem *ras_base, u32 *log) 702 { 703 void __iomem *addr; 704 u32 *log_addr; 705 int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32); 706 707 addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET; 708 log_addr = log; 709 710 for (i = 0; i < log_u32_size; i++) { 711 *log_addr = readl(addr); 712 log_addr++; 713 addr += sizeof(u32); 714 } 715 } 716 717 /* 718 * Log the state of the RAS status registers and prepare them to log the 719 * next error status. Return 1 if reset needed. 720 */ 721 static bool __cxl_handle_ras(struct cxl_dev_state *cxlds, 722 void __iomem *ras_base) 723 { 724 u32 hl[CXL_HEADERLOG_SIZE_U32]; 725 void __iomem *addr; 726 u32 status; 727 u32 fe; 728 729 if (!ras_base) 730 return false; 731 732 addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; 733 status = readl(addr); 734 if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK)) 735 return false; 736 737 /* If multiple errors, log header points to first error from ctrl reg */ 738 if (hweight32(status) > 1) { 739 void __iomem *rcc_addr = 740 ras_base + CXL_RAS_CAP_CONTROL_OFFSET; 741 742 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, 743 readl(rcc_addr))); 744 } else { 745 fe = status; 746 } 747 748 header_log_copy(ras_base, hl); 749 trace_cxl_aer_uncorrectable_error(cxlds->cxlmd, status, fe, hl); 750 writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr); 751 752 return true; 753 } 754 755 static bool cxl_handle_endpoint_ras(struct cxl_dev_state *cxlds) 756 { 757 return __cxl_handle_ras(cxlds, cxlds->regs.ras); 758 } 759 760 #ifdef CONFIG_PCIEAER_CXL 761 762 static void cxl_dport_map_rch_aer(struct cxl_dport *dport) 763 { 764 struct cxl_rcrb_info *ri = &dport->rcrb; 765 void __iomem *dport_aer = NULL; 766 resource_size_t aer_phys; 767 struct device *host; 768 769 if (dport->rch && ri->aer_cap) { 770 host = dport->reg_map.host; 771 aer_phys = ri->aer_cap + ri->base; 772 dport_aer = devm_cxl_iomap_block(host, aer_phys, 773 sizeof(struct aer_capability_regs)); 774 } 775 776 dport->regs.dport_aer = dport_aer; 777 } 778 779 static void cxl_dport_map_regs(struct cxl_dport *dport) 780 { 781 struct cxl_register_map *map = &dport->reg_map; 782 struct device *dev = dport->dport_dev; 783 784 if (!map->component_map.ras.valid) 785 dev_dbg(dev, "RAS registers not found\n"); 786 else if (cxl_map_component_regs(map, &dport->regs.component, 787 BIT(CXL_CM_CAP_CAP_ID_RAS))) 788 dev_dbg(dev, "Failed to map RAS capability.\n"); 789 790 if (dport->rch) 791 cxl_dport_map_rch_aer(dport); 792 } 793 794 static void cxl_disable_rch_root_ints(struct cxl_dport *dport) 795 { 796 void __iomem *aer_base = dport->regs.dport_aer; 797 struct pci_host_bridge *bridge; 798 u32 aer_cmd_mask, aer_cmd; 799 800 if (!aer_base) 801 return; 802 803 bridge = to_pci_host_bridge(dport->dport_dev); 804 805 /* 806 * Disable RCH root port command interrupts. 807 * CXL 3.0 12.2.1.1 - RCH Downstream Port-detected Errors 808 * 809 * This sequence may not be necessary. CXL spec states disabling 810 * the root cmd register's interrupts is required. But, PCI spec 811 * shows these are disabled by default on reset. 812 */ 813 if (bridge->native_aer) { 814 aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN | 815 PCI_ERR_ROOT_CMD_NONFATAL_EN | 816 PCI_ERR_ROOT_CMD_FATAL_EN); 817 aer_cmd = readl(aer_base + PCI_ERR_ROOT_COMMAND); 818 aer_cmd &= ~aer_cmd_mask; 819 writel(aer_cmd, aer_base + PCI_ERR_ROOT_COMMAND); 820 } 821 } 822 823 void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport) 824 { 825 struct device *dport_dev = dport->dport_dev; 826 struct pci_host_bridge *host_bridge; 827 828 host_bridge = to_pci_host_bridge(dport_dev); 829 if (host_bridge->native_aer) 830 dport->rcrb.aer_cap = cxl_rcrb_to_aer(dport_dev, dport->rcrb.base); 831 832 dport->reg_map.host = host; 833 cxl_dport_map_regs(dport); 834 835 if (dport->rch) 836 cxl_disable_rch_root_ints(dport); 837 } 838 EXPORT_SYMBOL_NS_GPL(cxl_setup_parent_dport, CXL); 839 840 static void cxl_handle_rdport_cor_ras(struct cxl_dev_state *cxlds, 841 struct cxl_dport *dport) 842 { 843 return __cxl_handle_cor_ras(cxlds, dport->regs.ras); 844 } 845 846 static bool cxl_handle_rdport_ras(struct cxl_dev_state *cxlds, 847 struct cxl_dport *dport) 848 { 849 return __cxl_handle_ras(cxlds, dport->regs.ras); 850 } 851 852 /* 853 * Copy the AER capability registers using 32 bit read accesses. 854 * This is necessary because RCRB AER capability is MMIO mapped. Clear the 855 * status after copying. 856 * 857 * @aer_base: base address of AER capability block in RCRB 858 * @aer_regs: destination for copying AER capability 859 */ 860 static bool cxl_rch_get_aer_info(void __iomem *aer_base, 861 struct aer_capability_regs *aer_regs) 862 { 863 int read_cnt = sizeof(struct aer_capability_regs) / sizeof(u32); 864 u32 *aer_regs_buf = (u32 *)aer_regs; 865 int n; 866 867 if (!aer_base) 868 return false; 869 870 /* Use readl() to guarantee 32-bit accesses */ 871 for (n = 0; n < read_cnt; n++) 872 aer_regs_buf[n] = readl(aer_base + n * sizeof(u32)); 873 874 writel(aer_regs->uncor_status, aer_base + PCI_ERR_UNCOR_STATUS); 875 writel(aer_regs->cor_status, aer_base + PCI_ERR_COR_STATUS); 876 877 return true; 878 } 879 880 /* Get AER severity. Return false if there is no error. */ 881 static bool cxl_rch_get_aer_severity(struct aer_capability_regs *aer_regs, 882 int *severity) 883 { 884 if (aer_regs->uncor_status & ~aer_regs->uncor_mask) { 885 if (aer_regs->uncor_status & PCI_ERR_ROOT_FATAL_RCV) 886 *severity = AER_FATAL; 887 else 888 *severity = AER_NONFATAL; 889 return true; 890 } 891 892 if (aer_regs->cor_status & ~aer_regs->cor_mask) { 893 *severity = AER_CORRECTABLE; 894 return true; 895 } 896 897 return false; 898 } 899 900 static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) 901 { 902 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 903 struct aer_capability_regs aer_regs; 904 struct cxl_dport *dport; 905 struct cxl_port *port; 906 int severity; 907 908 port = cxl_pci_find_port(pdev, &dport); 909 if (!port) 910 return; 911 912 put_device(&port->dev); 913 914 if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs)) 915 return; 916 917 if (!cxl_rch_get_aer_severity(&aer_regs, &severity)) 918 return; 919 920 pci_print_aer(pdev, severity, &aer_regs); 921 922 if (severity == AER_CORRECTABLE) 923 cxl_handle_rdport_cor_ras(cxlds, dport); 924 else 925 cxl_handle_rdport_ras(cxlds, dport); 926 } 927 928 #else 929 static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) { } 930 #endif 931 932 void cxl_cor_error_detected(struct pci_dev *pdev) 933 { 934 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 935 936 if (cxlds->rcd) 937 cxl_handle_rdport_errors(cxlds); 938 939 cxl_handle_endpoint_cor_ras(cxlds); 940 } 941 EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, CXL); 942 943 pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, 944 pci_channel_state_t state) 945 { 946 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 947 struct cxl_memdev *cxlmd = cxlds->cxlmd; 948 struct device *dev = &cxlmd->dev; 949 bool ue; 950 951 if (cxlds->rcd) 952 cxl_handle_rdport_errors(cxlds); 953 954 /* 955 * A frozen channel indicates an impending reset which is fatal to 956 * CXL.mem operation, and will likely crash the system. On the off 957 * chance the situation is recoverable dump the status of the RAS 958 * capability registers and bounce the active state of the memdev. 959 */ 960 ue = cxl_handle_endpoint_ras(cxlds); 961 962 switch (state) { 963 case pci_channel_io_normal: 964 if (ue) { 965 device_release_driver(dev); 966 return PCI_ERS_RESULT_NEED_RESET; 967 } 968 return PCI_ERS_RESULT_CAN_RECOVER; 969 case pci_channel_io_frozen: 970 dev_warn(&pdev->dev, 971 "%s: frozen state error detected, disable CXL.mem\n", 972 dev_name(dev)); 973 device_release_driver(dev); 974 return PCI_ERS_RESULT_NEED_RESET; 975 case pci_channel_io_perm_failure: 976 dev_warn(&pdev->dev, 977 "failure state error detected, request disconnect\n"); 978 return PCI_ERS_RESULT_DISCONNECT; 979 } 980 return PCI_ERS_RESULT_NEED_RESET; 981 } 982 EXPORT_SYMBOL_NS_GPL(cxl_error_detected, CXL); 983