1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2021 Intel Corporation. All rights reserved. */ 3 #include <linux/io-64-nonatomic-lo-hi.h> 4 #include <linux/device.h> 5 #include <linux/delay.h> 6 #include <linux/pci.h> 7 #include <linux/pci-doe.h> 8 #include <cxlpci.h> 9 #include <cxlmem.h> 10 #include <cxl.h> 11 #include "core.h" 12 #include "trace.h" 13 14 /** 15 * DOC: cxl core pci 16 * 17 * Compute Express Link protocols are layered on top of PCIe. CXL core provides 18 * a set of helpers for CXL interactions which occur via PCIe. 19 */ 20 21 static unsigned short media_ready_timeout = 60; 22 module_param(media_ready_timeout, ushort, 0644); 23 MODULE_PARM_DESC(media_ready_timeout, "seconds to wait for media ready"); 24 25 struct cxl_walk_context { 26 struct pci_bus *bus; 27 struct cxl_port *port; 28 int type; 29 int error; 30 int count; 31 }; 32 33 static int match_add_dports(struct pci_dev *pdev, void *data) 34 { 35 struct cxl_walk_context *ctx = data; 36 struct cxl_port *port = ctx->port; 37 int type = pci_pcie_type(pdev); 38 struct cxl_register_map map; 39 struct cxl_dport *dport; 40 u32 lnkcap, port_num; 41 int rc; 42 43 if (pdev->bus != ctx->bus) 44 return 0; 45 if (!pci_is_pcie(pdev)) 46 return 0; 47 if (type != ctx->type) 48 return 0; 49 if (pci_read_config_dword(pdev, pci_pcie_cap(pdev) + PCI_EXP_LNKCAP, 50 &lnkcap)) 51 return 0; 52 53 rc = cxl_find_regblock(pdev, CXL_REGLOC_RBI_COMPONENT, &map); 54 if (rc) 55 dev_dbg(&port->dev, "failed to find component registers\n"); 56 57 port_num = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap); 58 dport = devm_cxl_add_dport(port, &pdev->dev, port_num, map.resource); 59 if (IS_ERR(dport)) { 60 ctx->error = PTR_ERR(dport); 61 return PTR_ERR(dport); 62 } 63 ctx->count++; 64 65 return 0; 66 } 67 68 /** 69 * devm_cxl_port_enumerate_dports - enumerate downstream ports of the upstream port 70 * @port: cxl_port whose ->uport is the upstream of dports to be enumerated 71 * 72 * Returns a positive number of dports enumerated or a negative error 73 * code. 74 */ 75 int devm_cxl_port_enumerate_dports(struct cxl_port *port) 76 { 77 struct pci_bus *bus = cxl_port_to_pci_bus(port); 78 struct cxl_walk_context ctx; 79 int type; 80 81 if (!bus) 82 return -ENXIO; 83 84 if (pci_is_root_bus(bus)) 85 type = PCI_EXP_TYPE_ROOT_PORT; 86 else 87 type = PCI_EXP_TYPE_DOWNSTREAM; 88 89 ctx = (struct cxl_walk_context) { 90 .port = port, 91 .bus = bus, 92 .type = type, 93 }; 94 pci_walk_bus(bus, match_add_dports, &ctx); 95 96 if (ctx.count == 0) 97 return -ENODEV; 98 if (ctx.error) 99 return ctx.error; 100 return ctx.count; 101 } 102 EXPORT_SYMBOL_NS_GPL(devm_cxl_port_enumerate_dports, CXL); 103 104 /* 105 * Wait up to @media_ready_timeout for the device to report memory 106 * active. 107 */ 108 int cxl_await_media_ready(struct cxl_dev_state *cxlds) 109 { 110 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 111 int d = cxlds->cxl_dvsec; 112 bool active = false; 113 u64 md_status; 114 int rc, i; 115 116 for (i = media_ready_timeout; i; i--) { 117 u32 temp; 118 119 rc = pci_read_config_dword( 120 pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &temp); 121 if (rc) 122 return rc; 123 124 active = FIELD_GET(CXL_DVSEC_MEM_ACTIVE, temp); 125 if (active) 126 break; 127 msleep(1000); 128 } 129 130 if (!active) { 131 dev_err(&pdev->dev, 132 "timeout awaiting memory active after %d seconds\n", 133 media_ready_timeout); 134 return -ETIMEDOUT; 135 } 136 137 md_status = readq(cxlds->regs.memdev + CXLMDEV_STATUS_OFFSET); 138 if (!CXLMDEV_READY(md_status)) 139 return -EIO; 140 141 return 0; 142 } 143 EXPORT_SYMBOL_NS_GPL(cxl_await_media_ready, CXL); 144 145 static int wait_for_valid(struct cxl_dev_state *cxlds) 146 { 147 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 148 int d = cxlds->cxl_dvsec, rc; 149 u32 val; 150 151 /* 152 * Memory_Info_Valid: When set, indicates that the CXL Range 1 Size high 153 * and Size Low registers are valid. Must be set within 1 second of 154 * deassertion of reset to CXL device. Likely it is already set by the 155 * time this runs, but otherwise give a 1.5 second timeout in case of 156 * clock skew. 157 */ 158 rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val); 159 if (rc) 160 return rc; 161 162 if (val & CXL_DVSEC_MEM_INFO_VALID) 163 return 0; 164 165 msleep(1500); 166 167 rc = pci_read_config_dword(pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(0), &val); 168 if (rc) 169 return rc; 170 171 if (val & CXL_DVSEC_MEM_INFO_VALID) 172 return 0; 173 174 return -ETIMEDOUT; 175 } 176 177 static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val) 178 { 179 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 180 int d = cxlds->cxl_dvsec; 181 u16 ctrl; 182 int rc; 183 184 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl); 185 if (rc < 0) 186 return rc; 187 188 if ((ctrl & CXL_DVSEC_MEM_ENABLE) == val) 189 return 1; 190 ctrl &= ~CXL_DVSEC_MEM_ENABLE; 191 ctrl |= val; 192 193 rc = pci_write_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, ctrl); 194 if (rc < 0) 195 return rc; 196 197 return 0; 198 } 199 200 static void clear_mem_enable(void *cxlds) 201 { 202 cxl_set_mem_enable(cxlds, 0); 203 } 204 205 static int devm_cxl_enable_mem(struct device *host, struct cxl_dev_state *cxlds) 206 { 207 int rc; 208 209 rc = cxl_set_mem_enable(cxlds, CXL_DVSEC_MEM_ENABLE); 210 if (rc < 0) 211 return rc; 212 if (rc > 0) 213 return 0; 214 return devm_add_action_or_reset(host, clear_mem_enable, cxlds); 215 } 216 217 /* require dvsec ranges to be covered by a locked platform window */ 218 static int dvsec_range_allowed(struct device *dev, void *arg) 219 { 220 struct range *dev_range = arg; 221 struct cxl_decoder *cxld; 222 223 if (!is_root_decoder(dev)) 224 return 0; 225 226 cxld = to_cxl_decoder(dev); 227 228 if (!(cxld->flags & CXL_DECODER_F_LOCK)) 229 return 0; 230 if (!(cxld->flags & CXL_DECODER_F_RAM)) 231 return 0; 232 233 return range_contains(&cxld->hpa_range, dev_range); 234 } 235 236 static void disable_hdm(void *_cxlhdm) 237 { 238 u32 global_ctrl; 239 struct cxl_hdm *cxlhdm = _cxlhdm; 240 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 241 242 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 243 writel(global_ctrl & ~CXL_HDM_DECODER_ENABLE, 244 hdm + CXL_HDM_DECODER_CTRL_OFFSET); 245 } 246 247 static int devm_cxl_enable_hdm(struct device *host, struct cxl_hdm *cxlhdm) 248 { 249 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 250 u32 global_ctrl; 251 252 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 253 writel(global_ctrl | CXL_HDM_DECODER_ENABLE, 254 hdm + CXL_HDM_DECODER_CTRL_OFFSET); 255 256 return devm_add_action_or_reset(host, disable_hdm, cxlhdm); 257 } 258 259 static bool __cxl_hdm_decode_init(struct cxl_dev_state *cxlds, 260 struct cxl_hdm *cxlhdm, 261 struct cxl_endpoint_dvsec_info *info) 262 { 263 void __iomem *hdm = cxlhdm->regs.hdm_decoder; 264 struct cxl_port *port = cxlhdm->port; 265 struct device *dev = cxlds->dev; 266 struct cxl_port *root; 267 int i, rc, allowed; 268 u32 global_ctrl; 269 270 global_ctrl = readl(hdm + CXL_HDM_DECODER_CTRL_OFFSET); 271 272 /* 273 * If the HDM Decoder Capability is already enabled then assume 274 * that some other agent like platform firmware set it up. 275 */ 276 if (global_ctrl & CXL_HDM_DECODER_ENABLE) { 277 rc = devm_cxl_enable_mem(&port->dev, cxlds); 278 if (rc) 279 return false; 280 return true; 281 } 282 283 root = to_cxl_port(port->dev.parent); 284 while (!is_cxl_root(root) && is_cxl_port(root->dev.parent)) 285 root = to_cxl_port(root->dev.parent); 286 if (!is_cxl_root(root)) { 287 dev_err(dev, "Failed to acquire root port for HDM enable\n"); 288 return false; 289 } 290 291 for (i = 0, allowed = 0; info->mem_enabled && i < info->ranges; i++) { 292 struct device *cxld_dev; 293 294 cxld_dev = device_find_child(&root->dev, &info->dvsec_range[i], 295 dvsec_range_allowed); 296 if (!cxld_dev) { 297 dev_dbg(dev, "DVSEC Range%d denied by platform\n", i); 298 continue; 299 } 300 dev_dbg(dev, "DVSEC Range%d allowed by platform\n", i); 301 put_device(cxld_dev); 302 allowed++; 303 } 304 305 if (!allowed) { 306 cxl_set_mem_enable(cxlds, 0); 307 info->mem_enabled = 0; 308 } 309 310 /* 311 * Per CXL 2.0 Section 8.1.3.8.3 and 8.1.3.8.4 DVSEC CXL Range 1 Base 312 * [High,Low] when HDM operation is enabled the range register values 313 * are ignored by the device, but the spec also recommends matching the 314 * DVSEC Range 1,2 to HDM Decoder Range 0,1. So, non-zero info->ranges 315 * are expected even though Linux does not require or maintain that 316 * match. If at least one DVSEC range is enabled and allowed, skip HDM 317 * Decoder Capability Enable. 318 */ 319 if (info->mem_enabled) 320 return false; 321 322 rc = devm_cxl_enable_hdm(&port->dev, cxlhdm); 323 if (rc) 324 return false; 325 326 rc = devm_cxl_enable_mem(&port->dev, cxlds); 327 if (rc) 328 return false; 329 330 return true; 331 } 332 333 /** 334 * cxl_hdm_decode_init() - Setup HDM decoding for the endpoint 335 * @cxlds: Device state 336 * @cxlhdm: Mapped HDM decoder Capability 337 * 338 * Try to enable the endpoint's HDM Decoder Capability 339 */ 340 int cxl_hdm_decode_init(struct cxl_dev_state *cxlds, struct cxl_hdm *cxlhdm) 341 { 342 struct pci_dev *pdev = to_pci_dev(cxlds->dev); 343 struct cxl_endpoint_dvsec_info info = { 0 }; 344 int hdm_count, rc, i, ranges = 0; 345 struct device *dev = &pdev->dev; 346 int d = cxlds->cxl_dvsec; 347 u16 cap, ctrl; 348 349 if (!d) { 350 dev_dbg(dev, "No DVSEC Capability\n"); 351 return -ENXIO; 352 } 353 354 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CAP_OFFSET, &cap); 355 if (rc) 356 return rc; 357 358 rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl); 359 if (rc) 360 return rc; 361 362 if (!(cap & CXL_DVSEC_MEM_CAPABLE)) { 363 dev_dbg(dev, "Not MEM Capable\n"); 364 return -ENXIO; 365 } 366 367 /* 368 * It is not allowed by spec for MEM.capable to be set and have 0 legacy 369 * HDM decoders (values > 2 are also undefined as of CXL 2.0). As this 370 * driver is for a spec defined class code which must be CXL.mem 371 * capable, there is no point in continuing to enable CXL.mem. 372 */ 373 hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap); 374 if (!hdm_count || hdm_count > 2) 375 return -EINVAL; 376 377 rc = wait_for_valid(cxlds); 378 if (rc) { 379 dev_dbg(dev, "Failure awaiting MEM_INFO_VALID (%d)\n", rc); 380 return rc; 381 } 382 383 /* 384 * The current DVSEC values are moot if the memory capability is 385 * disabled, and they will remain moot after the HDM Decoder 386 * capability is enabled. 387 */ 388 info.mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl); 389 if (!info.mem_enabled) 390 goto hdm_init; 391 392 for (i = 0; i < hdm_count; i++) { 393 u64 base, size; 394 u32 temp; 395 396 rc = pci_read_config_dword( 397 pdev, d + CXL_DVSEC_RANGE_SIZE_HIGH(i), &temp); 398 if (rc) 399 return rc; 400 401 size = (u64)temp << 32; 402 403 rc = pci_read_config_dword( 404 pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(i), &temp); 405 if (rc) 406 return rc; 407 408 size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK; 409 410 rc = pci_read_config_dword( 411 pdev, d + CXL_DVSEC_RANGE_BASE_HIGH(i), &temp); 412 if (rc) 413 return rc; 414 415 base = (u64)temp << 32; 416 417 rc = pci_read_config_dword( 418 pdev, d + CXL_DVSEC_RANGE_BASE_LOW(i), &temp); 419 if (rc) 420 return rc; 421 422 base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK; 423 424 info.dvsec_range[i] = (struct range) { 425 .start = base, 426 .end = base + size - 1 427 }; 428 429 if (size) 430 ranges++; 431 } 432 433 info.ranges = ranges; 434 435 /* 436 * If DVSEC ranges are being used instead of HDM decoder registers there 437 * is no use in trying to manage those. 438 */ 439 hdm_init: 440 if (!__cxl_hdm_decode_init(cxlds, cxlhdm, &info)) { 441 dev_err(dev, 442 "Legacy range registers configuration prevents HDM operation.\n"); 443 return -EBUSY; 444 } 445 446 return 0; 447 } 448 EXPORT_SYMBOL_NS_GPL(cxl_hdm_decode_init, CXL); 449 450 #define CXL_DOE_TABLE_ACCESS_REQ_CODE 0x000000ff 451 #define CXL_DOE_TABLE_ACCESS_REQ_CODE_READ 0 452 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE 0x0000ff00 453 #define CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA 0 454 #define CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE 0xffff0000 455 #define CXL_DOE_TABLE_ACCESS_LAST_ENTRY 0xffff 456 #define CXL_DOE_PROTOCOL_TABLE_ACCESS 2 457 458 static struct pci_doe_mb *find_cdat_doe(struct device *uport) 459 { 460 struct cxl_memdev *cxlmd; 461 struct cxl_dev_state *cxlds; 462 unsigned long index; 463 void *entry; 464 465 cxlmd = to_cxl_memdev(uport); 466 cxlds = cxlmd->cxlds; 467 468 xa_for_each(&cxlds->doe_mbs, index, entry) { 469 struct pci_doe_mb *cur = entry; 470 471 if (pci_doe_supports_prot(cur, PCI_DVSEC_VENDOR_ID_CXL, 472 CXL_DOE_PROTOCOL_TABLE_ACCESS)) 473 return cur; 474 } 475 476 return NULL; 477 } 478 479 #define CDAT_DOE_REQ(entry_handle) \ 480 (FIELD_PREP(CXL_DOE_TABLE_ACCESS_REQ_CODE, \ 481 CXL_DOE_TABLE_ACCESS_REQ_CODE_READ) | \ 482 FIELD_PREP(CXL_DOE_TABLE_ACCESS_TABLE_TYPE, \ 483 CXL_DOE_TABLE_ACCESS_TABLE_TYPE_CDATA) | \ 484 FIELD_PREP(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, (entry_handle))) 485 486 static void cxl_doe_task_complete(struct pci_doe_task *task) 487 { 488 complete(task->private); 489 } 490 491 struct cdat_doe_task { 492 u32 request_pl; 493 u32 response_pl[32]; 494 struct completion c; 495 struct pci_doe_task task; 496 }; 497 498 #define DECLARE_CDAT_DOE_TASK(req, cdt) \ 499 struct cdat_doe_task cdt = { \ 500 .c = COMPLETION_INITIALIZER_ONSTACK(cdt.c), \ 501 .request_pl = req, \ 502 .task = { \ 503 .prot.vid = PCI_DVSEC_VENDOR_ID_CXL, \ 504 .prot.type = CXL_DOE_PROTOCOL_TABLE_ACCESS, \ 505 .request_pl = &cdt.request_pl, \ 506 .request_pl_sz = sizeof(cdt.request_pl), \ 507 .response_pl = cdt.response_pl, \ 508 .response_pl_sz = sizeof(cdt.response_pl), \ 509 .complete = cxl_doe_task_complete, \ 510 .private = &cdt.c, \ 511 } \ 512 } 513 514 static int cxl_cdat_get_length(struct device *dev, 515 struct pci_doe_mb *cdat_doe, 516 size_t *length) 517 { 518 DECLARE_CDAT_DOE_TASK(CDAT_DOE_REQ(0), t); 519 int rc; 520 521 rc = pci_doe_submit_task(cdat_doe, &t.task); 522 if (rc < 0) { 523 dev_err(dev, "DOE submit failed: %d", rc); 524 return rc; 525 } 526 wait_for_completion(&t.c); 527 if (t.task.rv < sizeof(u32)) 528 return -EIO; 529 530 *length = t.response_pl[1]; 531 dev_dbg(dev, "CDAT length %zu\n", *length); 532 533 return 0; 534 } 535 536 static int cxl_cdat_read_table(struct device *dev, 537 struct pci_doe_mb *cdat_doe, 538 struct cxl_cdat *cdat) 539 { 540 size_t length = cdat->length; 541 u32 *data = cdat->table; 542 int entry_handle = 0; 543 544 do { 545 DECLARE_CDAT_DOE_TASK(CDAT_DOE_REQ(entry_handle), t); 546 size_t entry_dw; 547 u32 *entry; 548 int rc; 549 550 rc = pci_doe_submit_task(cdat_doe, &t.task); 551 if (rc < 0) { 552 dev_err(dev, "DOE submit failed: %d", rc); 553 return rc; 554 } 555 wait_for_completion(&t.c); 556 /* 1 DW header + 1 DW data min */ 557 if (t.task.rv < (2 * sizeof(u32))) 558 return -EIO; 559 560 /* Get the CXL table access header entry handle */ 561 entry_handle = FIELD_GET(CXL_DOE_TABLE_ACCESS_ENTRY_HANDLE, 562 t.response_pl[0]); 563 entry = t.response_pl + 1; 564 entry_dw = t.task.rv / sizeof(u32); 565 /* Skip Header */ 566 entry_dw -= 1; 567 entry_dw = min(length / sizeof(u32), entry_dw); 568 /* Prevent length < 1 DW from causing a buffer overflow */ 569 if (entry_dw) { 570 memcpy(data, entry, entry_dw * sizeof(u32)); 571 length -= entry_dw * sizeof(u32); 572 data += entry_dw; 573 } 574 } while (entry_handle != CXL_DOE_TABLE_ACCESS_LAST_ENTRY); 575 576 return 0; 577 } 578 579 /** 580 * read_cdat_data - Read the CDAT data on this port 581 * @port: Port to read data from 582 * 583 * This call will sleep waiting for responses from the DOE mailbox. 584 */ 585 void read_cdat_data(struct cxl_port *port) 586 { 587 struct pci_doe_mb *cdat_doe; 588 struct device *dev = &port->dev; 589 struct device *uport = port->uport; 590 size_t cdat_length; 591 int rc; 592 593 cdat_doe = find_cdat_doe(uport); 594 if (!cdat_doe) { 595 dev_dbg(dev, "No CDAT mailbox\n"); 596 return; 597 } 598 599 port->cdat_available = true; 600 601 if (cxl_cdat_get_length(dev, cdat_doe, &cdat_length)) { 602 dev_dbg(dev, "No CDAT length\n"); 603 return; 604 } 605 606 port->cdat.table = devm_kzalloc(dev, cdat_length, GFP_KERNEL); 607 if (!port->cdat.table) 608 return; 609 610 port->cdat.length = cdat_length; 611 rc = cxl_cdat_read_table(dev, cdat_doe, &port->cdat); 612 if (rc) { 613 /* Don't leave table data allocated on error */ 614 devm_kfree(dev, port->cdat.table); 615 port->cdat.table = NULL; 616 port->cdat.length = 0; 617 dev_err(dev, "CDAT data read error\n"); 618 } 619 } 620 EXPORT_SYMBOL_NS_GPL(read_cdat_data, CXL); 621 622 void cxl_cor_error_detected(struct pci_dev *pdev) 623 { 624 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 625 struct cxl_memdev *cxlmd = cxlds->cxlmd; 626 struct device *dev = &cxlmd->dev; 627 void __iomem *addr; 628 u32 status; 629 630 if (!cxlds->regs.ras) 631 return; 632 633 addr = cxlds->regs.ras + CXL_RAS_CORRECTABLE_STATUS_OFFSET; 634 status = readl(addr); 635 if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) { 636 writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr); 637 trace_cxl_aer_correctable_error(dev, status); 638 } 639 } 640 EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, CXL); 641 642 /* CXL spec rev3.0 8.2.4.16.1 */ 643 static void header_log_copy(struct cxl_dev_state *cxlds, u32 *log) 644 { 645 void __iomem *addr; 646 u32 *log_addr; 647 int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32); 648 649 addr = cxlds->regs.ras + CXL_RAS_HEADER_LOG_OFFSET; 650 log_addr = log; 651 652 for (i = 0; i < log_u32_size; i++) { 653 *log_addr = readl(addr); 654 log_addr++; 655 addr += sizeof(u32); 656 } 657 } 658 659 /* 660 * Log the state of the RAS status registers and prepare them to log the 661 * next error status. Return 1 if reset needed. 662 */ 663 static bool cxl_report_and_clear(struct cxl_dev_state *cxlds) 664 { 665 struct cxl_memdev *cxlmd = cxlds->cxlmd; 666 struct device *dev = &cxlmd->dev; 667 u32 hl[CXL_HEADERLOG_SIZE_U32]; 668 void __iomem *addr; 669 u32 status; 670 u32 fe; 671 672 if (!cxlds->regs.ras) 673 return false; 674 675 addr = cxlds->regs.ras + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; 676 status = readl(addr); 677 if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK)) 678 return false; 679 680 /* If multiple errors, log header points to first error from ctrl reg */ 681 if (hweight32(status) > 1) { 682 addr = cxlds->regs.ras + CXL_RAS_CAP_CONTROL_OFFSET; 683 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, readl(addr))); 684 } else { 685 fe = status; 686 } 687 688 header_log_copy(cxlds, hl); 689 trace_cxl_aer_uncorrectable_error(dev, status, fe, hl); 690 writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr); 691 692 return true; 693 } 694 695 pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, 696 pci_channel_state_t state) 697 { 698 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 699 struct cxl_memdev *cxlmd = cxlds->cxlmd; 700 struct device *dev = &cxlmd->dev; 701 bool ue; 702 703 /* 704 * A frozen channel indicates an impending reset which is fatal to 705 * CXL.mem operation, and will likely crash the system. On the off 706 * chance the situation is recoverable dump the status of the RAS 707 * capability registers and bounce the active state of the memdev. 708 */ 709 ue = cxl_report_and_clear(cxlds); 710 711 switch (state) { 712 case pci_channel_io_normal: 713 if (ue) { 714 device_release_driver(dev); 715 return PCI_ERS_RESULT_NEED_RESET; 716 } 717 return PCI_ERS_RESULT_CAN_RECOVER; 718 case pci_channel_io_frozen: 719 dev_warn(&pdev->dev, 720 "%s: frozen state error detected, disable CXL.mem\n", 721 dev_name(dev)); 722 device_release_driver(dev); 723 return PCI_ERS_RESULT_NEED_RESET; 724 case pci_channel_io_perm_failure: 725 dev_warn(&pdev->dev, 726 "failure state error detected, request disconnect\n"); 727 return PCI_ERS_RESULT_DISCONNECT; 728 } 729 return PCI_ERS_RESULT_NEED_RESET; 730 } 731 EXPORT_SYMBOL_NS_GPL(cxl_error_detected, CXL); 732