1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implement the AER root port service driver. The driver registers an IRQ 4 * handler. When a root port triggers an AER interrupt, the IRQ handler 5 * collects Root Port status and schedules work. 6 * 7 * Copyright (C) 2006 Intel Corp. 8 * Tom Long Nguyen (tom.l.nguyen@intel.com) 9 * Zhang Yanmin (yanmin.zhang@intel.com) 10 * 11 * (C) Copyright 2009 Hewlett-Packard Development Company, L.P. 12 * Andrew Patterson <andrew.patterson@hp.com> 13 */ 14 15 #define pr_fmt(fmt) "AER: " fmt 16 #define dev_fmt pr_fmt 17 18 #include <linux/bitops.h> 19 #include <linux/cper.h> 20 #include <linux/dev_printk.h> 21 #include <linux/pci.h> 22 #include <linux/pci-acpi.h> 23 #include <linux/sched.h> 24 #include <linux/kernel.h> 25 #include <linux/errno.h> 26 #include <linux/pm.h> 27 #include <linux/init.h> 28 #include <linux/interrupt.h> 29 #include <linux/delay.h> 30 #include <linux/kfifo.h> 31 #include <linux/ratelimit.h> 32 #include <linux/slab.h> 33 #include <linux/vmcore_info.h> 34 #include <acpi/apei.h> 35 #include <acpi/ghes.h> 36 #include <ras/ras_event.h> 37 38 #include "../pci.h" 39 #include "portdrv.h" 40 41 #define aer_printk(level, pdev, fmt, arg...) \ 42 dev_printk(level, &(pdev)->dev, fmt, ##arg) 43 44 #define AER_ERROR_SOURCES_MAX 128 45 46 #define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */ 47 #define AER_MAX_TYPEOF_UNCOR_ERRS 32 /* as per PCI_ERR_UNCOR_STATUS*/ 48 49 struct aer_err_source { 50 u32 status; /* PCI_ERR_ROOT_STATUS */ 51 u32 id; /* PCI_ERR_ROOT_ERR_SRC */ 52 }; 53 54 struct aer_rpc { 55 struct pci_dev *rpd; /* Root Port device */ 56 DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX); 57 }; 58 59 /* AER info for the device */ 60 struct aer_info { 61 62 /* 63 * Fields for all AER capable devices. They indicate the errors 64 * "as seen by this device". Note that this may mean that if an 65 * Endpoint is causing problems, the AER counters may increment 66 * at its link partner (e.g. Root Port) because the errors will be 67 * "seen" by the link partner and not the problematic Endpoint 68 * itself (which may report all counters as 0 as it never saw any 69 * problems). 70 */ 71 /* Counters for different type of correctable errors */ 72 u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS]; 73 /* Counters for different type of fatal uncorrectable errors */ 74 u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 75 /* Counters for different type of nonfatal uncorrectable errors */ 76 u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 77 /* Total number of ERR_COR sent by this device */ 78 u64 dev_total_cor_errs; 79 /* Total number of ERR_FATAL sent by this device */ 80 u64 dev_total_fatal_errs; 81 /* Total number of ERR_NONFATAL sent by this device */ 82 u64 dev_total_nonfatal_errs; 83 84 /* 85 * Fields for Root Ports & Root Complex Event Collectors only; these 86 * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL 87 * messages received by the Root Port / Event Collector, INCLUDING the 88 * ones that are generated internally (by the Root Port itself) 89 */ 90 u64 rootport_total_cor_errs; 91 u64 rootport_total_fatal_errs; 92 u64 rootport_total_nonfatal_errs; 93 94 /* Ratelimits for errors */ 95 struct ratelimit_state correctable_ratelimit; 96 struct ratelimit_state nonfatal_ratelimit; 97 }; 98 99 #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \ 100 PCI_ERR_UNC_POISON_BLK | \ 101 PCI_ERR_UNC_ECRC| \ 102 PCI_ERR_UNC_UNSUP| \ 103 PCI_ERR_UNC_COMP_ABORT| \ 104 PCI_ERR_UNC_UNX_COMP| \ 105 PCI_ERR_UNC_ACSV | \ 106 PCI_ERR_UNC_MCBTLP | \ 107 PCI_ERR_UNC_ATOMEG | \ 108 PCI_ERR_UNC_DMWR_BLK | \ 109 PCI_ERR_UNC_XLAT_BLK | \ 110 PCI_ERR_UNC_TLPPRE | \ 111 PCI_ERR_UNC_MALF_TLP | \ 112 PCI_ERR_UNC_IDE_CHECK | \ 113 PCI_ERR_UNC_MISR_IDE | \ 114 PCI_ERR_UNC_PCRC_CHECK) 115 116 #define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \ 117 PCI_EXP_RTCTL_SENFEE| \ 118 PCI_EXP_RTCTL_SEFEE) 119 #define ROOT_PORT_INTR_ON_MESG_MASK (PCI_ERR_ROOT_CMD_COR_EN| \ 120 PCI_ERR_ROOT_CMD_NONFATAL_EN| \ 121 PCI_ERR_ROOT_CMD_FATAL_EN) 122 #define ERR_COR_ID(d) (d & 0xffff) 123 #define ERR_UNCOR_ID(d) (d >> 16) 124 125 #define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV | \ 126 PCI_ERR_ROOT_COR_RCV | \ 127 PCI_ERR_ROOT_MULTI_COR_RCV | \ 128 PCI_ERR_ROOT_MULTI_UNCOR_RCV) 129 130 static bool pcie_aer_disable; 131 static pci_ers_result_t aer_root_reset(struct pci_dev *dev); 132 133 void pci_no_aer(void) 134 { 135 pcie_aer_disable = true; 136 } 137 138 bool pci_aer_available(void) 139 { 140 return !pcie_aer_disable && pci_msi_enabled(); 141 } 142 143 #ifdef CONFIG_PCIE_ECRC 144 145 #define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */ 146 #define ECRC_POLICY_OFF 1 /* ECRC off for performance */ 147 #define ECRC_POLICY_ON 2 /* ECRC on for data integrity */ 148 149 static int ecrc_policy = ECRC_POLICY_DEFAULT; 150 151 static const char * const ecrc_policy_str[] = { 152 [ECRC_POLICY_DEFAULT] = "bios", 153 [ECRC_POLICY_OFF] = "off", 154 [ECRC_POLICY_ON] = "on" 155 }; 156 157 /** 158 * enable_ecrc_checking - enable PCIe ECRC checking for a device 159 * @dev: the PCI device 160 * 161 * Return: 0 on success, or negative on failure. 162 */ 163 static int enable_ecrc_checking(struct pci_dev *dev) 164 { 165 int aer = dev->aer_cap; 166 u32 reg32; 167 168 if (!aer) 169 return -ENODEV; 170 171 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 172 if (reg32 & PCI_ERR_CAP_ECRC_GENC) 173 reg32 |= PCI_ERR_CAP_ECRC_GENE; 174 if (reg32 & PCI_ERR_CAP_ECRC_CHKC) 175 reg32 |= PCI_ERR_CAP_ECRC_CHKE; 176 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 177 178 return 0; 179 } 180 181 /** 182 * disable_ecrc_checking - disable PCIe ECRC checking for a device 183 * @dev: the PCI device 184 * 185 * Return: 0 on success, or negative on failure. 186 */ 187 static int disable_ecrc_checking(struct pci_dev *dev) 188 { 189 int aer = dev->aer_cap; 190 u32 reg32; 191 192 if (!aer) 193 return -ENODEV; 194 195 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 196 reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE); 197 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 198 199 return 0; 200 } 201 202 /** 203 * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based 204 * on global policy 205 * @dev: the PCI device 206 */ 207 void pcie_set_ecrc_checking(struct pci_dev *dev) 208 { 209 if (!pcie_aer_is_native(dev)) 210 return; 211 212 switch (ecrc_policy) { 213 case ECRC_POLICY_DEFAULT: 214 return; 215 case ECRC_POLICY_OFF: 216 disable_ecrc_checking(dev); 217 break; 218 case ECRC_POLICY_ON: 219 enable_ecrc_checking(dev); 220 break; 221 default: 222 return; 223 } 224 } 225 226 /** 227 * pcie_ecrc_get_policy - parse kernel command-line ecrc option 228 * @str: ECRC policy from kernel command line to use 229 */ 230 void pcie_ecrc_get_policy(char *str) 231 { 232 int i; 233 234 i = match_string(ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), str); 235 if (i < 0) 236 return; 237 238 ecrc_policy = i; 239 } 240 #endif /* CONFIG_PCIE_ECRC */ 241 242 int pcie_aer_is_native(struct pci_dev *dev) 243 { 244 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 245 246 if (!dev->aer_cap) 247 return 0; 248 249 return pcie_ports_native || host->native_aer; 250 } 251 EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, "CXL"); 252 253 static int pci_enable_pcie_error_reporting(struct pci_dev *dev) 254 { 255 int rc; 256 257 if (!pcie_aer_is_native(dev)) 258 return -EIO; 259 260 rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS); 261 return pcibios_err_to_errno(rc); 262 } 263 264 int pci_aer_clear_nonfatal_status(struct pci_dev *dev) 265 { 266 int aer = dev->aer_cap; 267 u32 status, sev; 268 269 if (!pcie_aer_is_native(dev)) 270 return -EIO; 271 272 /* Clear status bits for ERR_NONFATAL errors only */ 273 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 274 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 275 status &= ~sev; 276 if (status) 277 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 278 279 return 0; 280 } 281 EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status); 282 283 void pci_aer_clear_fatal_status(struct pci_dev *dev) 284 { 285 int aer = dev->aer_cap; 286 u32 status, sev; 287 288 if (!pcie_aer_is_native(dev)) 289 return; 290 291 /* Clear status bits for ERR_FATAL errors only */ 292 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 293 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 294 status &= sev; 295 if (status) 296 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 297 } 298 299 /** 300 * pci_aer_raw_clear_status - Clear AER error registers. 301 * @dev: the PCI device 302 * 303 * Clear AER error status registers unconditionally, regardless of 304 * whether they're owned by firmware or the OS. 305 * 306 * Return: 0 on success, or negative on failure. 307 */ 308 int pci_aer_raw_clear_status(struct pci_dev *dev) 309 { 310 int aer = dev->aer_cap; 311 u32 status; 312 int port_type; 313 314 if (!aer) 315 return -EIO; 316 317 port_type = pci_pcie_type(dev); 318 if (port_type == PCI_EXP_TYPE_ROOT_PORT || 319 port_type == PCI_EXP_TYPE_RC_EC) { 320 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status); 321 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status); 322 } 323 324 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 325 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status); 326 327 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 328 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 329 330 return 0; 331 } 332 333 int pci_aer_clear_status(struct pci_dev *dev) 334 { 335 if (!pcie_aer_is_native(dev)) 336 return -EIO; 337 338 return pci_aer_raw_clear_status(dev); 339 } 340 341 void pci_save_aer_state(struct pci_dev *dev) 342 { 343 int aer = dev->aer_cap; 344 struct pci_cap_saved_state *save_state; 345 u32 *cap; 346 347 if (!aer) 348 return; 349 350 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 351 if (!save_state) 352 return; 353 354 cap = &save_state->cap.data[0]; 355 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++); 356 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++); 357 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++); 358 pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++); 359 if (pcie_cap_has_rtctl(dev)) 360 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++); 361 } 362 363 void pci_restore_aer_state(struct pci_dev *dev) 364 { 365 int aer = dev->aer_cap; 366 struct pci_cap_saved_state *save_state; 367 u32 *cap; 368 369 if (!aer) 370 return; 371 372 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 373 if (!save_state) 374 return; 375 376 cap = &save_state->cap.data[0]; 377 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++); 378 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++); 379 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++); 380 pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++); 381 if (pcie_cap_has_rtctl(dev)) 382 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++); 383 } 384 385 void pci_aer_init(struct pci_dev *dev) 386 { 387 int n; 388 389 dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); 390 if (!dev->aer_cap) 391 return; 392 393 dev->aer_info = kzalloc_obj(*dev->aer_info); 394 if (!dev->aer_info) { 395 dev->aer_cap = 0; 396 return; 397 } 398 399 ratelimit_state_init(&dev->aer_info->correctable_ratelimit, 400 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 401 ratelimit_state_init(&dev->aer_info->nonfatal_ratelimit, 402 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 403 404 /* 405 * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER, 406 * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event 407 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r6.0, sec 408 * 7.8.4.9). 409 */ 410 n = pcie_cap_has_rtctl(dev) ? 5 : 4; 411 pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, sizeof(u32) * n); 412 413 pci_aer_clear_status(dev); 414 415 if (pci_aer_available()) 416 pci_enable_pcie_error_reporting(dev); 417 418 pcie_set_ecrc_checking(dev); 419 } 420 421 void pci_aer_exit(struct pci_dev *dev) 422 { 423 kfree(dev->aer_info); 424 dev->aer_info = NULL; 425 } 426 427 #define AER_AGENT_RECEIVER 0 428 #define AER_AGENT_REQUESTER 1 429 #define AER_AGENT_COMPLETER 2 430 #define AER_AGENT_TRANSMITTER 3 431 432 #define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 433 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP)) 434 #define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \ 435 0 : PCI_ERR_UNC_COMP_ABORT) 436 #define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 437 (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0) 438 439 #define AER_GET_AGENT(t, e) \ 440 ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \ 441 (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \ 442 (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \ 443 AER_AGENT_RECEIVER) 444 445 #define AER_PHYSICAL_LAYER_ERROR 0 446 #define AER_DATA_LINK_LAYER_ERROR 1 447 #define AER_TRANSACTION_LAYER_ERROR 2 448 449 #define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 450 PCI_ERR_COR_RCVR : 0) 451 #define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 452 (PCI_ERR_COR_BAD_TLP| \ 453 PCI_ERR_COR_BAD_DLLP| \ 454 PCI_ERR_COR_REP_ROLL| \ 455 PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP) 456 457 #define AER_GET_LAYER_ERROR(t, e) \ 458 ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \ 459 (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ 460 AER_TRANSACTION_LAYER_ERROR) 461 462 /* 463 * AER error strings 464 */ 465 static const char * const aer_error_severity_string[] = { 466 "Uncorrectable (Non-Fatal)", 467 "Uncorrectable (Fatal)", 468 "Correctable" 469 }; 470 471 static const char *aer_error_layer[] = { 472 "Physical Layer", 473 "Data Link Layer", 474 "Transaction Layer" 475 }; 476 477 static const char *aer_correctable_error_string[] = { 478 "RxErr", /* Bit Position 0 */ 479 NULL, 480 NULL, 481 NULL, 482 NULL, 483 NULL, 484 "BadTLP", /* Bit Position 6 */ 485 "BadDLLP", /* Bit Position 7 */ 486 "Rollover", /* Bit Position 8 */ 487 NULL, 488 NULL, 489 NULL, 490 "Timeout", /* Bit Position 12 */ 491 "NonFatalErr", /* Bit Position 13 */ 492 "CorrIntErr", /* Bit Position 14 */ 493 "HeaderOF", /* Bit Position 15 */ 494 NULL, /* Bit Position 16 */ 495 NULL, /* Bit Position 17 */ 496 NULL, /* Bit Position 18 */ 497 NULL, /* Bit Position 19 */ 498 NULL, /* Bit Position 20 */ 499 NULL, /* Bit Position 21 */ 500 NULL, /* Bit Position 22 */ 501 NULL, /* Bit Position 23 */ 502 NULL, /* Bit Position 24 */ 503 NULL, /* Bit Position 25 */ 504 NULL, /* Bit Position 26 */ 505 NULL, /* Bit Position 27 */ 506 NULL, /* Bit Position 28 */ 507 NULL, /* Bit Position 29 */ 508 NULL, /* Bit Position 30 */ 509 NULL, /* Bit Position 31 */ 510 }; 511 512 static const char *aer_uncorrectable_error_string[] = { 513 "Undefined", /* Bit Position 0 */ 514 NULL, 515 NULL, 516 NULL, 517 "DLP", /* Bit Position 4 */ 518 "SDES", /* Bit Position 5 */ 519 NULL, 520 NULL, 521 NULL, 522 NULL, 523 NULL, 524 NULL, 525 "TLP", /* Bit Position 12 */ 526 "FCP", /* Bit Position 13 */ 527 "CmpltTO", /* Bit Position 14 */ 528 "CmpltAbrt", /* Bit Position 15 */ 529 "UnxCmplt", /* Bit Position 16 */ 530 "RxOF", /* Bit Position 17 */ 531 "MalfTLP", /* Bit Position 18 */ 532 "ECRC", /* Bit Position 19 */ 533 "UnsupReq", /* Bit Position 20 */ 534 "ACSViol", /* Bit Position 21 */ 535 "UncorrIntErr", /* Bit Position 22 */ 536 "BlockedTLP", /* Bit Position 23 */ 537 "AtomicOpBlocked", /* Bit Position 24 */ 538 "TLPBlockedErr", /* Bit Position 25 */ 539 "PoisonTLPBlocked", /* Bit Position 26 */ 540 "DMWrReqBlocked", /* Bit Position 27 */ 541 "IDECheck", /* Bit Position 28 */ 542 "MisIDETLP", /* Bit Position 29 */ 543 "PCRC_CHECK", /* Bit Position 30 */ 544 "TLPXlatBlocked", /* Bit Position 31 */ 545 }; 546 547 static const char *aer_agent_string[] = { 548 "Receiver ID", 549 "Requester ID", 550 "Completer ID", 551 "Transmitter ID" 552 }; 553 554 #define aer_stats_dev_attr(name, stats_array, strings_array, \ 555 total_string, total_field) \ 556 static ssize_t \ 557 name##_show(struct device *dev, struct device_attribute *attr, \ 558 char *buf) \ 559 { \ 560 unsigned int i; \ 561 struct pci_dev *pdev = to_pci_dev(dev); \ 562 u64 *stats = pdev->aer_info->stats_array; \ 563 size_t len = 0; \ 564 \ 565 for (i = 0; i < ARRAY_SIZE(pdev->aer_info->stats_array); i++) { \ 566 if (strings_array[i]) \ 567 len += sysfs_emit_at(buf, len, "%s %llu\n", \ 568 strings_array[i], \ 569 stats[i]); \ 570 else if (stats[i]) \ 571 len += sysfs_emit_at(buf, len, \ 572 #stats_array "_bit[%d] %llu\n",\ 573 i, stats[i]); \ 574 } \ 575 len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \ 576 pdev->aer_info->total_field); \ 577 return len; \ 578 } \ 579 static DEVICE_ATTR_RO(name) 580 581 aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs, 582 aer_correctable_error_string, "ERR_COR", 583 dev_total_cor_errs); 584 aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs, 585 aer_uncorrectable_error_string, "ERR_FATAL", 586 dev_total_fatal_errs); 587 aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs, 588 aer_uncorrectable_error_string, "ERR_NONFATAL", 589 dev_total_nonfatal_errs); 590 591 #define aer_stats_rootport_attr(name, field) \ 592 static ssize_t \ 593 name##_show(struct device *dev, struct device_attribute *attr, \ 594 char *buf) \ 595 { \ 596 struct pci_dev *pdev = to_pci_dev(dev); \ 597 return sysfs_emit(buf, "%llu\n", pdev->aer_info->field); \ 598 } \ 599 static DEVICE_ATTR_RO(name) 600 601 aer_stats_rootport_attr(aer_rootport_total_err_cor, 602 rootport_total_cor_errs); 603 aer_stats_rootport_attr(aer_rootport_total_err_fatal, 604 rootport_total_fatal_errs); 605 aer_stats_rootport_attr(aer_rootport_total_err_nonfatal, 606 rootport_total_nonfatal_errs); 607 608 static struct attribute *aer_stats_attrs[] __ro_after_init = { 609 &dev_attr_aer_dev_correctable.attr, 610 &dev_attr_aer_dev_fatal.attr, 611 &dev_attr_aer_dev_nonfatal.attr, 612 &dev_attr_aer_rootport_total_err_cor.attr, 613 &dev_attr_aer_rootport_total_err_fatal.attr, 614 &dev_attr_aer_rootport_total_err_nonfatal.attr, 615 NULL 616 }; 617 618 static umode_t aer_stats_attrs_are_visible(struct kobject *kobj, 619 struct attribute *a, int n) 620 { 621 struct device *dev = kobj_to_dev(kobj); 622 struct pci_dev *pdev = to_pci_dev(dev); 623 624 if (!pdev->aer_info) 625 return 0; 626 627 if ((a == &dev_attr_aer_rootport_total_err_cor.attr || 628 a == &dev_attr_aer_rootport_total_err_fatal.attr || 629 a == &dev_attr_aer_rootport_total_err_nonfatal.attr) && 630 ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && 631 (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_EC))) 632 return 0; 633 634 return a->mode; 635 } 636 637 const struct attribute_group aer_stats_attr_group = { 638 .attrs = aer_stats_attrs, 639 .is_visible = aer_stats_attrs_are_visible, 640 }; 641 642 /* 643 * Ratelimit interval 644 * <=0: disabled with ratelimit.interval = 0 645 * >0: enabled with ratelimit.interval in ms 646 */ 647 #define aer_ratelimit_interval_attr(name, ratelimit) \ 648 static ssize_t \ 649 name##_show(struct device *dev, struct device_attribute *attr, \ 650 char *buf) \ 651 { \ 652 struct pci_dev *pdev = to_pci_dev(dev); \ 653 \ 654 return sysfs_emit(buf, "%d\n", \ 655 pdev->aer_info->ratelimit.interval); \ 656 } \ 657 \ 658 static ssize_t \ 659 name##_store(struct device *dev, struct device_attribute *attr, \ 660 const char *buf, size_t count) \ 661 { \ 662 struct pci_dev *pdev = to_pci_dev(dev); \ 663 int interval; \ 664 \ 665 if (!capable(CAP_SYS_ADMIN)) \ 666 return -EPERM; \ 667 \ 668 if (kstrtoint(buf, 0, &interval) < 0) \ 669 return -EINVAL; \ 670 \ 671 if (interval <= 0) \ 672 interval = 0; \ 673 else \ 674 interval = msecs_to_jiffies(interval); \ 675 \ 676 pdev->aer_info->ratelimit.interval = interval; \ 677 \ 678 return count; \ 679 } \ 680 static DEVICE_ATTR_RW(name); 681 682 #define aer_ratelimit_burst_attr(name, ratelimit) \ 683 static ssize_t \ 684 name##_show(struct device *dev, struct device_attribute *attr, \ 685 char *buf) \ 686 { \ 687 struct pci_dev *pdev = to_pci_dev(dev); \ 688 \ 689 return sysfs_emit(buf, "%d\n", \ 690 pdev->aer_info->ratelimit.burst); \ 691 } \ 692 \ 693 static ssize_t \ 694 name##_store(struct device *dev, struct device_attribute *attr, \ 695 const char *buf, size_t count) \ 696 { \ 697 struct pci_dev *pdev = to_pci_dev(dev); \ 698 int burst; \ 699 \ 700 if (!capable(CAP_SYS_ADMIN)) \ 701 return -EPERM; \ 702 \ 703 if (kstrtoint(buf, 0, &burst) < 0) \ 704 return -EINVAL; \ 705 \ 706 pdev->aer_info->ratelimit.burst = burst; \ 707 \ 708 return count; \ 709 } \ 710 static DEVICE_ATTR_RW(name); 711 712 #define aer_ratelimit_attrs(name) \ 713 aer_ratelimit_interval_attr(name##_ratelimit_interval_ms, \ 714 name##_ratelimit) \ 715 aer_ratelimit_burst_attr(name##_ratelimit_burst, \ 716 name##_ratelimit) 717 718 aer_ratelimit_attrs(correctable) 719 aer_ratelimit_attrs(nonfatal) 720 721 static struct attribute *aer_attrs[] = { 722 &dev_attr_correctable_ratelimit_interval_ms.attr, 723 &dev_attr_correctable_ratelimit_burst.attr, 724 &dev_attr_nonfatal_ratelimit_interval_ms.attr, 725 &dev_attr_nonfatal_ratelimit_burst.attr, 726 NULL 727 }; 728 729 static umode_t aer_attrs_are_visible(struct kobject *kobj, 730 struct attribute *a, int n) 731 { 732 struct device *dev = kobj_to_dev(kobj); 733 struct pci_dev *pdev = to_pci_dev(dev); 734 735 if (!pdev->aer_info) 736 return 0; 737 738 return a->mode; 739 } 740 741 const struct attribute_group aer_attr_group = { 742 .name = "aer", 743 .attrs = aer_attrs, 744 .is_visible = aer_attrs_are_visible, 745 }; 746 747 static void pci_dev_aer_stats_incr(struct pci_dev *pdev, 748 struct aer_err_info *info) 749 { 750 unsigned long status = info->status & ~info->mask; 751 int i, max = -1; 752 u64 *counter = NULL; 753 struct aer_info *aer_info = pdev->aer_info; 754 755 if (!aer_info) 756 return; 757 758 switch (info->severity) { 759 case AER_CORRECTABLE: 760 aer_info->dev_total_cor_errs++; 761 counter = &aer_info->dev_cor_errs[0]; 762 max = AER_MAX_TYPEOF_COR_ERRS; 763 break; 764 case AER_NONFATAL: 765 aer_info->dev_total_nonfatal_errs++; 766 hwerr_log_error_type(HWERR_RECOV_PCI); 767 counter = &aer_info->dev_nonfatal_errs[0]; 768 max = AER_MAX_TYPEOF_UNCOR_ERRS; 769 break; 770 case AER_FATAL: 771 aer_info->dev_total_fatal_errs++; 772 counter = &aer_info->dev_fatal_errs[0]; 773 max = AER_MAX_TYPEOF_UNCOR_ERRS; 774 break; 775 } 776 777 for_each_set_bit(i, &status, max) 778 counter[i]++; 779 } 780 781 static void pci_rootport_aer_stats_incr(struct pci_dev *pdev, 782 struct aer_err_source *e_src) 783 { 784 struct aer_info *aer_info = pdev->aer_info; 785 786 if (!aer_info) 787 return; 788 789 if (e_src->status & PCI_ERR_ROOT_COR_RCV) 790 aer_info->rootport_total_cor_errs++; 791 792 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { 793 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) 794 aer_info->rootport_total_fatal_errs++; 795 else 796 aer_info->rootport_total_nonfatal_errs++; 797 } 798 } 799 800 static int aer_ratelimit(struct pci_dev *dev, unsigned int severity) 801 { 802 if (!dev->aer_info) 803 return 1; 804 805 switch (severity) { 806 case AER_NONFATAL: 807 return __ratelimit(&dev->aer_info->nonfatal_ratelimit); 808 case AER_CORRECTABLE: 809 return __ratelimit(&dev->aer_info->correctable_ratelimit); 810 default: 811 return 1; /* Don't ratelimit fatal errors */ 812 } 813 } 814 815 static bool tlp_header_logged(u32 status, u32 capctl) 816 { 817 /* Errors for which a header is always logged (PCIe r7.0 sec 6.2.7) */ 818 if (status & AER_LOG_TLP_MASKS) 819 return true; 820 821 /* Completion Timeout header is only logged on capable devices */ 822 if (status & PCI_ERR_UNC_COMP_TIME && 823 capctl & PCI_ERR_CAP_COMP_TIME_LOG) 824 return true; 825 826 return false; 827 } 828 829 static void __aer_print_error(struct pci_dev *dev, struct aer_err_info *info) 830 { 831 const char **strings; 832 unsigned long status = info->status & ~info->mask; 833 const char *level = info->level; 834 const char *errmsg; 835 int i; 836 837 if (info->severity == AER_CORRECTABLE) 838 strings = aer_correctable_error_string; 839 else 840 strings = aer_uncorrectable_error_string; 841 842 for_each_set_bit(i, &status, 32) { 843 errmsg = strings[i]; 844 if (!errmsg) 845 errmsg = "Unknown Error Bit"; 846 847 aer_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg, 848 info->first_error == i ? " (First)" : ""); 849 } 850 } 851 852 static void aer_print_source(struct pci_dev *dev, struct aer_err_info *info, 853 bool found) 854 { 855 u16 source = info->id; 856 857 pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d%s\n", 858 info->multi_error_valid ? "Multiple " : "", 859 aer_error_severity_string[info->severity], 860 pci_domain_nr(dev->bus), PCI_BUS_NUM(source), 861 PCI_SLOT(source), PCI_FUNC(source), 862 found ? "" : " (no details found"); 863 } 864 865 void aer_print_error(struct aer_err_info *info, int i) 866 { 867 struct pci_dev *dev; 868 int layer, agent, id; 869 const char *level = info->level; 870 const char *bus_type = aer_err_bus(info); 871 872 if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES)) 873 return; 874 875 dev = info->dev[i]; 876 id = pci_dev_id(dev); 877 878 pci_dev_aer_stats_incr(dev, info); 879 trace_aer_event(pci_name(dev), (info->status & ~info->mask), 880 info->severity, info->tlp_header_valid, &info->tlp, bus_type); 881 882 if (!info->ratelimit_print[i]) 883 return; 884 885 if (!info->status) { 886 pci_err(dev, "%s Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", 887 bus_type, aer_error_severity_string[info->severity]); 888 goto out; 889 } 890 891 layer = AER_GET_LAYER_ERROR(info->severity, info->status); 892 agent = AER_GET_AGENT(info->severity, info->status); 893 894 aer_printk(level, dev, "%s Bus Error: severity=%s, type=%s, (%s)\n", 895 bus_type, aer_error_severity_string[info->severity], 896 aer_error_layer[layer], aer_agent_string[agent]); 897 898 aer_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n", 899 dev->vendor, dev->device, info->status, info->mask); 900 901 __aer_print_error(dev, info); 902 903 if (info->tlp_header_valid) 904 pcie_print_tlp_log(dev, &info->tlp, level, dev_fmt(" ")); 905 906 out: 907 if (info->id && info->error_dev_num > 1 && info->id == id) 908 pci_err(dev, " Error of this Agent is reported first\n"); 909 } 910 911 #ifdef CONFIG_ACPI_APEI_PCIEAER 912 int cper_severity_to_aer(int cper_severity) 913 { 914 switch (cper_severity) { 915 case CPER_SEV_RECOVERABLE: 916 return AER_NONFATAL; 917 case CPER_SEV_FATAL: 918 return AER_FATAL; 919 default: 920 return AER_CORRECTABLE; 921 } 922 } 923 EXPORT_SYMBOL_GPL(cper_severity_to_aer); 924 #endif 925 926 void pci_print_aer(struct pci_dev *dev, int aer_severity, 927 struct aer_capability_regs *aer) 928 { 929 const char *bus_type; 930 int layer, agent, tlp_header_valid = 0; 931 u32 status, mask; 932 struct aer_err_info info = { 933 .severity = aer_severity, 934 .first_error = PCI_ERR_CAP_FEP(aer->cap_control), 935 }; 936 937 if (aer_severity == AER_CORRECTABLE) { 938 status = aer->cor_status; 939 mask = aer->cor_mask; 940 info.level = KERN_WARNING; 941 } else { 942 status = aer->uncor_status; 943 mask = aer->uncor_mask; 944 info.level = KERN_ERR; 945 tlp_header_valid = tlp_header_logged(status, aer->cap_control); 946 } 947 948 info.status = status; 949 info.mask = mask; 950 info.is_cxl = pcie_is_cxl(dev); 951 952 bus_type = aer_err_bus(&info); 953 954 pci_dev_aer_stats_incr(dev, &info); 955 trace_aer_event(pci_name(dev), (status & ~mask), aer_severity, 956 tlp_header_valid, &aer->header_log, bus_type); 957 958 if (!aer_ratelimit(dev, info.severity)) 959 return; 960 961 layer = AER_GET_LAYER_ERROR(aer_severity, status); 962 agent = AER_GET_AGENT(aer_severity, status); 963 964 aer_printk(info.level, dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", 965 status, mask); 966 __aer_print_error(dev, &info); 967 aer_printk(info.level, dev, "aer_layer=%s, aer_agent=%s\n", 968 aer_error_layer[layer], aer_agent_string[agent]); 969 970 if (aer_severity != AER_CORRECTABLE) 971 aer_printk(info.level, dev, "aer_uncor_severity: 0x%08x\n", 972 aer->uncor_severity); 973 974 if (tlp_header_valid) 975 pcie_print_tlp_log(dev, &aer->header_log, info.level, 976 dev_fmt(" ")); 977 } 978 EXPORT_SYMBOL_GPL(pci_print_aer); 979 980 /** 981 * add_error_device - list device to be handled 982 * @e_info: pointer to error info 983 * @dev: pointer to pci_dev to be added 984 */ 985 static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) 986 { 987 int i = e_info->error_dev_num; 988 989 if (i >= AER_MAX_MULTI_ERR_DEVICES) 990 return -ENOSPC; 991 992 e_info->dev[i] = pci_dev_get(dev); 993 e_info->error_dev_num++; 994 995 /* 996 * Ratelimit AER log messages. "dev" is either the source 997 * identified by the root's Error Source ID or it has an unmasked 998 * error logged in its own AER Capability. Messages are emitted 999 * when "ratelimit_print[i]" is non-zero. If we will print detail 1000 * for a downstream device, make sure we print the Error Source ID 1001 * from the root as well. 1002 */ 1003 if (aer_ratelimit(dev, e_info->severity)) { 1004 e_info->ratelimit_print[i] = 1; 1005 e_info->root_ratelimit_print = 1; 1006 } 1007 return 0; 1008 } 1009 1010 /** 1011 * is_error_source - check whether the device is source of reported error 1012 * @dev: pointer to pci_dev to be checked 1013 * @e_info: pointer to reported error info 1014 */ 1015 static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info) 1016 { 1017 int aer = dev->aer_cap; 1018 u32 status, mask; 1019 u16 reg16; 1020 1021 /* 1022 * When bus ID is equal to 0, it might be a bad ID 1023 * reported by Root Port. 1024 */ 1025 if ((PCI_BUS_NUM(e_info->id) != 0) && 1026 !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) { 1027 /* Device ID match? */ 1028 if (e_info->id == pci_dev_id(dev)) 1029 return true; 1030 1031 /* Continue ID comparing if there is no multiple error */ 1032 if (!e_info->multi_error_valid) 1033 return false; 1034 } 1035 1036 /* 1037 * When either 1038 * 1) bus ID is equal to 0. Some ports might lose the bus 1039 * ID of error source id; 1040 * 2) bus flag PCI_BUS_FLAGS_NO_AERSID is set 1041 * 3) There are multiple errors and prior ID comparing fails; 1042 * We check AER status registers to find possible reporter. 1043 */ 1044 1045 /* Check if AER is enabled */ 1046 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16); 1047 if (!(reg16 & PCI_EXP_AER_FLAGS)) 1048 return false; 1049 1050 if (!aer) 1051 return false; 1052 1053 /* Check if error is recorded */ 1054 if (e_info->severity == AER_CORRECTABLE) { 1055 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 1056 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1057 } else { 1058 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 1059 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1060 } 1061 if (status & ~mask) 1062 return true; 1063 1064 return false; 1065 } 1066 1067 static int find_device_iter(struct pci_dev *dev, void *data) 1068 { 1069 struct aer_err_info *e_info = (struct aer_err_info *)data; 1070 1071 if (is_error_source(dev, e_info)) { 1072 /* List this device */ 1073 if (add_error_device(e_info, dev)) { 1074 /* We cannot handle more... Stop iteration */ 1075 pci_err(dev, "Exceeded max supported (%d) devices with errors logged\n", 1076 AER_MAX_MULTI_ERR_DEVICES); 1077 return 1; 1078 } 1079 1080 /* If there is only a single error, stop iteration */ 1081 if (!e_info->multi_error_valid) 1082 return 1; 1083 } 1084 return 0; 1085 } 1086 1087 /** 1088 * find_source_device - search through device hierarchy for source device 1089 * @parent: pointer to Root Port pci_dev data structure 1090 * @e_info: including detailed error information such as ID 1091 * 1092 * Return: true if found. 1093 * 1094 * Invoked by DPC when error is detected at the Root Port. 1095 * Caller of this function must set id, severity, and multi_error_valid of 1096 * struct aer_err_info pointed by @e_info properly. This function must fill 1097 * e_info->error_dev_num and e_info->dev[], based on the given information. 1098 */ 1099 static bool find_source_device(struct pci_dev *parent, 1100 struct aer_err_info *e_info) 1101 { 1102 struct pci_dev *dev = parent; 1103 int result; 1104 1105 /* Must reset in this function */ 1106 e_info->error_dev_num = 0; 1107 1108 /* Is Root Port an agent that sends error message? */ 1109 result = find_device_iter(dev, e_info); 1110 if (result) 1111 return true; 1112 1113 if (pci_pcie_type(parent) == PCI_EXP_TYPE_RC_EC) 1114 pcie_walk_rcec(parent, find_device_iter, e_info); 1115 else 1116 pci_walk_bus(parent->subordinate, find_device_iter, e_info); 1117 1118 if (!e_info->error_dev_num) 1119 return false; 1120 return true; 1121 } 1122 1123 /** 1124 * pci_aer_unmask_internal_errors - unmask internal errors 1125 * @dev: pointer to the pci_dev data structure 1126 * 1127 * Unmask internal errors in the Uncorrectable and Correctable Error 1128 * Mask registers. 1129 * 1130 * Note: AER must be enabled and supported by the device which must be 1131 * checked in advance, e.g. with pcie_aer_is_native(). 1132 */ 1133 void pci_aer_unmask_internal_errors(struct pci_dev *dev) 1134 { 1135 int aer = dev->aer_cap; 1136 u32 mask; 1137 1138 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1139 mask &= ~PCI_ERR_UNC_INTN; 1140 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask); 1141 1142 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1143 mask &= ~PCI_ERR_COR_INTERNAL; 1144 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask); 1145 } 1146 1147 /* 1148 * Internal errors are too device-specific to enable generally, however for CXL 1149 * their behavior is standardized for conveying CXL protocol errors. 1150 */ 1151 EXPORT_SYMBOL_FOR_MODULES(pci_aer_unmask_internal_errors, "cxl_core"); 1152 1153 #ifdef CONFIG_CXL_RAS 1154 bool is_aer_internal_error(struct aer_err_info *info) 1155 { 1156 if (info->severity == AER_CORRECTABLE) 1157 return info->status & PCI_ERR_COR_INTERNAL; 1158 1159 return info->status & PCI_ERR_UNC_INTN; 1160 } 1161 #endif 1162 1163 /** 1164 * pci_aer_handle_error - handle logging error into an event log 1165 * @dev: pointer to pci_dev data structure of error source device 1166 * @info: comprehensive error information 1167 * 1168 * Invoked when an error being detected by Root Port. 1169 */ 1170 static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info) 1171 { 1172 int aer = dev->aer_cap; 1173 1174 if (info->severity == AER_CORRECTABLE) { 1175 /* 1176 * Correctable error does not need software intervention. 1177 * No need to go through error recovery process. 1178 */ 1179 if (aer) 1180 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1181 info->status); 1182 if (pcie_aer_is_native(dev)) { 1183 struct pci_driver *pdrv = dev->driver; 1184 1185 if (pdrv && pdrv->err_handler && 1186 pdrv->err_handler->cor_error_detected) 1187 pdrv->err_handler->cor_error_detected(dev); 1188 pcie_clear_device_status(dev); 1189 } 1190 } else if (info->severity == AER_NONFATAL) 1191 pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset); 1192 else if (info->severity == AER_FATAL) 1193 pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset); 1194 } 1195 1196 static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) 1197 { 1198 cxl_rch_handle_error(dev, info); 1199 pci_aer_handle_error(dev, info); 1200 pci_dev_put(dev); 1201 } 1202 1203 #ifdef CONFIG_ACPI_APEI_PCIEAER 1204 1205 #define AER_RECOVER_RING_SIZE 16 1206 1207 struct aer_recover_entry { 1208 u8 bus; 1209 u8 devfn; 1210 u16 domain; 1211 int severity; 1212 struct aer_capability_regs *regs; 1213 }; 1214 1215 static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry, 1216 AER_RECOVER_RING_SIZE); 1217 1218 static void aer_recover_work_func(struct work_struct *work) 1219 { 1220 struct aer_recover_entry entry; 1221 struct pci_dev *pdev; 1222 1223 while (kfifo_get(&aer_recover_ring, &entry)) { 1224 pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, 1225 entry.devfn); 1226 if (!pdev) { 1227 pr_err_ratelimited("%04x:%02x:%02x.%x: no pci_dev found\n", 1228 entry.domain, entry.bus, 1229 PCI_SLOT(entry.devfn), 1230 PCI_FUNC(entry.devfn)); 1231 continue; 1232 } 1233 pci_print_aer(pdev, entry.severity, entry.regs); 1234 1235 /* 1236 * Memory for aer_capability_regs(entry.regs) is being 1237 * allocated from the ghes_estatus_pool to protect it from 1238 * overwriting when multiple sections are present in the 1239 * error status. Thus free the same after processing the 1240 * data. 1241 */ 1242 ghes_estatus_pool_region_free((unsigned long)entry.regs, 1243 sizeof(struct aer_capability_regs)); 1244 1245 if (entry.severity == AER_NONFATAL) 1246 pcie_do_recovery(pdev, pci_channel_io_normal, 1247 aer_root_reset); 1248 else if (entry.severity == AER_FATAL) 1249 pcie_do_recovery(pdev, pci_channel_io_frozen, 1250 aer_root_reset); 1251 pci_dev_put(pdev); 1252 } 1253 } 1254 1255 /* 1256 * Mutual exclusion for writers of aer_recover_ring, reader side don't 1257 * need lock, because there is only one reader and lock is not needed 1258 * between reader and writer. 1259 */ 1260 static DEFINE_SPINLOCK(aer_recover_ring_lock); 1261 static DECLARE_WORK(aer_recover_work, aer_recover_work_func); 1262 1263 void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, 1264 int severity, struct aer_capability_regs *aer_regs) 1265 { 1266 struct aer_recover_entry entry = { 1267 .bus = bus, 1268 .devfn = devfn, 1269 .domain = domain, 1270 .severity = severity, 1271 .regs = aer_regs, 1272 }; 1273 1274 if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1, 1275 &aer_recover_ring_lock)) 1276 schedule_work(&aer_recover_work); 1277 else 1278 pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n", 1279 domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 1280 } 1281 EXPORT_SYMBOL_GPL(aer_recover_queue); 1282 #endif 1283 1284 /** 1285 * aer_get_device_error_info - read error status from dev and store it to info 1286 * @info: pointer to structure to store the error record 1287 * @i: index into info->dev[] 1288 * 1289 * Return: 1 on success, 0 on error. 1290 * 1291 * Note that @info is reused among all error devices. Clear fields properly. 1292 */ 1293 int aer_get_device_error_info(struct aer_err_info *info, int i) 1294 { 1295 struct pci_dev *dev; 1296 int type, aer; 1297 u32 aercc; 1298 1299 if (i >= AER_MAX_MULTI_ERR_DEVICES) 1300 return 0; 1301 1302 dev = info->dev[i]; 1303 aer = dev->aer_cap; 1304 type = pci_pcie_type(dev); 1305 1306 /* Must reset in this function */ 1307 info->status = 0; 1308 info->tlp_header_valid = 0; 1309 info->is_cxl = pcie_is_cxl(dev); 1310 1311 /* The device might not support AER */ 1312 if (!aer) 1313 return 0; 1314 1315 if (info->severity == AER_CORRECTABLE) { 1316 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1317 &info->status); 1318 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, 1319 &info->mask); 1320 if (!(info->status & ~info->mask)) 1321 return 0; 1322 } else if (type == PCI_EXP_TYPE_ROOT_PORT || 1323 type == PCI_EXP_TYPE_RC_EC || 1324 type == PCI_EXP_TYPE_DOWNSTREAM || 1325 info->severity == AER_NONFATAL) { 1326 1327 /* Link is still healthy for IO reads */ 1328 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, 1329 &info->status); 1330 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, 1331 &info->mask); 1332 if (!(info->status & ~info->mask)) 1333 return 0; 1334 1335 /* Get First Error Pointer */ 1336 pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc); 1337 info->first_error = PCI_ERR_CAP_FEP(aercc); 1338 1339 if (tlp_header_logged(info->status, aercc)) { 1340 info->tlp_header_valid = 1; 1341 pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG, 1342 aer + PCI_ERR_PREFIX_LOG, 1343 aer_tlp_log_len(dev, aercc), 1344 aercc & PCI_ERR_CAP_TLP_LOG_FLIT, 1345 &info->tlp); 1346 } 1347 } 1348 1349 return 1; 1350 } 1351 1352 static inline void aer_process_err_devices(struct aer_err_info *e_info) 1353 { 1354 int i; 1355 1356 /* Report all before handling them, to not lose records by reset etc. */ 1357 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1358 if (aer_get_device_error_info(e_info, i)) 1359 aer_print_error(e_info, i); 1360 } 1361 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1362 if (aer_get_device_error_info(e_info, i)) 1363 handle_error_source(e_info->dev[i], e_info); 1364 } 1365 } 1366 1367 /** 1368 * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error 1369 * detected by Root Port or RCEC 1370 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1371 * @info: pointer to AER error info 1372 */ 1373 static void aer_isr_one_error_type(struct pci_dev *root, 1374 struct aer_err_info *info) 1375 { 1376 bool found; 1377 1378 found = find_source_device(root, info); 1379 1380 /* 1381 * If we're going to log error messages, we've already set 1382 * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to 1383 * non-zero (which enables printing) because this is either an 1384 * ERR_FATAL or we found a device with an error logged in its AER 1385 * Capability. 1386 * 1387 * If we didn't find the Error Source device, at least log the 1388 * Requester ID from the ERR_* Message received by the Root Port or 1389 * RCEC, ratelimited by the RP or RCEC. 1390 */ 1391 if (info->root_ratelimit_print || 1392 (!found && aer_ratelimit(root, info->severity))) 1393 aer_print_source(root, info, found); 1394 1395 if (found) 1396 aer_process_err_devices(info); 1397 } 1398 1399 /** 1400 * aer_isr_one_error - consume error(s) signaled by an AER interrupt from 1401 * Root Port or RCEC 1402 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1403 * @e_src: pointer to an error source 1404 */ 1405 static void aer_isr_one_error(struct pci_dev *root, 1406 struct aer_err_source *e_src) 1407 { 1408 u32 status = e_src->status; 1409 1410 pci_rootport_aer_stats_incr(root, e_src); 1411 1412 /* 1413 * There is a possibility that both correctable error and 1414 * uncorrectable error being logged. Report correctable error first. 1415 */ 1416 if (status & PCI_ERR_ROOT_COR_RCV) { 1417 int multi = status & PCI_ERR_ROOT_MULTI_COR_RCV; 1418 struct aer_err_info e_info = { 1419 .id = ERR_COR_ID(e_src->id), 1420 .severity = AER_CORRECTABLE, 1421 .level = KERN_WARNING, 1422 .multi_error_valid = multi ? 1 : 0, 1423 }; 1424 1425 aer_isr_one_error_type(root, &e_info); 1426 } 1427 1428 if (status & PCI_ERR_ROOT_UNCOR_RCV) { 1429 int fatal = status & PCI_ERR_ROOT_FATAL_RCV; 1430 int multi = status & PCI_ERR_ROOT_MULTI_UNCOR_RCV; 1431 struct aer_err_info e_info = { 1432 .id = ERR_UNCOR_ID(e_src->id), 1433 .severity = fatal ? AER_FATAL : AER_NONFATAL, 1434 .level = KERN_ERR, 1435 .multi_error_valid = multi ? 1 : 0, 1436 }; 1437 1438 aer_isr_one_error_type(root, &e_info); 1439 } 1440 } 1441 1442 /** 1443 * aer_isr - consume errors detected by Root Port 1444 * @irq: IRQ assigned to Root Port 1445 * @context: pointer to Root Port data structure 1446 * 1447 * Invoked, as DPC, when Root Port records new detected error 1448 */ 1449 static irqreturn_t aer_isr(int irq, void *context) 1450 { 1451 struct pcie_device *dev = (struct pcie_device *)context; 1452 struct aer_rpc *rpc = get_service_data(dev); 1453 struct aer_err_source e_src; 1454 1455 if (kfifo_is_empty(&rpc->aer_fifo)) 1456 return IRQ_NONE; 1457 1458 while (kfifo_get(&rpc->aer_fifo, &e_src)) 1459 aer_isr_one_error(rpc->rpd, &e_src); 1460 return IRQ_HANDLED; 1461 } 1462 1463 /** 1464 * aer_irq - Root Port's ISR 1465 * @irq: IRQ assigned to Root Port 1466 * @context: pointer to Root Port data structure 1467 * 1468 * Invoked when Root Port detects AER messages. 1469 */ 1470 static irqreturn_t aer_irq(int irq, void *context) 1471 { 1472 struct pcie_device *pdev = (struct pcie_device *)context; 1473 struct aer_rpc *rpc = get_service_data(pdev); 1474 struct pci_dev *rp = rpc->rpd; 1475 int aer = rp->aer_cap; 1476 struct aer_err_source e_src = {}; 1477 1478 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status); 1479 if (!(e_src.status & AER_ERR_STATUS_MASK)) 1480 return IRQ_NONE; 1481 1482 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id); 1483 pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status); 1484 1485 if (!kfifo_put(&rpc->aer_fifo, e_src)) 1486 return IRQ_HANDLED; 1487 1488 return IRQ_WAKE_THREAD; 1489 } 1490 1491 static void aer_enable_irq(struct pci_dev *pdev) 1492 { 1493 int aer = pdev->aer_cap; 1494 u32 reg32; 1495 1496 /* Enable Root Port's interrupt in response to error messages */ 1497 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1498 reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; 1499 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1500 } 1501 1502 static void aer_disable_irq(struct pci_dev *pdev) 1503 { 1504 int aer = pdev->aer_cap; 1505 u32 reg32; 1506 1507 /* Disable Root Port's interrupt in response to error messages */ 1508 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1509 reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; 1510 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1511 } 1512 1513 static int clear_status_iter(struct pci_dev *dev, void *data) 1514 { 1515 u16 devctl; 1516 1517 /* Skip if pci_enable_pcie_error_reporting() hasn't been called yet */ 1518 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &devctl); 1519 if (!(devctl & PCI_EXP_AER_FLAGS)) 1520 return 0; 1521 1522 pci_aer_clear_status(dev); 1523 pcie_clear_device_status(dev); 1524 return 0; 1525 } 1526 1527 /** 1528 * aer_enable_rootport - enable Root Port's interrupts when receiving messages 1529 * @rpc: pointer to a Root Port data structure 1530 * 1531 * Invoked when PCIe bus loads AER service driver. 1532 */ 1533 static void aer_enable_rootport(struct aer_rpc *rpc) 1534 { 1535 struct pci_dev *pdev = rpc->rpd; 1536 int aer = pdev->aer_cap; 1537 u16 reg16; 1538 u32 reg32; 1539 1540 /* Clear PCIe Capability's Device Status */ 1541 pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, ®16); 1542 pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16); 1543 1544 /* Disable system error generation in response to error messages */ 1545 pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, 1546 SYSTEM_ERROR_INTR_ON_MESG_MASK); 1547 1548 /* Clear error status of this Root Port or RCEC */ 1549 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1550 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1551 1552 /* Clear error status of agents reporting to this Root Port or RCEC */ 1553 if (reg32 & AER_ERR_STATUS_MASK) { 1554 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_EC) 1555 pcie_walk_rcec(pdev, clear_status_iter, NULL); 1556 else if (pdev->subordinate) 1557 pci_walk_bus(pdev->subordinate, clear_status_iter, 1558 NULL); 1559 } 1560 1561 pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32); 1562 pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32); 1563 pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32); 1564 pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32); 1565 1566 aer_enable_irq(pdev); 1567 } 1568 1569 /** 1570 * aer_disable_rootport - disable Root Port's interrupts when receiving messages 1571 * @rpc: pointer to a Root Port data structure 1572 * 1573 * Invoked when PCIe bus unloads AER service driver. 1574 */ 1575 static void aer_disable_rootport(struct aer_rpc *rpc) 1576 { 1577 struct pci_dev *pdev = rpc->rpd; 1578 int aer = pdev->aer_cap; 1579 u32 reg32; 1580 1581 aer_disable_irq(pdev); 1582 1583 /* Clear Root's error status reg */ 1584 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1585 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1586 } 1587 1588 /** 1589 * aer_remove - clean up resources 1590 * @dev: pointer to the pcie_dev data structure 1591 * 1592 * Invoked when PCI Express bus unloads or AER probe fails. 1593 */ 1594 static void aer_remove(struct pcie_device *dev) 1595 { 1596 struct aer_rpc *rpc = get_service_data(dev); 1597 1598 aer_disable_rootport(rpc); 1599 } 1600 1601 /** 1602 * aer_probe - initialize resources 1603 * @dev: pointer to the pcie_dev data structure 1604 * 1605 * Invoked when PCI Express bus loads AER service driver. 1606 */ 1607 static int aer_probe(struct pcie_device *dev) 1608 { 1609 int status; 1610 struct aer_rpc *rpc; 1611 struct device *device = &dev->device; 1612 struct pci_dev *port = dev->port; 1613 1614 BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) < 1615 AER_MAX_TYPEOF_COR_ERRS); 1616 BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) < 1617 AER_MAX_TYPEOF_UNCOR_ERRS); 1618 1619 /* Limit to Root Ports or Root Complex Event Collectors */ 1620 if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) && 1621 (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT)) 1622 return -ENODEV; 1623 1624 rpc = devm_kzalloc(device, sizeof(struct aer_rpc), GFP_KERNEL); 1625 if (!rpc) 1626 return -ENOMEM; 1627 1628 rpc->rpd = port; 1629 INIT_KFIFO(rpc->aer_fifo); 1630 set_service_data(dev, rpc); 1631 1632 status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr, 1633 IRQF_SHARED, "aerdrv", dev); 1634 if (status) { 1635 pci_err(port, "request AER IRQ %d failed\n", dev->irq); 1636 return status; 1637 } 1638 1639 cxl_rch_enable_rcec(port); 1640 aer_enable_rootport(rpc); 1641 pci_info(port, "enabled with IRQ %d\n", dev->irq); 1642 return 0; 1643 } 1644 1645 static int aer_suspend(struct pcie_device *dev) 1646 { 1647 struct aer_rpc *rpc = get_service_data(dev); 1648 1649 aer_disable_rootport(rpc); 1650 return 0; 1651 } 1652 1653 static int aer_resume(struct pcie_device *dev) 1654 { 1655 struct aer_rpc *rpc = get_service_data(dev); 1656 1657 aer_enable_rootport(rpc); 1658 return 0; 1659 } 1660 1661 /** 1662 * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP 1663 * @dev: pointer to Root Port, RCEC, or RCiEP 1664 * 1665 * Invoked by Port Bus driver when performing reset. 1666 */ 1667 static pci_ers_result_t aer_root_reset(struct pci_dev *dev) 1668 { 1669 int type = pci_pcie_type(dev); 1670 struct pci_dev *root; 1671 int aer; 1672 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 1673 u32 reg32; 1674 int rc; 1675 1676 /* 1677 * Only Root Ports and RCECs have AER Root Command and Root Status 1678 * registers. If "dev" is an RCiEP, the relevant registers are in 1679 * the RCEC. 1680 */ 1681 if (type == PCI_EXP_TYPE_RC_END) 1682 root = dev->rcec; 1683 else 1684 root = pcie_find_root_port(dev); 1685 1686 /* 1687 * If the platform retained control of AER, an RCiEP may not have 1688 * an RCEC visible to us, so dev->rcec ("root") may be NULL. In 1689 * that case, firmware is responsible for these registers. 1690 */ 1691 aer = root ? root->aer_cap : 0; 1692 1693 if ((host->native_aer || pcie_ports_native) && aer) 1694 aer_disable_irq(root); 1695 1696 if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { 1697 rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET); 1698 if (!rc) 1699 pci_info(dev, "has been reset\n"); 1700 else 1701 pci_info(dev, "not reset (no FLR support: %d)\n", rc); 1702 } else { 1703 rc = pci_bus_error_reset(dev); 1704 pci_info(dev, "%s Port link has been reset (%d)\n", 1705 pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc); 1706 } 1707 1708 if ((host->native_aer || pcie_ports_native) && aer) { 1709 /* Clear Root Error Status */ 1710 pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, ®32); 1711 pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32); 1712 1713 aer_enable_irq(root); 1714 } 1715 1716 return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 1717 } 1718 1719 static struct pcie_port_service_driver aerdriver = { 1720 .name = "aer", 1721 .port_type = PCIE_ANY_PORT, 1722 .service = PCIE_PORT_SERVICE_AER, 1723 1724 .probe = aer_probe, 1725 .suspend = aer_suspend, 1726 .resume = aer_resume, 1727 .remove = aer_remove, 1728 }; 1729 1730 /** 1731 * pcie_aer_init - register AER service driver 1732 * 1733 * Invoked when AER service driver is loaded. 1734 */ 1735 int __init pcie_aer_init(void) 1736 { 1737 if (!pci_aer_available()) 1738 return -ENXIO; 1739 return pcie_port_service_register(&aerdriver); 1740 } 1741