1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implement the AER root port service driver. The driver registers an IRQ 4 * handler. When a root port triggers an AER interrupt, the IRQ handler 5 * collects Root Port status and schedules work. 6 * 7 * Copyright (C) 2006 Intel Corp. 8 * Tom Long Nguyen (tom.l.nguyen@intel.com) 9 * Zhang Yanmin (yanmin.zhang@intel.com) 10 * 11 * (C) Copyright 2009 Hewlett-Packard Development Company, L.P. 12 * Andrew Patterson <andrew.patterson@hp.com> 13 */ 14 15 #define pr_fmt(fmt) "AER: " fmt 16 #define dev_fmt pr_fmt 17 18 #include <linux/bitops.h> 19 #include <linux/cper.h> 20 #include <linux/dev_printk.h> 21 #include <linux/pci.h> 22 #include <linux/pci-acpi.h> 23 #include <linux/sched.h> 24 #include <linux/kernel.h> 25 #include <linux/errno.h> 26 #include <linux/pm.h> 27 #include <linux/init.h> 28 #include <linux/interrupt.h> 29 #include <linux/delay.h> 30 #include <linux/kfifo.h> 31 #include <linux/ratelimit.h> 32 #include <linux/slab.h> 33 #include <linux/vmcore_info.h> 34 #include <acpi/apei.h> 35 #include <acpi/ghes.h> 36 #include <ras/ras_event.h> 37 38 #include "../pci.h" 39 #include "portdrv.h" 40 41 #define aer_printk(level, pdev, fmt, arg...) \ 42 dev_printk(level, &(pdev)->dev, fmt, ##arg) 43 44 #define AER_ERROR_SOURCES_MAX 128 45 46 #define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */ 47 #define AER_MAX_TYPEOF_UNCOR_ERRS 32 /* as per PCI_ERR_UNCOR_STATUS*/ 48 49 struct aer_err_source { 50 u32 status; /* PCI_ERR_ROOT_STATUS */ 51 u32 id; /* PCI_ERR_ROOT_ERR_SRC */ 52 }; 53 54 struct aer_rpc { 55 struct pci_dev *rpd; /* Root Port device */ 56 DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX); 57 }; 58 59 /* AER info for the device */ 60 struct aer_info { 61 62 /* 63 * Fields for all AER capable devices. They indicate the errors 64 * "as seen by this device". Note that this may mean that if an 65 * Endpoint is causing problems, the AER counters may increment 66 * at its link partner (e.g. Root Port) because the errors will be 67 * "seen" by the link partner and not the problematic Endpoint 68 * itself (which may report all counters as 0 as it never saw any 69 * problems). 70 */ 71 /* Counters for different type of correctable errors */ 72 u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS]; 73 /* Counters for different type of fatal uncorrectable errors */ 74 u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 75 /* Counters for different type of nonfatal uncorrectable errors */ 76 u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 77 /* Total number of ERR_COR sent by this device */ 78 u64 dev_total_cor_errs; 79 /* Total number of ERR_FATAL sent by this device */ 80 u64 dev_total_fatal_errs; 81 /* Total number of ERR_NONFATAL sent by this device */ 82 u64 dev_total_nonfatal_errs; 83 84 /* 85 * Fields for Root Ports & Root Complex Event Collectors only; these 86 * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL 87 * messages received by the Root Port / Event Collector, INCLUDING the 88 * ones that are generated internally (by the Root Port itself) 89 */ 90 u64 rootport_total_cor_errs; 91 u64 rootport_total_fatal_errs; 92 u64 rootport_total_nonfatal_errs; 93 94 /* Ratelimits for errors */ 95 struct ratelimit_state correctable_ratelimit; 96 struct ratelimit_state nonfatal_ratelimit; 97 }; 98 99 #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \ 100 PCI_ERR_UNC_POISON_BLK | \ 101 PCI_ERR_UNC_ECRC| \ 102 PCI_ERR_UNC_UNSUP| \ 103 PCI_ERR_UNC_COMP_ABORT| \ 104 PCI_ERR_UNC_UNX_COMP| \ 105 PCI_ERR_UNC_ACSV | \ 106 PCI_ERR_UNC_MCBTLP | \ 107 PCI_ERR_UNC_ATOMEG | \ 108 PCI_ERR_UNC_DMWR_BLK | \ 109 PCI_ERR_UNC_XLAT_BLK | \ 110 PCI_ERR_UNC_TLPPRE | \ 111 PCI_ERR_UNC_MALF_TLP | \ 112 PCI_ERR_UNC_IDE_CHECK | \ 113 PCI_ERR_UNC_MISR_IDE | \ 114 PCI_ERR_UNC_PCRC_CHECK) 115 116 #define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \ 117 PCI_EXP_RTCTL_SENFEE| \ 118 PCI_EXP_RTCTL_SEFEE) 119 #define ROOT_PORT_INTR_ON_MESG_MASK (PCI_ERR_ROOT_CMD_COR_EN| \ 120 PCI_ERR_ROOT_CMD_NONFATAL_EN| \ 121 PCI_ERR_ROOT_CMD_FATAL_EN) 122 #define ERR_COR_ID(d) (d & 0xffff) 123 #define ERR_UNCOR_ID(d) (d >> 16) 124 125 #define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV | \ 126 PCI_ERR_ROOT_COR_RCV | \ 127 PCI_ERR_ROOT_MULTI_COR_RCV | \ 128 PCI_ERR_ROOT_MULTI_UNCOR_RCV) 129 130 static bool pcie_aer_disable; 131 static pci_ers_result_t aer_root_reset(struct pci_dev *dev); 132 133 void pci_no_aer(void) 134 { 135 pcie_aer_disable = true; 136 } 137 138 bool pci_aer_available(void) 139 { 140 return !pcie_aer_disable && pci_msi_enabled(); 141 } 142 143 #ifdef CONFIG_PCIE_ECRC 144 145 #define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */ 146 #define ECRC_POLICY_OFF 1 /* ECRC off for performance */ 147 #define ECRC_POLICY_ON 2 /* ECRC on for data integrity */ 148 149 static int ecrc_policy = ECRC_POLICY_DEFAULT; 150 151 static const char * const ecrc_policy_str[] = { 152 [ECRC_POLICY_DEFAULT] = "bios", 153 [ECRC_POLICY_OFF] = "off", 154 [ECRC_POLICY_ON] = "on" 155 }; 156 157 /** 158 * enable_ecrc_checking - enable PCIe ECRC checking for a device 159 * @dev: the PCI device 160 * 161 * Return: 0 on success, or negative on failure. 162 */ 163 static int enable_ecrc_checking(struct pci_dev *dev) 164 { 165 int aer = dev->aer_cap; 166 u32 reg32; 167 168 if (!aer) 169 return -ENODEV; 170 171 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 172 if (reg32 & PCI_ERR_CAP_ECRC_GENC) 173 reg32 |= PCI_ERR_CAP_ECRC_GENE; 174 if (reg32 & PCI_ERR_CAP_ECRC_CHKC) 175 reg32 |= PCI_ERR_CAP_ECRC_CHKE; 176 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 177 178 return 0; 179 } 180 181 /** 182 * disable_ecrc_checking - disable PCIe ECRC checking for a device 183 * @dev: the PCI device 184 * 185 * Return: 0 on success, or negative on failure. 186 */ 187 static int disable_ecrc_checking(struct pci_dev *dev) 188 { 189 int aer = dev->aer_cap; 190 u32 reg32; 191 192 if (!aer) 193 return -ENODEV; 194 195 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 196 reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE); 197 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 198 199 return 0; 200 } 201 202 /** 203 * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based 204 * on global policy 205 * @dev: the PCI device 206 */ 207 void pcie_set_ecrc_checking(struct pci_dev *dev) 208 { 209 if (!pcie_aer_is_native(dev)) 210 return; 211 212 switch (ecrc_policy) { 213 case ECRC_POLICY_DEFAULT: 214 return; 215 case ECRC_POLICY_OFF: 216 disable_ecrc_checking(dev); 217 break; 218 case ECRC_POLICY_ON: 219 enable_ecrc_checking(dev); 220 break; 221 default: 222 return; 223 } 224 } 225 226 /** 227 * pcie_ecrc_get_policy - parse kernel command-line ecrc option 228 * @str: ECRC policy from kernel command line to use 229 */ 230 void pcie_ecrc_get_policy(char *str) 231 { 232 int i; 233 234 i = match_string(ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), str); 235 if (i < 0) 236 return; 237 238 ecrc_policy = i; 239 } 240 #endif /* CONFIG_PCIE_ECRC */ 241 242 int pcie_aer_is_native(struct pci_dev *dev) 243 { 244 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 245 246 if (!dev->aer_cap) 247 return 0; 248 249 return pcie_ports_native || host->native_aer; 250 } 251 EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, "CXL"); 252 253 static int pci_enable_pcie_error_reporting(struct pci_dev *dev) 254 { 255 int rc; 256 257 if (!pcie_aer_is_native(dev)) 258 return -EIO; 259 260 rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS); 261 return pcibios_err_to_errno(rc); 262 } 263 264 int pci_aer_clear_nonfatal_status(struct pci_dev *dev) 265 { 266 int aer = dev->aer_cap; 267 u32 status, sev; 268 269 if (!pcie_aer_is_native(dev)) 270 return -EIO; 271 272 /* Clear status bits for ERR_NONFATAL errors only */ 273 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 274 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 275 status &= ~sev; 276 if (status) 277 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 278 279 return 0; 280 } 281 EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status); 282 283 void pci_aer_clear_fatal_status(struct pci_dev *dev) 284 { 285 int aer = dev->aer_cap; 286 u32 status, sev; 287 288 if (!pcie_aer_is_native(dev)) 289 return; 290 291 /* Clear status bits for ERR_FATAL errors only */ 292 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 293 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 294 status &= sev; 295 if (status) 296 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 297 } 298 299 /** 300 * pci_aer_raw_clear_status - Clear AER error registers. 301 * @dev: the PCI device 302 * 303 * Clear AER error status registers unconditionally, regardless of 304 * whether they're owned by firmware or the OS. 305 * 306 * Return: 0 on success, or negative on failure. 307 */ 308 int pci_aer_raw_clear_status(struct pci_dev *dev) 309 { 310 int aer = dev->aer_cap; 311 u32 status; 312 int port_type; 313 314 if (!aer) 315 return -EIO; 316 317 port_type = pci_pcie_type(dev); 318 if (port_type == PCI_EXP_TYPE_ROOT_PORT || 319 port_type == PCI_EXP_TYPE_RC_EC) { 320 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status); 321 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status); 322 } 323 324 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 325 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status); 326 327 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 328 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 329 330 return 0; 331 } 332 333 int pci_aer_clear_status(struct pci_dev *dev) 334 { 335 if (!pcie_aer_is_native(dev)) 336 return -EIO; 337 338 return pci_aer_raw_clear_status(dev); 339 } 340 341 void pci_save_aer_state(struct pci_dev *dev) 342 { 343 int aer = dev->aer_cap; 344 struct pci_cap_saved_state *save_state; 345 u32 *cap; 346 347 if (!aer) 348 return; 349 350 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 351 if (!save_state) 352 return; 353 354 cap = &save_state->cap.data[0]; 355 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++); 356 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++); 357 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++); 358 pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++); 359 if (pcie_cap_has_rtctl(dev)) 360 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++); 361 } 362 363 void pci_restore_aer_state(struct pci_dev *dev) 364 { 365 int aer = dev->aer_cap; 366 struct pci_cap_saved_state *save_state; 367 u32 *cap; 368 369 if (!aer) 370 return; 371 372 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 373 if (!save_state) 374 return; 375 376 cap = &save_state->cap.data[0]; 377 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++); 378 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++); 379 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++); 380 pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++); 381 if (pcie_cap_has_rtctl(dev)) 382 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++); 383 } 384 385 void pci_aer_init(struct pci_dev *dev) 386 { 387 int n; 388 389 dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); 390 if (!dev->aer_cap) 391 return; 392 393 dev->aer_info = kzalloc(sizeof(*dev->aer_info), GFP_KERNEL); 394 if (!dev->aer_info) { 395 dev->aer_cap = 0; 396 return; 397 } 398 399 ratelimit_state_init(&dev->aer_info->correctable_ratelimit, 400 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 401 ratelimit_state_init(&dev->aer_info->nonfatal_ratelimit, 402 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 403 404 /* 405 * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER, 406 * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event 407 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r6.0, sec 408 * 7.8.4.9). 409 */ 410 n = pcie_cap_has_rtctl(dev) ? 5 : 4; 411 pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, sizeof(u32) * n); 412 413 pci_aer_clear_status(dev); 414 415 if (pci_aer_available()) 416 pci_enable_pcie_error_reporting(dev); 417 418 pcie_set_ecrc_checking(dev); 419 } 420 421 void pci_aer_exit(struct pci_dev *dev) 422 { 423 kfree(dev->aer_info); 424 dev->aer_info = NULL; 425 } 426 427 #define AER_AGENT_RECEIVER 0 428 #define AER_AGENT_REQUESTER 1 429 #define AER_AGENT_COMPLETER 2 430 #define AER_AGENT_TRANSMITTER 3 431 432 #define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 433 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP)) 434 #define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \ 435 0 : PCI_ERR_UNC_COMP_ABORT) 436 #define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 437 (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0) 438 439 #define AER_GET_AGENT(t, e) \ 440 ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \ 441 (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \ 442 (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \ 443 AER_AGENT_RECEIVER) 444 445 #define AER_PHYSICAL_LAYER_ERROR 0 446 #define AER_DATA_LINK_LAYER_ERROR 1 447 #define AER_TRANSACTION_LAYER_ERROR 2 448 449 #define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 450 PCI_ERR_COR_RCVR : 0) 451 #define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 452 (PCI_ERR_COR_BAD_TLP| \ 453 PCI_ERR_COR_BAD_DLLP| \ 454 PCI_ERR_COR_REP_ROLL| \ 455 PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP) 456 457 #define AER_GET_LAYER_ERROR(t, e) \ 458 ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \ 459 (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ 460 AER_TRANSACTION_LAYER_ERROR) 461 462 /* 463 * AER error strings 464 */ 465 static const char * const aer_error_severity_string[] = { 466 "Uncorrectable (Non-Fatal)", 467 "Uncorrectable (Fatal)", 468 "Correctable" 469 }; 470 471 static const char *aer_error_layer[] = { 472 "Physical Layer", 473 "Data Link Layer", 474 "Transaction Layer" 475 }; 476 477 static const char *aer_correctable_error_string[] = { 478 "RxErr", /* Bit Position 0 */ 479 NULL, 480 NULL, 481 NULL, 482 NULL, 483 NULL, 484 "BadTLP", /* Bit Position 6 */ 485 "BadDLLP", /* Bit Position 7 */ 486 "Rollover", /* Bit Position 8 */ 487 NULL, 488 NULL, 489 NULL, 490 "Timeout", /* Bit Position 12 */ 491 "NonFatalErr", /* Bit Position 13 */ 492 "CorrIntErr", /* Bit Position 14 */ 493 "HeaderOF", /* Bit Position 15 */ 494 NULL, /* Bit Position 16 */ 495 NULL, /* Bit Position 17 */ 496 NULL, /* Bit Position 18 */ 497 NULL, /* Bit Position 19 */ 498 NULL, /* Bit Position 20 */ 499 NULL, /* Bit Position 21 */ 500 NULL, /* Bit Position 22 */ 501 NULL, /* Bit Position 23 */ 502 NULL, /* Bit Position 24 */ 503 NULL, /* Bit Position 25 */ 504 NULL, /* Bit Position 26 */ 505 NULL, /* Bit Position 27 */ 506 NULL, /* Bit Position 28 */ 507 NULL, /* Bit Position 29 */ 508 NULL, /* Bit Position 30 */ 509 NULL, /* Bit Position 31 */ 510 }; 511 512 static const char *aer_uncorrectable_error_string[] = { 513 "Undefined", /* Bit Position 0 */ 514 NULL, 515 NULL, 516 NULL, 517 "DLP", /* Bit Position 4 */ 518 "SDES", /* Bit Position 5 */ 519 NULL, 520 NULL, 521 NULL, 522 NULL, 523 NULL, 524 NULL, 525 "TLP", /* Bit Position 12 */ 526 "FCP", /* Bit Position 13 */ 527 "CmpltTO", /* Bit Position 14 */ 528 "CmpltAbrt", /* Bit Position 15 */ 529 "UnxCmplt", /* Bit Position 16 */ 530 "RxOF", /* Bit Position 17 */ 531 "MalfTLP", /* Bit Position 18 */ 532 "ECRC", /* Bit Position 19 */ 533 "UnsupReq", /* Bit Position 20 */ 534 "ACSViol", /* Bit Position 21 */ 535 "UncorrIntErr", /* Bit Position 22 */ 536 "BlockedTLP", /* Bit Position 23 */ 537 "AtomicOpBlocked", /* Bit Position 24 */ 538 "TLPBlockedErr", /* Bit Position 25 */ 539 "PoisonTLPBlocked", /* Bit Position 26 */ 540 "DMWrReqBlocked", /* Bit Position 27 */ 541 "IDECheck", /* Bit Position 28 */ 542 "MisIDETLP", /* Bit Position 29 */ 543 "PCRC_CHECK", /* Bit Position 30 */ 544 "TLPXlatBlocked", /* Bit Position 31 */ 545 }; 546 547 static const char *aer_agent_string[] = { 548 "Receiver ID", 549 "Requester ID", 550 "Completer ID", 551 "Transmitter ID" 552 }; 553 554 #define aer_stats_dev_attr(name, stats_array, strings_array, \ 555 total_string, total_field) \ 556 static ssize_t \ 557 name##_show(struct device *dev, struct device_attribute *attr, \ 558 char *buf) \ 559 { \ 560 unsigned int i; \ 561 struct pci_dev *pdev = to_pci_dev(dev); \ 562 u64 *stats = pdev->aer_info->stats_array; \ 563 size_t len = 0; \ 564 \ 565 for (i = 0; i < ARRAY_SIZE(pdev->aer_info->stats_array); i++) { \ 566 if (strings_array[i]) \ 567 len += sysfs_emit_at(buf, len, "%s %llu\n", \ 568 strings_array[i], \ 569 stats[i]); \ 570 else if (stats[i]) \ 571 len += sysfs_emit_at(buf, len, \ 572 #stats_array "_bit[%d] %llu\n",\ 573 i, stats[i]); \ 574 } \ 575 len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \ 576 pdev->aer_info->total_field); \ 577 return len; \ 578 } \ 579 static DEVICE_ATTR_RO(name) 580 581 aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs, 582 aer_correctable_error_string, "ERR_COR", 583 dev_total_cor_errs); 584 aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs, 585 aer_uncorrectable_error_string, "ERR_FATAL", 586 dev_total_fatal_errs); 587 aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs, 588 aer_uncorrectable_error_string, "ERR_NONFATAL", 589 dev_total_nonfatal_errs); 590 591 #define aer_stats_rootport_attr(name, field) \ 592 static ssize_t \ 593 name##_show(struct device *dev, struct device_attribute *attr, \ 594 char *buf) \ 595 { \ 596 struct pci_dev *pdev = to_pci_dev(dev); \ 597 return sysfs_emit(buf, "%llu\n", pdev->aer_info->field); \ 598 } \ 599 static DEVICE_ATTR_RO(name) 600 601 aer_stats_rootport_attr(aer_rootport_total_err_cor, 602 rootport_total_cor_errs); 603 aer_stats_rootport_attr(aer_rootport_total_err_fatal, 604 rootport_total_fatal_errs); 605 aer_stats_rootport_attr(aer_rootport_total_err_nonfatal, 606 rootport_total_nonfatal_errs); 607 608 static struct attribute *aer_stats_attrs[] __ro_after_init = { 609 &dev_attr_aer_dev_correctable.attr, 610 &dev_attr_aer_dev_fatal.attr, 611 &dev_attr_aer_dev_nonfatal.attr, 612 &dev_attr_aer_rootport_total_err_cor.attr, 613 &dev_attr_aer_rootport_total_err_fatal.attr, 614 &dev_attr_aer_rootport_total_err_nonfatal.attr, 615 NULL 616 }; 617 618 static umode_t aer_stats_attrs_are_visible(struct kobject *kobj, 619 struct attribute *a, int n) 620 { 621 struct device *dev = kobj_to_dev(kobj); 622 struct pci_dev *pdev = to_pci_dev(dev); 623 624 if (!pdev->aer_info) 625 return 0; 626 627 if ((a == &dev_attr_aer_rootport_total_err_cor.attr || 628 a == &dev_attr_aer_rootport_total_err_fatal.attr || 629 a == &dev_attr_aer_rootport_total_err_nonfatal.attr) && 630 ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && 631 (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_EC))) 632 return 0; 633 634 return a->mode; 635 } 636 637 const struct attribute_group aer_stats_attr_group = { 638 .attrs = aer_stats_attrs, 639 .is_visible = aer_stats_attrs_are_visible, 640 }; 641 642 /* 643 * Ratelimit interval 644 * <=0: disabled with ratelimit.interval = 0 645 * >0: enabled with ratelimit.interval in ms 646 */ 647 #define aer_ratelimit_interval_attr(name, ratelimit) \ 648 static ssize_t \ 649 name##_show(struct device *dev, struct device_attribute *attr, \ 650 char *buf) \ 651 { \ 652 struct pci_dev *pdev = to_pci_dev(dev); \ 653 \ 654 return sysfs_emit(buf, "%d\n", \ 655 pdev->aer_info->ratelimit.interval); \ 656 } \ 657 \ 658 static ssize_t \ 659 name##_store(struct device *dev, struct device_attribute *attr, \ 660 const char *buf, size_t count) \ 661 { \ 662 struct pci_dev *pdev = to_pci_dev(dev); \ 663 int interval; \ 664 \ 665 if (!capable(CAP_SYS_ADMIN)) \ 666 return -EPERM; \ 667 \ 668 if (kstrtoint(buf, 0, &interval) < 0) \ 669 return -EINVAL; \ 670 \ 671 if (interval <= 0) \ 672 interval = 0; \ 673 else \ 674 interval = msecs_to_jiffies(interval); \ 675 \ 676 pdev->aer_info->ratelimit.interval = interval; \ 677 \ 678 return count; \ 679 } \ 680 static DEVICE_ATTR_RW(name); 681 682 #define aer_ratelimit_burst_attr(name, ratelimit) \ 683 static ssize_t \ 684 name##_show(struct device *dev, struct device_attribute *attr, \ 685 char *buf) \ 686 { \ 687 struct pci_dev *pdev = to_pci_dev(dev); \ 688 \ 689 return sysfs_emit(buf, "%d\n", \ 690 pdev->aer_info->ratelimit.burst); \ 691 } \ 692 \ 693 static ssize_t \ 694 name##_store(struct device *dev, struct device_attribute *attr, \ 695 const char *buf, size_t count) \ 696 { \ 697 struct pci_dev *pdev = to_pci_dev(dev); \ 698 int burst; \ 699 \ 700 if (!capable(CAP_SYS_ADMIN)) \ 701 return -EPERM; \ 702 \ 703 if (kstrtoint(buf, 0, &burst) < 0) \ 704 return -EINVAL; \ 705 \ 706 pdev->aer_info->ratelimit.burst = burst; \ 707 \ 708 return count; \ 709 } \ 710 static DEVICE_ATTR_RW(name); 711 712 #define aer_ratelimit_attrs(name) \ 713 aer_ratelimit_interval_attr(name##_ratelimit_interval_ms, \ 714 name##_ratelimit) \ 715 aer_ratelimit_burst_attr(name##_ratelimit_burst, \ 716 name##_ratelimit) 717 718 aer_ratelimit_attrs(correctable) 719 aer_ratelimit_attrs(nonfatal) 720 721 static struct attribute *aer_attrs[] = { 722 &dev_attr_correctable_ratelimit_interval_ms.attr, 723 &dev_attr_correctable_ratelimit_burst.attr, 724 &dev_attr_nonfatal_ratelimit_interval_ms.attr, 725 &dev_attr_nonfatal_ratelimit_burst.attr, 726 NULL 727 }; 728 729 static umode_t aer_attrs_are_visible(struct kobject *kobj, 730 struct attribute *a, int n) 731 { 732 struct device *dev = kobj_to_dev(kobj); 733 struct pci_dev *pdev = to_pci_dev(dev); 734 735 if (!pdev->aer_info) 736 return 0; 737 738 return a->mode; 739 } 740 741 const struct attribute_group aer_attr_group = { 742 .name = "aer", 743 .attrs = aer_attrs, 744 .is_visible = aer_attrs_are_visible, 745 }; 746 747 static void pci_dev_aer_stats_incr(struct pci_dev *pdev, 748 struct aer_err_info *info) 749 { 750 unsigned long status = info->status & ~info->mask; 751 int i, max = -1; 752 u64 *counter = NULL; 753 struct aer_info *aer_info = pdev->aer_info; 754 755 if (!aer_info) 756 return; 757 758 switch (info->severity) { 759 case AER_CORRECTABLE: 760 aer_info->dev_total_cor_errs++; 761 counter = &aer_info->dev_cor_errs[0]; 762 max = AER_MAX_TYPEOF_COR_ERRS; 763 break; 764 case AER_NONFATAL: 765 aer_info->dev_total_nonfatal_errs++; 766 hwerr_log_error_type(HWERR_RECOV_PCI); 767 counter = &aer_info->dev_nonfatal_errs[0]; 768 max = AER_MAX_TYPEOF_UNCOR_ERRS; 769 break; 770 case AER_FATAL: 771 aer_info->dev_total_fatal_errs++; 772 counter = &aer_info->dev_fatal_errs[0]; 773 max = AER_MAX_TYPEOF_UNCOR_ERRS; 774 break; 775 } 776 777 for_each_set_bit(i, &status, max) 778 counter[i]++; 779 } 780 781 static void pci_rootport_aer_stats_incr(struct pci_dev *pdev, 782 struct aer_err_source *e_src) 783 { 784 struct aer_info *aer_info = pdev->aer_info; 785 786 if (!aer_info) 787 return; 788 789 if (e_src->status & PCI_ERR_ROOT_COR_RCV) 790 aer_info->rootport_total_cor_errs++; 791 792 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { 793 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) 794 aer_info->rootport_total_fatal_errs++; 795 else 796 aer_info->rootport_total_nonfatal_errs++; 797 } 798 } 799 800 static int aer_ratelimit(struct pci_dev *dev, unsigned int severity) 801 { 802 if (!dev->aer_info) 803 return 1; 804 805 switch (severity) { 806 case AER_NONFATAL: 807 return __ratelimit(&dev->aer_info->nonfatal_ratelimit); 808 case AER_CORRECTABLE: 809 return __ratelimit(&dev->aer_info->correctable_ratelimit); 810 default: 811 return 1; /* Don't ratelimit fatal errors */ 812 } 813 } 814 815 static bool tlp_header_logged(u32 status, u32 capctl) 816 { 817 /* Errors for which a header is always logged (PCIe r7.0 sec 6.2.7) */ 818 if (status & AER_LOG_TLP_MASKS) 819 return true; 820 821 /* Completion Timeout header is only logged on capable devices */ 822 if (status & PCI_ERR_UNC_COMP_TIME && 823 capctl & PCI_ERR_CAP_COMP_TIME_LOG) 824 return true; 825 826 return false; 827 } 828 829 static void __aer_print_error(struct pci_dev *dev, struct aer_err_info *info) 830 { 831 const char **strings; 832 unsigned long status = info->status & ~info->mask; 833 const char *level = info->level; 834 const char *errmsg; 835 int i; 836 837 if (info->severity == AER_CORRECTABLE) 838 strings = aer_correctable_error_string; 839 else 840 strings = aer_uncorrectable_error_string; 841 842 for_each_set_bit(i, &status, 32) { 843 errmsg = strings[i]; 844 if (!errmsg) 845 errmsg = "Unknown Error Bit"; 846 847 aer_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg, 848 info->first_error == i ? " (First)" : ""); 849 } 850 } 851 852 static void aer_print_source(struct pci_dev *dev, struct aer_err_info *info, 853 bool found) 854 { 855 u16 source = info->id; 856 857 pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d%s\n", 858 info->multi_error_valid ? "Multiple " : "", 859 aer_error_severity_string[info->severity], 860 pci_domain_nr(dev->bus), PCI_BUS_NUM(source), 861 PCI_SLOT(source), PCI_FUNC(source), 862 found ? "" : " (no details found"); 863 } 864 865 void aer_print_error(struct aer_err_info *info, int i) 866 { 867 struct pci_dev *dev; 868 int layer, agent, id; 869 const char *level = info->level; 870 const char *bus_type = aer_err_bus(info); 871 872 if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES)) 873 return; 874 875 dev = info->dev[i]; 876 id = pci_dev_id(dev); 877 878 pci_dev_aer_stats_incr(dev, info); 879 trace_aer_event(pci_name(dev), (info->status & ~info->mask), 880 info->severity, info->tlp_header_valid, &info->tlp, bus_type); 881 882 if (!info->ratelimit_print[i]) 883 return; 884 885 if (!info->status) { 886 pci_err(dev, "%s Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", 887 bus_type, aer_error_severity_string[info->severity]); 888 goto out; 889 } 890 891 layer = AER_GET_LAYER_ERROR(info->severity, info->status); 892 agent = AER_GET_AGENT(info->severity, info->status); 893 894 aer_printk(level, dev, "%s Bus Error: severity=%s, type=%s, (%s)\n", 895 bus_type, aer_error_severity_string[info->severity], 896 aer_error_layer[layer], aer_agent_string[agent]); 897 898 aer_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n", 899 dev->vendor, dev->device, info->status, info->mask); 900 901 __aer_print_error(dev, info); 902 903 if (info->tlp_header_valid) 904 pcie_print_tlp_log(dev, &info->tlp, level, dev_fmt(" ")); 905 906 out: 907 if (info->id && info->error_dev_num > 1 && info->id == id) 908 pci_err(dev, " Error of this Agent is reported first\n"); 909 } 910 911 #ifdef CONFIG_ACPI_APEI_PCIEAER 912 int cper_severity_to_aer(int cper_severity) 913 { 914 switch (cper_severity) { 915 case CPER_SEV_RECOVERABLE: 916 return AER_NONFATAL; 917 case CPER_SEV_FATAL: 918 return AER_FATAL; 919 default: 920 return AER_CORRECTABLE; 921 } 922 } 923 EXPORT_SYMBOL_GPL(cper_severity_to_aer); 924 #endif 925 926 void pci_print_aer(struct pci_dev *dev, int aer_severity, 927 struct aer_capability_regs *aer) 928 { 929 const char *bus_type; 930 int layer, agent, tlp_header_valid = 0; 931 u32 status, mask; 932 struct aer_err_info info = { 933 .severity = aer_severity, 934 .first_error = PCI_ERR_CAP_FEP(aer->cap_control), 935 }; 936 937 if (aer_severity == AER_CORRECTABLE) { 938 status = aer->cor_status; 939 mask = aer->cor_mask; 940 info.level = KERN_WARNING; 941 } else { 942 status = aer->uncor_status; 943 mask = aer->uncor_mask; 944 info.level = KERN_ERR; 945 tlp_header_valid = tlp_header_logged(status, aer->cap_control); 946 } 947 948 info.status = status; 949 info.mask = mask; 950 info.is_cxl = pcie_is_cxl(dev); 951 952 bus_type = aer_err_bus(&info); 953 954 pci_dev_aer_stats_incr(dev, &info); 955 trace_aer_event(pci_name(dev), (status & ~mask), aer_severity, 956 tlp_header_valid, &aer->header_log, bus_type); 957 958 if (!aer_ratelimit(dev, info.severity)) 959 return; 960 961 layer = AER_GET_LAYER_ERROR(aer_severity, status); 962 agent = AER_GET_AGENT(aer_severity, status); 963 964 aer_printk(info.level, dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", 965 status, mask); 966 __aer_print_error(dev, &info); 967 aer_printk(info.level, dev, "aer_layer=%s, aer_agent=%s\n", 968 aer_error_layer[layer], aer_agent_string[agent]); 969 970 if (aer_severity != AER_CORRECTABLE) 971 aer_printk(info.level, dev, "aer_uncor_severity: 0x%08x\n", 972 aer->uncor_severity); 973 974 if (tlp_header_valid) 975 pcie_print_tlp_log(dev, &aer->header_log, info.level, 976 dev_fmt(" ")); 977 } 978 EXPORT_SYMBOL_GPL(pci_print_aer); 979 980 /** 981 * add_error_device - list device to be handled 982 * @e_info: pointer to error info 983 * @dev: pointer to pci_dev to be added 984 */ 985 static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) 986 { 987 int i = e_info->error_dev_num; 988 989 if (i >= AER_MAX_MULTI_ERR_DEVICES) 990 return -ENOSPC; 991 992 e_info->dev[i] = pci_dev_get(dev); 993 e_info->error_dev_num++; 994 995 /* 996 * Ratelimit AER log messages. "dev" is either the source 997 * identified by the root's Error Source ID or it has an unmasked 998 * error logged in its own AER Capability. Messages are emitted 999 * when "ratelimit_print[i]" is non-zero. If we will print detail 1000 * for a downstream device, make sure we print the Error Source ID 1001 * from the root as well. 1002 */ 1003 if (aer_ratelimit(dev, e_info->severity)) { 1004 e_info->ratelimit_print[i] = 1; 1005 e_info->root_ratelimit_print = 1; 1006 } 1007 return 0; 1008 } 1009 1010 /** 1011 * is_error_source - check whether the device is source of reported error 1012 * @dev: pointer to pci_dev to be checked 1013 * @e_info: pointer to reported error info 1014 */ 1015 static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info) 1016 { 1017 int aer = dev->aer_cap; 1018 u32 status, mask; 1019 u16 reg16; 1020 1021 /* 1022 * When bus ID is equal to 0, it might be a bad ID 1023 * reported by Root Port. 1024 */ 1025 if ((PCI_BUS_NUM(e_info->id) != 0) && 1026 !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) { 1027 /* Device ID match? */ 1028 if (e_info->id == pci_dev_id(dev)) 1029 return true; 1030 1031 /* Continue ID comparing if there is no multiple error */ 1032 if (!e_info->multi_error_valid) 1033 return false; 1034 } 1035 1036 /* 1037 * When either 1038 * 1) bus ID is equal to 0. Some ports might lose the bus 1039 * ID of error source id; 1040 * 2) bus flag PCI_BUS_FLAGS_NO_AERSID is set 1041 * 3) There are multiple errors and prior ID comparing fails; 1042 * We check AER status registers to find possible reporter. 1043 */ 1044 if (atomic_read(&dev->enable_cnt) == 0) 1045 return false; 1046 1047 /* Check if AER is enabled */ 1048 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16); 1049 if (!(reg16 & PCI_EXP_AER_FLAGS)) 1050 return false; 1051 1052 if (!aer) 1053 return false; 1054 1055 /* Check if error is recorded */ 1056 if (e_info->severity == AER_CORRECTABLE) { 1057 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 1058 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1059 } else { 1060 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 1061 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1062 } 1063 if (status & ~mask) 1064 return true; 1065 1066 return false; 1067 } 1068 1069 static int find_device_iter(struct pci_dev *dev, void *data) 1070 { 1071 struct aer_err_info *e_info = (struct aer_err_info *)data; 1072 1073 if (is_error_source(dev, e_info)) { 1074 /* List this device */ 1075 if (add_error_device(e_info, dev)) { 1076 /* We cannot handle more... Stop iteration */ 1077 pci_err(dev, "Exceeded max supported (%d) devices with errors logged\n", 1078 AER_MAX_MULTI_ERR_DEVICES); 1079 return 1; 1080 } 1081 1082 /* If there is only a single error, stop iteration */ 1083 if (!e_info->multi_error_valid) 1084 return 1; 1085 } 1086 return 0; 1087 } 1088 1089 /** 1090 * find_source_device - search through device hierarchy for source device 1091 * @parent: pointer to Root Port pci_dev data structure 1092 * @e_info: including detailed error information such as ID 1093 * 1094 * Return: true if found. 1095 * 1096 * Invoked by DPC when error is detected at the Root Port. 1097 * Caller of this function must set id, severity, and multi_error_valid of 1098 * struct aer_err_info pointed by @e_info properly. This function must fill 1099 * e_info->error_dev_num and e_info->dev[], based on the given information. 1100 */ 1101 static bool find_source_device(struct pci_dev *parent, 1102 struct aer_err_info *e_info) 1103 { 1104 struct pci_dev *dev = parent; 1105 int result; 1106 1107 /* Must reset in this function */ 1108 e_info->error_dev_num = 0; 1109 1110 /* Is Root Port an agent that sends error message? */ 1111 result = find_device_iter(dev, e_info); 1112 if (result) 1113 return true; 1114 1115 if (pci_pcie_type(parent) == PCI_EXP_TYPE_RC_EC) 1116 pcie_walk_rcec(parent, find_device_iter, e_info); 1117 else 1118 pci_walk_bus(parent->subordinate, find_device_iter, e_info); 1119 1120 if (!e_info->error_dev_num) 1121 return false; 1122 return true; 1123 } 1124 1125 /** 1126 * pci_aer_unmask_internal_errors - unmask internal errors 1127 * @dev: pointer to the pci_dev data structure 1128 * 1129 * Unmask internal errors in the Uncorrectable and Correctable Error 1130 * Mask registers. 1131 * 1132 * Note: AER must be enabled and supported by the device which must be 1133 * checked in advance, e.g. with pcie_aer_is_native(). 1134 */ 1135 void pci_aer_unmask_internal_errors(struct pci_dev *dev) 1136 { 1137 int aer = dev->aer_cap; 1138 u32 mask; 1139 1140 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1141 mask &= ~PCI_ERR_UNC_INTN; 1142 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask); 1143 1144 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1145 mask &= ~PCI_ERR_COR_INTERNAL; 1146 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask); 1147 } 1148 1149 /* 1150 * Internal errors are too device-specific to enable generally, however for CXL 1151 * their behavior is standardized for conveying CXL protocol errors. 1152 */ 1153 EXPORT_SYMBOL_FOR_MODULES(pci_aer_unmask_internal_errors, "cxl_core"); 1154 1155 #ifdef CONFIG_CXL_RAS 1156 bool is_aer_internal_error(struct aer_err_info *info) 1157 { 1158 if (info->severity == AER_CORRECTABLE) 1159 return info->status & PCI_ERR_COR_INTERNAL; 1160 1161 return info->status & PCI_ERR_UNC_INTN; 1162 } 1163 #endif 1164 1165 /** 1166 * pci_aer_handle_error - handle logging error into an event log 1167 * @dev: pointer to pci_dev data structure of error source device 1168 * @info: comprehensive error information 1169 * 1170 * Invoked when an error being detected by Root Port. 1171 */ 1172 static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info) 1173 { 1174 int aer = dev->aer_cap; 1175 1176 if (info->severity == AER_CORRECTABLE) { 1177 /* 1178 * Correctable error does not need software intervention. 1179 * No need to go through error recovery process. 1180 */ 1181 if (aer) 1182 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1183 info->status); 1184 if (pcie_aer_is_native(dev)) { 1185 struct pci_driver *pdrv = dev->driver; 1186 1187 if (pdrv && pdrv->err_handler && 1188 pdrv->err_handler->cor_error_detected) 1189 pdrv->err_handler->cor_error_detected(dev); 1190 pcie_clear_device_status(dev); 1191 } 1192 } else if (info->severity == AER_NONFATAL) 1193 pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset); 1194 else if (info->severity == AER_FATAL) 1195 pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset); 1196 } 1197 1198 static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) 1199 { 1200 cxl_rch_handle_error(dev, info); 1201 pci_aer_handle_error(dev, info); 1202 pci_dev_put(dev); 1203 } 1204 1205 #ifdef CONFIG_ACPI_APEI_PCIEAER 1206 1207 #define AER_RECOVER_RING_SIZE 16 1208 1209 struct aer_recover_entry { 1210 u8 bus; 1211 u8 devfn; 1212 u16 domain; 1213 int severity; 1214 struct aer_capability_regs *regs; 1215 }; 1216 1217 static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry, 1218 AER_RECOVER_RING_SIZE); 1219 1220 static void aer_recover_work_func(struct work_struct *work) 1221 { 1222 struct aer_recover_entry entry; 1223 struct pci_dev *pdev; 1224 1225 while (kfifo_get(&aer_recover_ring, &entry)) { 1226 pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, 1227 entry.devfn); 1228 if (!pdev) { 1229 pr_err_ratelimited("%04x:%02x:%02x.%x: no pci_dev found\n", 1230 entry.domain, entry.bus, 1231 PCI_SLOT(entry.devfn), 1232 PCI_FUNC(entry.devfn)); 1233 continue; 1234 } 1235 pci_print_aer(pdev, entry.severity, entry.regs); 1236 1237 /* 1238 * Memory for aer_capability_regs(entry.regs) is being 1239 * allocated from the ghes_estatus_pool to protect it from 1240 * overwriting when multiple sections are present in the 1241 * error status. Thus free the same after processing the 1242 * data. 1243 */ 1244 ghes_estatus_pool_region_free((unsigned long)entry.regs, 1245 sizeof(struct aer_capability_regs)); 1246 1247 if (entry.severity == AER_NONFATAL) 1248 pcie_do_recovery(pdev, pci_channel_io_normal, 1249 aer_root_reset); 1250 else if (entry.severity == AER_FATAL) 1251 pcie_do_recovery(pdev, pci_channel_io_frozen, 1252 aer_root_reset); 1253 pci_dev_put(pdev); 1254 } 1255 } 1256 1257 /* 1258 * Mutual exclusion for writers of aer_recover_ring, reader side don't 1259 * need lock, because there is only one reader and lock is not needed 1260 * between reader and writer. 1261 */ 1262 static DEFINE_SPINLOCK(aer_recover_ring_lock); 1263 static DECLARE_WORK(aer_recover_work, aer_recover_work_func); 1264 1265 void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, 1266 int severity, struct aer_capability_regs *aer_regs) 1267 { 1268 struct aer_recover_entry entry = { 1269 .bus = bus, 1270 .devfn = devfn, 1271 .domain = domain, 1272 .severity = severity, 1273 .regs = aer_regs, 1274 }; 1275 1276 if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1, 1277 &aer_recover_ring_lock)) 1278 schedule_work(&aer_recover_work); 1279 else 1280 pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n", 1281 domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 1282 } 1283 EXPORT_SYMBOL_GPL(aer_recover_queue); 1284 #endif 1285 1286 /** 1287 * aer_get_device_error_info - read error status from dev and store it to info 1288 * @info: pointer to structure to store the error record 1289 * @i: index into info->dev[] 1290 * 1291 * Return: 1 on success, 0 on error. 1292 * 1293 * Note that @info is reused among all error devices. Clear fields properly. 1294 */ 1295 int aer_get_device_error_info(struct aer_err_info *info, int i) 1296 { 1297 struct pci_dev *dev; 1298 int type, aer; 1299 u32 aercc; 1300 1301 if (i >= AER_MAX_MULTI_ERR_DEVICES) 1302 return 0; 1303 1304 dev = info->dev[i]; 1305 aer = dev->aer_cap; 1306 type = pci_pcie_type(dev); 1307 1308 /* Must reset in this function */ 1309 info->status = 0; 1310 info->tlp_header_valid = 0; 1311 info->is_cxl = pcie_is_cxl(dev); 1312 1313 /* The device might not support AER */ 1314 if (!aer) 1315 return 0; 1316 1317 if (info->severity == AER_CORRECTABLE) { 1318 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1319 &info->status); 1320 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, 1321 &info->mask); 1322 if (!(info->status & ~info->mask)) 1323 return 0; 1324 } else if (type == PCI_EXP_TYPE_ROOT_PORT || 1325 type == PCI_EXP_TYPE_RC_EC || 1326 type == PCI_EXP_TYPE_DOWNSTREAM || 1327 info->severity == AER_NONFATAL) { 1328 1329 /* Link is still healthy for IO reads */ 1330 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, 1331 &info->status); 1332 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, 1333 &info->mask); 1334 if (!(info->status & ~info->mask)) 1335 return 0; 1336 1337 /* Get First Error Pointer */ 1338 pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc); 1339 info->first_error = PCI_ERR_CAP_FEP(aercc); 1340 1341 if (tlp_header_logged(info->status, aercc)) { 1342 info->tlp_header_valid = 1; 1343 pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG, 1344 aer + PCI_ERR_PREFIX_LOG, 1345 aer_tlp_log_len(dev, aercc), 1346 aercc & PCI_ERR_CAP_TLP_LOG_FLIT, 1347 &info->tlp); 1348 } 1349 } 1350 1351 return 1; 1352 } 1353 1354 static inline void aer_process_err_devices(struct aer_err_info *e_info) 1355 { 1356 int i; 1357 1358 /* Report all before handling them, to not lose records by reset etc. */ 1359 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1360 if (aer_get_device_error_info(e_info, i)) 1361 aer_print_error(e_info, i); 1362 } 1363 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1364 if (aer_get_device_error_info(e_info, i)) 1365 handle_error_source(e_info->dev[i], e_info); 1366 } 1367 } 1368 1369 /** 1370 * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error 1371 * detected by Root Port or RCEC 1372 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1373 * @info: pointer to AER error info 1374 */ 1375 static void aer_isr_one_error_type(struct pci_dev *root, 1376 struct aer_err_info *info) 1377 { 1378 bool found; 1379 1380 found = find_source_device(root, info); 1381 1382 /* 1383 * If we're going to log error messages, we've already set 1384 * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to 1385 * non-zero (which enables printing) because this is either an 1386 * ERR_FATAL or we found a device with an error logged in its AER 1387 * Capability. 1388 * 1389 * If we didn't find the Error Source device, at least log the 1390 * Requester ID from the ERR_* Message received by the Root Port or 1391 * RCEC, ratelimited by the RP or RCEC. 1392 */ 1393 if (info->root_ratelimit_print || 1394 (!found && aer_ratelimit(root, info->severity))) 1395 aer_print_source(root, info, found); 1396 1397 if (found) 1398 aer_process_err_devices(info); 1399 } 1400 1401 /** 1402 * aer_isr_one_error - consume error(s) signaled by an AER interrupt from 1403 * Root Port or RCEC 1404 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1405 * @e_src: pointer to an error source 1406 */ 1407 static void aer_isr_one_error(struct pci_dev *root, 1408 struct aer_err_source *e_src) 1409 { 1410 u32 status = e_src->status; 1411 1412 pci_rootport_aer_stats_incr(root, e_src); 1413 1414 /* 1415 * There is a possibility that both correctable error and 1416 * uncorrectable error being logged. Report correctable error first. 1417 */ 1418 if (status & PCI_ERR_ROOT_COR_RCV) { 1419 int multi = status & PCI_ERR_ROOT_MULTI_COR_RCV; 1420 struct aer_err_info e_info = { 1421 .id = ERR_COR_ID(e_src->id), 1422 .severity = AER_CORRECTABLE, 1423 .level = KERN_WARNING, 1424 .multi_error_valid = multi ? 1 : 0, 1425 }; 1426 1427 aer_isr_one_error_type(root, &e_info); 1428 } 1429 1430 if (status & PCI_ERR_ROOT_UNCOR_RCV) { 1431 int fatal = status & PCI_ERR_ROOT_FATAL_RCV; 1432 int multi = status & PCI_ERR_ROOT_MULTI_UNCOR_RCV; 1433 struct aer_err_info e_info = { 1434 .id = ERR_UNCOR_ID(e_src->id), 1435 .severity = fatal ? AER_FATAL : AER_NONFATAL, 1436 .level = KERN_ERR, 1437 .multi_error_valid = multi ? 1 : 0, 1438 }; 1439 1440 aer_isr_one_error_type(root, &e_info); 1441 } 1442 } 1443 1444 /** 1445 * aer_isr - consume errors detected by Root Port 1446 * @irq: IRQ assigned to Root Port 1447 * @context: pointer to Root Port data structure 1448 * 1449 * Invoked, as DPC, when Root Port records new detected error 1450 */ 1451 static irqreturn_t aer_isr(int irq, void *context) 1452 { 1453 struct pcie_device *dev = (struct pcie_device *)context; 1454 struct aer_rpc *rpc = get_service_data(dev); 1455 struct aer_err_source e_src; 1456 1457 if (kfifo_is_empty(&rpc->aer_fifo)) 1458 return IRQ_NONE; 1459 1460 while (kfifo_get(&rpc->aer_fifo, &e_src)) 1461 aer_isr_one_error(rpc->rpd, &e_src); 1462 return IRQ_HANDLED; 1463 } 1464 1465 /** 1466 * aer_irq - Root Port's ISR 1467 * @irq: IRQ assigned to Root Port 1468 * @context: pointer to Root Port data structure 1469 * 1470 * Invoked when Root Port detects AER messages. 1471 */ 1472 static irqreturn_t aer_irq(int irq, void *context) 1473 { 1474 struct pcie_device *pdev = (struct pcie_device *)context; 1475 struct aer_rpc *rpc = get_service_data(pdev); 1476 struct pci_dev *rp = rpc->rpd; 1477 int aer = rp->aer_cap; 1478 struct aer_err_source e_src = {}; 1479 1480 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status); 1481 if (!(e_src.status & AER_ERR_STATUS_MASK)) 1482 return IRQ_NONE; 1483 1484 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id); 1485 pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status); 1486 1487 if (!kfifo_put(&rpc->aer_fifo, e_src)) 1488 return IRQ_HANDLED; 1489 1490 return IRQ_WAKE_THREAD; 1491 } 1492 1493 static void aer_enable_irq(struct pci_dev *pdev) 1494 { 1495 int aer = pdev->aer_cap; 1496 u32 reg32; 1497 1498 /* Enable Root Port's interrupt in response to error messages */ 1499 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1500 reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; 1501 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1502 } 1503 1504 static void aer_disable_irq(struct pci_dev *pdev) 1505 { 1506 int aer = pdev->aer_cap; 1507 u32 reg32; 1508 1509 /* Disable Root Port's interrupt in response to error messages */ 1510 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1511 reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; 1512 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1513 } 1514 1515 static int clear_status_iter(struct pci_dev *dev, void *data) 1516 { 1517 u16 devctl; 1518 1519 /* Skip if pci_enable_pcie_error_reporting() hasn't been called yet */ 1520 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, &devctl); 1521 if (!(devctl & PCI_EXP_AER_FLAGS)) 1522 return 0; 1523 1524 pci_aer_clear_status(dev); 1525 pcie_clear_device_status(dev); 1526 return 0; 1527 } 1528 1529 /** 1530 * aer_enable_rootport - enable Root Port's interrupts when receiving messages 1531 * @rpc: pointer to a Root Port data structure 1532 * 1533 * Invoked when PCIe bus loads AER service driver. 1534 */ 1535 static void aer_enable_rootport(struct aer_rpc *rpc) 1536 { 1537 struct pci_dev *pdev = rpc->rpd; 1538 int aer = pdev->aer_cap; 1539 u16 reg16; 1540 u32 reg32; 1541 1542 /* Clear PCIe Capability's Device Status */ 1543 pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, ®16); 1544 pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16); 1545 1546 /* Disable system error generation in response to error messages */ 1547 pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, 1548 SYSTEM_ERROR_INTR_ON_MESG_MASK); 1549 1550 /* Clear error status of this Root Port or RCEC */ 1551 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1552 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1553 1554 /* Clear error status of agents reporting to this Root Port or RCEC */ 1555 if (reg32 & AER_ERR_STATUS_MASK) { 1556 if (pci_pcie_type(pdev) == PCI_EXP_TYPE_RC_EC) 1557 pcie_walk_rcec(pdev, clear_status_iter, NULL); 1558 else if (pdev->subordinate) 1559 pci_walk_bus(pdev->subordinate, clear_status_iter, 1560 NULL); 1561 } 1562 1563 pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32); 1564 pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32); 1565 pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32); 1566 pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32); 1567 1568 aer_enable_irq(pdev); 1569 } 1570 1571 /** 1572 * aer_disable_rootport - disable Root Port's interrupts when receiving messages 1573 * @rpc: pointer to a Root Port data structure 1574 * 1575 * Invoked when PCIe bus unloads AER service driver. 1576 */ 1577 static void aer_disable_rootport(struct aer_rpc *rpc) 1578 { 1579 struct pci_dev *pdev = rpc->rpd; 1580 int aer = pdev->aer_cap; 1581 u32 reg32; 1582 1583 aer_disable_irq(pdev); 1584 1585 /* Clear Root's error status reg */ 1586 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1587 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1588 } 1589 1590 /** 1591 * aer_remove - clean up resources 1592 * @dev: pointer to the pcie_dev data structure 1593 * 1594 * Invoked when PCI Express bus unloads or AER probe fails. 1595 */ 1596 static void aer_remove(struct pcie_device *dev) 1597 { 1598 struct aer_rpc *rpc = get_service_data(dev); 1599 1600 aer_disable_rootport(rpc); 1601 } 1602 1603 /** 1604 * aer_probe - initialize resources 1605 * @dev: pointer to the pcie_dev data structure 1606 * 1607 * Invoked when PCI Express bus loads AER service driver. 1608 */ 1609 static int aer_probe(struct pcie_device *dev) 1610 { 1611 int status; 1612 struct aer_rpc *rpc; 1613 struct device *device = &dev->device; 1614 struct pci_dev *port = dev->port; 1615 1616 BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) < 1617 AER_MAX_TYPEOF_COR_ERRS); 1618 BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) < 1619 AER_MAX_TYPEOF_UNCOR_ERRS); 1620 1621 /* Limit to Root Ports or Root Complex Event Collectors */ 1622 if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) && 1623 (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT)) 1624 return -ENODEV; 1625 1626 rpc = devm_kzalloc(device, sizeof(struct aer_rpc), GFP_KERNEL); 1627 if (!rpc) 1628 return -ENOMEM; 1629 1630 rpc->rpd = port; 1631 INIT_KFIFO(rpc->aer_fifo); 1632 set_service_data(dev, rpc); 1633 1634 status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr, 1635 IRQF_SHARED, "aerdrv", dev); 1636 if (status) { 1637 pci_err(port, "request AER IRQ %d failed\n", dev->irq); 1638 return status; 1639 } 1640 1641 cxl_rch_enable_rcec(port); 1642 aer_enable_rootport(rpc); 1643 pci_info(port, "enabled with IRQ %d\n", dev->irq); 1644 return 0; 1645 } 1646 1647 static int aer_suspend(struct pcie_device *dev) 1648 { 1649 struct aer_rpc *rpc = get_service_data(dev); 1650 1651 aer_disable_rootport(rpc); 1652 return 0; 1653 } 1654 1655 static int aer_resume(struct pcie_device *dev) 1656 { 1657 struct aer_rpc *rpc = get_service_data(dev); 1658 1659 aer_enable_rootport(rpc); 1660 return 0; 1661 } 1662 1663 /** 1664 * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP 1665 * @dev: pointer to Root Port, RCEC, or RCiEP 1666 * 1667 * Invoked by Port Bus driver when performing reset. 1668 */ 1669 static pci_ers_result_t aer_root_reset(struct pci_dev *dev) 1670 { 1671 int type = pci_pcie_type(dev); 1672 struct pci_dev *root; 1673 int aer; 1674 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 1675 u32 reg32; 1676 int rc; 1677 1678 /* 1679 * Only Root Ports and RCECs have AER Root Command and Root Status 1680 * registers. If "dev" is an RCiEP, the relevant registers are in 1681 * the RCEC. 1682 */ 1683 if (type == PCI_EXP_TYPE_RC_END) 1684 root = dev->rcec; 1685 else 1686 root = pcie_find_root_port(dev); 1687 1688 /* 1689 * If the platform retained control of AER, an RCiEP may not have 1690 * an RCEC visible to us, so dev->rcec ("root") may be NULL. In 1691 * that case, firmware is responsible for these registers. 1692 */ 1693 aer = root ? root->aer_cap : 0; 1694 1695 if ((host->native_aer || pcie_ports_native) && aer) 1696 aer_disable_irq(root); 1697 1698 if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { 1699 rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET); 1700 if (!rc) 1701 pci_info(dev, "has been reset\n"); 1702 else 1703 pci_info(dev, "not reset (no FLR support: %d)\n", rc); 1704 } else { 1705 rc = pci_bus_error_reset(dev); 1706 pci_info(dev, "%s Port link has been reset (%d)\n", 1707 pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc); 1708 } 1709 1710 if ((host->native_aer || pcie_ports_native) && aer) { 1711 /* Clear Root Error Status */ 1712 pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, ®32); 1713 pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32); 1714 1715 aer_enable_irq(root); 1716 } 1717 1718 return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 1719 } 1720 1721 static struct pcie_port_service_driver aerdriver = { 1722 .name = "aer", 1723 .port_type = PCIE_ANY_PORT, 1724 .service = PCIE_PORT_SERVICE_AER, 1725 1726 .probe = aer_probe, 1727 .suspend = aer_suspend, 1728 .resume = aer_resume, 1729 .remove = aer_remove, 1730 }; 1731 1732 /** 1733 * pcie_aer_init - register AER service driver 1734 * 1735 * Invoked when AER service driver is loaded. 1736 */ 1737 int __init pcie_aer_init(void) 1738 { 1739 if (!pci_aer_available()) 1740 return -ENXIO; 1741 return pcie_port_service_register(&aerdriver); 1742 } 1743