1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implement the AER root port service driver. The driver registers an IRQ 4 * handler. When a root port triggers an AER interrupt, the IRQ handler 5 * collects Root Port status and schedules work. 6 * 7 * Copyright (C) 2006 Intel Corp. 8 * Tom Long Nguyen (tom.l.nguyen@intel.com) 9 * Zhang Yanmin (yanmin.zhang@intel.com) 10 * 11 * (C) Copyright 2009 Hewlett-Packard Development Company, L.P. 12 * Andrew Patterson <andrew.patterson@hp.com> 13 */ 14 15 #define pr_fmt(fmt) "AER: " fmt 16 #define dev_fmt pr_fmt 17 18 #include <linux/bitops.h> 19 #include <linux/cper.h> 20 #include <linux/dev_printk.h> 21 #include <linux/pci.h> 22 #include <linux/pci-acpi.h> 23 #include <linux/sched.h> 24 #include <linux/kernel.h> 25 #include <linux/errno.h> 26 #include <linux/pm.h> 27 #include <linux/init.h> 28 #include <linux/interrupt.h> 29 #include <linux/delay.h> 30 #include <linux/kfifo.h> 31 #include <linux/ratelimit.h> 32 #include <linux/slab.h> 33 #include <linux/vmcore_info.h> 34 #include <acpi/apei.h> 35 #include <acpi/ghes.h> 36 #include <ras/ras_event.h> 37 38 #include "../pci.h" 39 #include "portdrv.h" 40 41 #define aer_printk(level, pdev, fmt, arg...) \ 42 dev_printk(level, &(pdev)->dev, fmt, ##arg) 43 44 #define AER_ERROR_SOURCES_MAX 128 45 46 #define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */ 47 #define AER_MAX_TYPEOF_UNCOR_ERRS 32 /* as per PCI_ERR_UNCOR_STATUS*/ 48 49 struct aer_err_source { 50 u32 status; /* PCI_ERR_ROOT_STATUS */ 51 u32 id; /* PCI_ERR_ROOT_ERR_SRC */ 52 }; 53 54 struct aer_rpc { 55 struct pci_dev *rpd; /* Root Port device */ 56 DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX); 57 }; 58 59 /* AER info for the device */ 60 struct aer_info { 61 62 /* 63 * Fields for all AER capable devices. They indicate the errors 64 * "as seen by this device". Note that this may mean that if an 65 * Endpoint is causing problems, the AER counters may increment 66 * at its link partner (e.g. Root Port) because the errors will be 67 * "seen" by the link partner and not the problematic Endpoint 68 * itself (which may report all counters as 0 as it never saw any 69 * problems). 70 */ 71 /* Counters for different type of correctable errors */ 72 u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS]; 73 /* Counters for different type of fatal uncorrectable errors */ 74 u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 75 /* Counters for different type of nonfatal uncorrectable errors */ 76 u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 77 /* Total number of ERR_COR sent by this device */ 78 u64 dev_total_cor_errs; 79 /* Total number of ERR_FATAL sent by this device */ 80 u64 dev_total_fatal_errs; 81 /* Total number of ERR_NONFATAL sent by this device */ 82 u64 dev_total_nonfatal_errs; 83 84 /* 85 * Fields for Root Ports & Root Complex Event Collectors only; these 86 * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL 87 * messages received by the Root Port / Event Collector, INCLUDING the 88 * ones that are generated internally (by the Root Port itself) 89 */ 90 u64 rootport_total_cor_errs; 91 u64 rootport_total_fatal_errs; 92 u64 rootport_total_nonfatal_errs; 93 94 /* Ratelimits for errors */ 95 struct ratelimit_state correctable_ratelimit; 96 struct ratelimit_state nonfatal_ratelimit; 97 }; 98 99 #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \ 100 PCI_ERR_UNC_POISON_BLK | \ 101 PCI_ERR_UNC_ECRC| \ 102 PCI_ERR_UNC_UNSUP| \ 103 PCI_ERR_UNC_COMP_ABORT| \ 104 PCI_ERR_UNC_UNX_COMP| \ 105 PCI_ERR_UNC_ACSV | \ 106 PCI_ERR_UNC_MCBTLP | \ 107 PCI_ERR_UNC_ATOMEG | \ 108 PCI_ERR_UNC_DMWR_BLK | \ 109 PCI_ERR_UNC_XLAT_BLK | \ 110 PCI_ERR_UNC_TLPPRE | \ 111 PCI_ERR_UNC_MALF_TLP | \ 112 PCI_ERR_UNC_IDE_CHECK | \ 113 PCI_ERR_UNC_MISR_IDE | \ 114 PCI_ERR_UNC_PCRC_CHECK) 115 116 #define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \ 117 PCI_EXP_RTCTL_SENFEE| \ 118 PCI_EXP_RTCTL_SEFEE) 119 #define ROOT_PORT_INTR_ON_MESG_MASK (PCI_ERR_ROOT_CMD_COR_EN| \ 120 PCI_ERR_ROOT_CMD_NONFATAL_EN| \ 121 PCI_ERR_ROOT_CMD_FATAL_EN) 122 #define ERR_COR_ID(d) (d & 0xffff) 123 #define ERR_UNCOR_ID(d) (d >> 16) 124 125 #define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV | \ 126 PCI_ERR_ROOT_COR_RCV | \ 127 PCI_ERR_ROOT_MULTI_COR_RCV | \ 128 PCI_ERR_ROOT_MULTI_UNCOR_RCV) 129 130 static bool pcie_aer_disable; 131 static pci_ers_result_t aer_root_reset(struct pci_dev *dev); 132 133 void pci_no_aer(void) 134 { 135 pcie_aer_disable = true; 136 } 137 138 bool pci_aer_available(void) 139 { 140 return !pcie_aer_disable && pci_msi_enabled(); 141 } 142 143 #ifdef CONFIG_PCIE_ECRC 144 145 #define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */ 146 #define ECRC_POLICY_OFF 1 /* ECRC off for performance */ 147 #define ECRC_POLICY_ON 2 /* ECRC on for data integrity */ 148 149 static int ecrc_policy = ECRC_POLICY_DEFAULT; 150 151 static const char * const ecrc_policy_str[] = { 152 [ECRC_POLICY_DEFAULT] = "bios", 153 [ECRC_POLICY_OFF] = "off", 154 [ECRC_POLICY_ON] = "on" 155 }; 156 157 /** 158 * enable_ecrc_checking - enable PCIe ECRC checking for a device 159 * @dev: the PCI device 160 * 161 * Return: 0 on success, or negative on failure. 162 */ 163 static int enable_ecrc_checking(struct pci_dev *dev) 164 { 165 int aer = dev->aer_cap; 166 u32 reg32; 167 168 if (!aer) 169 return -ENODEV; 170 171 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 172 if (reg32 & PCI_ERR_CAP_ECRC_GENC) 173 reg32 |= PCI_ERR_CAP_ECRC_GENE; 174 if (reg32 & PCI_ERR_CAP_ECRC_CHKC) 175 reg32 |= PCI_ERR_CAP_ECRC_CHKE; 176 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 177 178 return 0; 179 } 180 181 /** 182 * disable_ecrc_checking - disable PCIe ECRC checking for a device 183 * @dev: the PCI device 184 * 185 * Return: 0 on success, or negative on failure. 186 */ 187 static int disable_ecrc_checking(struct pci_dev *dev) 188 { 189 int aer = dev->aer_cap; 190 u32 reg32; 191 192 if (!aer) 193 return -ENODEV; 194 195 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 196 reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE); 197 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 198 199 return 0; 200 } 201 202 /** 203 * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based 204 * on global policy 205 * @dev: the PCI device 206 */ 207 void pcie_set_ecrc_checking(struct pci_dev *dev) 208 { 209 if (!pcie_aer_is_native(dev)) 210 return; 211 212 switch (ecrc_policy) { 213 case ECRC_POLICY_DEFAULT: 214 return; 215 case ECRC_POLICY_OFF: 216 disable_ecrc_checking(dev); 217 break; 218 case ECRC_POLICY_ON: 219 enable_ecrc_checking(dev); 220 break; 221 default: 222 return; 223 } 224 } 225 226 /** 227 * pcie_ecrc_get_policy - parse kernel command-line ecrc option 228 * @str: ECRC policy from kernel command line to use 229 */ 230 void pcie_ecrc_get_policy(char *str) 231 { 232 int i; 233 234 i = match_string(ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), str); 235 if (i < 0) 236 return; 237 238 ecrc_policy = i; 239 } 240 #endif /* CONFIG_PCIE_ECRC */ 241 242 #define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ 243 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) 244 245 int pcie_aer_is_native(struct pci_dev *dev) 246 { 247 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 248 249 if (!dev->aer_cap) 250 return 0; 251 252 return pcie_ports_native || host->native_aer; 253 } 254 EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, "CXL"); 255 256 static int pci_enable_pcie_error_reporting(struct pci_dev *dev) 257 { 258 int rc; 259 260 if (!pcie_aer_is_native(dev)) 261 return -EIO; 262 263 rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS); 264 return pcibios_err_to_errno(rc); 265 } 266 267 int pci_aer_clear_nonfatal_status(struct pci_dev *dev) 268 { 269 int aer = dev->aer_cap; 270 u32 status, sev; 271 272 if (!pcie_aer_is_native(dev)) 273 return -EIO; 274 275 /* Clear status bits for ERR_NONFATAL errors only */ 276 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 277 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 278 status &= ~sev; 279 if (status) 280 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 281 282 return 0; 283 } 284 EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status); 285 286 void pci_aer_clear_fatal_status(struct pci_dev *dev) 287 { 288 int aer = dev->aer_cap; 289 u32 status, sev; 290 291 if (!pcie_aer_is_native(dev)) 292 return; 293 294 /* Clear status bits for ERR_FATAL errors only */ 295 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 296 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 297 status &= sev; 298 if (status) 299 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 300 } 301 302 /** 303 * pci_aer_raw_clear_status - Clear AER error registers. 304 * @dev: the PCI device 305 * 306 * Clear AER error status registers unconditionally, regardless of 307 * whether they're owned by firmware or the OS. 308 * 309 * Return: 0 on success, or negative on failure. 310 */ 311 int pci_aer_raw_clear_status(struct pci_dev *dev) 312 { 313 int aer = dev->aer_cap; 314 u32 status; 315 int port_type; 316 317 if (!aer) 318 return -EIO; 319 320 port_type = pci_pcie_type(dev); 321 if (port_type == PCI_EXP_TYPE_ROOT_PORT || 322 port_type == PCI_EXP_TYPE_RC_EC) { 323 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status); 324 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status); 325 } 326 327 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 328 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status); 329 330 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 331 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 332 333 return 0; 334 } 335 336 int pci_aer_clear_status(struct pci_dev *dev) 337 { 338 if (!pcie_aer_is_native(dev)) 339 return -EIO; 340 341 return pci_aer_raw_clear_status(dev); 342 } 343 344 void pci_save_aer_state(struct pci_dev *dev) 345 { 346 int aer = dev->aer_cap; 347 struct pci_cap_saved_state *save_state; 348 u32 *cap; 349 350 if (!aer) 351 return; 352 353 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 354 if (!save_state) 355 return; 356 357 cap = &save_state->cap.data[0]; 358 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++); 359 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++); 360 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++); 361 pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++); 362 if (pcie_cap_has_rtctl(dev)) 363 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++); 364 } 365 366 void pci_restore_aer_state(struct pci_dev *dev) 367 { 368 int aer = dev->aer_cap; 369 struct pci_cap_saved_state *save_state; 370 u32 *cap; 371 372 if (!aer) 373 return; 374 375 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 376 if (!save_state) 377 return; 378 379 cap = &save_state->cap.data[0]; 380 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++); 381 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++); 382 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++); 383 pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++); 384 if (pcie_cap_has_rtctl(dev)) 385 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++); 386 } 387 388 void pci_aer_init(struct pci_dev *dev) 389 { 390 int n; 391 392 dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); 393 if (!dev->aer_cap) 394 return; 395 396 dev->aer_info = kzalloc(sizeof(*dev->aer_info), GFP_KERNEL); 397 if (!dev->aer_info) { 398 dev->aer_cap = 0; 399 return; 400 } 401 402 ratelimit_state_init(&dev->aer_info->correctable_ratelimit, 403 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 404 ratelimit_state_init(&dev->aer_info->nonfatal_ratelimit, 405 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 406 407 /* 408 * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER, 409 * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event 410 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r6.0, sec 411 * 7.8.4.9). 412 */ 413 n = pcie_cap_has_rtctl(dev) ? 5 : 4; 414 pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, sizeof(u32) * n); 415 416 pci_aer_clear_status(dev); 417 418 if (pci_aer_available()) 419 pci_enable_pcie_error_reporting(dev); 420 421 pcie_set_ecrc_checking(dev); 422 } 423 424 void pci_aer_exit(struct pci_dev *dev) 425 { 426 kfree(dev->aer_info); 427 dev->aer_info = NULL; 428 } 429 430 #define AER_AGENT_RECEIVER 0 431 #define AER_AGENT_REQUESTER 1 432 #define AER_AGENT_COMPLETER 2 433 #define AER_AGENT_TRANSMITTER 3 434 435 #define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 436 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP)) 437 #define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \ 438 0 : PCI_ERR_UNC_COMP_ABORT) 439 #define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 440 (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0) 441 442 #define AER_GET_AGENT(t, e) \ 443 ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \ 444 (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \ 445 (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \ 446 AER_AGENT_RECEIVER) 447 448 #define AER_PHYSICAL_LAYER_ERROR 0 449 #define AER_DATA_LINK_LAYER_ERROR 1 450 #define AER_TRANSACTION_LAYER_ERROR 2 451 452 #define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 453 PCI_ERR_COR_RCVR : 0) 454 #define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 455 (PCI_ERR_COR_BAD_TLP| \ 456 PCI_ERR_COR_BAD_DLLP| \ 457 PCI_ERR_COR_REP_ROLL| \ 458 PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP) 459 460 #define AER_GET_LAYER_ERROR(t, e) \ 461 ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \ 462 (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ 463 AER_TRANSACTION_LAYER_ERROR) 464 465 /* 466 * AER error strings 467 */ 468 static const char * const aer_error_severity_string[] = { 469 "Uncorrectable (Non-Fatal)", 470 "Uncorrectable (Fatal)", 471 "Correctable" 472 }; 473 474 static const char *aer_error_layer[] = { 475 "Physical Layer", 476 "Data Link Layer", 477 "Transaction Layer" 478 }; 479 480 static const char *aer_correctable_error_string[] = { 481 "RxErr", /* Bit Position 0 */ 482 NULL, 483 NULL, 484 NULL, 485 NULL, 486 NULL, 487 "BadTLP", /* Bit Position 6 */ 488 "BadDLLP", /* Bit Position 7 */ 489 "Rollover", /* Bit Position 8 */ 490 NULL, 491 NULL, 492 NULL, 493 "Timeout", /* Bit Position 12 */ 494 "NonFatalErr", /* Bit Position 13 */ 495 "CorrIntErr", /* Bit Position 14 */ 496 "HeaderOF", /* Bit Position 15 */ 497 NULL, /* Bit Position 16 */ 498 NULL, /* Bit Position 17 */ 499 NULL, /* Bit Position 18 */ 500 NULL, /* Bit Position 19 */ 501 NULL, /* Bit Position 20 */ 502 NULL, /* Bit Position 21 */ 503 NULL, /* Bit Position 22 */ 504 NULL, /* Bit Position 23 */ 505 NULL, /* Bit Position 24 */ 506 NULL, /* Bit Position 25 */ 507 NULL, /* Bit Position 26 */ 508 NULL, /* Bit Position 27 */ 509 NULL, /* Bit Position 28 */ 510 NULL, /* Bit Position 29 */ 511 NULL, /* Bit Position 30 */ 512 NULL, /* Bit Position 31 */ 513 }; 514 515 static const char *aer_uncorrectable_error_string[] = { 516 "Undefined", /* Bit Position 0 */ 517 NULL, 518 NULL, 519 NULL, 520 "DLP", /* Bit Position 4 */ 521 "SDES", /* Bit Position 5 */ 522 NULL, 523 NULL, 524 NULL, 525 NULL, 526 NULL, 527 NULL, 528 "TLP", /* Bit Position 12 */ 529 "FCP", /* Bit Position 13 */ 530 "CmpltTO", /* Bit Position 14 */ 531 "CmpltAbrt", /* Bit Position 15 */ 532 "UnxCmplt", /* Bit Position 16 */ 533 "RxOF", /* Bit Position 17 */ 534 "MalfTLP", /* Bit Position 18 */ 535 "ECRC", /* Bit Position 19 */ 536 "UnsupReq", /* Bit Position 20 */ 537 "ACSViol", /* Bit Position 21 */ 538 "UncorrIntErr", /* Bit Position 22 */ 539 "BlockedTLP", /* Bit Position 23 */ 540 "AtomicOpBlocked", /* Bit Position 24 */ 541 "TLPBlockedErr", /* Bit Position 25 */ 542 "PoisonTLPBlocked", /* Bit Position 26 */ 543 "DMWrReqBlocked", /* Bit Position 27 */ 544 "IDECheck", /* Bit Position 28 */ 545 "MisIDETLP", /* Bit Position 29 */ 546 "PCRC_CHECK", /* Bit Position 30 */ 547 "TLPXlatBlocked", /* Bit Position 31 */ 548 }; 549 550 static const char *aer_agent_string[] = { 551 "Receiver ID", 552 "Requester ID", 553 "Completer ID", 554 "Transmitter ID" 555 }; 556 557 #define aer_stats_dev_attr(name, stats_array, strings_array, \ 558 total_string, total_field) \ 559 static ssize_t \ 560 name##_show(struct device *dev, struct device_attribute *attr, \ 561 char *buf) \ 562 { \ 563 unsigned int i; \ 564 struct pci_dev *pdev = to_pci_dev(dev); \ 565 u64 *stats = pdev->aer_info->stats_array; \ 566 size_t len = 0; \ 567 \ 568 for (i = 0; i < ARRAY_SIZE(pdev->aer_info->stats_array); i++) { \ 569 if (strings_array[i]) \ 570 len += sysfs_emit_at(buf, len, "%s %llu\n", \ 571 strings_array[i], \ 572 stats[i]); \ 573 else if (stats[i]) \ 574 len += sysfs_emit_at(buf, len, \ 575 #stats_array "_bit[%d] %llu\n",\ 576 i, stats[i]); \ 577 } \ 578 len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \ 579 pdev->aer_info->total_field); \ 580 return len; \ 581 } \ 582 static DEVICE_ATTR_RO(name) 583 584 aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs, 585 aer_correctable_error_string, "ERR_COR", 586 dev_total_cor_errs); 587 aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs, 588 aer_uncorrectable_error_string, "ERR_FATAL", 589 dev_total_fatal_errs); 590 aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs, 591 aer_uncorrectable_error_string, "ERR_NONFATAL", 592 dev_total_nonfatal_errs); 593 594 #define aer_stats_rootport_attr(name, field) \ 595 static ssize_t \ 596 name##_show(struct device *dev, struct device_attribute *attr, \ 597 char *buf) \ 598 { \ 599 struct pci_dev *pdev = to_pci_dev(dev); \ 600 return sysfs_emit(buf, "%llu\n", pdev->aer_info->field); \ 601 } \ 602 static DEVICE_ATTR_RO(name) 603 604 aer_stats_rootport_attr(aer_rootport_total_err_cor, 605 rootport_total_cor_errs); 606 aer_stats_rootport_attr(aer_rootport_total_err_fatal, 607 rootport_total_fatal_errs); 608 aer_stats_rootport_attr(aer_rootport_total_err_nonfatal, 609 rootport_total_nonfatal_errs); 610 611 static struct attribute *aer_stats_attrs[] __ro_after_init = { 612 &dev_attr_aer_dev_correctable.attr, 613 &dev_attr_aer_dev_fatal.attr, 614 &dev_attr_aer_dev_nonfatal.attr, 615 &dev_attr_aer_rootport_total_err_cor.attr, 616 &dev_attr_aer_rootport_total_err_fatal.attr, 617 &dev_attr_aer_rootport_total_err_nonfatal.attr, 618 NULL 619 }; 620 621 static umode_t aer_stats_attrs_are_visible(struct kobject *kobj, 622 struct attribute *a, int n) 623 { 624 struct device *dev = kobj_to_dev(kobj); 625 struct pci_dev *pdev = to_pci_dev(dev); 626 627 if (!pdev->aer_info) 628 return 0; 629 630 if ((a == &dev_attr_aer_rootport_total_err_cor.attr || 631 a == &dev_attr_aer_rootport_total_err_fatal.attr || 632 a == &dev_attr_aer_rootport_total_err_nonfatal.attr) && 633 ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && 634 (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_EC))) 635 return 0; 636 637 return a->mode; 638 } 639 640 const struct attribute_group aer_stats_attr_group = { 641 .attrs = aer_stats_attrs, 642 .is_visible = aer_stats_attrs_are_visible, 643 }; 644 645 /* 646 * Ratelimit interval 647 * <=0: disabled with ratelimit.interval = 0 648 * >0: enabled with ratelimit.interval in ms 649 */ 650 #define aer_ratelimit_interval_attr(name, ratelimit) \ 651 static ssize_t \ 652 name##_show(struct device *dev, struct device_attribute *attr, \ 653 char *buf) \ 654 { \ 655 struct pci_dev *pdev = to_pci_dev(dev); \ 656 \ 657 return sysfs_emit(buf, "%d\n", \ 658 pdev->aer_info->ratelimit.interval); \ 659 } \ 660 \ 661 static ssize_t \ 662 name##_store(struct device *dev, struct device_attribute *attr, \ 663 const char *buf, size_t count) \ 664 { \ 665 struct pci_dev *pdev = to_pci_dev(dev); \ 666 int interval; \ 667 \ 668 if (!capable(CAP_SYS_ADMIN)) \ 669 return -EPERM; \ 670 \ 671 if (kstrtoint(buf, 0, &interval) < 0) \ 672 return -EINVAL; \ 673 \ 674 if (interval <= 0) \ 675 interval = 0; \ 676 else \ 677 interval = msecs_to_jiffies(interval); \ 678 \ 679 pdev->aer_info->ratelimit.interval = interval; \ 680 \ 681 return count; \ 682 } \ 683 static DEVICE_ATTR_RW(name); 684 685 #define aer_ratelimit_burst_attr(name, ratelimit) \ 686 static ssize_t \ 687 name##_show(struct device *dev, struct device_attribute *attr, \ 688 char *buf) \ 689 { \ 690 struct pci_dev *pdev = to_pci_dev(dev); \ 691 \ 692 return sysfs_emit(buf, "%d\n", \ 693 pdev->aer_info->ratelimit.burst); \ 694 } \ 695 \ 696 static ssize_t \ 697 name##_store(struct device *dev, struct device_attribute *attr, \ 698 const char *buf, size_t count) \ 699 { \ 700 struct pci_dev *pdev = to_pci_dev(dev); \ 701 int burst; \ 702 \ 703 if (!capable(CAP_SYS_ADMIN)) \ 704 return -EPERM; \ 705 \ 706 if (kstrtoint(buf, 0, &burst) < 0) \ 707 return -EINVAL; \ 708 \ 709 pdev->aer_info->ratelimit.burst = burst; \ 710 \ 711 return count; \ 712 } \ 713 static DEVICE_ATTR_RW(name); 714 715 #define aer_ratelimit_attrs(name) \ 716 aer_ratelimit_interval_attr(name##_ratelimit_interval_ms, \ 717 name##_ratelimit) \ 718 aer_ratelimit_burst_attr(name##_ratelimit_burst, \ 719 name##_ratelimit) 720 721 aer_ratelimit_attrs(correctable) 722 aer_ratelimit_attrs(nonfatal) 723 724 static struct attribute *aer_attrs[] = { 725 &dev_attr_correctable_ratelimit_interval_ms.attr, 726 &dev_attr_correctable_ratelimit_burst.attr, 727 &dev_attr_nonfatal_ratelimit_interval_ms.attr, 728 &dev_attr_nonfatal_ratelimit_burst.attr, 729 NULL 730 }; 731 732 static umode_t aer_attrs_are_visible(struct kobject *kobj, 733 struct attribute *a, int n) 734 { 735 struct device *dev = kobj_to_dev(kobj); 736 struct pci_dev *pdev = to_pci_dev(dev); 737 738 if (!pdev->aer_info) 739 return 0; 740 741 return a->mode; 742 } 743 744 const struct attribute_group aer_attr_group = { 745 .name = "aer", 746 .attrs = aer_attrs, 747 .is_visible = aer_attrs_are_visible, 748 }; 749 750 static void pci_dev_aer_stats_incr(struct pci_dev *pdev, 751 struct aer_err_info *info) 752 { 753 unsigned long status = info->status & ~info->mask; 754 int i, max = -1; 755 u64 *counter = NULL; 756 struct aer_info *aer_info = pdev->aer_info; 757 758 if (!aer_info) 759 return; 760 761 switch (info->severity) { 762 case AER_CORRECTABLE: 763 aer_info->dev_total_cor_errs++; 764 counter = &aer_info->dev_cor_errs[0]; 765 max = AER_MAX_TYPEOF_COR_ERRS; 766 break; 767 case AER_NONFATAL: 768 aer_info->dev_total_nonfatal_errs++; 769 hwerr_log_error_type(HWERR_RECOV_PCI); 770 counter = &aer_info->dev_nonfatal_errs[0]; 771 max = AER_MAX_TYPEOF_UNCOR_ERRS; 772 break; 773 case AER_FATAL: 774 aer_info->dev_total_fatal_errs++; 775 counter = &aer_info->dev_fatal_errs[0]; 776 max = AER_MAX_TYPEOF_UNCOR_ERRS; 777 break; 778 } 779 780 for_each_set_bit(i, &status, max) 781 counter[i]++; 782 } 783 784 static void pci_rootport_aer_stats_incr(struct pci_dev *pdev, 785 struct aer_err_source *e_src) 786 { 787 struct aer_info *aer_info = pdev->aer_info; 788 789 if (!aer_info) 790 return; 791 792 if (e_src->status & PCI_ERR_ROOT_COR_RCV) 793 aer_info->rootport_total_cor_errs++; 794 795 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { 796 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) 797 aer_info->rootport_total_fatal_errs++; 798 else 799 aer_info->rootport_total_nonfatal_errs++; 800 } 801 } 802 803 static int aer_ratelimit(struct pci_dev *dev, unsigned int severity) 804 { 805 if (!dev->aer_info) 806 return 1; 807 808 switch (severity) { 809 case AER_NONFATAL: 810 return __ratelimit(&dev->aer_info->nonfatal_ratelimit); 811 case AER_CORRECTABLE: 812 return __ratelimit(&dev->aer_info->correctable_ratelimit); 813 default: 814 return 1; /* Don't ratelimit fatal errors */ 815 } 816 } 817 818 static bool tlp_header_logged(u32 status, u32 capctl) 819 { 820 /* Errors for which a header is always logged (PCIe r7.0 sec 6.2.7) */ 821 if (status & AER_LOG_TLP_MASKS) 822 return true; 823 824 /* Completion Timeout header is only logged on capable devices */ 825 if (status & PCI_ERR_UNC_COMP_TIME && 826 capctl & PCI_ERR_CAP_COMP_TIME_LOG) 827 return true; 828 829 return false; 830 } 831 832 static void __aer_print_error(struct pci_dev *dev, struct aer_err_info *info) 833 { 834 const char **strings; 835 unsigned long status = info->status & ~info->mask; 836 const char *level = info->level; 837 const char *errmsg; 838 int i; 839 840 if (info->severity == AER_CORRECTABLE) 841 strings = aer_correctable_error_string; 842 else 843 strings = aer_uncorrectable_error_string; 844 845 for_each_set_bit(i, &status, 32) { 846 errmsg = strings[i]; 847 if (!errmsg) 848 errmsg = "Unknown Error Bit"; 849 850 aer_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg, 851 info->first_error == i ? " (First)" : ""); 852 } 853 } 854 855 static void aer_print_source(struct pci_dev *dev, struct aer_err_info *info, 856 bool found) 857 { 858 u16 source = info->id; 859 860 pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d%s\n", 861 info->multi_error_valid ? "Multiple " : "", 862 aer_error_severity_string[info->severity], 863 pci_domain_nr(dev->bus), PCI_BUS_NUM(source), 864 PCI_SLOT(source), PCI_FUNC(source), 865 found ? "" : " (no details found"); 866 } 867 868 void aer_print_error(struct aer_err_info *info, int i) 869 { 870 struct pci_dev *dev; 871 int layer, agent, id; 872 const char *level = info->level; 873 874 if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES)) 875 return; 876 877 dev = info->dev[i]; 878 id = pci_dev_id(dev); 879 880 pci_dev_aer_stats_incr(dev, info); 881 trace_aer_event(pci_name(dev), (info->status & ~info->mask), 882 info->severity, info->tlp_header_valid, &info->tlp); 883 884 if (!info->ratelimit_print[i]) 885 return; 886 887 if (!info->status) { 888 pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", 889 aer_error_severity_string[info->severity]); 890 goto out; 891 } 892 893 layer = AER_GET_LAYER_ERROR(info->severity, info->status); 894 agent = AER_GET_AGENT(info->severity, info->status); 895 896 aer_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n", 897 aer_error_severity_string[info->severity], 898 aer_error_layer[layer], aer_agent_string[agent]); 899 900 aer_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n", 901 dev->vendor, dev->device, info->status, info->mask); 902 903 __aer_print_error(dev, info); 904 905 if (info->tlp_header_valid) 906 pcie_print_tlp_log(dev, &info->tlp, level, dev_fmt(" ")); 907 908 out: 909 if (info->id && info->error_dev_num > 1 && info->id == id) 910 pci_err(dev, " Error of this Agent is reported first\n"); 911 } 912 913 #ifdef CONFIG_ACPI_APEI_PCIEAER 914 int cper_severity_to_aer(int cper_severity) 915 { 916 switch (cper_severity) { 917 case CPER_SEV_RECOVERABLE: 918 return AER_NONFATAL; 919 case CPER_SEV_FATAL: 920 return AER_FATAL; 921 default: 922 return AER_CORRECTABLE; 923 } 924 } 925 EXPORT_SYMBOL_GPL(cper_severity_to_aer); 926 #endif 927 928 void pci_print_aer(struct pci_dev *dev, int aer_severity, 929 struct aer_capability_regs *aer) 930 { 931 int layer, agent, tlp_header_valid = 0; 932 u32 status, mask; 933 struct aer_err_info info = { 934 .severity = aer_severity, 935 .first_error = PCI_ERR_CAP_FEP(aer->cap_control), 936 }; 937 938 if (aer_severity == AER_CORRECTABLE) { 939 status = aer->cor_status; 940 mask = aer->cor_mask; 941 info.level = KERN_WARNING; 942 } else { 943 status = aer->uncor_status; 944 mask = aer->uncor_mask; 945 info.level = KERN_ERR; 946 tlp_header_valid = tlp_header_logged(status, aer->cap_control); 947 } 948 949 info.status = status; 950 info.mask = mask; 951 952 pci_dev_aer_stats_incr(dev, &info); 953 trace_aer_event(pci_name(dev), (status & ~mask), 954 aer_severity, tlp_header_valid, &aer->header_log); 955 956 if (!aer_ratelimit(dev, info.severity)) 957 return; 958 959 layer = AER_GET_LAYER_ERROR(aer_severity, status); 960 agent = AER_GET_AGENT(aer_severity, status); 961 962 aer_printk(info.level, dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", 963 status, mask); 964 __aer_print_error(dev, &info); 965 aer_printk(info.level, dev, "aer_layer=%s, aer_agent=%s\n", 966 aer_error_layer[layer], aer_agent_string[agent]); 967 968 if (aer_severity != AER_CORRECTABLE) 969 aer_printk(info.level, dev, "aer_uncor_severity: 0x%08x\n", 970 aer->uncor_severity); 971 972 if (tlp_header_valid) 973 pcie_print_tlp_log(dev, &aer->header_log, info.level, 974 dev_fmt(" ")); 975 } 976 EXPORT_SYMBOL_NS_GPL(pci_print_aer, "CXL"); 977 978 /** 979 * add_error_device - list device to be handled 980 * @e_info: pointer to error info 981 * @dev: pointer to pci_dev to be added 982 */ 983 static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) 984 { 985 int i = e_info->error_dev_num; 986 987 if (i >= AER_MAX_MULTI_ERR_DEVICES) 988 return -ENOSPC; 989 990 e_info->dev[i] = pci_dev_get(dev); 991 e_info->error_dev_num++; 992 993 /* 994 * Ratelimit AER log messages. "dev" is either the source 995 * identified by the root's Error Source ID or it has an unmasked 996 * error logged in its own AER Capability. Messages are emitted 997 * when "ratelimit_print[i]" is non-zero. If we will print detail 998 * for a downstream device, make sure we print the Error Source ID 999 * from the root as well. 1000 */ 1001 if (aer_ratelimit(dev, e_info->severity)) { 1002 e_info->ratelimit_print[i] = 1; 1003 e_info->root_ratelimit_print = 1; 1004 } 1005 return 0; 1006 } 1007 1008 /** 1009 * is_error_source - check whether the device is source of reported error 1010 * @dev: pointer to pci_dev to be checked 1011 * @e_info: pointer to reported error info 1012 */ 1013 static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info) 1014 { 1015 int aer = dev->aer_cap; 1016 u32 status, mask; 1017 u16 reg16; 1018 1019 /* 1020 * When bus ID is equal to 0, it might be a bad ID 1021 * reported by Root Port. 1022 */ 1023 if ((PCI_BUS_NUM(e_info->id) != 0) && 1024 !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) { 1025 /* Device ID match? */ 1026 if (e_info->id == pci_dev_id(dev)) 1027 return true; 1028 1029 /* Continue ID comparing if there is no multiple error */ 1030 if (!e_info->multi_error_valid) 1031 return false; 1032 } 1033 1034 /* 1035 * When either 1036 * 1) bus ID is equal to 0. Some ports might lose the bus 1037 * ID of error source id; 1038 * 2) bus flag PCI_BUS_FLAGS_NO_AERSID is set 1039 * 3) There are multiple errors and prior ID comparing fails; 1040 * We check AER status registers to find possible reporter. 1041 */ 1042 if (atomic_read(&dev->enable_cnt) == 0) 1043 return false; 1044 1045 /* Check if AER is enabled */ 1046 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16); 1047 if (!(reg16 & PCI_EXP_AER_FLAGS)) 1048 return false; 1049 1050 if (!aer) 1051 return false; 1052 1053 /* Check if error is recorded */ 1054 if (e_info->severity == AER_CORRECTABLE) { 1055 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 1056 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1057 } else { 1058 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 1059 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1060 } 1061 if (status & ~mask) 1062 return true; 1063 1064 return false; 1065 } 1066 1067 static int find_device_iter(struct pci_dev *dev, void *data) 1068 { 1069 struct aer_err_info *e_info = (struct aer_err_info *)data; 1070 1071 if (is_error_source(dev, e_info)) { 1072 /* List this device */ 1073 if (add_error_device(e_info, dev)) { 1074 /* We cannot handle more... Stop iteration */ 1075 pci_err(dev, "Exceeded max supported (%d) devices with errors logged\n", 1076 AER_MAX_MULTI_ERR_DEVICES); 1077 return 1; 1078 } 1079 1080 /* If there is only a single error, stop iteration */ 1081 if (!e_info->multi_error_valid) 1082 return 1; 1083 } 1084 return 0; 1085 } 1086 1087 /** 1088 * find_source_device - search through device hierarchy for source device 1089 * @parent: pointer to Root Port pci_dev data structure 1090 * @e_info: including detailed error information such as ID 1091 * 1092 * Return: true if found. 1093 * 1094 * Invoked by DPC when error is detected at the Root Port. 1095 * Caller of this function must set id, severity, and multi_error_valid of 1096 * struct aer_err_info pointed by @e_info properly. This function must fill 1097 * e_info->error_dev_num and e_info->dev[], based on the given information. 1098 */ 1099 static bool find_source_device(struct pci_dev *parent, 1100 struct aer_err_info *e_info) 1101 { 1102 struct pci_dev *dev = parent; 1103 int result; 1104 1105 /* Must reset in this function */ 1106 e_info->error_dev_num = 0; 1107 1108 /* Is Root Port an agent that sends error message? */ 1109 result = find_device_iter(dev, e_info); 1110 if (result) 1111 return true; 1112 1113 if (pci_pcie_type(parent) == PCI_EXP_TYPE_RC_EC) 1114 pcie_walk_rcec(parent, find_device_iter, e_info); 1115 else 1116 pci_walk_bus(parent->subordinate, find_device_iter, e_info); 1117 1118 if (!e_info->error_dev_num) 1119 return false; 1120 return true; 1121 } 1122 1123 #ifdef CONFIG_PCIEAER_CXL 1124 1125 /** 1126 * pci_aer_unmask_internal_errors - unmask internal errors 1127 * @dev: pointer to the pci_dev data structure 1128 * 1129 * Unmask internal errors in the Uncorrectable and Correctable Error 1130 * Mask registers. 1131 * 1132 * Note: AER must be enabled and supported by the device which must be 1133 * checked in advance, e.g. with pcie_aer_is_native(). 1134 */ 1135 static void pci_aer_unmask_internal_errors(struct pci_dev *dev) 1136 { 1137 int aer = dev->aer_cap; 1138 u32 mask; 1139 1140 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1141 mask &= ~PCI_ERR_UNC_INTN; 1142 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask); 1143 1144 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1145 mask &= ~PCI_ERR_COR_INTERNAL; 1146 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask); 1147 } 1148 1149 static bool is_cxl_mem_dev(struct pci_dev *dev) 1150 { 1151 /* 1152 * The capability, status, and control fields in Device 0, 1153 * Function 0 DVSEC control the CXL functionality of the 1154 * entire device (CXL 3.0, 8.1.3). 1155 */ 1156 if (dev->devfn != PCI_DEVFN(0, 0)) 1157 return false; 1158 1159 /* 1160 * CXL Memory Devices must have the 502h class code set (CXL 1161 * 3.0, 8.1.12.1). 1162 */ 1163 if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL) 1164 return false; 1165 1166 return true; 1167 } 1168 1169 static bool cxl_error_is_native(struct pci_dev *dev) 1170 { 1171 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 1172 1173 return (pcie_ports_native || host->native_aer); 1174 } 1175 1176 static bool is_internal_error(struct aer_err_info *info) 1177 { 1178 if (info->severity == AER_CORRECTABLE) 1179 return info->status & PCI_ERR_COR_INTERNAL; 1180 1181 return info->status & PCI_ERR_UNC_INTN; 1182 } 1183 1184 static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data) 1185 { 1186 struct aer_err_info *info = (struct aer_err_info *)data; 1187 const struct pci_error_handlers *err_handler; 1188 1189 if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev)) 1190 return 0; 1191 1192 /* Protect dev->driver */ 1193 device_lock(&dev->dev); 1194 1195 err_handler = dev->driver ? dev->driver->err_handler : NULL; 1196 if (!err_handler) 1197 goto out; 1198 1199 if (info->severity == AER_CORRECTABLE) { 1200 if (err_handler->cor_error_detected) 1201 err_handler->cor_error_detected(dev); 1202 } else if (err_handler->error_detected) { 1203 if (info->severity == AER_NONFATAL) 1204 err_handler->error_detected(dev, pci_channel_io_normal); 1205 else if (info->severity == AER_FATAL) 1206 err_handler->error_detected(dev, pci_channel_io_frozen); 1207 } 1208 out: 1209 device_unlock(&dev->dev); 1210 return 0; 1211 } 1212 1213 static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) 1214 { 1215 /* 1216 * Internal errors of an RCEC indicate an AER error in an 1217 * RCH's downstream port. Check and handle them in the CXL.mem 1218 * device driver. 1219 */ 1220 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC && 1221 is_internal_error(info)) 1222 pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info); 1223 } 1224 1225 static int handles_cxl_error_iter(struct pci_dev *dev, void *data) 1226 { 1227 bool *handles_cxl = data; 1228 1229 if (!*handles_cxl) 1230 *handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev); 1231 1232 /* Non-zero terminates iteration */ 1233 return *handles_cxl; 1234 } 1235 1236 static bool handles_cxl_errors(struct pci_dev *rcec) 1237 { 1238 bool handles_cxl = false; 1239 1240 if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC && 1241 pcie_aer_is_native(rcec)) 1242 pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl); 1243 1244 return handles_cxl; 1245 } 1246 1247 static void cxl_rch_enable_rcec(struct pci_dev *rcec) 1248 { 1249 if (!handles_cxl_errors(rcec)) 1250 return; 1251 1252 pci_aer_unmask_internal_errors(rcec); 1253 pci_info(rcec, "CXL: Internal errors unmasked"); 1254 } 1255 1256 #else 1257 static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { } 1258 static inline void cxl_rch_handle_error(struct pci_dev *dev, 1259 struct aer_err_info *info) { } 1260 #endif 1261 1262 /** 1263 * pci_aer_handle_error - handle logging error into an event log 1264 * @dev: pointer to pci_dev data structure of error source device 1265 * @info: comprehensive error information 1266 * 1267 * Invoked when an error being detected by Root Port. 1268 */ 1269 static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info) 1270 { 1271 int aer = dev->aer_cap; 1272 1273 if (info->severity == AER_CORRECTABLE) { 1274 /* 1275 * Correctable error does not need software intervention. 1276 * No need to go through error recovery process. 1277 */ 1278 if (aer) 1279 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1280 info->status); 1281 if (pcie_aer_is_native(dev)) { 1282 struct pci_driver *pdrv = dev->driver; 1283 1284 if (pdrv && pdrv->err_handler && 1285 pdrv->err_handler->cor_error_detected) 1286 pdrv->err_handler->cor_error_detected(dev); 1287 pcie_clear_device_status(dev); 1288 } 1289 } else if (info->severity == AER_NONFATAL) 1290 pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset); 1291 else if (info->severity == AER_FATAL) 1292 pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset); 1293 } 1294 1295 static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) 1296 { 1297 cxl_rch_handle_error(dev, info); 1298 pci_aer_handle_error(dev, info); 1299 pci_dev_put(dev); 1300 } 1301 1302 #ifdef CONFIG_ACPI_APEI_PCIEAER 1303 1304 #define AER_RECOVER_RING_SIZE 16 1305 1306 struct aer_recover_entry { 1307 u8 bus; 1308 u8 devfn; 1309 u16 domain; 1310 int severity; 1311 struct aer_capability_regs *regs; 1312 }; 1313 1314 static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry, 1315 AER_RECOVER_RING_SIZE); 1316 1317 static void aer_recover_work_func(struct work_struct *work) 1318 { 1319 struct aer_recover_entry entry; 1320 struct pci_dev *pdev; 1321 1322 while (kfifo_get(&aer_recover_ring, &entry)) { 1323 pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, 1324 entry.devfn); 1325 if (!pdev) { 1326 pr_err_ratelimited("%04x:%02x:%02x.%x: no pci_dev found\n", 1327 entry.domain, entry.bus, 1328 PCI_SLOT(entry.devfn), 1329 PCI_FUNC(entry.devfn)); 1330 continue; 1331 } 1332 pci_print_aer(pdev, entry.severity, entry.regs); 1333 1334 /* 1335 * Memory for aer_capability_regs(entry.regs) is being 1336 * allocated from the ghes_estatus_pool to protect it from 1337 * overwriting when multiple sections are present in the 1338 * error status. Thus free the same after processing the 1339 * data. 1340 */ 1341 ghes_estatus_pool_region_free((unsigned long)entry.regs, 1342 sizeof(struct aer_capability_regs)); 1343 1344 if (entry.severity == AER_NONFATAL) 1345 pcie_do_recovery(pdev, pci_channel_io_normal, 1346 aer_root_reset); 1347 else if (entry.severity == AER_FATAL) 1348 pcie_do_recovery(pdev, pci_channel_io_frozen, 1349 aer_root_reset); 1350 pci_dev_put(pdev); 1351 } 1352 } 1353 1354 /* 1355 * Mutual exclusion for writers of aer_recover_ring, reader side don't 1356 * need lock, because there is only one reader and lock is not needed 1357 * between reader and writer. 1358 */ 1359 static DEFINE_SPINLOCK(aer_recover_ring_lock); 1360 static DECLARE_WORK(aer_recover_work, aer_recover_work_func); 1361 1362 void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, 1363 int severity, struct aer_capability_regs *aer_regs) 1364 { 1365 struct aer_recover_entry entry = { 1366 .bus = bus, 1367 .devfn = devfn, 1368 .domain = domain, 1369 .severity = severity, 1370 .regs = aer_regs, 1371 }; 1372 1373 if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1, 1374 &aer_recover_ring_lock)) 1375 schedule_work(&aer_recover_work); 1376 else 1377 pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n", 1378 domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 1379 } 1380 EXPORT_SYMBOL_GPL(aer_recover_queue); 1381 #endif 1382 1383 /** 1384 * aer_get_device_error_info - read error status from dev and store it to info 1385 * @info: pointer to structure to store the error record 1386 * @i: index into info->dev[] 1387 * 1388 * Return: 1 on success, 0 on error. 1389 * 1390 * Note that @info is reused among all error devices. Clear fields properly. 1391 */ 1392 int aer_get_device_error_info(struct aer_err_info *info, int i) 1393 { 1394 struct pci_dev *dev; 1395 int type, aer; 1396 u32 aercc; 1397 1398 if (i >= AER_MAX_MULTI_ERR_DEVICES) 1399 return 0; 1400 1401 dev = info->dev[i]; 1402 aer = dev->aer_cap; 1403 type = pci_pcie_type(dev); 1404 1405 /* Must reset in this function */ 1406 info->status = 0; 1407 info->tlp_header_valid = 0; 1408 1409 /* The device might not support AER */ 1410 if (!aer) 1411 return 0; 1412 1413 if (info->severity == AER_CORRECTABLE) { 1414 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1415 &info->status); 1416 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, 1417 &info->mask); 1418 if (!(info->status & ~info->mask)) 1419 return 0; 1420 } else if (type == PCI_EXP_TYPE_ROOT_PORT || 1421 type == PCI_EXP_TYPE_RC_EC || 1422 type == PCI_EXP_TYPE_DOWNSTREAM || 1423 info->severity == AER_NONFATAL) { 1424 1425 /* Link is still healthy for IO reads */ 1426 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, 1427 &info->status); 1428 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, 1429 &info->mask); 1430 if (!(info->status & ~info->mask)) 1431 return 0; 1432 1433 /* Get First Error Pointer */ 1434 pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc); 1435 info->first_error = PCI_ERR_CAP_FEP(aercc); 1436 1437 if (tlp_header_logged(info->status, aercc)) { 1438 info->tlp_header_valid = 1; 1439 pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG, 1440 aer + PCI_ERR_PREFIX_LOG, 1441 aer_tlp_log_len(dev, aercc), 1442 aercc & PCI_ERR_CAP_TLP_LOG_FLIT, 1443 &info->tlp); 1444 } 1445 } 1446 1447 return 1; 1448 } 1449 1450 static inline void aer_process_err_devices(struct aer_err_info *e_info) 1451 { 1452 int i; 1453 1454 /* Report all before handling them, to not lose records by reset etc. */ 1455 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1456 if (aer_get_device_error_info(e_info, i)) 1457 aer_print_error(e_info, i); 1458 } 1459 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1460 if (aer_get_device_error_info(e_info, i)) 1461 handle_error_source(e_info->dev[i], e_info); 1462 } 1463 } 1464 1465 /** 1466 * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error 1467 * detected by Root Port or RCEC 1468 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1469 * @info: pointer to AER error info 1470 */ 1471 static void aer_isr_one_error_type(struct pci_dev *root, 1472 struct aer_err_info *info) 1473 { 1474 bool found; 1475 1476 found = find_source_device(root, info); 1477 1478 /* 1479 * If we're going to log error messages, we've already set 1480 * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to 1481 * non-zero (which enables printing) because this is either an 1482 * ERR_FATAL or we found a device with an error logged in its AER 1483 * Capability. 1484 * 1485 * If we didn't find the Error Source device, at least log the 1486 * Requester ID from the ERR_* Message received by the Root Port or 1487 * RCEC, ratelimited by the RP or RCEC. 1488 */ 1489 if (info->root_ratelimit_print || 1490 (!found && aer_ratelimit(root, info->severity))) 1491 aer_print_source(root, info, found); 1492 1493 if (found) 1494 aer_process_err_devices(info); 1495 } 1496 1497 /** 1498 * aer_isr_one_error - consume error(s) signaled by an AER interrupt from 1499 * Root Port or RCEC 1500 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1501 * @e_src: pointer to an error source 1502 */ 1503 static void aer_isr_one_error(struct pci_dev *root, 1504 struct aer_err_source *e_src) 1505 { 1506 u32 status = e_src->status; 1507 1508 pci_rootport_aer_stats_incr(root, e_src); 1509 1510 /* 1511 * There is a possibility that both correctable error and 1512 * uncorrectable error being logged. Report correctable error first. 1513 */ 1514 if (status & PCI_ERR_ROOT_COR_RCV) { 1515 int multi = status & PCI_ERR_ROOT_MULTI_COR_RCV; 1516 struct aer_err_info e_info = { 1517 .id = ERR_COR_ID(e_src->id), 1518 .severity = AER_CORRECTABLE, 1519 .level = KERN_WARNING, 1520 .multi_error_valid = multi ? 1 : 0, 1521 }; 1522 1523 aer_isr_one_error_type(root, &e_info); 1524 } 1525 1526 if (status & PCI_ERR_ROOT_UNCOR_RCV) { 1527 int fatal = status & PCI_ERR_ROOT_FATAL_RCV; 1528 int multi = status & PCI_ERR_ROOT_MULTI_UNCOR_RCV; 1529 struct aer_err_info e_info = { 1530 .id = ERR_UNCOR_ID(e_src->id), 1531 .severity = fatal ? AER_FATAL : AER_NONFATAL, 1532 .level = KERN_ERR, 1533 .multi_error_valid = multi ? 1 : 0, 1534 }; 1535 1536 aer_isr_one_error_type(root, &e_info); 1537 } 1538 } 1539 1540 /** 1541 * aer_isr - consume errors detected by Root Port 1542 * @irq: IRQ assigned to Root Port 1543 * @context: pointer to Root Port data structure 1544 * 1545 * Invoked, as DPC, when Root Port records new detected error 1546 */ 1547 static irqreturn_t aer_isr(int irq, void *context) 1548 { 1549 struct pcie_device *dev = (struct pcie_device *)context; 1550 struct aer_rpc *rpc = get_service_data(dev); 1551 struct aer_err_source e_src; 1552 1553 if (kfifo_is_empty(&rpc->aer_fifo)) 1554 return IRQ_NONE; 1555 1556 while (kfifo_get(&rpc->aer_fifo, &e_src)) 1557 aer_isr_one_error(rpc->rpd, &e_src); 1558 return IRQ_HANDLED; 1559 } 1560 1561 /** 1562 * aer_irq - Root Port's ISR 1563 * @irq: IRQ assigned to Root Port 1564 * @context: pointer to Root Port data structure 1565 * 1566 * Invoked when Root Port detects AER messages. 1567 */ 1568 static irqreturn_t aer_irq(int irq, void *context) 1569 { 1570 struct pcie_device *pdev = (struct pcie_device *)context; 1571 struct aer_rpc *rpc = get_service_data(pdev); 1572 struct pci_dev *rp = rpc->rpd; 1573 int aer = rp->aer_cap; 1574 struct aer_err_source e_src = {}; 1575 1576 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status); 1577 if (!(e_src.status & AER_ERR_STATUS_MASK)) 1578 return IRQ_NONE; 1579 1580 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id); 1581 pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status); 1582 1583 if (!kfifo_put(&rpc->aer_fifo, e_src)) 1584 return IRQ_HANDLED; 1585 1586 return IRQ_WAKE_THREAD; 1587 } 1588 1589 static void aer_enable_irq(struct pci_dev *pdev) 1590 { 1591 int aer = pdev->aer_cap; 1592 u32 reg32; 1593 1594 /* Enable Root Port's interrupt in response to error messages */ 1595 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1596 reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; 1597 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1598 } 1599 1600 static void aer_disable_irq(struct pci_dev *pdev) 1601 { 1602 int aer = pdev->aer_cap; 1603 u32 reg32; 1604 1605 /* Disable Root Port's interrupt in response to error messages */ 1606 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1607 reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; 1608 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1609 } 1610 1611 /** 1612 * aer_enable_rootport - enable Root Port's interrupts when receiving messages 1613 * @rpc: pointer to a Root Port data structure 1614 * 1615 * Invoked when PCIe bus loads AER service driver. 1616 */ 1617 static void aer_enable_rootport(struct aer_rpc *rpc) 1618 { 1619 struct pci_dev *pdev = rpc->rpd; 1620 int aer = pdev->aer_cap; 1621 u16 reg16; 1622 u32 reg32; 1623 1624 /* Clear PCIe Capability's Device Status */ 1625 pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, ®16); 1626 pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16); 1627 1628 /* Disable system error generation in response to error messages */ 1629 pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, 1630 SYSTEM_ERROR_INTR_ON_MESG_MASK); 1631 1632 /* Clear error status */ 1633 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1634 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1635 pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32); 1636 pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32); 1637 pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32); 1638 pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32); 1639 1640 aer_enable_irq(pdev); 1641 } 1642 1643 /** 1644 * aer_disable_rootport - disable Root Port's interrupts when receiving messages 1645 * @rpc: pointer to a Root Port data structure 1646 * 1647 * Invoked when PCIe bus unloads AER service driver. 1648 */ 1649 static void aer_disable_rootport(struct aer_rpc *rpc) 1650 { 1651 struct pci_dev *pdev = rpc->rpd; 1652 int aer = pdev->aer_cap; 1653 u32 reg32; 1654 1655 aer_disable_irq(pdev); 1656 1657 /* Clear Root's error status reg */ 1658 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1659 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1660 } 1661 1662 /** 1663 * aer_remove - clean up resources 1664 * @dev: pointer to the pcie_dev data structure 1665 * 1666 * Invoked when PCI Express bus unloads or AER probe fails. 1667 */ 1668 static void aer_remove(struct pcie_device *dev) 1669 { 1670 struct aer_rpc *rpc = get_service_data(dev); 1671 1672 aer_disable_rootport(rpc); 1673 } 1674 1675 /** 1676 * aer_probe - initialize resources 1677 * @dev: pointer to the pcie_dev data structure 1678 * 1679 * Invoked when PCI Express bus loads AER service driver. 1680 */ 1681 static int aer_probe(struct pcie_device *dev) 1682 { 1683 int status; 1684 struct aer_rpc *rpc; 1685 struct device *device = &dev->device; 1686 struct pci_dev *port = dev->port; 1687 1688 BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) < 1689 AER_MAX_TYPEOF_COR_ERRS); 1690 BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) < 1691 AER_MAX_TYPEOF_UNCOR_ERRS); 1692 1693 /* Limit to Root Ports or Root Complex Event Collectors */ 1694 if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) && 1695 (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT)) 1696 return -ENODEV; 1697 1698 rpc = devm_kzalloc(device, sizeof(struct aer_rpc), GFP_KERNEL); 1699 if (!rpc) 1700 return -ENOMEM; 1701 1702 rpc->rpd = port; 1703 INIT_KFIFO(rpc->aer_fifo); 1704 set_service_data(dev, rpc); 1705 1706 status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr, 1707 IRQF_SHARED, "aerdrv", dev); 1708 if (status) { 1709 pci_err(port, "request AER IRQ %d failed\n", dev->irq); 1710 return status; 1711 } 1712 1713 cxl_rch_enable_rcec(port); 1714 aer_enable_rootport(rpc); 1715 pci_info(port, "enabled with IRQ %d\n", dev->irq); 1716 return 0; 1717 } 1718 1719 static int aer_suspend(struct pcie_device *dev) 1720 { 1721 struct aer_rpc *rpc = get_service_data(dev); 1722 1723 aer_disable_rootport(rpc); 1724 return 0; 1725 } 1726 1727 static int aer_resume(struct pcie_device *dev) 1728 { 1729 struct aer_rpc *rpc = get_service_data(dev); 1730 1731 aer_enable_rootport(rpc); 1732 return 0; 1733 } 1734 1735 /** 1736 * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP 1737 * @dev: pointer to Root Port, RCEC, or RCiEP 1738 * 1739 * Invoked by Port Bus driver when performing reset. 1740 */ 1741 static pci_ers_result_t aer_root_reset(struct pci_dev *dev) 1742 { 1743 int type = pci_pcie_type(dev); 1744 struct pci_dev *root; 1745 int aer; 1746 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 1747 u32 reg32; 1748 int rc; 1749 1750 /* 1751 * Only Root Ports and RCECs have AER Root Command and Root Status 1752 * registers. If "dev" is an RCiEP, the relevant registers are in 1753 * the RCEC. 1754 */ 1755 if (type == PCI_EXP_TYPE_RC_END) 1756 root = dev->rcec; 1757 else 1758 root = pcie_find_root_port(dev); 1759 1760 /* 1761 * If the platform retained control of AER, an RCiEP may not have 1762 * an RCEC visible to us, so dev->rcec ("root") may be NULL. In 1763 * that case, firmware is responsible for these registers. 1764 */ 1765 aer = root ? root->aer_cap : 0; 1766 1767 if ((host->native_aer || pcie_ports_native) && aer) 1768 aer_disable_irq(root); 1769 1770 if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { 1771 rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET); 1772 if (!rc) 1773 pci_info(dev, "has been reset\n"); 1774 else 1775 pci_info(dev, "not reset (no FLR support: %d)\n", rc); 1776 } else { 1777 rc = pci_bus_error_reset(dev); 1778 pci_info(dev, "%s Port link has been reset (%d)\n", 1779 pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc); 1780 } 1781 1782 if ((host->native_aer || pcie_ports_native) && aer) { 1783 /* Clear Root Error Status */ 1784 pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, ®32); 1785 pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32); 1786 1787 aer_enable_irq(root); 1788 } 1789 1790 return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 1791 } 1792 1793 static struct pcie_port_service_driver aerdriver = { 1794 .name = "aer", 1795 .port_type = PCIE_ANY_PORT, 1796 .service = PCIE_PORT_SERVICE_AER, 1797 1798 .probe = aer_probe, 1799 .suspend = aer_suspend, 1800 .resume = aer_resume, 1801 .remove = aer_remove, 1802 }; 1803 1804 /** 1805 * pcie_aer_init - register AER service driver 1806 * 1807 * Invoked when AER service driver is loaded. 1808 */ 1809 int __init pcie_aer_init(void) 1810 { 1811 if (!pci_aer_available()) 1812 return -ENXIO; 1813 return pcie_port_service_register(&aerdriver); 1814 } 1815