1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implement the AER root port service driver. The driver registers an IRQ 4 * handler. When a root port triggers an AER interrupt, the IRQ handler 5 * collects Root Port status and schedules work. 6 * 7 * Copyright (C) 2006 Intel Corp. 8 * Tom Long Nguyen (tom.l.nguyen@intel.com) 9 * Zhang Yanmin (yanmin.zhang@intel.com) 10 * 11 * (C) Copyright 2009 Hewlett-Packard Development Company, L.P. 12 * Andrew Patterson <andrew.patterson@hp.com> 13 */ 14 15 #define pr_fmt(fmt) "AER: " fmt 16 #define dev_fmt pr_fmt 17 18 #include <linux/bitops.h> 19 #include <linux/cper.h> 20 #include <linux/dev_printk.h> 21 #include <linux/pci.h> 22 #include <linux/pci-acpi.h> 23 #include <linux/sched.h> 24 #include <linux/kernel.h> 25 #include <linux/errno.h> 26 #include <linux/pm.h> 27 #include <linux/init.h> 28 #include <linux/interrupt.h> 29 #include <linux/delay.h> 30 #include <linux/kfifo.h> 31 #include <linux/ratelimit.h> 32 #include <linux/slab.h> 33 #include <acpi/apei.h> 34 #include <acpi/ghes.h> 35 #include <ras/ras_event.h> 36 37 #include "../pci.h" 38 #include "portdrv.h" 39 40 #define aer_printk(level, pdev, fmt, arg...) \ 41 dev_printk(level, &(pdev)->dev, fmt, ##arg) 42 43 #define AER_ERROR_SOURCES_MAX 128 44 45 #define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */ 46 #define AER_MAX_TYPEOF_UNCOR_ERRS 32 /* as per PCI_ERR_UNCOR_STATUS*/ 47 48 struct aer_err_source { 49 u32 status; /* PCI_ERR_ROOT_STATUS */ 50 u32 id; /* PCI_ERR_ROOT_ERR_SRC */ 51 }; 52 53 struct aer_rpc { 54 struct pci_dev *rpd; /* Root Port device */ 55 DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX); 56 }; 57 58 /* AER info for the device */ 59 struct aer_info { 60 61 /* 62 * Fields for all AER capable devices. They indicate the errors 63 * "as seen by this device". Note that this may mean that if an 64 * Endpoint is causing problems, the AER counters may increment 65 * at its link partner (e.g. Root Port) because the errors will be 66 * "seen" by the link partner and not the problematic Endpoint 67 * itself (which may report all counters as 0 as it never saw any 68 * problems). 69 */ 70 /* Counters for different type of correctable errors */ 71 u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS]; 72 /* Counters for different type of fatal uncorrectable errors */ 73 u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 74 /* Counters for different type of nonfatal uncorrectable errors */ 75 u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 76 /* Total number of ERR_COR sent by this device */ 77 u64 dev_total_cor_errs; 78 /* Total number of ERR_FATAL sent by this device */ 79 u64 dev_total_fatal_errs; 80 /* Total number of ERR_NONFATAL sent by this device */ 81 u64 dev_total_nonfatal_errs; 82 83 /* 84 * Fields for Root Ports & Root Complex Event Collectors only; these 85 * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL 86 * messages received by the Root Port / Event Collector, INCLUDING the 87 * ones that are generated internally (by the Root Port itself) 88 */ 89 u64 rootport_total_cor_errs; 90 u64 rootport_total_fatal_errs; 91 u64 rootport_total_nonfatal_errs; 92 93 /* Ratelimits for errors */ 94 struct ratelimit_state correctable_ratelimit; 95 struct ratelimit_state nonfatal_ratelimit; 96 }; 97 98 #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \ 99 PCI_ERR_UNC_POISON_BLK | \ 100 PCI_ERR_UNC_ECRC| \ 101 PCI_ERR_UNC_UNSUP| \ 102 PCI_ERR_UNC_COMP_ABORT| \ 103 PCI_ERR_UNC_UNX_COMP| \ 104 PCI_ERR_UNC_ACSV | \ 105 PCI_ERR_UNC_MCBTLP | \ 106 PCI_ERR_UNC_ATOMEG | \ 107 PCI_ERR_UNC_DMWR_BLK | \ 108 PCI_ERR_UNC_XLAT_BLK | \ 109 PCI_ERR_UNC_TLPPRE | \ 110 PCI_ERR_UNC_MALF_TLP | \ 111 PCI_ERR_UNC_IDE_CHECK | \ 112 PCI_ERR_UNC_MISR_IDE | \ 113 PCI_ERR_UNC_PCRC_CHECK) 114 115 #define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \ 116 PCI_EXP_RTCTL_SENFEE| \ 117 PCI_EXP_RTCTL_SEFEE) 118 #define ROOT_PORT_INTR_ON_MESG_MASK (PCI_ERR_ROOT_CMD_COR_EN| \ 119 PCI_ERR_ROOT_CMD_NONFATAL_EN| \ 120 PCI_ERR_ROOT_CMD_FATAL_EN) 121 #define ERR_COR_ID(d) (d & 0xffff) 122 #define ERR_UNCOR_ID(d) (d >> 16) 123 124 #define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV | \ 125 PCI_ERR_ROOT_COR_RCV | \ 126 PCI_ERR_ROOT_MULTI_COR_RCV | \ 127 PCI_ERR_ROOT_MULTI_UNCOR_RCV) 128 129 static bool pcie_aer_disable; 130 static pci_ers_result_t aer_root_reset(struct pci_dev *dev); 131 132 void pci_no_aer(void) 133 { 134 pcie_aer_disable = true; 135 } 136 137 bool pci_aer_available(void) 138 { 139 return !pcie_aer_disable && pci_msi_enabled(); 140 } 141 142 #ifdef CONFIG_PCIE_ECRC 143 144 #define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */ 145 #define ECRC_POLICY_OFF 1 /* ECRC off for performance */ 146 #define ECRC_POLICY_ON 2 /* ECRC on for data integrity */ 147 148 static int ecrc_policy = ECRC_POLICY_DEFAULT; 149 150 static const char * const ecrc_policy_str[] = { 151 [ECRC_POLICY_DEFAULT] = "bios", 152 [ECRC_POLICY_OFF] = "off", 153 [ECRC_POLICY_ON] = "on" 154 }; 155 156 /** 157 * enable_ecrc_checking - enable PCIe ECRC checking for a device 158 * @dev: the PCI device 159 * 160 * Return: 0 on success, or negative on failure. 161 */ 162 static int enable_ecrc_checking(struct pci_dev *dev) 163 { 164 int aer = dev->aer_cap; 165 u32 reg32; 166 167 if (!aer) 168 return -ENODEV; 169 170 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 171 if (reg32 & PCI_ERR_CAP_ECRC_GENC) 172 reg32 |= PCI_ERR_CAP_ECRC_GENE; 173 if (reg32 & PCI_ERR_CAP_ECRC_CHKC) 174 reg32 |= PCI_ERR_CAP_ECRC_CHKE; 175 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 176 177 return 0; 178 } 179 180 /** 181 * disable_ecrc_checking - disable PCIe ECRC checking for a device 182 * @dev: the PCI device 183 * 184 * Return: 0 on success, or negative on failure. 185 */ 186 static int disable_ecrc_checking(struct pci_dev *dev) 187 { 188 int aer = dev->aer_cap; 189 u32 reg32; 190 191 if (!aer) 192 return -ENODEV; 193 194 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 195 reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE); 196 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 197 198 return 0; 199 } 200 201 /** 202 * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based 203 * on global policy 204 * @dev: the PCI device 205 */ 206 void pcie_set_ecrc_checking(struct pci_dev *dev) 207 { 208 if (!pcie_aer_is_native(dev)) 209 return; 210 211 switch (ecrc_policy) { 212 case ECRC_POLICY_DEFAULT: 213 return; 214 case ECRC_POLICY_OFF: 215 disable_ecrc_checking(dev); 216 break; 217 case ECRC_POLICY_ON: 218 enable_ecrc_checking(dev); 219 break; 220 default: 221 return; 222 } 223 } 224 225 /** 226 * pcie_ecrc_get_policy - parse kernel command-line ecrc option 227 * @str: ECRC policy from kernel command line to use 228 */ 229 void pcie_ecrc_get_policy(char *str) 230 { 231 int i; 232 233 i = match_string(ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), str); 234 if (i < 0) 235 return; 236 237 ecrc_policy = i; 238 } 239 #endif /* CONFIG_PCIE_ECRC */ 240 241 #define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ 242 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) 243 244 int pcie_aer_is_native(struct pci_dev *dev) 245 { 246 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 247 248 if (!dev->aer_cap) 249 return 0; 250 251 return pcie_ports_native || host->native_aer; 252 } 253 EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, "CXL"); 254 255 static int pci_enable_pcie_error_reporting(struct pci_dev *dev) 256 { 257 int rc; 258 259 if (!pcie_aer_is_native(dev)) 260 return -EIO; 261 262 rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS); 263 return pcibios_err_to_errno(rc); 264 } 265 266 int pci_aer_clear_nonfatal_status(struct pci_dev *dev) 267 { 268 int aer = dev->aer_cap; 269 u32 status, sev; 270 271 if (!pcie_aer_is_native(dev)) 272 return -EIO; 273 274 /* Clear status bits for ERR_NONFATAL errors only */ 275 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 276 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 277 status &= ~sev; 278 if (status) 279 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 280 281 return 0; 282 } 283 EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status); 284 285 void pci_aer_clear_fatal_status(struct pci_dev *dev) 286 { 287 int aer = dev->aer_cap; 288 u32 status, sev; 289 290 if (!pcie_aer_is_native(dev)) 291 return; 292 293 /* Clear status bits for ERR_FATAL errors only */ 294 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 295 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 296 status &= sev; 297 if (status) 298 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 299 } 300 301 /** 302 * pci_aer_raw_clear_status - Clear AER error registers. 303 * @dev: the PCI device 304 * 305 * Clear AER error status registers unconditionally, regardless of 306 * whether they're owned by firmware or the OS. 307 * 308 * Return: 0 on success, or negative on failure. 309 */ 310 int pci_aer_raw_clear_status(struct pci_dev *dev) 311 { 312 int aer = dev->aer_cap; 313 u32 status; 314 int port_type; 315 316 if (!aer) 317 return -EIO; 318 319 port_type = pci_pcie_type(dev); 320 if (port_type == PCI_EXP_TYPE_ROOT_PORT || 321 port_type == PCI_EXP_TYPE_RC_EC) { 322 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status); 323 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status); 324 } 325 326 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 327 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status); 328 329 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 330 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 331 332 return 0; 333 } 334 335 int pci_aer_clear_status(struct pci_dev *dev) 336 { 337 if (!pcie_aer_is_native(dev)) 338 return -EIO; 339 340 return pci_aer_raw_clear_status(dev); 341 } 342 343 void pci_save_aer_state(struct pci_dev *dev) 344 { 345 int aer = dev->aer_cap; 346 struct pci_cap_saved_state *save_state; 347 u32 *cap; 348 349 if (!aer) 350 return; 351 352 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 353 if (!save_state) 354 return; 355 356 cap = &save_state->cap.data[0]; 357 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++); 358 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++); 359 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++); 360 pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++); 361 if (pcie_cap_has_rtctl(dev)) 362 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++); 363 } 364 365 void pci_restore_aer_state(struct pci_dev *dev) 366 { 367 int aer = dev->aer_cap; 368 struct pci_cap_saved_state *save_state; 369 u32 *cap; 370 371 if (!aer) 372 return; 373 374 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 375 if (!save_state) 376 return; 377 378 cap = &save_state->cap.data[0]; 379 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++); 380 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++); 381 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++); 382 pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++); 383 if (pcie_cap_has_rtctl(dev)) 384 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++); 385 } 386 387 void pci_aer_init(struct pci_dev *dev) 388 { 389 int n; 390 391 dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); 392 if (!dev->aer_cap) 393 return; 394 395 dev->aer_info = kzalloc(sizeof(*dev->aer_info), GFP_KERNEL); 396 if (!dev->aer_info) { 397 dev->aer_cap = 0; 398 return; 399 } 400 401 ratelimit_state_init(&dev->aer_info->correctable_ratelimit, 402 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 403 ratelimit_state_init(&dev->aer_info->nonfatal_ratelimit, 404 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 405 406 /* 407 * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER, 408 * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event 409 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r6.0, sec 410 * 7.8.4.9). 411 */ 412 n = pcie_cap_has_rtctl(dev) ? 5 : 4; 413 pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, sizeof(u32) * n); 414 415 pci_aer_clear_status(dev); 416 417 if (pci_aer_available()) 418 pci_enable_pcie_error_reporting(dev); 419 420 pcie_set_ecrc_checking(dev); 421 } 422 423 void pci_aer_exit(struct pci_dev *dev) 424 { 425 kfree(dev->aer_info); 426 dev->aer_info = NULL; 427 } 428 429 #define AER_AGENT_RECEIVER 0 430 #define AER_AGENT_REQUESTER 1 431 #define AER_AGENT_COMPLETER 2 432 #define AER_AGENT_TRANSMITTER 3 433 434 #define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 435 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP)) 436 #define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \ 437 0 : PCI_ERR_UNC_COMP_ABORT) 438 #define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 439 (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0) 440 441 #define AER_GET_AGENT(t, e) \ 442 ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \ 443 (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \ 444 (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \ 445 AER_AGENT_RECEIVER) 446 447 #define AER_PHYSICAL_LAYER_ERROR 0 448 #define AER_DATA_LINK_LAYER_ERROR 1 449 #define AER_TRANSACTION_LAYER_ERROR 2 450 451 #define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 452 PCI_ERR_COR_RCVR : 0) 453 #define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 454 (PCI_ERR_COR_BAD_TLP| \ 455 PCI_ERR_COR_BAD_DLLP| \ 456 PCI_ERR_COR_REP_ROLL| \ 457 PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP) 458 459 #define AER_GET_LAYER_ERROR(t, e) \ 460 ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \ 461 (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ 462 AER_TRANSACTION_LAYER_ERROR) 463 464 /* 465 * AER error strings 466 */ 467 static const char * const aer_error_severity_string[] = { 468 "Uncorrectable (Non-Fatal)", 469 "Uncorrectable (Fatal)", 470 "Correctable" 471 }; 472 473 static const char *aer_error_layer[] = { 474 "Physical Layer", 475 "Data Link Layer", 476 "Transaction Layer" 477 }; 478 479 static const char *aer_correctable_error_string[] = { 480 "RxErr", /* Bit Position 0 */ 481 NULL, 482 NULL, 483 NULL, 484 NULL, 485 NULL, 486 "BadTLP", /* Bit Position 6 */ 487 "BadDLLP", /* Bit Position 7 */ 488 "Rollover", /* Bit Position 8 */ 489 NULL, 490 NULL, 491 NULL, 492 "Timeout", /* Bit Position 12 */ 493 "NonFatalErr", /* Bit Position 13 */ 494 "CorrIntErr", /* Bit Position 14 */ 495 "HeaderOF", /* Bit Position 15 */ 496 NULL, /* Bit Position 16 */ 497 NULL, /* Bit Position 17 */ 498 NULL, /* Bit Position 18 */ 499 NULL, /* Bit Position 19 */ 500 NULL, /* Bit Position 20 */ 501 NULL, /* Bit Position 21 */ 502 NULL, /* Bit Position 22 */ 503 NULL, /* Bit Position 23 */ 504 NULL, /* Bit Position 24 */ 505 NULL, /* Bit Position 25 */ 506 NULL, /* Bit Position 26 */ 507 NULL, /* Bit Position 27 */ 508 NULL, /* Bit Position 28 */ 509 NULL, /* Bit Position 29 */ 510 NULL, /* Bit Position 30 */ 511 NULL, /* Bit Position 31 */ 512 }; 513 514 static const char *aer_uncorrectable_error_string[] = { 515 "Undefined", /* Bit Position 0 */ 516 NULL, 517 NULL, 518 NULL, 519 "DLP", /* Bit Position 4 */ 520 "SDES", /* Bit Position 5 */ 521 NULL, 522 NULL, 523 NULL, 524 NULL, 525 NULL, 526 NULL, 527 "TLP", /* Bit Position 12 */ 528 "FCP", /* Bit Position 13 */ 529 "CmpltTO", /* Bit Position 14 */ 530 "CmpltAbrt", /* Bit Position 15 */ 531 "UnxCmplt", /* Bit Position 16 */ 532 "RxOF", /* Bit Position 17 */ 533 "MalfTLP", /* Bit Position 18 */ 534 "ECRC", /* Bit Position 19 */ 535 "UnsupReq", /* Bit Position 20 */ 536 "ACSViol", /* Bit Position 21 */ 537 "UncorrIntErr", /* Bit Position 22 */ 538 "BlockedTLP", /* Bit Position 23 */ 539 "AtomicOpBlocked", /* Bit Position 24 */ 540 "TLPBlockedErr", /* Bit Position 25 */ 541 "PoisonTLPBlocked", /* Bit Position 26 */ 542 "DMWrReqBlocked", /* Bit Position 27 */ 543 "IDECheck", /* Bit Position 28 */ 544 "MisIDETLP", /* Bit Position 29 */ 545 "PCRC_CHECK", /* Bit Position 30 */ 546 "TLPXlatBlocked", /* Bit Position 31 */ 547 }; 548 549 static const char *aer_agent_string[] = { 550 "Receiver ID", 551 "Requester ID", 552 "Completer ID", 553 "Transmitter ID" 554 }; 555 556 #define aer_stats_dev_attr(name, stats_array, strings_array, \ 557 total_string, total_field) \ 558 static ssize_t \ 559 name##_show(struct device *dev, struct device_attribute *attr, \ 560 char *buf) \ 561 { \ 562 unsigned int i; \ 563 struct pci_dev *pdev = to_pci_dev(dev); \ 564 u64 *stats = pdev->aer_info->stats_array; \ 565 size_t len = 0; \ 566 \ 567 for (i = 0; i < ARRAY_SIZE(pdev->aer_info->stats_array); i++) { \ 568 if (strings_array[i]) \ 569 len += sysfs_emit_at(buf, len, "%s %llu\n", \ 570 strings_array[i], \ 571 stats[i]); \ 572 else if (stats[i]) \ 573 len += sysfs_emit_at(buf, len, \ 574 #stats_array "_bit[%d] %llu\n",\ 575 i, stats[i]); \ 576 } \ 577 len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \ 578 pdev->aer_info->total_field); \ 579 return len; \ 580 } \ 581 static DEVICE_ATTR_RO(name) 582 583 aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs, 584 aer_correctable_error_string, "ERR_COR", 585 dev_total_cor_errs); 586 aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs, 587 aer_uncorrectable_error_string, "ERR_FATAL", 588 dev_total_fatal_errs); 589 aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs, 590 aer_uncorrectable_error_string, "ERR_NONFATAL", 591 dev_total_nonfatal_errs); 592 593 #define aer_stats_rootport_attr(name, field) \ 594 static ssize_t \ 595 name##_show(struct device *dev, struct device_attribute *attr, \ 596 char *buf) \ 597 { \ 598 struct pci_dev *pdev = to_pci_dev(dev); \ 599 return sysfs_emit(buf, "%llu\n", pdev->aer_info->field); \ 600 } \ 601 static DEVICE_ATTR_RO(name) 602 603 aer_stats_rootport_attr(aer_rootport_total_err_cor, 604 rootport_total_cor_errs); 605 aer_stats_rootport_attr(aer_rootport_total_err_fatal, 606 rootport_total_fatal_errs); 607 aer_stats_rootport_attr(aer_rootport_total_err_nonfatal, 608 rootport_total_nonfatal_errs); 609 610 static struct attribute *aer_stats_attrs[] __ro_after_init = { 611 &dev_attr_aer_dev_correctable.attr, 612 &dev_attr_aer_dev_fatal.attr, 613 &dev_attr_aer_dev_nonfatal.attr, 614 &dev_attr_aer_rootport_total_err_cor.attr, 615 &dev_attr_aer_rootport_total_err_fatal.attr, 616 &dev_attr_aer_rootport_total_err_nonfatal.attr, 617 NULL 618 }; 619 620 static umode_t aer_stats_attrs_are_visible(struct kobject *kobj, 621 struct attribute *a, int n) 622 { 623 struct device *dev = kobj_to_dev(kobj); 624 struct pci_dev *pdev = to_pci_dev(dev); 625 626 if (!pdev->aer_info) 627 return 0; 628 629 if ((a == &dev_attr_aer_rootport_total_err_cor.attr || 630 a == &dev_attr_aer_rootport_total_err_fatal.attr || 631 a == &dev_attr_aer_rootport_total_err_nonfatal.attr) && 632 ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && 633 (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_EC))) 634 return 0; 635 636 return a->mode; 637 } 638 639 const struct attribute_group aer_stats_attr_group = { 640 .attrs = aer_stats_attrs, 641 .is_visible = aer_stats_attrs_are_visible, 642 }; 643 644 /* 645 * Ratelimit interval 646 * <=0: disabled with ratelimit.interval = 0 647 * >0: enabled with ratelimit.interval in ms 648 */ 649 #define aer_ratelimit_interval_attr(name, ratelimit) \ 650 static ssize_t \ 651 name##_show(struct device *dev, struct device_attribute *attr, \ 652 char *buf) \ 653 { \ 654 struct pci_dev *pdev = to_pci_dev(dev); \ 655 \ 656 return sysfs_emit(buf, "%d\n", \ 657 pdev->aer_info->ratelimit.interval); \ 658 } \ 659 \ 660 static ssize_t \ 661 name##_store(struct device *dev, struct device_attribute *attr, \ 662 const char *buf, size_t count) \ 663 { \ 664 struct pci_dev *pdev = to_pci_dev(dev); \ 665 int interval; \ 666 \ 667 if (!capable(CAP_SYS_ADMIN)) \ 668 return -EPERM; \ 669 \ 670 if (kstrtoint(buf, 0, &interval) < 0) \ 671 return -EINVAL; \ 672 \ 673 if (interval <= 0) \ 674 interval = 0; \ 675 else \ 676 interval = msecs_to_jiffies(interval); \ 677 \ 678 pdev->aer_info->ratelimit.interval = interval; \ 679 \ 680 return count; \ 681 } \ 682 static DEVICE_ATTR_RW(name); 683 684 #define aer_ratelimit_burst_attr(name, ratelimit) \ 685 static ssize_t \ 686 name##_show(struct device *dev, struct device_attribute *attr, \ 687 char *buf) \ 688 { \ 689 struct pci_dev *pdev = to_pci_dev(dev); \ 690 \ 691 return sysfs_emit(buf, "%d\n", \ 692 pdev->aer_info->ratelimit.burst); \ 693 } \ 694 \ 695 static ssize_t \ 696 name##_store(struct device *dev, struct device_attribute *attr, \ 697 const char *buf, size_t count) \ 698 { \ 699 struct pci_dev *pdev = to_pci_dev(dev); \ 700 int burst; \ 701 \ 702 if (!capable(CAP_SYS_ADMIN)) \ 703 return -EPERM; \ 704 \ 705 if (kstrtoint(buf, 0, &burst) < 0) \ 706 return -EINVAL; \ 707 \ 708 pdev->aer_info->ratelimit.burst = burst; \ 709 \ 710 return count; \ 711 } \ 712 static DEVICE_ATTR_RW(name); 713 714 #define aer_ratelimit_attrs(name) \ 715 aer_ratelimit_interval_attr(name##_ratelimit_interval_ms, \ 716 name##_ratelimit) \ 717 aer_ratelimit_burst_attr(name##_ratelimit_burst, \ 718 name##_ratelimit) 719 720 aer_ratelimit_attrs(correctable) 721 aer_ratelimit_attrs(nonfatal) 722 723 static struct attribute *aer_attrs[] = { 724 &dev_attr_correctable_ratelimit_interval_ms.attr, 725 &dev_attr_correctable_ratelimit_burst.attr, 726 &dev_attr_nonfatal_ratelimit_interval_ms.attr, 727 &dev_attr_nonfatal_ratelimit_burst.attr, 728 NULL 729 }; 730 731 static umode_t aer_attrs_are_visible(struct kobject *kobj, 732 struct attribute *a, int n) 733 { 734 struct device *dev = kobj_to_dev(kobj); 735 struct pci_dev *pdev = to_pci_dev(dev); 736 737 if (!pdev->aer_info) 738 return 0; 739 740 return a->mode; 741 } 742 743 const struct attribute_group aer_attr_group = { 744 .name = "aer", 745 .attrs = aer_attrs, 746 .is_visible = aer_attrs_are_visible, 747 }; 748 749 static void pci_dev_aer_stats_incr(struct pci_dev *pdev, 750 struct aer_err_info *info) 751 { 752 unsigned long status = info->status & ~info->mask; 753 int i, max = -1; 754 u64 *counter = NULL; 755 struct aer_info *aer_info = pdev->aer_info; 756 757 if (!aer_info) 758 return; 759 760 switch (info->severity) { 761 case AER_CORRECTABLE: 762 aer_info->dev_total_cor_errs++; 763 counter = &aer_info->dev_cor_errs[0]; 764 max = AER_MAX_TYPEOF_COR_ERRS; 765 break; 766 case AER_NONFATAL: 767 aer_info->dev_total_nonfatal_errs++; 768 counter = &aer_info->dev_nonfatal_errs[0]; 769 max = AER_MAX_TYPEOF_UNCOR_ERRS; 770 break; 771 case AER_FATAL: 772 aer_info->dev_total_fatal_errs++; 773 counter = &aer_info->dev_fatal_errs[0]; 774 max = AER_MAX_TYPEOF_UNCOR_ERRS; 775 break; 776 } 777 778 for_each_set_bit(i, &status, max) 779 counter[i]++; 780 } 781 782 static void pci_rootport_aer_stats_incr(struct pci_dev *pdev, 783 struct aer_err_source *e_src) 784 { 785 struct aer_info *aer_info = pdev->aer_info; 786 787 if (!aer_info) 788 return; 789 790 if (e_src->status & PCI_ERR_ROOT_COR_RCV) 791 aer_info->rootport_total_cor_errs++; 792 793 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { 794 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) 795 aer_info->rootport_total_fatal_errs++; 796 else 797 aer_info->rootport_total_nonfatal_errs++; 798 } 799 } 800 801 static int aer_ratelimit(struct pci_dev *dev, unsigned int severity) 802 { 803 if (!dev->aer_info) 804 return 1; 805 806 switch (severity) { 807 case AER_NONFATAL: 808 return __ratelimit(&dev->aer_info->nonfatal_ratelimit); 809 case AER_CORRECTABLE: 810 return __ratelimit(&dev->aer_info->correctable_ratelimit); 811 default: 812 return 1; /* Don't ratelimit fatal errors */ 813 } 814 } 815 816 static bool tlp_header_logged(u32 status, u32 capctl) 817 { 818 /* Errors for which a header is always logged (PCIe r7.0 sec 6.2.7) */ 819 if (status & AER_LOG_TLP_MASKS) 820 return true; 821 822 /* Completion Timeout header is only logged on capable devices */ 823 if (status & PCI_ERR_UNC_COMP_TIME && 824 capctl & PCI_ERR_CAP_COMP_TIME_LOG) 825 return true; 826 827 return false; 828 } 829 830 static void __aer_print_error(struct pci_dev *dev, struct aer_err_info *info) 831 { 832 const char **strings; 833 unsigned long status = info->status & ~info->mask; 834 const char *level = info->level; 835 const char *errmsg; 836 int i; 837 838 if (info->severity == AER_CORRECTABLE) 839 strings = aer_correctable_error_string; 840 else 841 strings = aer_uncorrectable_error_string; 842 843 for_each_set_bit(i, &status, 32) { 844 errmsg = strings[i]; 845 if (!errmsg) 846 errmsg = "Unknown Error Bit"; 847 848 aer_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg, 849 info->first_error == i ? " (First)" : ""); 850 } 851 } 852 853 static void aer_print_source(struct pci_dev *dev, struct aer_err_info *info, 854 bool found) 855 { 856 u16 source = info->id; 857 858 pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d%s\n", 859 info->multi_error_valid ? "Multiple " : "", 860 aer_error_severity_string[info->severity], 861 pci_domain_nr(dev->bus), PCI_BUS_NUM(source), 862 PCI_SLOT(source), PCI_FUNC(source), 863 found ? "" : " (no details found"); 864 } 865 866 void aer_print_error(struct aer_err_info *info, int i) 867 { 868 struct pci_dev *dev; 869 int layer, agent, id; 870 const char *level = info->level; 871 872 if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES)) 873 return; 874 875 dev = info->dev[i]; 876 id = pci_dev_id(dev); 877 878 pci_dev_aer_stats_incr(dev, info); 879 trace_aer_event(pci_name(dev), (info->status & ~info->mask), 880 info->severity, info->tlp_header_valid, &info->tlp); 881 882 if (!info->ratelimit_print[i]) 883 return; 884 885 if (!info->status) { 886 pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", 887 aer_error_severity_string[info->severity]); 888 goto out; 889 } 890 891 layer = AER_GET_LAYER_ERROR(info->severity, info->status); 892 agent = AER_GET_AGENT(info->severity, info->status); 893 894 aer_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n", 895 aer_error_severity_string[info->severity], 896 aer_error_layer[layer], aer_agent_string[agent]); 897 898 aer_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n", 899 dev->vendor, dev->device, info->status, info->mask); 900 901 __aer_print_error(dev, info); 902 903 if (info->tlp_header_valid) 904 pcie_print_tlp_log(dev, &info->tlp, level, dev_fmt(" ")); 905 906 out: 907 if (info->id && info->error_dev_num > 1 && info->id == id) 908 pci_err(dev, " Error of this Agent is reported first\n"); 909 } 910 911 #ifdef CONFIG_ACPI_APEI_PCIEAER 912 int cper_severity_to_aer(int cper_severity) 913 { 914 switch (cper_severity) { 915 case CPER_SEV_RECOVERABLE: 916 return AER_NONFATAL; 917 case CPER_SEV_FATAL: 918 return AER_FATAL; 919 default: 920 return AER_CORRECTABLE; 921 } 922 } 923 EXPORT_SYMBOL_GPL(cper_severity_to_aer); 924 #endif 925 926 void pci_print_aer(struct pci_dev *dev, int aer_severity, 927 struct aer_capability_regs *aer) 928 { 929 int layer, agent, tlp_header_valid = 0; 930 u32 status, mask; 931 struct aer_err_info info = { 932 .severity = aer_severity, 933 .first_error = PCI_ERR_CAP_FEP(aer->cap_control), 934 }; 935 936 if (aer_severity == AER_CORRECTABLE) { 937 status = aer->cor_status; 938 mask = aer->cor_mask; 939 info.level = KERN_WARNING; 940 } else { 941 status = aer->uncor_status; 942 mask = aer->uncor_mask; 943 info.level = KERN_ERR; 944 tlp_header_valid = tlp_header_logged(status, aer->cap_control); 945 } 946 947 info.status = status; 948 info.mask = mask; 949 950 pci_dev_aer_stats_incr(dev, &info); 951 trace_aer_event(pci_name(dev), (status & ~mask), 952 aer_severity, tlp_header_valid, &aer->header_log); 953 954 if (!aer_ratelimit(dev, info.severity)) 955 return; 956 957 layer = AER_GET_LAYER_ERROR(aer_severity, status); 958 agent = AER_GET_AGENT(aer_severity, status); 959 960 aer_printk(info.level, dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", 961 status, mask); 962 __aer_print_error(dev, &info); 963 aer_printk(info.level, dev, "aer_layer=%s, aer_agent=%s\n", 964 aer_error_layer[layer], aer_agent_string[agent]); 965 966 if (aer_severity != AER_CORRECTABLE) 967 aer_printk(info.level, dev, "aer_uncor_severity: 0x%08x\n", 968 aer->uncor_severity); 969 970 if (tlp_header_valid) 971 pcie_print_tlp_log(dev, &aer->header_log, info.level, 972 dev_fmt(" ")); 973 } 974 EXPORT_SYMBOL_NS_GPL(pci_print_aer, "CXL"); 975 976 /** 977 * add_error_device - list device to be handled 978 * @e_info: pointer to error info 979 * @dev: pointer to pci_dev to be added 980 */ 981 static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) 982 { 983 int i = e_info->error_dev_num; 984 985 if (i >= AER_MAX_MULTI_ERR_DEVICES) 986 return -ENOSPC; 987 988 e_info->dev[i] = pci_dev_get(dev); 989 e_info->error_dev_num++; 990 991 /* 992 * Ratelimit AER log messages. "dev" is either the source 993 * identified by the root's Error Source ID or it has an unmasked 994 * error logged in its own AER Capability. Messages are emitted 995 * when "ratelimit_print[i]" is non-zero. If we will print detail 996 * for a downstream device, make sure we print the Error Source ID 997 * from the root as well. 998 */ 999 if (aer_ratelimit(dev, e_info->severity)) { 1000 e_info->ratelimit_print[i] = 1; 1001 e_info->root_ratelimit_print = 1; 1002 } 1003 return 0; 1004 } 1005 1006 /** 1007 * is_error_source - check whether the device is source of reported error 1008 * @dev: pointer to pci_dev to be checked 1009 * @e_info: pointer to reported error info 1010 */ 1011 static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info) 1012 { 1013 int aer = dev->aer_cap; 1014 u32 status, mask; 1015 u16 reg16; 1016 1017 /* 1018 * When bus ID is equal to 0, it might be a bad ID 1019 * reported by Root Port. 1020 */ 1021 if ((PCI_BUS_NUM(e_info->id) != 0) && 1022 !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) { 1023 /* Device ID match? */ 1024 if (e_info->id == pci_dev_id(dev)) 1025 return true; 1026 1027 /* Continue ID comparing if there is no multiple error */ 1028 if (!e_info->multi_error_valid) 1029 return false; 1030 } 1031 1032 /* 1033 * When either 1034 * 1) bus ID is equal to 0. Some ports might lose the bus 1035 * ID of error source id; 1036 * 2) bus flag PCI_BUS_FLAGS_NO_AERSID is set 1037 * 3) There are multiple errors and prior ID comparing fails; 1038 * We check AER status registers to find possible reporter. 1039 */ 1040 if (atomic_read(&dev->enable_cnt) == 0) 1041 return false; 1042 1043 /* Check if AER is enabled */ 1044 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16); 1045 if (!(reg16 & PCI_EXP_AER_FLAGS)) 1046 return false; 1047 1048 if (!aer) 1049 return false; 1050 1051 /* Check if error is recorded */ 1052 if (e_info->severity == AER_CORRECTABLE) { 1053 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 1054 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1055 } else { 1056 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 1057 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1058 } 1059 if (status & ~mask) 1060 return true; 1061 1062 return false; 1063 } 1064 1065 static int find_device_iter(struct pci_dev *dev, void *data) 1066 { 1067 struct aer_err_info *e_info = (struct aer_err_info *)data; 1068 1069 if (is_error_source(dev, e_info)) { 1070 /* List this device */ 1071 if (add_error_device(e_info, dev)) { 1072 /* We cannot handle more... Stop iteration */ 1073 pci_err(dev, "Exceeded max supported (%d) devices with errors logged\n", 1074 AER_MAX_MULTI_ERR_DEVICES); 1075 return 1; 1076 } 1077 1078 /* If there is only a single error, stop iteration */ 1079 if (!e_info->multi_error_valid) 1080 return 1; 1081 } 1082 return 0; 1083 } 1084 1085 /** 1086 * find_source_device - search through device hierarchy for source device 1087 * @parent: pointer to Root Port pci_dev data structure 1088 * @e_info: including detailed error information such as ID 1089 * 1090 * Return: true if found. 1091 * 1092 * Invoked by DPC when error is detected at the Root Port. 1093 * Caller of this function must set id, severity, and multi_error_valid of 1094 * struct aer_err_info pointed by @e_info properly. This function must fill 1095 * e_info->error_dev_num and e_info->dev[], based on the given information. 1096 */ 1097 static bool find_source_device(struct pci_dev *parent, 1098 struct aer_err_info *e_info) 1099 { 1100 struct pci_dev *dev = parent; 1101 int result; 1102 1103 /* Must reset in this function */ 1104 e_info->error_dev_num = 0; 1105 1106 /* Is Root Port an agent that sends error message? */ 1107 result = find_device_iter(dev, e_info); 1108 if (result) 1109 return true; 1110 1111 if (pci_pcie_type(parent) == PCI_EXP_TYPE_RC_EC) 1112 pcie_walk_rcec(parent, find_device_iter, e_info); 1113 else 1114 pci_walk_bus(parent->subordinate, find_device_iter, e_info); 1115 1116 if (!e_info->error_dev_num) 1117 return false; 1118 return true; 1119 } 1120 1121 #ifdef CONFIG_PCIEAER_CXL 1122 1123 /** 1124 * pci_aer_unmask_internal_errors - unmask internal errors 1125 * @dev: pointer to the pci_dev data structure 1126 * 1127 * Unmask internal errors in the Uncorrectable and Correctable Error 1128 * Mask registers. 1129 * 1130 * Note: AER must be enabled and supported by the device which must be 1131 * checked in advance, e.g. with pcie_aer_is_native(). 1132 */ 1133 static void pci_aer_unmask_internal_errors(struct pci_dev *dev) 1134 { 1135 int aer = dev->aer_cap; 1136 u32 mask; 1137 1138 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1139 mask &= ~PCI_ERR_UNC_INTN; 1140 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask); 1141 1142 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1143 mask &= ~PCI_ERR_COR_INTERNAL; 1144 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask); 1145 } 1146 1147 static bool is_cxl_mem_dev(struct pci_dev *dev) 1148 { 1149 /* 1150 * The capability, status, and control fields in Device 0, 1151 * Function 0 DVSEC control the CXL functionality of the 1152 * entire device (CXL 3.0, 8.1.3). 1153 */ 1154 if (dev->devfn != PCI_DEVFN(0, 0)) 1155 return false; 1156 1157 /* 1158 * CXL Memory Devices must have the 502h class code set (CXL 1159 * 3.0, 8.1.12.1). 1160 */ 1161 if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL) 1162 return false; 1163 1164 return true; 1165 } 1166 1167 static bool cxl_error_is_native(struct pci_dev *dev) 1168 { 1169 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 1170 1171 return (pcie_ports_native || host->native_aer); 1172 } 1173 1174 static bool is_internal_error(struct aer_err_info *info) 1175 { 1176 if (info->severity == AER_CORRECTABLE) 1177 return info->status & PCI_ERR_COR_INTERNAL; 1178 1179 return info->status & PCI_ERR_UNC_INTN; 1180 } 1181 1182 static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data) 1183 { 1184 struct aer_err_info *info = (struct aer_err_info *)data; 1185 const struct pci_error_handlers *err_handler; 1186 1187 if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev)) 1188 return 0; 1189 1190 /* Protect dev->driver */ 1191 device_lock(&dev->dev); 1192 1193 err_handler = dev->driver ? dev->driver->err_handler : NULL; 1194 if (!err_handler) 1195 goto out; 1196 1197 if (info->severity == AER_CORRECTABLE) { 1198 if (err_handler->cor_error_detected) 1199 err_handler->cor_error_detected(dev); 1200 } else if (err_handler->error_detected) { 1201 if (info->severity == AER_NONFATAL) 1202 err_handler->error_detected(dev, pci_channel_io_normal); 1203 else if (info->severity == AER_FATAL) 1204 err_handler->error_detected(dev, pci_channel_io_frozen); 1205 } 1206 out: 1207 device_unlock(&dev->dev); 1208 return 0; 1209 } 1210 1211 static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) 1212 { 1213 /* 1214 * Internal errors of an RCEC indicate an AER error in an 1215 * RCH's downstream port. Check and handle them in the CXL.mem 1216 * device driver. 1217 */ 1218 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC && 1219 is_internal_error(info)) 1220 pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info); 1221 } 1222 1223 static int handles_cxl_error_iter(struct pci_dev *dev, void *data) 1224 { 1225 bool *handles_cxl = data; 1226 1227 if (!*handles_cxl) 1228 *handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev); 1229 1230 /* Non-zero terminates iteration */ 1231 return *handles_cxl; 1232 } 1233 1234 static bool handles_cxl_errors(struct pci_dev *rcec) 1235 { 1236 bool handles_cxl = false; 1237 1238 if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC && 1239 pcie_aer_is_native(rcec)) 1240 pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl); 1241 1242 return handles_cxl; 1243 } 1244 1245 static void cxl_rch_enable_rcec(struct pci_dev *rcec) 1246 { 1247 if (!handles_cxl_errors(rcec)) 1248 return; 1249 1250 pci_aer_unmask_internal_errors(rcec); 1251 pci_info(rcec, "CXL: Internal errors unmasked"); 1252 } 1253 1254 #else 1255 static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { } 1256 static inline void cxl_rch_handle_error(struct pci_dev *dev, 1257 struct aer_err_info *info) { } 1258 #endif 1259 1260 /** 1261 * pci_aer_handle_error - handle logging error into an event log 1262 * @dev: pointer to pci_dev data structure of error source device 1263 * @info: comprehensive error information 1264 * 1265 * Invoked when an error being detected by Root Port. 1266 */ 1267 static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info) 1268 { 1269 int aer = dev->aer_cap; 1270 1271 if (info->severity == AER_CORRECTABLE) { 1272 /* 1273 * Correctable error does not need software intervention. 1274 * No need to go through error recovery process. 1275 */ 1276 if (aer) 1277 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1278 info->status); 1279 if (pcie_aer_is_native(dev)) { 1280 struct pci_driver *pdrv = dev->driver; 1281 1282 if (pdrv && pdrv->err_handler && 1283 pdrv->err_handler->cor_error_detected) 1284 pdrv->err_handler->cor_error_detected(dev); 1285 pcie_clear_device_status(dev); 1286 } 1287 } else if (info->severity == AER_NONFATAL) 1288 pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset); 1289 else if (info->severity == AER_FATAL) 1290 pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset); 1291 } 1292 1293 static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) 1294 { 1295 cxl_rch_handle_error(dev, info); 1296 pci_aer_handle_error(dev, info); 1297 pci_dev_put(dev); 1298 } 1299 1300 #ifdef CONFIG_ACPI_APEI_PCIEAER 1301 1302 #define AER_RECOVER_RING_SIZE 16 1303 1304 struct aer_recover_entry { 1305 u8 bus; 1306 u8 devfn; 1307 u16 domain; 1308 int severity; 1309 struct aer_capability_regs *regs; 1310 }; 1311 1312 static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry, 1313 AER_RECOVER_RING_SIZE); 1314 1315 static void aer_recover_work_func(struct work_struct *work) 1316 { 1317 struct aer_recover_entry entry; 1318 struct pci_dev *pdev; 1319 1320 while (kfifo_get(&aer_recover_ring, &entry)) { 1321 pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, 1322 entry.devfn); 1323 if (!pdev) { 1324 pr_err_ratelimited("%04x:%02x:%02x.%x: no pci_dev found\n", 1325 entry.domain, entry.bus, 1326 PCI_SLOT(entry.devfn), 1327 PCI_FUNC(entry.devfn)); 1328 continue; 1329 } 1330 pci_print_aer(pdev, entry.severity, entry.regs); 1331 1332 /* 1333 * Memory for aer_capability_regs(entry.regs) is being 1334 * allocated from the ghes_estatus_pool to protect it from 1335 * overwriting when multiple sections are present in the 1336 * error status. Thus free the same after processing the 1337 * data. 1338 */ 1339 ghes_estatus_pool_region_free((unsigned long)entry.regs, 1340 sizeof(struct aer_capability_regs)); 1341 1342 if (entry.severity == AER_NONFATAL) 1343 pcie_do_recovery(pdev, pci_channel_io_normal, 1344 aer_root_reset); 1345 else if (entry.severity == AER_FATAL) 1346 pcie_do_recovery(pdev, pci_channel_io_frozen, 1347 aer_root_reset); 1348 pci_dev_put(pdev); 1349 } 1350 } 1351 1352 /* 1353 * Mutual exclusion for writers of aer_recover_ring, reader side don't 1354 * need lock, because there is only one reader and lock is not needed 1355 * between reader and writer. 1356 */ 1357 static DEFINE_SPINLOCK(aer_recover_ring_lock); 1358 static DECLARE_WORK(aer_recover_work, aer_recover_work_func); 1359 1360 void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, 1361 int severity, struct aer_capability_regs *aer_regs) 1362 { 1363 struct aer_recover_entry entry = { 1364 .bus = bus, 1365 .devfn = devfn, 1366 .domain = domain, 1367 .severity = severity, 1368 .regs = aer_regs, 1369 }; 1370 1371 if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1, 1372 &aer_recover_ring_lock)) 1373 schedule_work(&aer_recover_work); 1374 else 1375 pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n", 1376 domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 1377 } 1378 EXPORT_SYMBOL_GPL(aer_recover_queue); 1379 #endif 1380 1381 /** 1382 * aer_get_device_error_info - read error status from dev and store it to info 1383 * @info: pointer to structure to store the error record 1384 * @i: index into info->dev[] 1385 * 1386 * Return: 1 on success, 0 on error. 1387 * 1388 * Note that @info is reused among all error devices. Clear fields properly. 1389 */ 1390 int aer_get_device_error_info(struct aer_err_info *info, int i) 1391 { 1392 struct pci_dev *dev; 1393 int type, aer; 1394 u32 aercc; 1395 1396 if (i >= AER_MAX_MULTI_ERR_DEVICES) 1397 return 0; 1398 1399 dev = info->dev[i]; 1400 aer = dev->aer_cap; 1401 type = pci_pcie_type(dev); 1402 1403 /* Must reset in this function */ 1404 info->status = 0; 1405 info->tlp_header_valid = 0; 1406 1407 /* The device might not support AER */ 1408 if (!aer) 1409 return 0; 1410 1411 if (info->severity == AER_CORRECTABLE) { 1412 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1413 &info->status); 1414 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, 1415 &info->mask); 1416 if (!(info->status & ~info->mask)) 1417 return 0; 1418 } else if (type == PCI_EXP_TYPE_ROOT_PORT || 1419 type == PCI_EXP_TYPE_RC_EC || 1420 type == PCI_EXP_TYPE_DOWNSTREAM || 1421 info->severity == AER_NONFATAL) { 1422 1423 /* Link is still healthy for IO reads */ 1424 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, 1425 &info->status); 1426 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, 1427 &info->mask); 1428 if (!(info->status & ~info->mask)) 1429 return 0; 1430 1431 /* Get First Error Pointer */ 1432 pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc); 1433 info->first_error = PCI_ERR_CAP_FEP(aercc); 1434 1435 if (tlp_header_logged(info->status, aercc)) { 1436 info->tlp_header_valid = 1; 1437 pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG, 1438 aer + PCI_ERR_PREFIX_LOG, 1439 aer_tlp_log_len(dev, aercc), 1440 aercc & PCI_ERR_CAP_TLP_LOG_FLIT, 1441 &info->tlp); 1442 } 1443 } 1444 1445 return 1; 1446 } 1447 1448 static inline void aer_process_err_devices(struct aer_err_info *e_info) 1449 { 1450 int i; 1451 1452 /* Report all before handling them, to not lose records by reset etc. */ 1453 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1454 if (aer_get_device_error_info(e_info, i)) 1455 aer_print_error(e_info, i); 1456 } 1457 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1458 if (aer_get_device_error_info(e_info, i)) 1459 handle_error_source(e_info->dev[i], e_info); 1460 } 1461 } 1462 1463 /** 1464 * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error 1465 * detected by Root Port or RCEC 1466 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1467 * @info: pointer to AER error info 1468 */ 1469 static void aer_isr_one_error_type(struct pci_dev *root, 1470 struct aer_err_info *info) 1471 { 1472 bool found; 1473 1474 found = find_source_device(root, info); 1475 1476 /* 1477 * If we're going to log error messages, we've already set 1478 * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to 1479 * non-zero (which enables printing) because this is either an 1480 * ERR_FATAL or we found a device with an error logged in its AER 1481 * Capability. 1482 * 1483 * If we didn't find the Error Source device, at least log the 1484 * Requester ID from the ERR_* Message received by the Root Port or 1485 * RCEC, ratelimited by the RP or RCEC. 1486 */ 1487 if (info->root_ratelimit_print || 1488 (!found && aer_ratelimit(root, info->severity))) 1489 aer_print_source(root, info, found); 1490 1491 if (found) 1492 aer_process_err_devices(info); 1493 } 1494 1495 /** 1496 * aer_isr_one_error - consume error(s) signaled by an AER interrupt from 1497 * Root Port or RCEC 1498 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1499 * @e_src: pointer to an error source 1500 */ 1501 static void aer_isr_one_error(struct pci_dev *root, 1502 struct aer_err_source *e_src) 1503 { 1504 u32 status = e_src->status; 1505 1506 pci_rootport_aer_stats_incr(root, e_src); 1507 1508 /* 1509 * There is a possibility that both correctable error and 1510 * uncorrectable error being logged. Report correctable error first. 1511 */ 1512 if (status & PCI_ERR_ROOT_COR_RCV) { 1513 int multi = status & PCI_ERR_ROOT_MULTI_COR_RCV; 1514 struct aer_err_info e_info = { 1515 .id = ERR_COR_ID(e_src->id), 1516 .severity = AER_CORRECTABLE, 1517 .level = KERN_WARNING, 1518 .multi_error_valid = multi ? 1 : 0, 1519 }; 1520 1521 aer_isr_one_error_type(root, &e_info); 1522 } 1523 1524 if (status & PCI_ERR_ROOT_UNCOR_RCV) { 1525 int fatal = status & PCI_ERR_ROOT_FATAL_RCV; 1526 int multi = status & PCI_ERR_ROOT_MULTI_UNCOR_RCV; 1527 struct aer_err_info e_info = { 1528 .id = ERR_UNCOR_ID(e_src->id), 1529 .severity = fatal ? AER_FATAL : AER_NONFATAL, 1530 .level = KERN_ERR, 1531 .multi_error_valid = multi ? 1 : 0, 1532 }; 1533 1534 aer_isr_one_error_type(root, &e_info); 1535 } 1536 } 1537 1538 /** 1539 * aer_isr - consume errors detected by Root Port 1540 * @irq: IRQ assigned to Root Port 1541 * @context: pointer to Root Port data structure 1542 * 1543 * Invoked, as DPC, when Root Port records new detected error 1544 */ 1545 static irqreturn_t aer_isr(int irq, void *context) 1546 { 1547 struct pcie_device *dev = (struct pcie_device *)context; 1548 struct aer_rpc *rpc = get_service_data(dev); 1549 struct aer_err_source e_src; 1550 1551 if (kfifo_is_empty(&rpc->aer_fifo)) 1552 return IRQ_NONE; 1553 1554 while (kfifo_get(&rpc->aer_fifo, &e_src)) 1555 aer_isr_one_error(rpc->rpd, &e_src); 1556 return IRQ_HANDLED; 1557 } 1558 1559 /** 1560 * aer_irq - Root Port's ISR 1561 * @irq: IRQ assigned to Root Port 1562 * @context: pointer to Root Port data structure 1563 * 1564 * Invoked when Root Port detects AER messages. 1565 */ 1566 static irqreturn_t aer_irq(int irq, void *context) 1567 { 1568 struct pcie_device *pdev = (struct pcie_device *)context; 1569 struct aer_rpc *rpc = get_service_data(pdev); 1570 struct pci_dev *rp = rpc->rpd; 1571 int aer = rp->aer_cap; 1572 struct aer_err_source e_src = {}; 1573 1574 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status); 1575 if (!(e_src.status & AER_ERR_STATUS_MASK)) 1576 return IRQ_NONE; 1577 1578 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id); 1579 pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status); 1580 1581 if (!kfifo_put(&rpc->aer_fifo, e_src)) 1582 return IRQ_HANDLED; 1583 1584 return IRQ_WAKE_THREAD; 1585 } 1586 1587 static void aer_enable_irq(struct pci_dev *pdev) 1588 { 1589 int aer = pdev->aer_cap; 1590 u32 reg32; 1591 1592 /* Enable Root Port's interrupt in response to error messages */ 1593 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1594 reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; 1595 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1596 } 1597 1598 static void aer_disable_irq(struct pci_dev *pdev) 1599 { 1600 int aer = pdev->aer_cap; 1601 u32 reg32; 1602 1603 /* Disable Root Port's interrupt in response to error messages */ 1604 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1605 reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; 1606 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1607 } 1608 1609 /** 1610 * aer_enable_rootport - enable Root Port's interrupts when receiving messages 1611 * @rpc: pointer to a Root Port data structure 1612 * 1613 * Invoked when PCIe bus loads AER service driver. 1614 */ 1615 static void aer_enable_rootport(struct aer_rpc *rpc) 1616 { 1617 struct pci_dev *pdev = rpc->rpd; 1618 int aer = pdev->aer_cap; 1619 u16 reg16; 1620 u32 reg32; 1621 1622 /* Clear PCIe Capability's Device Status */ 1623 pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, ®16); 1624 pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16); 1625 1626 /* Disable system error generation in response to error messages */ 1627 pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, 1628 SYSTEM_ERROR_INTR_ON_MESG_MASK); 1629 1630 /* Clear error status */ 1631 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1632 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1633 pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32); 1634 pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32); 1635 pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32); 1636 pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32); 1637 1638 aer_enable_irq(pdev); 1639 } 1640 1641 /** 1642 * aer_disable_rootport - disable Root Port's interrupts when receiving messages 1643 * @rpc: pointer to a Root Port data structure 1644 * 1645 * Invoked when PCIe bus unloads AER service driver. 1646 */ 1647 static void aer_disable_rootport(struct aer_rpc *rpc) 1648 { 1649 struct pci_dev *pdev = rpc->rpd; 1650 int aer = pdev->aer_cap; 1651 u32 reg32; 1652 1653 aer_disable_irq(pdev); 1654 1655 /* Clear Root's error status reg */ 1656 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1657 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1658 } 1659 1660 /** 1661 * aer_remove - clean up resources 1662 * @dev: pointer to the pcie_dev data structure 1663 * 1664 * Invoked when PCI Express bus unloads or AER probe fails. 1665 */ 1666 static void aer_remove(struct pcie_device *dev) 1667 { 1668 struct aer_rpc *rpc = get_service_data(dev); 1669 1670 aer_disable_rootport(rpc); 1671 } 1672 1673 /** 1674 * aer_probe - initialize resources 1675 * @dev: pointer to the pcie_dev data structure 1676 * 1677 * Invoked when PCI Express bus loads AER service driver. 1678 */ 1679 static int aer_probe(struct pcie_device *dev) 1680 { 1681 int status; 1682 struct aer_rpc *rpc; 1683 struct device *device = &dev->device; 1684 struct pci_dev *port = dev->port; 1685 1686 BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) < 1687 AER_MAX_TYPEOF_COR_ERRS); 1688 BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) < 1689 AER_MAX_TYPEOF_UNCOR_ERRS); 1690 1691 /* Limit to Root Ports or Root Complex Event Collectors */ 1692 if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) && 1693 (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT)) 1694 return -ENODEV; 1695 1696 rpc = devm_kzalloc(device, sizeof(struct aer_rpc), GFP_KERNEL); 1697 if (!rpc) 1698 return -ENOMEM; 1699 1700 rpc->rpd = port; 1701 INIT_KFIFO(rpc->aer_fifo); 1702 set_service_data(dev, rpc); 1703 1704 status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr, 1705 IRQF_SHARED, "aerdrv", dev); 1706 if (status) { 1707 pci_err(port, "request AER IRQ %d failed\n", dev->irq); 1708 return status; 1709 } 1710 1711 cxl_rch_enable_rcec(port); 1712 aer_enable_rootport(rpc); 1713 pci_info(port, "enabled with IRQ %d\n", dev->irq); 1714 return 0; 1715 } 1716 1717 static int aer_suspend(struct pcie_device *dev) 1718 { 1719 struct aer_rpc *rpc = get_service_data(dev); 1720 1721 aer_disable_rootport(rpc); 1722 return 0; 1723 } 1724 1725 static int aer_resume(struct pcie_device *dev) 1726 { 1727 struct aer_rpc *rpc = get_service_data(dev); 1728 1729 aer_enable_rootport(rpc); 1730 return 0; 1731 } 1732 1733 /** 1734 * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP 1735 * @dev: pointer to Root Port, RCEC, or RCiEP 1736 * 1737 * Invoked by Port Bus driver when performing reset. 1738 */ 1739 static pci_ers_result_t aer_root_reset(struct pci_dev *dev) 1740 { 1741 int type = pci_pcie_type(dev); 1742 struct pci_dev *root; 1743 int aer; 1744 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 1745 u32 reg32; 1746 int rc; 1747 1748 /* 1749 * Only Root Ports and RCECs have AER Root Command and Root Status 1750 * registers. If "dev" is an RCiEP, the relevant registers are in 1751 * the RCEC. 1752 */ 1753 if (type == PCI_EXP_TYPE_RC_END) 1754 root = dev->rcec; 1755 else 1756 root = pcie_find_root_port(dev); 1757 1758 /* 1759 * If the platform retained control of AER, an RCiEP may not have 1760 * an RCEC visible to us, so dev->rcec ("root") may be NULL. In 1761 * that case, firmware is responsible for these registers. 1762 */ 1763 aer = root ? root->aer_cap : 0; 1764 1765 if ((host->native_aer || pcie_ports_native) && aer) 1766 aer_disable_irq(root); 1767 1768 if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { 1769 rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET); 1770 if (!rc) 1771 pci_info(dev, "has been reset\n"); 1772 else 1773 pci_info(dev, "not reset (no FLR support: %d)\n", rc); 1774 } else { 1775 rc = pci_bus_error_reset(dev); 1776 pci_info(dev, "%s Port link has been reset (%d)\n", 1777 pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc); 1778 } 1779 1780 if ((host->native_aer || pcie_ports_native) && aer) { 1781 /* Clear Root Error Status */ 1782 pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, ®32); 1783 pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32); 1784 1785 aer_enable_irq(root); 1786 } 1787 1788 return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 1789 } 1790 1791 static struct pcie_port_service_driver aerdriver = { 1792 .name = "aer", 1793 .port_type = PCIE_ANY_PORT, 1794 .service = PCIE_PORT_SERVICE_AER, 1795 1796 .probe = aer_probe, 1797 .suspend = aer_suspend, 1798 .resume = aer_resume, 1799 .remove = aer_remove, 1800 }; 1801 1802 /** 1803 * pcie_aer_init - register AER service driver 1804 * 1805 * Invoked when AER service driver is loaded. 1806 */ 1807 int __init pcie_aer_init(void) 1808 { 1809 if (!pci_aer_available()) 1810 return -ENXIO; 1811 return pcie_port_service_register(&aerdriver); 1812 } 1813