1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implement the AER root port service driver. The driver registers an IRQ 4 * handler. When a root port triggers an AER interrupt, the IRQ handler 5 * collects Root Port status and schedules work. 6 * 7 * Copyright (C) 2006 Intel Corp. 8 * Tom Long Nguyen (tom.l.nguyen@intel.com) 9 * Zhang Yanmin (yanmin.zhang@intel.com) 10 * 11 * (C) Copyright 2009 Hewlett-Packard Development Company, L.P. 12 * Andrew Patterson <andrew.patterson@hp.com> 13 */ 14 15 #define pr_fmt(fmt) "AER: " fmt 16 #define dev_fmt pr_fmt 17 18 #include <linux/bitops.h> 19 #include <linux/cper.h> 20 #include <linux/dev_printk.h> 21 #include <linux/pci.h> 22 #include <linux/pci-acpi.h> 23 #include <linux/sched.h> 24 #include <linux/kernel.h> 25 #include <linux/errno.h> 26 #include <linux/pm.h> 27 #include <linux/init.h> 28 #include <linux/interrupt.h> 29 #include <linux/delay.h> 30 #include <linux/kfifo.h> 31 #include <linux/ratelimit.h> 32 #include <linux/slab.h> 33 #include <acpi/apei.h> 34 #include <acpi/ghes.h> 35 #include <ras/ras_event.h> 36 37 #include "../pci.h" 38 #include "portdrv.h" 39 40 #define aer_printk(level, pdev, fmt, arg...) \ 41 dev_printk(level, &(pdev)->dev, fmt, ##arg) 42 43 #define AER_ERROR_SOURCES_MAX 128 44 45 #define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */ 46 #define AER_MAX_TYPEOF_UNCOR_ERRS 27 /* as per PCI_ERR_UNCOR_STATUS*/ 47 48 struct aer_err_source { 49 u32 status; /* PCI_ERR_ROOT_STATUS */ 50 u32 id; /* PCI_ERR_ROOT_ERR_SRC */ 51 }; 52 53 struct aer_rpc { 54 struct pci_dev *rpd; /* Root Port device */ 55 DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX); 56 }; 57 58 /* AER info for the device */ 59 struct aer_info { 60 61 /* 62 * Fields for all AER capable devices. They indicate the errors 63 * "as seen by this device". Note that this may mean that if an 64 * Endpoint is causing problems, the AER counters may increment 65 * at its link partner (e.g. Root Port) because the errors will be 66 * "seen" by the link partner and not the problematic Endpoint 67 * itself (which may report all counters as 0 as it never saw any 68 * problems). 69 */ 70 /* Counters for different type of correctable errors */ 71 u64 dev_cor_errs[AER_MAX_TYPEOF_COR_ERRS]; 72 /* Counters for different type of fatal uncorrectable errors */ 73 u64 dev_fatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 74 /* Counters for different type of nonfatal uncorrectable errors */ 75 u64 dev_nonfatal_errs[AER_MAX_TYPEOF_UNCOR_ERRS]; 76 /* Total number of ERR_COR sent by this device */ 77 u64 dev_total_cor_errs; 78 /* Total number of ERR_FATAL sent by this device */ 79 u64 dev_total_fatal_errs; 80 /* Total number of ERR_NONFATAL sent by this device */ 81 u64 dev_total_nonfatal_errs; 82 83 /* 84 * Fields for Root Ports & Root Complex Event Collectors only; these 85 * indicate the total number of ERR_COR, ERR_FATAL, and ERR_NONFATAL 86 * messages received by the Root Port / Event Collector, INCLUDING the 87 * ones that are generated internally (by the Root Port itself) 88 */ 89 u64 rootport_total_cor_errs; 90 u64 rootport_total_fatal_errs; 91 u64 rootport_total_nonfatal_errs; 92 93 /* Ratelimits for errors */ 94 struct ratelimit_state correctable_ratelimit; 95 struct ratelimit_state nonfatal_ratelimit; 96 }; 97 98 #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \ 99 PCI_ERR_UNC_ECRC| \ 100 PCI_ERR_UNC_UNSUP| \ 101 PCI_ERR_UNC_COMP_ABORT| \ 102 PCI_ERR_UNC_UNX_COMP| \ 103 PCI_ERR_UNC_MALF_TLP) 104 105 #define SYSTEM_ERROR_INTR_ON_MESG_MASK (PCI_EXP_RTCTL_SECEE| \ 106 PCI_EXP_RTCTL_SENFEE| \ 107 PCI_EXP_RTCTL_SEFEE) 108 #define ROOT_PORT_INTR_ON_MESG_MASK (PCI_ERR_ROOT_CMD_COR_EN| \ 109 PCI_ERR_ROOT_CMD_NONFATAL_EN| \ 110 PCI_ERR_ROOT_CMD_FATAL_EN) 111 #define ERR_COR_ID(d) (d & 0xffff) 112 #define ERR_UNCOR_ID(d) (d >> 16) 113 114 #define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV | \ 115 PCI_ERR_ROOT_COR_RCV | \ 116 PCI_ERR_ROOT_MULTI_COR_RCV | \ 117 PCI_ERR_ROOT_MULTI_UNCOR_RCV) 118 119 static bool pcie_aer_disable; 120 static pci_ers_result_t aer_root_reset(struct pci_dev *dev); 121 122 void pci_no_aer(void) 123 { 124 pcie_aer_disable = true; 125 } 126 127 bool pci_aer_available(void) 128 { 129 return !pcie_aer_disable && pci_msi_enabled(); 130 } 131 132 #ifdef CONFIG_PCIE_ECRC 133 134 #define ECRC_POLICY_DEFAULT 0 /* ECRC set by BIOS */ 135 #define ECRC_POLICY_OFF 1 /* ECRC off for performance */ 136 #define ECRC_POLICY_ON 2 /* ECRC on for data integrity */ 137 138 static int ecrc_policy = ECRC_POLICY_DEFAULT; 139 140 static const char * const ecrc_policy_str[] = { 141 [ECRC_POLICY_DEFAULT] = "bios", 142 [ECRC_POLICY_OFF] = "off", 143 [ECRC_POLICY_ON] = "on" 144 }; 145 146 /** 147 * enable_ecrc_checking - enable PCIe ECRC checking for a device 148 * @dev: the PCI device 149 * 150 * Return: 0 on success, or negative on failure. 151 */ 152 static int enable_ecrc_checking(struct pci_dev *dev) 153 { 154 int aer = dev->aer_cap; 155 u32 reg32; 156 157 if (!aer) 158 return -ENODEV; 159 160 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 161 if (reg32 & PCI_ERR_CAP_ECRC_GENC) 162 reg32 |= PCI_ERR_CAP_ECRC_GENE; 163 if (reg32 & PCI_ERR_CAP_ECRC_CHKC) 164 reg32 |= PCI_ERR_CAP_ECRC_CHKE; 165 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 166 167 return 0; 168 } 169 170 /** 171 * disable_ecrc_checking - disable PCIe ECRC checking for a device 172 * @dev: the PCI device 173 * 174 * Return: 0 on success, or negative on failure. 175 */ 176 static int disable_ecrc_checking(struct pci_dev *dev) 177 { 178 int aer = dev->aer_cap; 179 u32 reg32; 180 181 if (!aer) 182 return -ENODEV; 183 184 pci_read_config_dword(dev, aer + PCI_ERR_CAP, ®32); 185 reg32 &= ~(PCI_ERR_CAP_ECRC_GENE | PCI_ERR_CAP_ECRC_CHKE); 186 pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32); 187 188 return 0; 189 } 190 191 /** 192 * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based 193 * on global policy 194 * @dev: the PCI device 195 */ 196 void pcie_set_ecrc_checking(struct pci_dev *dev) 197 { 198 if (!pcie_aer_is_native(dev)) 199 return; 200 201 switch (ecrc_policy) { 202 case ECRC_POLICY_DEFAULT: 203 return; 204 case ECRC_POLICY_OFF: 205 disable_ecrc_checking(dev); 206 break; 207 case ECRC_POLICY_ON: 208 enable_ecrc_checking(dev); 209 break; 210 default: 211 return; 212 } 213 } 214 215 /** 216 * pcie_ecrc_get_policy - parse kernel command-line ecrc option 217 * @str: ECRC policy from kernel command line to use 218 */ 219 void pcie_ecrc_get_policy(char *str) 220 { 221 int i; 222 223 i = match_string(ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), str); 224 if (i < 0) 225 return; 226 227 ecrc_policy = i; 228 } 229 #endif /* CONFIG_PCIE_ECRC */ 230 231 #define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE | PCI_EXP_DEVCTL_NFERE | \ 232 PCI_EXP_DEVCTL_FERE | PCI_EXP_DEVCTL_URRE) 233 234 int pcie_aer_is_native(struct pci_dev *dev) 235 { 236 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 237 238 if (!dev->aer_cap) 239 return 0; 240 241 return pcie_ports_native || host->native_aer; 242 } 243 EXPORT_SYMBOL_NS_GPL(pcie_aer_is_native, "CXL"); 244 245 static int pci_enable_pcie_error_reporting(struct pci_dev *dev) 246 { 247 int rc; 248 249 if (!pcie_aer_is_native(dev)) 250 return -EIO; 251 252 rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS); 253 return pcibios_err_to_errno(rc); 254 } 255 256 int pci_aer_clear_nonfatal_status(struct pci_dev *dev) 257 { 258 int aer = dev->aer_cap; 259 u32 status, sev; 260 261 if (!pcie_aer_is_native(dev)) 262 return -EIO; 263 264 /* Clear status bits for ERR_NONFATAL errors only */ 265 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 266 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 267 status &= ~sev; 268 if (status) 269 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 270 271 return 0; 272 } 273 EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status); 274 275 void pci_aer_clear_fatal_status(struct pci_dev *dev) 276 { 277 int aer = dev->aer_cap; 278 u32 status, sev; 279 280 if (!pcie_aer_is_native(dev)) 281 return; 282 283 /* Clear status bits for ERR_FATAL errors only */ 284 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 285 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev); 286 status &= sev; 287 if (status) 288 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 289 } 290 291 /** 292 * pci_aer_raw_clear_status - Clear AER error registers. 293 * @dev: the PCI device 294 * 295 * Clear AER error status registers unconditionally, regardless of 296 * whether they're owned by firmware or the OS. 297 * 298 * Return: 0 on success, or negative on failure. 299 */ 300 int pci_aer_raw_clear_status(struct pci_dev *dev) 301 { 302 int aer = dev->aer_cap; 303 u32 status; 304 int port_type; 305 306 if (!aer) 307 return -EIO; 308 309 port_type = pci_pcie_type(dev); 310 if (port_type == PCI_EXP_TYPE_ROOT_PORT || 311 port_type == PCI_EXP_TYPE_RC_EC) { 312 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status); 313 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status); 314 } 315 316 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 317 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status); 318 319 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 320 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status); 321 322 return 0; 323 } 324 325 int pci_aer_clear_status(struct pci_dev *dev) 326 { 327 if (!pcie_aer_is_native(dev)) 328 return -EIO; 329 330 return pci_aer_raw_clear_status(dev); 331 } 332 333 void pci_save_aer_state(struct pci_dev *dev) 334 { 335 int aer = dev->aer_cap; 336 struct pci_cap_saved_state *save_state; 337 u32 *cap; 338 339 if (!aer) 340 return; 341 342 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 343 if (!save_state) 344 return; 345 346 cap = &save_state->cap.data[0]; 347 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++); 348 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++); 349 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++); 350 pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++); 351 if (pcie_cap_has_rtctl(dev)) 352 pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++); 353 } 354 355 void pci_restore_aer_state(struct pci_dev *dev) 356 { 357 int aer = dev->aer_cap; 358 struct pci_cap_saved_state *save_state; 359 u32 *cap; 360 361 if (!aer) 362 return; 363 364 save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR); 365 if (!save_state) 366 return; 367 368 cap = &save_state->cap.data[0]; 369 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++); 370 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++); 371 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++); 372 pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++); 373 if (pcie_cap_has_rtctl(dev)) 374 pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++); 375 } 376 377 void pci_aer_init(struct pci_dev *dev) 378 { 379 int n; 380 381 dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR); 382 if (!dev->aer_cap) 383 return; 384 385 dev->aer_info = kzalloc(sizeof(*dev->aer_info), GFP_KERNEL); 386 387 ratelimit_state_init(&dev->aer_info->correctable_ratelimit, 388 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 389 ratelimit_state_init(&dev->aer_info->nonfatal_ratelimit, 390 DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); 391 392 /* 393 * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER, 394 * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event 395 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r6.0, sec 396 * 7.8.4.9). 397 */ 398 n = pcie_cap_has_rtctl(dev) ? 5 : 4; 399 pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, sizeof(u32) * n); 400 401 pci_aer_clear_status(dev); 402 403 if (pci_aer_available()) 404 pci_enable_pcie_error_reporting(dev); 405 406 pcie_set_ecrc_checking(dev); 407 } 408 409 void pci_aer_exit(struct pci_dev *dev) 410 { 411 kfree(dev->aer_info); 412 dev->aer_info = NULL; 413 } 414 415 #define AER_AGENT_RECEIVER 0 416 #define AER_AGENT_REQUESTER 1 417 #define AER_AGENT_COMPLETER 2 418 #define AER_AGENT_TRANSMITTER 3 419 420 #define AER_AGENT_REQUESTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 421 0 : (PCI_ERR_UNC_COMP_TIME|PCI_ERR_UNC_UNSUP)) 422 #define AER_AGENT_COMPLETER_MASK(t) ((t == AER_CORRECTABLE) ? \ 423 0 : PCI_ERR_UNC_COMP_ABORT) 424 #define AER_AGENT_TRANSMITTER_MASK(t) ((t == AER_CORRECTABLE) ? \ 425 (PCI_ERR_COR_REP_ROLL|PCI_ERR_COR_REP_TIMER) : 0) 426 427 #define AER_GET_AGENT(t, e) \ 428 ((e & AER_AGENT_COMPLETER_MASK(t)) ? AER_AGENT_COMPLETER : \ 429 (e & AER_AGENT_REQUESTER_MASK(t)) ? AER_AGENT_REQUESTER : \ 430 (e & AER_AGENT_TRANSMITTER_MASK(t)) ? AER_AGENT_TRANSMITTER : \ 431 AER_AGENT_RECEIVER) 432 433 #define AER_PHYSICAL_LAYER_ERROR 0 434 #define AER_DATA_LINK_LAYER_ERROR 1 435 #define AER_TRANSACTION_LAYER_ERROR 2 436 437 #define AER_PHYSICAL_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 438 PCI_ERR_COR_RCVR : 0) 439 #define AER_DATA_LINK_LAYER_ERROR_MASK(t) ((t == AER_CORRECTABLE) ? \ 440 (PCI_ERR_COR_BAD_TLP| \ 441 PCI_ERR_COR_BAD_DLLP| \ 442 PCI_ERR_COR_REP_ROLL| \ 443 PCI_ERR_COR_REP_TIMER) : PCI_ERR_UNC_DLP) 444 445 #define AER_GET_LAYER_ERROR(t, e) \ 446 ((e & AER_PHYSICAL_LAYER_ERROR_MASK(t)) ? AER_PHYSICAL_LAYER_ERROR : \ 447 (e & AER_DATA_LINK_LAYER_ERROR_MASK(t)) ? AER_DATA_LINK_LAYER_ERROR : \ 448 AER_TRANSACTION_LAYER_ERROR) 449 450 /* 451 * AER error strings 452 */ 453 static const char * const aer_error_severity_string[] = { 454 "Uncorrectable (Non-Fatal)", 455 "Uncorrectable (Fatal)", 456 "Correctable" 457 }; 458 459 static const char *aer_error_layer[] = { 460 "Physical Layer", 461 "Data Link Layer", 462 "Transaction Layer" 463 }; 464 465 static const char *aer_correctable_error_string[] = { 466 "RxErr", /* Bit Position 0 */ 467 NULL, 468 NULL, 469 NULL, 470 NULL, 471 NULL, 472 "BadTLP", /* Bit Position 6 */ 473 "BadDLLP", /* Bit Position 7 */ 474 "Rollover", /* Bit Position 8 */ 475 NULL, 476 NULL, 477 NULL, 478 "Timeout", /* Bit Position 12 */ 479 "NonFatalErr", /* Bit Position 13 */ 480 "CorrIntErr", /* Bit Position 14 */ 481 "HeaderOF", /* Bit Position 15 */ 482 NULL, /* Bit Position 16 */ 483 NULL, /* Bit Position 17 */ 484 NULL, /* Bit Position 18 */ 485 NULL, /* Bit Position 19 */ 486 NULL, /* Bit Position 20 */ 487 NULL, /* Bit Position 21 */ 488 NULL, /* Bit Position 22 */ 489 NULL, /* Bit Position 23 */ 490 NULL, /* Bit Position 24 */ 491 NULL, /* Bit Position 25 */ 492 NULL, /* Bit Position 26 */ 493 NULL, /* Bit Position 27 */ 494 NULL, /* Bit Position 28 */ 495 NULL, /* Bit Position 29 */ 496 NULL, /* Bit Position 30 */ 497 NULL, /* Bit Position 31 */ 498 }; 499 500 static const char *aer_uncorrectable_error_string[] = { 501 "Undefined", /* Bit Position 0 */ 502 NULL, 503 NULL, 504 NULL, 505 "DLP", /* Bit Position 4 */ 506 "SDES", /* Bit Position 5 */ 507 NULL, 508 NULL, 509 NULL, 510 NULL, 511 NULL, 512 NULL, 513 "TLP", /* Bit Position 12 */ 514 "FCP", /* Bit Position 13 */ 515 "CmpltTO", /* Bit Position 14 */ 516 "CmpltAbrt", /* Bit Position 15 */ 517 "UnxCmplt", /* Bit Position 16 */ 518 "RxOF", /* Bit Position 17 */ 519 "MalfTLP", /* Bit Position 18 */ 520 "ECRC", /* Bit Position 19 */ 521 "UnsupReq", /* Bit Position 20 */ 522 "ACSViol", /* Bit Position 21 */ 523 "UncorrIntErr", /* Bit Position 22 */ 524 "BlockedTLP", /* Bit Position 23 */ 525 "AtomicOpBlocked", /* Bit Position 24 */ 526 "TLPBlockedErr", /* Bit Position 25 */ 527 "PoisonTLPBlocked", /* Bit Position 26 */ 528 NULL, /* Bit Position 27 */ 529 NULL, /* Bit Position 28 */ 530 NULL, /* Bit Position 29 */ 531 NULL, /* Bit Position 30 */ 532 NULL, /* Bit Position 31 */ 533 }; 534 535 static const char *aer_agent_string[] = { 536 "Receiver ID", 537 "Requester ID", 538 "Completer ID", 539 "Transmitter ID" 540 }; 541 542 #define aer_stats_dev_attr(name, stats_array, strings_array, \ 543 total_string, total_field) \ 544 static ssize_t \ 545 name##_show(struct device *dev, struct device_attribute *attr, \ 546 char *buf) \ 547 { \ 548 unsigned int i; \ 549 struct pci_dev *pdev = to_pci_dev(dev); \ 550 u64 *stats = pdev->aer_info->stats_array; \ 551 size_t len = 0; \ 552 \ 553 for (i = 0; i < ARRAY_SIZE(pdev->aer_info->stats_array); i++) { \ 554 if (strings_array[i]) \ 555 len += sysfs_emit_at(buf, len, "%s %llu\n", \ 556 strings_array[i], \ 557 stats[i]); \ 558 else if (stats[i]) \ 559 len += sysfs_emit_at(buf, len, \ 560 #stats_array "_bit[%d] %llu\n",\ 561 i, stats[i]); \ 562 } \ 563 len += sysfs_emit_at(buf, len, "TOTAL_%s %llu\n", total_string, \ 564 pdev->aer_info->total_field); \ 565 return len; \ 566 } \ 567 static DEVICE_ATTR_RO(name) 568 569 aer_stats_dev_attr(aer_dev_correctable, dev_cor_errs, 570 aer_correctable_error_string, "ERR_COR", 571 dev_total_cor_errs); 572 aer_stats_dev_attr(aer_dev_fatal, dev_fatal_errs, 573 aer_uncorrectable_error_string, "ERR_FATAL", 574 dev_total_fatal_errs); 575 aer_stats_dev_attr(aer_dev_nonfatal, dev_nonfatal_errs, 576 aer_uncorrectable_error_string, "ERR_NONFATAL", 577 dev_total_nonfatal_errs); 578 579 #define aer_stats_rootport_attr(name, field) \ 580 static ssize_t \ 581 name##_show(struct device *dev, struct device_attribute *attr, \ 582 char *buf) \ 583 { \ 584 struct pci_dev *pdev = to_pci_dev(dev); \ 585 return sysfs_emit(buf, "%llu\n", pdev->aer_info->field); \ 586 } \ 587 static DEVICE_ATTR_RO(name) 588 589 aer_stats_rootport_attr(aer_rootport_total_err_cor, 590 rootport_total_cor_errs); 591 aer_stats_rootport_attr(aer_rootport_total_err_fatal, 592 rootport_total_fatal_errs); 593 aer_stats_rootport_attr(aer_rootport_total_err_nonfatal, 594 rootport_total_nonfatal_errs); 595 596 static struct attribute *aer_stats_attrs[] __ro_after_init = { 597 &dev_attr_aer_dev_correctable.attr, 598 &dev_attr_aer_dev_fatal.attr, 599 &dev_attr_aer_dev_nonfatal.attr, 600 &dev_attr_aer_rootport_total_err_cor.attr, 601 &dev_attr_aer_rootport_total_err_fatal.attr, 602 &dev_attr_aer_rootport_total_err_nonfatal.attr, 603 NULL 604 }; 605 606 static umode_t aer_stats_attrs_are_visible(struct kobject *kobj, 607 struct attribute *a, int n) 608 { 609 struct device *dev = kobj_to_dev(kobj); 610 struct pci_dev *pdev = to_pci_dev(dev); 611 612 if (!pdev->aer_info) 613 return 0; 614 615 if ((a == &dev_attr_aer_rootport_total_err_cor.attr || 616 a == &dev_attr_aer_rootport_total_err_fatal.attr || 617 a == &dev_attr_aer_rootport_total_err_nonfatal.attr) && 618 ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) && 619 (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_EC))) 620 return 0; 621 622 return a->mode; 623 } 624 625 const struct attribute_group aer_stats_attr_group = { 626 .attrs = aer_stats_attrs, 627 .is_visible = aer_stats_attrs_are_visible, 628 }; 629 630 /* 631 * Ratelimit interval 632 * <=0: disabled with ratelimit.interval = 0 633 * >0: enabled with ratelimit.interval in ms 634 */ 635 #define aer_ratelimit_interval_attr(name, ratelimit) \ 636 static ssize_t \ 637 name##_show(struct device *dev, struct device_attribute *attr, \ 638 char *buf) \ 639 { \ 640 struct pci_dev *pdev = to_pci_dev(dev); \ 641 \ 642 return sysfs_emit(buf, "%d\n", \ 643 pdev->aer_info->ratelimit.interval); \ 644 } \ 645 \ 646 static ssize_t \ 647 name##_store(struct device *dev, struct device_attribute *attr, \ 648 const char *buf, size_t count) \ 649 { \ 650 struct pci_dev *pdev = to_pci_dev(dev); \ 651 int interval; \ 652 \ 653 if (!capable(CAP_SYS_ADMIN)) \ 654 return -EPERM; \ 655 \ 656 if (kstrtoint(buf, 0, &interval) < 0) \ 657 return -EINVAL; \ 658 \ 659 if (interval <= 0) \ 660 interval = 0; \ 661 else \ 662 interval = msecs_to_jiffies(interval); \ 663 \ 664 pdev->aer_info->ratelimit.interval = interval; \ 665 \ 666 return count; \ 667 } \ 668 static DEVICE_ATTR_RW(name); 669 670 #define aer_ratelimit_burst_attr(name, ratelimit) \ 671 static ssize_t \ 672 name##_show(struct device *dev, struct device_attribute *attr, \ 673 char *buf) \ 674 { \ 675 struct pci_dev *pdev = to_pci_dev(dev); \ 676 \ 677 return sysfs_emit(buf, "%d\n", \ 678 pdev->aer_info->ratelimit.burst); \ 679 } \ 680 \ 681 static ssize_t \ 682 name##_store(struct device *dev, struct device_attribute *attr, \ 683 const char *buf, size_t count) \ 684 { \ 685 struct pci_dev *pdev = to_pci_dev(dev); \ 686 int burst; \ 687 \ 688 if (!capable(CAP_SYS_ADMIN)) \ 689 return -EPERM; \ 690 \ 691 if (kstrtoint(buf, 0, &burst) < 0) \ 692 return -EINVAL; \ 693 \ 694 pdev->aer_info->ratelimit.burst = burst; \ 695 \ 696 return count; \ 697 } \ 698 static DEVICE_ATTR_RW(name); 699 700 #define aer_ratelimit_attrs(name) \ 701 aer_ratelimit_interval_attr(name##_ratelimit_interval_ms, \ 702 name##_ratelimit) \ 703 aer_ratelimit_burst_attr(name##_ratelimit_burst, \ 704 name##_ratelimit) 705 706 aer_ratelimit_attrs(correctable) 707 aer_ratelimit_attrs(nonfatal) 708 709 static struct attribute *aer_attrs[] = { 710 &dev_attr_correctable_ratelimit_interval_ms.attr, 711 &dev_attr_correctable_ratelimit_burst.attr, 712 &dev_attr_nonfatal_ratelimit_interval_ms.attr, 713 &dev_attr_nonfatal_ratelimit_burst.attr, 714 NULL 715 }; 716 717 static umode_t aer_attrs_are_visible(struct kobject *kobj, 718 struct attribute *a, int n) 719 { 720 struct device *dev = kobj_to_dev(kobj); 721 struct pci_dev *pdev = to_pci_dev(dev); 722 723 if (!pdev->aer_info) 724 return 0; 725 726 return a->mode; 727 } 728 729 const struct attribute_group aer_attr_group = { 730 .name = "aer", 731 .attrs = aer_attrs, 732 .is_visible = aer_attrs_are_visible, 733 }; 734 735 static void pci_dev_aer_stats_incr(struct pci_dev *pdev, 736 struct aer_err_info *info) 737 { 738 unsigned long status = info->status & ~info->mask; 739 int i, max = -1; 740 u64 *counter = NULL; 741 struct aer_info *aer_info = pdev->aer_info; 742 743 if (!aer_info) 744 return; 745 746 switch (info->severity) { 747 case AER_CORRECTABLE: 748 aer_info->dev_total_cor_errs++; 749 counter = &aer_info->dev_cor_errs[0]; 750 max = AER_MAX_TYPEOF_COR_ERRS; 751 break; 752 case AER_NONFATAL: 753 aer_info->dev_total_nonfatal_errs++; 754 counter = &aer_info->dev_nonfatal_errs[0]; 755 max = AER_MAX_TYPEOF_UNCOR_ERRS; 756 break; 757 case AER_FATAL: 758 aer_info->dev_total_fatal_errs++; 759 counter = &aer_info->dev_fatal_errs[0]; 760 max = AER_MAX_TYPEOF_UNCOR_ERRS; 761 break; 762 } 763 764 for_each_set_bit(i, &status, max) 765 counter[i]++; 766 } 767 768 static void pci_rootport_aer_stats_incr(struct pci_dev *pdev, 769 struct aer_err_source *e_src) 770 { 771 struct aer_info *aer_info = pdev->aer_info; 772 773 if (!aer_info) 774 return; 775 776 if (e_src->status & PCI_ERR_ROOT_COR_RCV) 777 aer_info->rootport_total_cor_errs++; 778 779 if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { 780 if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) 781 aer_info->rootport_total_fatal_errs++; 782 else 783 aer_info->rootport_total_nonfatal_errs++; 784 } 785 } 786 787 static int aer_ratelimit(struct pci_dev *dev, unsigned int severity) 788 { 789 switch (severity) { 790 case AER_NONFATAL: 791 return __ratelimit(&dev->aer_info->nonfatal_ratelimit); 792 case AER_CORRECTABLE: 793 return __ratelimit(&dev->aer_info->correctable_ratelimit); 794 default: 795 return 1; /* Don't ratelimit fatal errors */ 796 } 797 } 798 799 static void __aer_print_error(struct pci_dev *dev, struct aer_err_info *info) 800 { 801 const char **strings; 802 unsigned long status = info->status & ~info->mask; 803 const char *level = info->level; 804 const char *errmsg; 805 int i; 806 807 if (info->severity == AER_CORRECTABLE) 808 strings = aer_correctable_error_string; 809 else 810 strings = aer_uncorrectable_error_string; 811 812 for_each_set_bit(i, &status, 32) { 813 errmsg = strings[i]; 814 if (!errmsg) 815 errmsg = "Unknown Error Bit"; 816 817 aer_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg, 818 info->first_error == i ? " (First)" : ""); 819 } 820 } 821 822 static void aer_print_source(struct pci_dev *dev, struct aer_err_info *info, 823 bool found) 824 { 825 u16 source = info->id; 826 827 pci_info(dev, "%s%s error message received from %04x:%02x:%02x.%d%s\n", 828 info->multi_error_valid ? "Multiple " : "", 829 aer_error_severity_string[info->severity], 830 pci_domain_nr(dev->bus), PCI_BUS_NUM(source), 831 PCI_SLOT(source), PCI_FUNC(source), 832 found ? "" : " (no details found"); 833 } 834 835 void aer_print_error(struct aer_err_info *info, int i) 836 { 837 struct pci_dev *dev; 838 int layer, agent, id; 839 const char *level = info->level; 840 841 if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES)) 842 return; 843 844 dev = info->dev[i]; 845 id = pci_dev_id(dev); 846 847 pci_dev_aer_stats_incr(dev, info); 848 trace_aer_event(pci_name(dev), (info->status & ~info->mask), 849 info->severity, info->tlp_header_valid, &info->tlp); 850 851 if (!info->ratelimit_print[i]) 852 return; 853 854 if (!info->status) { 855 pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", 856 aer_error_severity_string[info->severity]); 857 goto out; 858 } 859 860 layer = AER_GET_LAYER_ERROR(info->severity, info->status); 861 agent = AER_GET_AGENT(info->severity, info->status); 862 863 aer_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n", 864 aer_error_severity_string[info->severity], 865 aer_error_layer[layer], aer_agent_string[agent]); 866 867 aer_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n", 868 dev->vendor, dev->device, info->status, info->mask); 869 870 __aer_print_error(dev, info); 871 872 if (info->tlp_header_valid) 873 pcie_print_tlp_log(dev, &info->tlp, level, dev_fmt(" ")); 874 875 out: 876 if (info->id && info->error_dev_num > 1 && info->id == id) 877 pci_err(dev, " Error of this Agent is reported first\n"); 878 } 879 880 #ifdef CONFIG_ACPI_APEI_PCIEAER 881 int cper_severity_to_aer(int cper_severity) 882 { 883 switch (cper_severity) { 884 case CPER_SEV_RECOVERABLE: 885 return AER_NONFATAL; 886 case CPER_SEV_FATAL: 887 return AER_FATAL; 888 default: 889 return AER_CORRECTABLE; 890 } 891 } 892 EXPORT_SYMBOL_GPL(cper_severity_to_aer); 893 #endif 894 895 void pci_print_aer(struct pci_dev *dev, int aer_severity, 896 struct aer_capability_regs *aer) 897 { 898 int layer, agent, tlp_header_valid = 0; 899 u32 status, mask; 900 struct aer_err_info info = { 901 .severity = aer_severity, 902 .first_error = PCI_ERR_CAP_FEP(aer->cap_control), 903 }; 904 905 if (aer_severity == AER_CORRECTABLE) { 906 status = aer->cor_status; 907 mask = aer->cor_mask; 908 info.level = KERN_WARNING; 909 } else { 910 status = aer->uncor_status; 911 mask = aer->uncor_mask; 912 info.level = KERN_ERR; 913 tlp_header_valid = status & AER_LOG_TLP_MASKS; 914 } 915 916 info.status = status; 917 info.mask = mask; 918 919 pci_dev_aer_stats_incr(dev, &info); 920 trace_aer_event(pci_name(dev), (status & ~mask), 921 aer_severity, tlp_header_valid, &aer->header_log); 922 923 if (!aer_ratelimit(dev, info.severity)) 924 return; 925 926 layer = AER_GET_LAYER_ERROR(aer_severity, status); 927 agent = AER_GET_AGENT(aer_severity, status); 928 929 aer_printk(info.level, dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", 930 status, mask); 931 __aer_print_error(dev, &info); 932 aer_printk(info.level, dev, "aer_layer=%s, aer_agent=%s\n", 933 aer_error_layer[layer], aer_agent_string[agent]); 934 935 if (aer_severity != AER_CORRECTABLE) 936 aer_printk(info.level, dev, "aer_uncor_severity: 0x%08x\n", 937 aer->uncor_severity); 938 939 if (tlp_header_valid) 940 pcie_print_tlp_log(dev, &aer->header_log, info.level, 941 dev_fmt(" ")); 942 } 943 EXPORT_SYMBOL_NS_GPL(pci_print_aer, "CXL"); 944 945 /** 946 * add_error_device - list device to be handled 947 * @e_info: pointer to error info 948 * @dev: pointer to pci_dev to be added 949 */ 950 static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) 951 { 952 int i = e_info->error_dev_num; 953 954 if (i >= AER_MAX_MULTI_ERR_DEVICES) 955 return -ENOSPC; 956 957 e_info->dev[i] = pci_dev_get(dev); 958 e_info->error_dev_num++; 959 960 /* 961 * Ratelimit AER log messages. "dev" is either the source 962 * identified by the root's Error Source ID or it has an unmasked 963 * error logged in its own AER Capability. Messages are emitted 964 * when "ratelimit_print[i]" is non-zero. If we will print detail 965 * for a downstream device, make sure we print the Error Source ID 966 * from the root as well. 967 */ 968 if (aer_ratelimit(dev, e_info->severity)) { 969 e_info->ratelimit_print[i] = 1; 970 e_info->root_ratelimit_print = 1; 971 } 972 return 0; 973 } 974 975 /** 976 * is_error_source - check whether the device is source of reported error 977 * @dev: pointer to pci_dev to be checked 978 * @e_info: pointer to reported error info 979 */ 980 static bool is_error_source(struct pci_dev *dev, struct aer_err_info *e_info) 981 { 982 int aer = dev->aer_cap; 983 u32 status, mask; 984 u16 reg16; 985 986 /* 987 * When bus ID is equal to 0, it might be a bad ID 988 * reported by Root Port. 989 */ 990 if ((PCI_BUS_NUM(e_info->id) != 0) && 991 !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) { 992 /* Device ID match? */ 993 if (e_info->id == pci_dev_id(dev)) 994 return true; 995 996 /* Continue ID comparing if there is no multiple error */ 997 if (!e_info->multi_error_valid) 998 return false; 999 } 1000 1001 /* 1002 * When either 1003 * 1) bus ID is equal to 0. Some ports might lose the bus 1004 * ID of error source id; 1005 * 2) bus flag PCI_BUS_FLAGS_NO_AERSID is set 1006 * 3) There are multiple errors and prior ID comparing fails; 1007 * We check AER status registers to find possible reporter. 1008 */ 1009 if (atomic_read(&dev->enable_cnt) == 0) 1010 return false; 1011 1012 /* Check if AER is enabled */ 1013 pcie_capability_read_word(dev, PCI_EXP_DEVCTL, ®16); 1014 if (!(reg16 & PCI_EXP_AER_FLAGS)) 1015 return false; 1016 1017 if (!aer) 1018 return false; 1019 1020 /* Check if error is recorded */ 1021 if (e_info->severity == AER_CORRECTABLE) { 1022 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status); 1023 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1024 } else { 1025 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status); 1026 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1027 } 1028 if (status & ~mask) 1029 return true; 1030 1031 return false; 1032 } 1033 1034 static int find_device_iter(struct pci_dev *dev, void *data) 1035 { 1036 struct aer_err_info *e_info = (struct aer_err_info *)data; 1037 1038 if (is_error_source(dev, e_info)) { 1039 /* List this device */ 1040 if (add_error_device(e_info, dev)) { 1041 /* We cannot handle more... Stop iteration */ 1042 pci_err(dev, "Exceeded max supported (%d) devices with errors logged\n", 1043 AER_MAX_MULTI_ERR_DEVICES); 1044 return 1; 1045 } 1046 1047 /* If there is only a single error, stop iteration */ 1048 if (!e_info->multi_error_valid) 1049 return 1; 1050 } 1051 return 0; 1052 } 1053 1054 /** 1055 * find_source_device - search through device hierarchy for source device 1056 * @parent: pointer to Root Port pci_dev data structure 1057 * @e_info: including detailed error information such as ID 1058 * 1059 * Return: true if found. 1060 * 1061 * Invoked by DPC when error is detected at the Root Port. 1062 * Caller of this function must set id, severity, and multi_error_valid of 1063 * struct aer_err_info pointed by @e_info properly. This function must fill 1064 * e_info->error_dev_num and e_info->dev[], based on the given information. 1065 */ 1066 static bool find_source_device(struct pci_dev *parent, 1067 struct aer_err_info *e_info) 1068 { 1069 struct pci_dev *dev = parent; 1070 int result; 1071 1072 /* Must reset in this function */ 1073 e_info->error_dev_num = 0; 1074 1075 /* Is Root Port an agent that sends error message? */ 1076 result = find_device_iter(dev, e_info); 1077 if (result) 1078 return true; 1079 1080 if (pci_pcie_type(parent) == PCI_EXP_TYPE_RC_EC) 1081 pcie_walk_rcec(parent, find_device_iter, e_info); 1082 else 1083 pci_walk_bus(parent->subordinate, find_device_iter, e_info); 1084 1085 if (!e_info->error_dev_num) 1086 return false; 1087 return true; 1088 } 1089 1090 #ifdef CONFIG_PCIEAER_CXL 1091 1092 /** 1093 * pci_aer_unmask_internal_errors - unmask internal errors 1094 * @dev: pointer to the pci_dev data structure 1095 * 1096 * Unmask internal errors in the Uncorrectable and Correctable Error 1097 * Mask registers. 1098 * 1099 * Note: AER must be enabled and supported by the device which must be 1100 * checked in advance, e.g. with pcie_aer_is_native(). 1101 */ 1102 static void pci_aer_unmask_internal_errors(struct pci_dev *dev) 1103 { 1104 int aer = dev->aer_cap; 1105 u32 mask; 1106 1107 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask); 1108 mask &= ~PCI_ERR_UNC_INTN; 1109 pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, mask); 1110 1111 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask); 1112 mask &= ~PCI_ERR_COR_INTERNAL; 1113 pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask); 1114 } 1115 1116 static bool is_cxl_mem_dev(struct pci_dev *dev) 1117 { 1118 /* 1119 * The capability, status, and control fields in Device 0, 1120 * Function 0 DVSEC control the CXL functionality of the 1121 * entire device (CXL 3.0, 8.1.3). 1122 */ 1123 if (dev->devfn != PCI_DEVFN(0, 0)) 1124 return false; 1125 1126 /* 1127 * CXL Memory Devices must have the 502h class code set (CXL 1128 * 3.0, 8.1.12.1). 1129 */ 1130 if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL) 1131 return false; 1132 1133 return true; 1134 } 1135 1136 static bool cxl_error_is_native(struct pci_dev *dev) 1137 { 1138 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 1139 1140 return (pcie_ports_native || host->native_aer); 1141 } 1142 1143 static bool is_internal_error(struct aer_err_info *info) 1144 { 1145 if (info->severity == AER_CORRECTABLE) 1146 return info->status & PCI_ERR_COR_INTERNAL; 1147 1148 return info->status & PCI_ERR_UNC_INTN; 1149 } 1150 1151 static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data) 1152 { 1153 struct aer_err_info *info = (struct aer_err_info *)data; 1154 const struct pci_error_handlers *err_handler; 1155 1156 if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev)) 1157 return 0; 1158 1159 /* Protect dev->driver */ 1160 device_lock(&dev->dev); 1161 1162 err_handler = dev->driver ? dev->driver->err_handler : NULL; 1163 if (!err_handler) 1164 goto out; 1165 1166 if (info->severity == AER_CORRECTABLE) { 1167 if (err_handler->cor_error_detected) 1168 err_handler->cor_error_detected(dev); 1169 } else if (err_handler->error_detected) { 1170 if (info->severity == AER_NONFATAL) 1171 err_handler->error_detected(dev, pci_channel_io_normal); 1172 else if (info->severity == AER_FATAL) 1173 err_handler->error_detected(dev, pci_channel_io_frozen); 1174 } 1175 out: 1176 device_unlock(&dev->dev); 1177 return 0; 1178 } 1179 1180 static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) 1181 { 1182 /* 1183 * Internal errors of an RCEC indicate an AER error in an 1184 * RCH's downstream port. Check and handle them in the CXL.mem 1185 * device driver. 1186 */ 1187 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC && 1188 is_internal_error(info)) 1189 pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info); 1190 } 1191 1192 static int handles_cxl_error_iter(struct pci_dev *dev, void *data) 1193 { 1194 bool *handles_cxl = data; 1195 1196 if (!*handles_cxl) 1197 *handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev); 1198 1199 /* Non-zero terminates iteration */ 1200 return *handles_cxl; 1201 } 1202 1203 static bool handles_cxl_errors(struct pci_dev *rcec) 1204 { 1205 bool handles_cxl = false; 1206 1207 if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC && 1208 pcie_aer_is_native(rcec)) 1209 pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl); 1210 1211 return handles_cxl; 1212 } 1213 1214 static void cxl_rch_enable_rcec(struct pci_dev *rcec) 1215 { 1216 if (!handles_cxl_errors(rcec)) 1217 return; 1218 1219 pci_aer_unmask_internal_errors(rcec); 1220 pci_info(rcec, "CXL: Internal errors unmasked"); 1221 } 1222 1223 #else 1224 static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { } 1225 static inline void cxl_rch_handle_error(struct pci_dev *dev, 1226 struct aer_err_info *info) { } 1227 #endif 1228 1229 /** 1230 * pci_aer_handle_error - handle logging error into an event log 1231 * @dev: pointer to pci_dev data structure of error source device 1232 * @info: comprehensive error information 1233 * 1234 * Invoked when an error being detected by Root Port. 1235 */ 1236 static void pci_aer_handle_error(struct pci_dev *dev, struct aer_err_info *info) 1237 { 1238 int aer = dev->aer_cap; 1239 1240 if (info->severity == AER_CORRECTABLE) { 1241 /* 1242 * Correctable error does not need software intervention. 1243 * No need to go through error recovery process. 1244 */ 1245 if (aer) 1246 pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1247 info->status); 1248 if (pcie_aer_is_native(dev)) { 1249 struct pci_driver *pdrv = dev->driver; 1250 1251 if (pdrv && pdrv->err_handler && 1252 pdrv->err_handler->cor_error_detected) 1253 pdrv->err_handler->cor_error_detected(dev); 1254 pcie_clear_device_status(dev); 1255 } 1256 } else if (info->severity == AER_NONFATAL) 1257 pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset); 1258 else if (info->severity == AER_FATAL) 1259 pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset); 1260 } 1261 1262 static void handle_error_source(struct pci_dev *dev, struct aer_err_info *info) 1263 { 1264 cxl_rch_handle_error(dev, info); 1265 pci_aer_handle_error(dev, info); 1266 pci_dev_put(dev); 1267 } 1268 1269 #ifdef CONFIG_ACPI_APEI_PCIEAER 1270 1271 #define AER_RECOVER_RING_SIZE 16 1272 1273 struct aer_recover_entry { 1274 u8 bus; 1275 u8 devfn; 1276 u16 domain; 1277 int severity; 1278 struct aer_capability_regs *regs; 1279 }; 1280 1281 static DEFINE_KFIFO(aer_recover_ring, struct aer_recover_entry, 1282 AER_RECOVER_RING_SIZE); 1283 1284 static void aer_recover_work_func(struct work_struct *work) 1285 { 1286 struct aer_recover_entry entry; 1287 struct pci_dev *pdev; 1288 1289 while (kfifo_get(&aer_recover_ring, &entry)) { 1290 pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, 1291 entry.devfn); 1292 if (!pdev) { 1293 pr_err_ratelimited("%04x:%02x:%02x.%x: no pci_dev found\n", 1294 entry.domain, entry.bus, 1295 PCI_SLOT(entry.devfn), 1296 PCI_FUNC(entry.devfn)); 1297 continue; 1298 } 1299 pci_print_aer(pdev, entry.severity, entry.regs); 1300 1301 /* 1302 * Memory for aer_capability_regs(entry.regs) is being 1303 * allocated from the ghes_estatus_pool to protect it from 1304 * overwriting when multiple sections are present in the 1305 * error status. Thus free the same after processing the 1306 * data. 1307 */ 1308 ghes_estatus_pool_region_free((unsigned long)entry.regs, 1309 sizeof(struct aer_capability_regs)); 1310 1311 if (entry.severity == AER_NONFATAL) 1312 pcie_do_recovery(pdev, pci_channel_io_normal, 1313 aer_root_reset); 1314 else if (entry.severity == AER_FATAL) 1315 pcie_do_recovery(pdev, pci_channel_io_frozen, 1316 aer_root_reset); 1317 pci_dev_put(pdev); 1318 } 1319 } 1320 1321 /* 1322 * Mutual exclusion for writers of aer_recover_ring, reader side don't 1323 * need lock, because there is only one reader and lock is not needed 1324 * between reader and writer. 1325 */ 1326 static DEFINE_SPINLOCK(aer_recover_ring_lock); 1327 static DECLARE_WORK(aer_recover_work, aer_recover_work_func); 1328 1329 void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, 1330 int severity, struct aer_capability_regs *aer_regs) 1331 { 1332 struct aer_recover_entry entry = { 1333 .bus = bus, 1334 .devfn = devfn, 1335 .domain = domain, 1336 .severity = severity, 1337 .regs = aer_regs, 1338 }; 1339 1340 if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1, 1341 &aer_recover_ring_lock)) 1342 schedule_work(&aer_recover_work); 1343 else 1344 pr_err("buffer overflow in recovery for %04x:%02x:%02x.%x\n", 1345 domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 1346 } 1347 EXPORT_SYMBOL_GPL(aer_recover_queue); 1348 #endif 1349 1350 /** 1351 * aer_get_device_error_info - read error status from dev and store it to info 1352 * @info: pointer to structure to store the error record 1353 * @i: index into info->dev[] 1354 * 1355 * Return: 1 on success, 0 on error. 1356 * 1357 * Note that @info is reused among all error devices. Clear fields properly. 1358 */ 1359 int aer_get_device_error_info(struct aer_err_info *info, int i) 1360 { 1361 struct pci_dev *dev; 1362 int type, aer; 1363 u32 aercc; 1364 1365 if (i >= AER_MAX_MULTI_ERR_DEVICES) 1366 return 0; 1367 1368 dev = info->dev[i]; 1369 aer = dev->aer_cap; 1370 type = pci_pcie_type(dev); 1371 1372 /* Must reset in this function */ 1373 info->status = 0; 1374 info->tlp_header_valid = 0; 1375 1376 /* The device might not support AER */ 1377 if (!aer) 1378 return 0; 1379 1380 if (info->severity == AER_CORRECTABLE) { 1381 pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, 1382 &info->status); 1383 pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, 1384 &info->mask); 1385 if (!(info->status & ~info->mask)) 1386 return 0; 1387 } else if (type == PCI_EXP_TYPE_ROOT_PORT || 1388 type == PCI_EXP_TYPE_RC_EC || 1389 type == PCI_EXP_TYPE_DOWNSTREAM || 1390 info->severity == AER_NONFATAL) { 1391 1392 /* Link is still healthy for IO reads */ 1393 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, 1394 &info->status); 1395 pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, 1396 &info->mask); 1397 if (!(info->status & ~info->mask)) 1398 return 0; 1399 1400 /* Get First Error Pointer */ 1401 pci_read_config_dword(dev, aer + PCI_ERR_CAP, &aercc); 1402 info->first_error = PCI_ERR_CAP_FEP(aercc); 1403 1404 if (info->status & AER_LOG_TLP_MASKS) { 1405 info->tlp_header_valid = 1; 1406 pcie_read_tlp_log(dev, aer + PCI_ERR_HEADER_LOG, 1407 aer + PCI_ERR_PREFIX_LOG, 1408 aer_tlp_log_len(dev, aercc), 1409 aercc & PCI_ERR_CAP_TLP_LOG_FLIT, 1410 &info->tlp); 1411 } 1412 } 1413 1414 return 1; 1415 } 1416 1417 static inline void aer_process_err_devices(struct aer_err_info *e_info) 1418 { 1419 int i; 1420 1421 /* Report all before handling them, to not lose records by reset etc. */ 1422 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1423 if (aer_get_device_error_info(e_info, i)) 1424 aer_print_error(e_info, i); 1425 } 1426 for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { 1427 if (aer_get_device_error_info(e_info, i)) 1428 handle_error_source(e_info->dev[i], e_info); 1429 } 1430 } 1431 1432 /** 1433 * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error 1434 * detected by Root Port or RCEC 1435 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1436 * @info: pointer to AER error info 1437 */ 1438 static void aer_isr_one_error_type(struct pci_dev *root, 1439 struct aer_err_info *info) 1440 { 1441 bool found; 1442 1443 found = find_source_device(root, info); 1444 1445 /* 1446 * If we're going to log error messages, we've already set 1447 * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to 1448 * non-zero (which enables printing) because this is either an 1449 * ERR_FATAL or we found a device with an error logged in its AER 1450 * Capability. 1451 * 1452 * If we didn't find the Error Source device, at least log the 1453 * Requester ID from the ERR_* Message received by the Root Port or 1454 * RCEC, ratelimited by the RP or RCEC. 1455 */ 1456 if (info->root_ratelimit_print || 1457 (!found && aer_ratelimit(root, info->severity))) 1458 aer_print_source(root, info, found); 1459 1460 if (found) 1461 aer_process_err_devices(info); 1462 } 1463 1464 /** 1465 * aer_isr_one_error - consume error(s) signaled by an AER interrupt from 1466 * Root Port or RCEC 1467 * @root: pointer to Root Port or RCEC that signaled AER interrupt 1468 * @e_src: pointer to an error source 1469 */ 1470 static void aer_isr_one_error(struct pci_dev *root, 1471 struct aer_err_source *e_src) 1472 { 1473 u32 status = e_src->status; 1474 1475 pci_rootport_aer_stats_incr(root, e_src); 1476 1477 /* 1478 * There is a possibility that both correctable error and 1479 * uncorrectable error being logged. Report correctable error first. 1480 */ 1481 if (status & PCI_ERR_ROOT_COR_RCV) { 1482 int multi = status & PCI_ERR_ROOT_MULTI_COR_RCV; 1483 struct aer_err_info e_info = { 1484 .id = ERR_COR_ID(e_src->id), 1485 .severity = AER_CORRECTABLE, 1486 .level = KERN_WARNING, 1487 .multi_error_valid = multi ? 1 : 0, 1488 }; 1489 1490 aer_isr_one_error_type(root, &e_info); 1491 } 1492 1493 if (status & PCI_ERR_ROOT_UNCOR_RCV) { 1494 int fatal = status & PCI_ERR_ROOT_FATAL_RCV; 1495 int multi = status & PCI_ERR_ROOT_MULTI_UNCOR_RCV; 1496 struct aer_err_info e_info = { 1497 .id = ERR_UNCOR_ID(e_src->id), 1498 .severity = fatal ? AER_FATAL : AER_NONFATAL, 1499 .level = KERN_ERR, 1500 .multi_error_valid = multi ? 1 : 0, 1501 }; 1502 1503 aer_isr_one_error_type(root, &e_info); 1504 } 1505 } 1506 1507 /** 1508 * aer_isr - consume errors detected by Root Port 1509 * @irq: IRQ assigned to Root Port 1510 * @context: pointer to Root Port data structure 1511 * 1512 * Invoked, as DPC, when Root Port records new detected error 1513 */ 1514 static irqreturn_t aer_isr(int irq, void *context) 1515 { 1516 struct pcie_device *dev = (struct pcie_device *)context; 1517 struct aer_rpc *rpc = get_service_data(dev); 1518 struct aer_err_source e_src; 1519 1520 if (kfifo_is_empty(&rpc->aer_fifo)) 1521 return IRQ_NONE; 1522 1523 while (kfifo_get(&rpc->aer_fifo, &e_src)) 1524 aer_isr_one_error(rpc->rpd, &e_src); 1525 return IRQ_HANDLED; 1526 } 1527 1528 /** 1529 * aer_irq - Root Port's ISR 1530 * @irq: IRQ assigned to Root Port 1531 * @context: pointer to Root Port data structure 1532 * 1533 * Invoked when Root Port detects AER messages. 1534 */ 1535 static irqreturn_t aer_irq(int irq, void *context) 1536 { 1537 struct pcie_device *pdev = (struct pcie_device *)context; 1538 struct aer_rpc *rpc = get_service_data(pdev); 1539 struct pci_dev *rp = rpc->rpd; 1540 int aer = rp->aer_cap; 1541 struct aer_err_source e_src = {}; 1542 1543 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status); 1544 if (!(e_src.status & AER_ERR_STATUS_MASK)) 1545 return IRQ_NONE; 1546 1547 pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id); 1548 pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status); 1549 1550 if (!kfifo_put(&rpc->aer_fifo, e_src)) 1551 return IRQ_HANDLED; 1552 1553 return IRQ_WAKE_THREAD; 1554 } 1555 1556 static void aer_enable_irq(struct pci_dev *pdev) 1557 { 1558 int aer = pdev->aer_cap; 1559 u32 reg32; 1560 1561 /* Enable Root Port's interrupt in response to error messages */ 1562 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1563 reg32 |= ROOT_PORT_INTR_ON_MESG_MASK; 1564 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1565 } 1566 1567 static void aer_disable_irq(struct pci_dev *pdev) 1568 { 1569 int aer = pdev->aer_cap; 1570 u32 reg32; 1571 1572 /* Disable Root Port's interrupt in response to error messages */ 1573 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, ®32); 1574 reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK; 1575 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32); 1576 } 1577 1578 /** 1579 * aer_enable_rootport - enable Root Port's interrupts when receiving messages 1580 * @rpc: pointer to a Root Port data structure 1581 * 1582 * Invoked when PCIe bus loads AER service driver. 1583 */ 1584 static void aer_enable_rootport(struct aer_rpc *rpc) 1585 { 1586 struct pci_dev *pdev = rpc->rpd; 1587 int aer = pdev->aer_cap; 1588 u16 reg16; 1589 u32 reg32; 1590 1591 /* Clear PCIe Capability's Device Status */ 1592 pcie_capability_read_word(pdev, PCI_EXP_DEVSTA, ®16); 1593 pcie_capability_write_word(pdev, PCI_EXP_DEVSTA, reg16); 1594 1595 /* Disable system error generation in response to error messages */ 1596 pcie_capability_clear_word(pdev, PCI_EXP_RTCTL, 1597 SYSTEM_ERROR_INTR_ON_MESG_MASK); 1598 1599 /* Clear error status */ 1600 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1601 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1602 pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, ®32); 1603 pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32); 1604 pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, ®32); 1605 pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32); 1606 1607 aer_enable_irq(pdev); 1608 } 1609 1610 /** 1611 * aer_disable_rootport - disable Root Port's interrupts when receiving messages 1612 * @rpc: pointer to a Root Port data structure 1613 * 1614 * Invoked when PCIe bus unloads AER service driver. 1615 */ 1616 static void aer_disable_rootport(struct aer_rpc *rpc) 1617 { 1618 struct pci_dev *pdev = rpc->rpd; 1619 int aer = pdev->aer_cap; 1620 u32 reg32; 1621 1622 aer_disable_irq(pdev); 1623 1624 /* Clear Root's error status reg */ 1625 pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, ®32); 1626 pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32); 1627 } 1628 1629 /** 1630 * aer_remove - clean up resources 1631 * @dev: pointer to the pcie_dev data structure 1632 * 1633 * Invoked when PCI Express bus unloads or AER probe fails. 1634 */ 1635 static void aer_remove(struct pcie_device *dev) 1636 { 1637 struct aer_rpc *rpc = get_service_data(dev); 1638 1639 aer_disable_rootport(rpc); 1640 } 1641 1642 /** 1643 * aer_probe - initialize resources 1644 * @dev: pointer to the pcie_dev data structure 1645 * 1646 * Invoked when PCI Express bus loads AER service driver. 1647 */ 1648 static int aer_probe(struct pcie_device *dev) 1649 { 1650 int status; 1651 struct aer_rpc *rpc; 1652 struct device *device = &dev->device; 1653 struct pci_dev *port = dev->port; 1654 1655 BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) < 1656 AER_MAX_TYPEOF_COR_ERRS); 1657 BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) < 1658 AER_MAX_TYPEOF_UNCOR_ERRS); 1659 1660 /* Limit to Root Ports or Root Complex Event Collectors */ 1661 if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) && 1662 (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT)) 1663 return -ENODEV; 1664 1665 rpc = devm_kzalloc(device, sizeof(struct aer_rpc), GFP_KERNEL); 1666 if (!rpc) 1667 return -ENOMEM; 1668 1669 rpc->rpd = port; 1670 INIT_KFIFO(rpc->aer_fifo); 1671 set_service_data(dev, rpc); 1672 1673 status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr, 1674 IRQF_SHARED, "aerdrv", dev); 1675 if (status) { 1676 pci_err(port, "request AER IRQ %d failed\n", dev->irq); 1677 return status; 1678 } 1679 1680 cxl_rch_enable_rcec(port); 1681 aer_enable_rootport(rpc); 1682 pci_info(port, "enabled with IRQ %d\n", dev->irq); 1683 return 0; 1684 } 1685 1686 static int aer_suspend(struct pcie_device *dev) 1687 { 1688 struct aer_rpc *rpc = get_service_data(dev); 1689 1690 aer_disable_rootport(rpc); 1691 return 0; 1692 } 1693 1694 static int aer_resume(struct pcie_device *dev) 1695 { 1696 struct aer_rpc *rpc = get_service_data(dev); 1697 1698 aer_enable_rootport(rpc); 1699 return 0; 1700 } 1701 1702 /** 1703 * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP 1704 * @dev: pointer to Root Port, RCEC, or RCiEP 1705 * 1706 * Invoked by Port Bus driver when performing reset. 1707 */ 1708 static pci_ers_result_t aer_root_reset(struct pci_dev *dev) 1709 { 1710 int type = pci_pcie_type(dev); 1711 struct pci_dev *root; 1712 int aer; 1713 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus); 1714 u32 reg32; 1715 int rc; 1716 1717 /* 1718 * Only Root Ports and RCECs have AER Root Command and Root Status 1719 * registers. If "dev" is an RCiEP, the relevant registers are in 1720 * the RCEC. 1721 */ 1722 if (type == PCI_EXP_TYPE_RC_END) 1723 root = dev->rcec; 1724 else 1725 root = pcie_find_root_port(dev); 1726 1727 /* 1728 * If the platform retained control of AER, an RCiEP may not have 1729 * an RCEC visible to us, so dev->rcec ("root") may be NULL. In 1730 * that case, firmware is responsible for these registers. 1731 */ 1732 aer = root ? root->aer_cap : 0; 1733 1734 if ((host->native_aer || pcie_ports_native) && aer) 1735 aer_disable_irq(root); 1736 1737 if (type == PCI_EXP_TYPE_RC_EC || type == PCI_EXP_TYPE_RC_END) { 1738 rc = pcie_reset_flr(dev, PCI_RESET_DO_RESET); 1739 if (!rc) 1740 pci_info(dev, "has been reset\n"); 1741 else 1742 pci_info(dev, "not reset (no FLR support: %d)\n", rc); 1743 } else { 1744 rc = pci_bus_error_reset(dev); 1745 pci_info(dev, "%s Port link has been reset (%d)\n", 1746 pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc); 1747 } 1748 1749 if ((host->native_aer || pcie_ports_native) && aer) { 1750 /* Clear Root Error Status */ 1751 pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, ®32); 1752 pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32); 1753 1754 aer_enable_irq(root); 1755 } 1756 1757 return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 1758 } 1759 1760 static struct pcie_port_service_driver aerdriver = { 1761 .name = "aer", 1762 .port_type = PCIE_ANY_PORT, 1763 .service = PCIE_PORT_SERVICE_AER, 1764 1765 .probe = aer_probe, 1766 .suspend = aer_suspend, 1767 .resume = aer_resume, 1768 .remove = aer_remove, 1769 }; 1770 1771 /** 1772 * pcie_aer_init - register AER service driver 1773 * 1774 * Invoked when AER service driver is loaded. 1775 */ 1776 int __init pcie_aer_init(void) 1777 { 1778 if (!pci_aer_available()) 1779 return -ENXIO; 1780 return pcie_port_service_register(&aerdriver); 1781 } 1782