1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright IBM Corporation 2001, 2005, 2006 4 * Copyright Dave Engebretsen & Todd Inglett 2001 5 * Copyright Linas Vepstas 2005, 2006 6 * Copyright 2001-2012 IBM Corporation. 7 * 8 * Please address comments and feedback to Linas Vepstas <linas@austin.ibm.com> 9 */ 10 11 #include <linux/delay.h> 12 #include <linux/sched.h> 13 #include <linux/init.h> 14 #include <linux/list.h> 15 #include <linux/pci.h> 16 #include <linux/iommu.h> 17 #include <linux/proc_fs.h> 18 #include <linux/rbtree.h> 19 #include <linux/reboot.h> 20 #include <linux/seq_file.h> 21 #include <linux/spinlock.h> 22 #include <linux/export.h> 23 #include <linux/of.h> 24 25 #include <linux/atomic.h> 26 #include <asm/debugfs.h> 27 #include <asm/eeh.h> 28 #include <asm/eeh_event.h> 29 #include <asm/io.h> 30 #include <asm/iommu.h> 31 #include <asm/machdep.h> 32 #include <asm/ppc-pci.h> 33 #include <asm/rtas.h> 34 #include <asm/pte-walk.h> 35 36 37 /** Overview: 38 * EEH, or "Enhanced Error Handling" is a PCI bridge technology for 39 * dealing with PCI bus errors that can't be dealt with within the 40 * usual PCI framework, except by check-stopping the CPU. Systems 41 * that are designed for high-availability/reliability cannot afford 42 * to crash due to a "mere" PCI error, thus the need for EEH. 43 * An EEH-capable bridge operates by converting a detected error 44 * into a "slot freeze", taking the PCI adapter off-line, making 45 * the slot behave, from the OS'es point of view, as if the slot 46 * were "empty": all reads return 0xff's and all writes are silently 47 * ignored. EEH slot isolation events can be triggered by parity 48 * errors on the address or data busses (e.g. during posted writes), 49 * which in turn might be caused by low voltage on the bus, dust, 50 * vibration, humidity, radioactivity or plain-old failed hardware. 51 * 52 * Note, however, that one of the leading causes of EEH slot 53 * freeze events are buggy device drivers, buggy device microcode, 54 * or buggy device hardware. This is because any attempt by the 55 * device to bus-master data to a memory address that is not 56 * assigned to the device will trigger a slot freeze. (The idea 57 * is to prevent devices-gone-wild from corrupting system memory). 58 * Buggy hardware/drivers will have a miserable time co-existing 59 * with EEH. 60 * 61 * Ideally, a PCI device driver, when suspecting that an isolation 62 * event has occurred (e.g. by reading 0xff's), will then ask EEH 63 * whether this is the case, and then take appropriate steps to 64 * reset the PCI slot, the PCI device, and then resume operations. 65 * However, until that day, the checking is done here, with the 66 * eeh_check_failure() routine embedded in the MMIO macros. If 67 * the slot is found to be isolated, an "EEH Event" is synthesized 68 * and sent out for processing. 69 */ 70 71 /* If a device driver keeps reading an MMIO register in an interrupt 72 * handler after a slot isolation event, it might be broken. 73 * This sets the threshold for how many read attempts we allow 74 * before printing an error message. 75 */ 76 #define EEH_MAX_FAILS 2100000 77 78 /* Time to wait for a PCI slot to report status, in milliseconds */ 79 #define PCI_BUS_RESET_WAIT_MSEC (5*60*1000) 80 81 /* 82 * EEH probe mode support, which is part of the flags, 83 * is to support multiple platforms for EEH. Some platforms 84 * like pSeries do PCI emunation based on device tree. 85 * However, other platforms like powernv probe PCI devices 86 * from hardware. The flag is used to distinguish that. 87 * In addition, struct eeh_ops::probe would be invoked for 88 * particular OF node or PCI device so that the corresponding 89 * PE would be created there. 90 */ 91 int eeh_subsystem_flags; 92 EXPORT_SYMBOL(eeh_subsystem_flags); 93 94 /* 95 * EEH allowed maximal frozen times. If one particular PE's 96 * frozen count in last hour exceeds this limit, the PE will 97 * be forced to be offline permanently. 98 */ 99 u32 eeh_max_freezes = 5; 100 101 /* 102 * Controls whether a recovery event should be scheduled when an 103 * isolated device is discovered. This is only really useful for 104 * debugging problems with the EEH core. 105 */ 106 bool eeh_debugfs_no_recover; 107 108 /* Platform dependent EEH operations */ 109 struct eeh_ops *eeh_ops = NULL; 110 111 /* Lock to avoid races due to multiple reports of an error */ 112 DEFINE_RAW_SPINLOCK(confirm_error_lock); 113 EXPORT_SYMBOL_GPL(confirm_error_lock); 114 115 /* Lock to protect passed flags */ 116 static DEFINE_MUTEX(eeh_dev_mutex); 117 118 /* Buffer for reporting pci register dumps. Its here in BSS, and 119 * not dynamically alloced, so that it ends up in RMO where RTAS 120 * can access it. 121 */ 122 #define EEH_PCI_REGS_LOG_LEN 8192 123 static unsigned char pci_regs_buf[EEH_PCI_REGS_LOG_LEN]; 124 125 /* 126 * The struct is used to maintain the EEH global statistic 127 * information. Besides, the EEH global statistics will be 128 * exported to user space through procfs 129 */ 130 struct eeh_stats { 131 u64 no_device; /* PCI device not found */ 132 u64 no_dn; /* OF node not found */ 133 u64 no_cfg_addr; /* Config address not found */ 134 u64 ignored_check; /* EEH check skipped */ 135 u64 total_mmio_ffs; /* Total EEH checks */ 136 u64 false_positives; /* Unnecessary EEH checks */ 137 u64 slot_resets; /* PE reset */ 138 }; 139 140 static struct eeh_stats eeh_stats; 141 142 static int __init eeh_setup(char *str) 143 { 144 if (!strcmp(str, "off")) 145 eeh_add_flag(EEH_FORCE_DISABLED); 146 else if (!strcmp(str, "early_log")) 147 eeh_add_flag(EEH_EARLY_DUMP_LOG); 148 149 return 1; 150 } 151 __setup("eeh=", eeh_setup); 152 153 void eeh_show_enabled(void) 154 { 155 if (eeh_has_flag(EEH_FORCE_DISABLED)) 156 pr_info("EEH: Recovery disabled by kernel parameter.\n"); 157 else if (eeh_has_flag(EEH_ENABLED)) 158 pr_info("EEH: Capable adapter found: recovery enabled.\n"); 159 else 160 pr_info("EEH: No capable adapters found: recovery disabled.\n"); 161 } 162 163 /* 164 * This routine captures assorted PCI configuration space data 165 * for the indicated PCI device, and puts them into a buffer 166 * for RTAS error logging. 167 */ 168 static size_t eeh_dump_dev_log(struct eeh_dev *edev, char *buf, size_t len) 169 { 170 u32 cfg; 171 int cap, i; 172 int n = 0, l = 0; 173 char buffer[128]; 174 175 n += scnprintf(buf+n, len-n, "%04x:%02x:%02x.%01x\n", 176 edev->pe->phb->global_number, edev->bdfn >> 8, 177 PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn)); 178 pr_warn("EEH: of node=%04x:%02x:%02x.%01x\n", 179 edev->pe->phb->global_number, edev->bdfn >> 8, 180 PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn)); 181 182 eeh_ops->read_config(edev, PCI_VENDOR_ID, 4, &cfg); 183 n += scnprintf(buf+n, len-n, "dev/vend:%08x\n", cfg); 184 pr_warn("EEH: PCI device/vendor: %08x\n", cfg); 185 186 eeh_ops->read_config(edev, PCI_COMMAND, 4, &cfg); 187 n += scnprintf(buf+n, len-n, "cmd/stat:%x\n", cfg); 188 pr_warn("EEH: PCI cmd/status register: %08x\n", cfg); 189 190 /* Gather bridge-specific registers */ 191 if (edev->mode & EEH_DEV_BRIDGE) { 192 eeh_ops->read_config(edev, PCI_SEC_STATUS, 2, &cfg); 193 n += scnprintf(buf+n, len-n, "sec stat:%x\n", cfg); 194 pr_warn("EEH: Bridge secondary status: %04x\n", cfg); 195 196 eeh_ops->read_config(edev, PCI_BRIDGE_CONTROL, 2, &cfg); 197 n += scnprintf(buf+n, len-n, "brdg ctl:%x\n", cfg); 198 pr_warn("EEH: Bridge control: %04x\n", cfg); 199 } 200 201 /* Dump out the PCI-X command and status regs */ 202 cap = edev->pcix_cap; 203 if (cap) { 204 eeh_ops->read_config(edev, cap, 4, &cfg); 205 n += scnprintf(buf+n, len-n, "pcix-cmd:%x\n", cfg); 206 pr_warn("EEH: PCI-X cmd: %08x\n", cfg); 207 208 eeh_ops->read_config(edev, cap+4, 4, &cfg); 209 n += scnprintf(buf+n, len-n, "pcix-stat:%x\n", cfg); 210 pr_warn("EEH: PCI-X status: %08x\n", cfg); 211 } 212 213 /* If PCI-E capable, dump PCI-E cap 10 */ 214 cap = edev->pcie_cap; 215 if (cap) { 216 n += scnprintf(buf+n, len-n, "pci-e cap10:\n"); 217 pr_warn("EEH: PCI-E capabilities and status follow:\n"); 218 219 for (i=0; i<=8; i++) { 220 eeh_ops->read_config(edev, cap+4*i, 4, &cfg); 221 n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); 222 223 if ((i % 4) == 0) { 224 if (i != 0) 225 pr_warn("%s\n", buffer); 226 227 l = scnprintf(buffer, sizeof(buffer), 228 "EEH: PCI-E %02x: %08x ", 229 4*i, cfg); 230 } else { 231 l += scnprintf(buffer+l, sizeof(buffer)-l, 232 "%08x ", cfg); 233 } 234 235 } 236 237 pr_warn("%s\n", buffer); 238 } 239 240 /* If AER capable, dump it */ 241 cap = edev->aer_cap; 242 if (cap) { 243 n += scnprintf(buf+n, len-n, "pci-e AER:\n"); 244 pr_warn("EEH: PCI-E AER capability register set follows:\n"); 245 246 for (i=0; i<=13; i++) { 247 eeh_ops->read_config(edev, cap+4*i, 4, &cfg); 248 n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg); 249 250 if ((i % 4) == 0) { 251 if (i != 0) 252 pr_warn("%s\n", buffer); 253 254 l = scnprintf(buffer, sizeof(buffer), 255 "EEH: PCI-E AER %02x: %08x ", 256 4*i, cfg); 257 } else { 258 l += scnprintf(buffer+l, sizeof(buffer)-l, 259 "%08x ", cfg); 260 } 261 } 262 263 pr_warn("%s\n", buffer); 264 } 265 266 return n; 267 } 268 269 static void *eeh_dump_pe_log(struct eeh_pe *pe, void *flag) 270 { 271 struct eeh_dev *edev, *tmp; 272 size_t *plen = flag; 273 274 eeh_pe_for_each_dev(pe, edev, tmp) 275 *plen += eeh_dump_dev_log(edev, pci_regs_buf + *plen, 276 EEH_PCI_REGS_LOG_LEN - *plen); 277 278 return NULL; 279 } 280 281 /** 282 * eeh_slot_error_detail - Generate combined log including driver log and error log 283 * @pe: EEH PE 284 * @severity: temporary or permanent error log 285 * 286 * This routine should be called to generate the combined log, which 287 * is comprised of driver log and error log. The driver log is figured 288 * out from the config space of the corresponding PCI device, while 289 * the error log is fetched through platform dependent function call. 290 */ 291 void eeh_slot_error_detail(struct eeh_pe *pe, int severity) 292 { 293 size_t loglen = 0; 294 295 /* 296 * When the PHB is fenced or dead, it's pointless to collect 297 * the data from PCI config space because it should return 298 * 0xFF's. For ER, we still retrieve the data from the PCI 299 * config space. 300 * 301 * For pHyp, we have to enable IO for log retrieval. Otherwise, 302 * 0xFF's is always returned from PCI config space. 303 * 304 * When the @severity is EEH_LOG_PERM, the PE is going to be 305 * removed. Prior to that, the drivers for devices included in 306 * the PE will be closed. The drivers rely on working IO path 307 * to bring the devices to quiet state. Otherwise, PCI traffic 308 * from those devices after they are removed is like to cause 309 * another unexpected EEH error. 310 */ 311 if (!(pe->type & EEH_PE_PHB)) { 312 if (eeh_has_flag(EEH_ENABLE_IO_FOR_LOG) || 313 severity == EEH_LOG_PERM) 314 eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); 315 316 /* 317 * The config space of some PCI devices can't be accessed 318 * when their PEs are in frozen state. Otherwise, fenced 319 * PHB might be seen. Those PEs are identified with flag 320 * EEH_PE_CFG_RESTRICTED, indicating EEH_PE_CFG_BLOCKED 321 * is set automatically when the PE is put to EEH_PE_ISOLATED. 322 * 323 * Restoring BARs possibly triggers PCI config access in 324 * (OPAL) firmware and then causes fenced PHB. If the 325 * PCI config is blocked with flag EEH_PE_CFG_BLOCKED, it's 326 * pointless to restore BARs and dump config space. 327 */ 328 eeh_ops->configure_bridge(pe); 329 if (!(pe->state & EEH_PE_CFG_BLOCKED)) { 330 eeh_pe_restore_bars(pe); 331 332 pci_regs_buf[0] = 0; 333 eeh_pe_traverse(pe, eeh_dump_pe_log, &loglen); 334 } 335 } 336 337 eeh_ops->get_log(pe, severity, pci_regs_buf, loglen); 338 } 339 340 /** 341 * eeh_token_to_phys - Convert EEH address token to phys address 342 * @token: I/O token, should be address in the form 0xA.... 343 * 344 * This routine should be called to convert virtual I/O address 345 * to physical one. 346 */ 347 static inline unsigned long eeh_token_to_phys(unsigned long token) 348 { 349 pte_t *ptep; 350 unsigned long pa; 351 int hugepage_shift; 352 353 /* 354 * We won't find hugepages here(this is iomem). Hence we are not 355 * worried about _PAGE_SPLITTING/collapse. Also we will not hit 356 * page table free, because of init_mm. 357 */ 358 ptep = find_init_mm_pte(token, &hugepage_shift); 359 if (!ptep) 360 return token; 361 362 pa = pte_pfn(*ptep); 363 364 /* On radix we can do hugepage mappings for io, so handle that */ 365 if (hugepage_shift) { 366 pa <<= hugepage_shift; 367 pa |= token & ((1ul << hugepage_shift) - 1); 368 } else { 369 pa <<= PAGE_SHIFT; 370 pa |= token & (PAGE_SIZE - 1); 371 } 372 373 return pa; 374 } 375 376 /* 377 * On PowerNV platform, we might already have fenced PHB there. 378 * For that case, it's meaningless to recover frozen PE. Intead, 379 * We have to handle fenced PHB firstly. 380 */ 381 static int eeh_phb_check_failure(struct eeh_pe *pe) 382 { 383 struct eeh_pe *phb_pe; 384 unsigned long flags; 385 int ret; 386 387 if (!eeh_has_flag(EEH_PROBE_MODE_DEV)) 388 return -EPERM; 389 390 /* Find the PHB PE */ 391 phb_pe = eeh_phb_pe_get(pe->phb); 392 if (!phb_pe) { 393 pr_warn("%s Can't find PE for PHB#%x\n", 394 __func__, pe->phb->global_number); 395 return -EEXIST; 396 } 397 398 /* If the PHB has been in problematic state */ 399 eeh_serialize_lock(&flags); 400 if (phb_pe->state & EEH_PE_ISOLATED) { 401 ret = 0; 402 goto out; 403 } 404 405 /* Check PHB state */ 406 ret = eeh_ops->get_state(phb_pe, NULL); 407 if ((ret < 0) || 408 (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) { 409 ret = 0; 410 goto out; 411 } 412 413 /* Isolate the PHB and send event */ 414 eeh_pe_mark_isolated(phb_pe); 415 eeh_serialize_unlock(flags); 416 417 pr_debug("EEH: PHB#%x failure detected, location: %s\n", 418 phb_pe->phb->global_number, eeh_pe_loc_get(phb_pe)); 419 eeh_send_failure_event(phb_pe); 420 return 1; 421 out: 422 eeh_serialize_unlock(flags); 423 return ret; 424 } 425 426 /** 427 * eeh_dev_check_failure - Check if all 1's data is due to EEH slot freeze 428 * @edev: eeh device 429 * 430 * Check for an EEH failure for the given device node. Call this 431 * routine if the result of a read was all 0xff's and you want to 432 * find out if this is due to an EEH slot freeze. This routine 433 * will query firmware for the EEH status. 434 * 435 * Returns 0 if there has not been an EEH error; otherwise returns 436 * a non-zero value and queues up a slot isolation event notification. 437 * 438 * It is safe to call this routine in an interrupt context. 439 */ 440 int eeh_dev_check_failure(struct eeh_dev *edev) 441 { 442 int ret; 443 unsigned long flags; 444 struct device_node *dn; 445 struct pci_dev *dev; 446 struct eeh_pe *pe, *parent_pe; 447 int rc = 0; 448 const char *location = NULL; 449 450 eeh_stats.total_mmio_ffs++; 451 452 if (!eeh_enabled()) 453 return 0; 454 455 if (!edev) { 456 eeh_stats.no_dn++; 457 return 0; 458 } 459 dev = eeh_dev_to_pci_dev(edev); 460 pe = eeh_dev_to_pe(edev); 461 462 /* Access to IO BARs might get this far and still not want checking. */ 463 if (!pe) { 464 eeh_stats.ignored_check++; 465 eeh_edev_dbg(edev, "Ignored check\n"); 466 return 0; 467 } 468 469 /* 470 * On PowerNV platform, we might already have fenced PHB 471 * there and we need take care of that firstly. 472 */ 473 ret = eeh_phb_check_failure(pe); 474 if (ret > 0) 475 return ret; 476 477 /* 478 * If the PE isn't owned by us, we shouldn't check the 479 * state. Instead, let the owner handle it if the PE has 480 * been frozen. 481 */ 482 if (eeh_pe_passed(pe)) 483 return 0; 484 485 /* If we already have a pending isolation event for this 486 * slot, we know it's bad already, we don't need to check. 487 * Do this checking under a lock; as multiple PCI devices 488 * in one slot might report errors simultaneously, and we 489 * only want one error recovery routine running. 490 */ 491 eeh_serialize_lock(&flags); 492 rc = 1; 493 if (pe->state & EEH_PE_ISOLATED) { 494 pe->check_count++; 495 if (pe->check_count == EEH_MAX_FAILS) { 496 dn = pci_device_to_OF_node(dev); 497 if (dn) 498 location = of_get_property(dn, "ibm,loc-code", 499 NULL); 500 eeh_edev_err(edev, "%d reads ignored for recovering device at location=%s driver=%s\n", 501 pe->check_count, 502 location ? location : "unknown", 503 eeh_driver_name(dev)); 504 eeh_edev_err(edev, "Might be infinite loop in %s driver\n", 505 eeh_driver_name(dev)); 506 dump_stack(); 507 } 508 goto dn_unlock; 509 } 510 511 /* 512 * Now test for an EEH failure. This is VERY expensive. 513 * Note that the eeh_config_addr may be a parent device 514 * in the case of a device behind a bridge, or it may be 515 * function zero of a multi-function device. 516 * In any case they must share a common PHB. 517 */ 518 ret = eeh_ops->get_state(pe, NULL); 519 520 /* Note that config-io to empty slots may fail; 521 * they are empty when they don't have children. 522 * We will punt with the following conditions: Failure to get 523 * PE's state, EEH not support and Permanently unavailable 524 * state, PE is in good state. 525 */ 526 if ((ret < 0) || 527 (ret == EEH_STATE_NOT_SUPPORT) || eeh_state_active(ret)) { 528 eeh_stats.false_positives++; 529 pe->false_positives++; 530 rc = 0; 531 goto dn_unlock; 532 } 533 534 /* 535 * It should be corner case that the parent PE has been 536 * put into frozen state as well. We should take care 537 * that at first. 538 */ 539 parent_pe = pe->parent; 540 while (parent_pe) { 541 /* Hit the ceiling ? */ 542 if (parent_pe->type & EEH_PE_PHB) 543 break; 544 545 /* Frozen parent PE ? */ 546 ret = eeh_ops->get_state(parent_pe, NULL); 547 if (ret > 0 && !eeh_state_active(ret)) { 548 pe = parent_pe; 549 pr_err("EEH: Failure of PHB#%x-PE#%x will be handled at parent PHB#%x-PE#%x.\n", 550 pe->phb->global_number, pe->addr, 551 pe->phb->global_number, parent_pe->addr); 552 } 553 554 /* Next parent level */ 555 parent_pe = parent_pe->parent; 556 } 557 558 eeh_stats.slot_resets++; 559 560 /* Avoid repeated reports of this failure, including problems 561 * with other functions on this device, and functions under 562 * bridges. 563 */ 564 eeh_pe_mark_isolated(pe); 565 eeh_serialize_unlock(flags); 566 567 /* Most EEH events are due to device driver bugs. Having 568 * a stack trace will help the device-driver authors figure 569 * out what happened. So print that out. 570 */ 571 pr_debug("EEH: %s: Frozen PHB#%x-PE#%x detected\n", 572 __func__, pe->phb->global_number, pe->addr); 573 eeh_send_failure_event(pe); 574 575 return 1; 576 577 dn_unlock: 578 eeh_serialize_unlock(flags); 579 return rc; 580 } 581 582 EXPORT_SYMBOL_GPL(eeh_dev_check_failure); 583 584 /** 585 * eeh_check_failure - Check if all 1's data is due to EEH slot freeze 586 * @token: I/O address 587 * 588 * Check for an EEH failure at the given I/O address. Call this 589 * routine if the result of a read was all 0xff's and you want to 590 * find out if this is due to an EEH slot freeze event. This routine 591 * will query firmware for the EEH status. 592 * 593 * Note this routine is safe to call in an interrupt context. 594 */ 595 int eeh_check_failure(const volatile void __iomem *token) 596 { 597 unsigned long addr; 598 struct eeh_dev *edev; 599 600 /* Finding the phys addr + pci device; this is pretty quick. */ 601 addr = eeh_token_to_phys((unsigned long __force) token); 602 edev = eeh_addr_cache_get_dev(addr); 603 if (!edev) { 604 eeh_stats.no_device++; 605 return 0; 606 } 607 608 return eeh_dev_check_failure(edev); 609 } 610 EXPORT_SYMBOL(eeh_check_failure); 611 612 613 /** 614 * eeh_pci_enable - Enable MMIO or DMA transfers for this slot 615 * @pe: EEH PE 616 * 617 * This routine should be called to reenable frozen MMIO or DMA 618 * so that it would work correctly again. It's useful while doing 619 * recovery or log collection on the indicated device. 620 */ 621 int eeh_pci_enable(struct eeh_pe *pe, int function) 622 { 623 int active_flag, rc; 624 625 /* 626 * pHyp doesn't allow to enable IO or DMA on unfrozen PE. 627 * Also, it's pointless to enable them on unfrozen PE. So 628 * we have to check before enabling IO or DMA. 629 */ 630 switch (function) { 631 case EEH_OPT_THAW_MMIO: 632 active_flag = EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED; 633 break; 634 case EEH_OPT_THAW_DMA: 635 active_flag = EEH_STATE_DMA_ACTIVE; 636 break; 637 case EEH_OPT_DISABLE: 638 case EEH_OPT_ENABLE: 639 case EEH_OPT_FREEZE_PE: 640 active_flag = 0; 641 break; 642 default: 643 pr_warn("%s: Invalid function %d\n", 644 __func__, function); 645 return -EINVAL; 646 } 647 648 /* 649 * Check if IO or DMA has been enabled before 650 * enabling them. 651 */ 652 if (active_flag) { 653 rc = eeh_ops->get_state(pe, NULL); 654 if (rc < 0) 655 return rc; 656 657 /* Needn't enable it at all */ 658 if (rc == EEH_STATE_NOT_SUPPORT) 659 return 0; 660 661 /* It's already enabled */ 662 if (rc & active_flag) 663 return 0; 664 } 665 666 667 /* Issue the request */ 668 rc = eeh_ops->set_option(pe, function); 669 if (rc) 670 pr_warn("%s: Unexpected state change %d on " 671 "PHB#%x-PE#%x, err=%d\n", 672 __func__, function, pe->phb->global_number, 673 pe->addr, rc); 674 675 /* Check if the request is finished successfully */ 676 if (active_flag) { 677 rc = eeh_wait_state(pe, PCI_BUS_RESET_WAIT_MSEC); 678 if (rc < 0) 679 return rc; 680 681 if (rc & active_flag) 682 return 0; 683 684 return -EIO; 685 } 686 687 return rc; 688 } 689 690 static void eeh_disable_and_save_dev_state(struct eeh_dev *edev, 691 void *userdata) 692 { 693 struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); 694 struct pci_dev *dev = userdata; 695 696 /* 697 * The caller should have disabled and saved the 698 * state for the specified device 699 */ 700 if (!pdev || pdev == dev) 701 return; 702 703 /* Ensure we have D0 power state */ 704 pci_set_power_state(pdev, PCI_D0); 705 706 /* Save device state */ 707 pci_save_state(pdev); 708 709 /* 710 * Disable device to avoid any DMA traffic and 711 * interrupt from the device 712 */ 713 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 714 } 715 716 static void eeh_restore_dev_state(struct eeh_dev *edev, void *userdata) 717 { 718 struct pci_dev *pdev = eeh_dev_to_pci_dev(edev); 719 struct pci_dev *dev = userdata; 720 721 if (!pdev) 722 return; 723 724 /* Apply customization from firmware */ 725 if (eeh_ops->restore_config) 726 eeh_ops->restore_config(edev); 727 728 /* The caller should restore state for the specified device */ 729 if (pdev != dev) 730 pci_restore_state(pdev); 731 } 732 733 /** 734 * pcibios_set_pcie_reset_state - Set PCI-E reset state 735 * @dev: pci device struct 736 * @state: reset state to enter 737 * 738 * Return value: 739 * 0 if success 740 */ 741 int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state) 742 { 743 struct eeh_dev *edev = pci_dev_to_eeh_dev(dev); 744 struct eeh_pe *pe = eeh_dev_to_pe(edev); 745 746 if (!pe) { 747 pr_err("%s: No PE found on PCI device %s\n", 748 __func__, pci_name(dev)); 749 return -EINVAL; 750 } 751 752 switch (state) { 753 case pcie_deassert_reset: 754 eeh_ops->reset(pe, EEH_RESET_DEACTIVATE); 755 eeh_unfreeze_pe(pe); 756 if (!(pe->type & EEH_PE_VF)) 757 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 758 eeh_pe_dev_traverse(pe, eeh_restore_dev_state, dev); 759 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true); 760 break; 761 case pcie_hot_reset: 762 eeh_pe_mark_isolated(pe); 763 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 764 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 765 eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); 766 if (!(pe->type & EEH_PE_VF)) 767 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 768 eeh_ops->reset(pe, EEH_RESET_HOT); 769 break; 770 case pcie_warm_reset: 771 eeh_pe_mark_isolated(pe); 772 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, true); 773 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 774 eeh_pe_dev_traverse(pe, eeh_disable_and_save_dev_state, dev); 775 if (!(pe->type & EEH_PE_VF)) 776 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 777 eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL); 778 break; 779 default: 780 eeh_pe_state_clear(pe, EEH_PE_ISOLATED | EEH_PE_CFG_BLOCKED, true); 781 return -EINVAL; 782 }; 783 784 return 0; 785 } 786 787 /** 788 * eeh_set_pe_freset - Check the required reset for the indicated device 789 * @data: EEH device 790 * @flag: return value 791 * 792 * Each device might have its preferred reset type: fundamental or 793 * hot reset. The routine is used to collected the information for 794 * the indicated device and its children so that the bunch of the 795 * devices could be reset properly. 796 */ 797 static void eeh_set_dev_freset(struct eeh_dev *edev, void *flag) 798 { 799 struct pci_dev *dev; 800 unsigned int *freset = (unsigned int *)flag; 801 802 dev = eeh_dev_to_pci_dev(edev); 803 if (dev) 804 *freset |= dev->needs_freset; 805 } 806 807 static void eeh_pe_refreeze_passed(struct eeh_pe *root) 808 { 809 struct eeh_pe *pe; 810 int state; 811 812 eeh_for_each_pe(root, pe) { 813 if (eeh_pe_passed(pe)) { 814 state = eeh_ops->get_state(pe, NULL); 815 if (state & 816 (EEH_STATE_MMIO_ACTIVE | EEH_STATE_MMIO_ENABLED)) { 817 pr_info("EEH: Passed-through PE PHB#%x-PE#%x was thawed by reset, re-freezing for safety.\n", 818 pe->phb->global_number, pe->addr); 819 eeh_pe_set_option(pe, EEH_OPT_FREEZE_PE); 820 } 821 } 822 } 823 } 824 825 /** 826 * eeh_pe_reset_full - Complete a full reset process on the indicated PE 827 * @pe: EEH PE 828 * 829 * This function executes a full reset procedure on a PE, including setting 830 * the appropriate flags, performing a fundamental or hot reset, and then 831 * deactivating the reset status. It is designed to be used within the EEH 832 * subsystem, as opposed to eeh_pe_reset which is exported to drivers and 833 * only performs a single operation at a time. 834 * 835 * This function will attempt to reset a PE three times before failing. 836 */ 837 int eeh_pe_reset_full(struct eeh_pe *pe, bool include_passed) 838 { 839 int reset_state = (EEH_PE_RESET | EEH_PE_CFG_BLOCKED); 840 int type = EEH_RESET_HOT; 841 unsigned int freset = 0; 842 int i, state = 0, ret; 843 844 /* 845 * Determine the type of reset to perform - hot or fundamental. 846 * Hot reset is the default operation, unless any device under the 847 * PE requires a fundamental reset. 848 */ 849 eeh_pe_dev_traverse(pe, eeh_set_dev_freset, &freset); 850 851 if (freset) 852 type = EEH_RESET_FUNDAMENTAL; 853 854 /* Mark the PE as in reset state and block config space accesses */ 855 eeh_pe_state_mark(pe, reset_state); 856 857 /* Make three attempts at resetting the bus */ 858 for (i = 0; i < 3; i++) { 859 ret = eeh_pe_reset(pe, type, include_passed); 860 if (!ret) 861 ret = eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, 862 include_passed); 863 if (ret) { 864 ret = -EIO; 865 pr_warn("EEH: Failure %d resetting PHB#%x-PE#%x (attempt %d)\n\n", 866 state, pe->phb->global_number, pe->addr, i + 1); 867 continue; 868 } 869 if (i) 870 pr_warn("EEH: PHB#%x-PE#%x: Successful reset (attempt %d)\n", 871 pe->phb->global_number, pe->addr, i + 1); 872 873 /* Wait until the PE is in a functioning state */ 874 state = eeh_wait_state(pe, PCI_BUS_RESET_WAIT_MSEC); 875 if (state < 0) { 876 pr_warn("EEH: Unrecoverable slot failure on PHB#%x-PE#%x", 877 pe->phb->global_number, pe->addr); 878 ret = -ENOTRECOVERABLE; 879 break; 880 } 881 if (eeh_state_active(state)) 882 break; 883 else 884 pr_warn("EEH: PHB#%x-PE#%x: Slot inactive after reset: 0x%x (attempt %d)\n", 885 pe->phb->global_number, pe->addr, state, i + 1); 886 } 887 888 /* Resetting the PE may have unfrozen child PEs. If those PEs have been 889 * (potentially) passed through to a guest, re-freeze them: 890 */ 891 if (!include_passed) 892 eeh_pe_refreeze_passed(pe); 893 894 eeh_pe_state_clear(pe, reset_state, true); 895 return ret; 896 } 897 898 /** 899 * eeh_save_bars - Save device bars 900 * @edev: PCI device associated EEH device 901 * 902 * Save the values of the device bars. Unlike the restore 903 * routine, this routine is *not* recursive. This is because 904 * PCI devices are added individually; but, for the restore, 905 * an entire slot is reset at a time. 906 */ 907 void eeh_save_bars(struct eeh_dev *edev) 908 { 909 int i; 910 911 if (!edev) 912 return; 913 914 for (i = 0; i < 16; i++) 915 eeh_ops->read_config(edev, i * 4, 4, &edev->config_space[i]); 916 917 /* 918 * For PCI bridges including root port, we need enable bus 919 * master explicitly. Otherwise, it can't fetch IODA table 920 * entries correctly. So we cache the bit in advance so that 921 * we can restore it after reset, either PHB range or PE range. 922 */ 923 if (edev->mode & EEH_DEV_BRIDGE) 924 edev->config_space[1] |= PCI_COMMAND_MASTER; 925 } 926 927 static int eeh_reboot_notifier(struct notifier_block *nb, 928 unsigned long action, void *unused) 929 { 930 eeh_clear_flag(EEH_ENABLED); 931 return NOTIFY_DONE; 932 } 933 934 static struct notifier_block eeh_reboot_nb = { 935 .notifier_call = eeh_reboot_notifier, 936 }; 937 938 static int eeh_device_notifier(struct notifier_block *nb, 939 unsigned long action, void *data) 940 { 941 struct device *dev = data; 942 943 switch (action) { 944 /* 945 * Note: It's not possible to perform EEH device addition (i.e. 946 * {pseries,pnv}_pcibios_bus_add_device()) here because it depends on 947 * the device's resources, which have not yet been set up. 948 */ 949 case BUS_NOTIFY_DEL_DEVICE: 950 eeh_remove_device(to_pci_dev(dev)); 951 break; 952 default: 953 break; 954 } 955 return NOTIFY_DONE; 956 } 957 958 static struct notifier_block eeh_device_nb = { 959 .notifier_call = eeh_device_notifier, 960 }; 961 962 /** 963 * eeh_init - System wide EEH initialization 964 * 965 * It's the platform's job to call this from an arch_initcall(). 966 */ 967 int eeh_init(struct eeh_ops *ops) 968 { 969 struct pci_controller *hose, *tmp; 970 int ret = 0; 971 972 /* the platform should only initialise EEH once */ 973 if (WARN_ON(eeh_ops)) 974 return -EEXIST; 975 if (WARN_ON(!ops)) 976 return -ENOENT; 977 eeh_ops = ops; 978 979 /* Register reboot notifier */ 980 ret = register_reboot_notifier(&eeh_reboot_nb); 981 if (ret) { 982 pr_warn("%s: Failed to register reboot notifier (%d)\n", 983 __func__, ret); 984 return ret; 985 } 986 987 ret = bus_register_notifier(&pci_bus_type, &eeh_device_nb); 988 if (ret) { 989 pr_warn("%s: Failed to register bus notifier (%d)\n", 990 __func__, ret); 991 return ret; 992 } 993 994 /* Initialize PHB PEs */ 995 list_for_each_entry_safe(hose, tmp, &hose_list, list_node) 996 eeh_phb_pe_create(hose); 997 998 eeh_addr_cache_init(); 999 1000 /* Initialize EEH event */ 1001 return eeh_event_init(); 1002 } 1003 1004 /** 1005 * eeh_probe_device() - Perform EEH initialization for the indicated pci device 1006 * @dev: pci device for which to set up EEH 1007 * 1008 * This routine must be used to complete EEH initialization for PCI 1009 * devices that were added after system boot (e.g. hotplug, dlpar). 1010 */ 1011 void eeh_probe_device(struct pci_dev *dev) 1012 { 1013 struct eeh_dev *edev; 1014 1015 pr_debug("EEH: Adding device %s\n", pci_name(dev)); 1016 1017 /* 1018 * pci_dev_to_eeh_dev() can only work if eeh_probe_dev() was 1019 * already called for this device. 1020 */ 1021 if (WARN_ON_ONCE(pci_dev_to_eeh_dev(dev))) { 1022 pci_dbg(dev, "Already bound to an eeh_dev!\n"); 1023 return; 1024 } 1025 1026 edev = eeh_ops->probe(dev); 1027 if (!edev) { 1028 pr_debug("EEH: Adding device failed\n"); 1029 return; 1030 } 1031 1032 /* 1033 * FIXME: We rely on pcibios_release_device() to remove the 1034 * existing EEH state. The release function is only called if 1035 * the pci_dev's refcount drops to zero so if something is 1036 * keeping a ref to a device (e.g. a filesystem) we need to 1037 * remove the old EEH state. 1038 * 1039 * FIXME: HEY MA, LOOK AT ME, NO LOCKING! 1040 */ 1041 if (edev->pdev && edev->pdev != dev) { 1042 eeh_pe_tree_remove(edev); 1043 eeh_addr_cache_rmv_dev(edev->pdev); 1044 eeh_sysfs_remove_device(edev->pdev); 1045 1046 /* 1047 * We definitely should have the PCI device removed 1048 * though it wasn't correctly. So we needn't call 1049 * into error handler afterwards. 1050 */ 1051 edev->mode |= EEH_DEV_NO_HANDLER; 1052 } 1053 1054 /* bind the pdev and the edev together */ 1055 edev->pdev = dev; 1056 dev->dev.archdata.edev = edev; 1057 eeh_addr_cache_insert_dev(dev); 1058 eeh_sysfs_add_device(dev); 1059 } 1060 1061 /** 1062 * eeh_remove_device - Undo EEH setup for the indicated pci device 1063 * @dev: pci device to be removed 1064 * 1065 * This routine should be called when a device is removed from 1066 * a running system (e.g. by hotplug or dlpar). It unregisters 1067 * the PCI device from the EEH subsystem. I/O errors affecting 1068 * this device will no longer be detected after this call; thus, 1069 * i/o errors affecting this slot may leave this device unusable. 1070 */ 1071 void eeh_remove_device(struct pci_dev *dev) 1072 { 1073 struct eeh_dev *edev; 1074 1075 if (!dev || !eeh_enabled()) 1076 return; 1077 edev = pci_dev_to_eeh_dev(dev); 1078 1079 /* Unregister the device with the EEH/PCI address search system */ 1080 dev_dbg(&dev->dev, "EEH: Removing device\n"); 1081 1082 if (!edev || !edev->pdev || !edev->pe) { 1083 dev_dbg(&dev->dev, "EEH: Device not referenced!\n"); 1084 return; 1085 } 1086 1087 /* 1088 * During the hotplug for EEH error recovery, we need the EEH 1089 * device attached to the parent PE in order for BAR restore 1090 * a bit later. So we keep it for BAR restore and remove it 1091 * from the parent PE during the BAR resotre. 1092 */ 1093 edev->pdev = NULL; 1094 1095 /* 1096 * eeh_sysfs_remove_device() uses pci_dev_to_eeh_dev() so we need to 1097 * remove the sysfs files before clearing dev.archdata.edev 1098 */ 1099 if (edev->mode & EEH_DEV_SYSFS) 1100 eeh_sysfs_remove_device(dev); 1101 1102 /* 1103 * We're removing from the PCI subsystem, that means 1104 * the PCI device driver can't support EEH or not 1105 * well. So we rely on hotplug completely to do recovery 1106 * for the specific PCI device. 1107 */ 1108 edev->mode |= EEH_DEV_NO_HANDLER; 1109 1110 eeh_addr_cache_rmv_dev(dev); 1111 1112 /* 1113 * The flag "in_error" is used to trace EEH devices for VFs 1114 * in error state or not. It's set in eeh_report_error(). If 1115 * it's not set, eeh_report_{reset,resume}() won't be called 1116 * for the VF EEH device. 1117 */ 1118 edev->in_error = false; 1119 dev->dev.archdata.edev = NULL; 1120 if (!(edev->pe->state & EEH_PE_KEEP)) 1121 eeh_pe_tree_remove(edev); 1122 else 1123 edev->mode |= EEH_DEV_DISCONNECTED; 1124 } 1125 1126 int eeh_unfreeze_pe(struct eeh_pe *pe) 1127 { 1128 int ret; 1129 1130 ret = eeh_pci_enable(pe, EEH_OPT_THAW_MMIO); 1131 if (ret) { 1132 pr_warn("%s: Failure %d enabling IO on PHB#%x-PE#%x\n", 1133 __func__, ret, pe->phb->global_number, pe->addr); 1134 return ret; 1135 } 1136 1137 ret = eeh_pci_enable(pe, EEH_OPT_THAW_DMA); 1138 if (ret) { 1139 pr_warn("%s: Failure %d enabling DMA on PHB#%x-PE#%x\n", 1140 __func__, ret, pe->phb->global_number, pe->addr); 1141 return ret; 1142 } 1143 1144 return ret; 1145 } 1146 1147 1148 static struct pci_device_id eeh_reset_ids[] = { 1149 { PCI_DEVICE(0x19a2, 0x0710) }, /* Emulex, BE */ 1150 { PCI_DEVICE(0x10df, 0xe220) }, /* Emulex, Lancer */ 1151 { PCI_DEVICE(0x14e4, 0x1657) }, /* Broadcom BCM5719 */ 1152 { 0 } 1153 }; 1154 1155 static int eeh_pe_change_owner(struct eeh_pe *pe) 1156 { 1157 struct eeh_dev *edev, *tmp; 1158 struct pci_dev *pdev; 1159 struct pci_device_id *id; 1160 int ret; 1161 1162 /* Check PE state */ 1163 ret = eeh_ops->get_state(pe, NULL); 1164 if (ret < 0 || ret == EEH_STATE_NOT_SUPPORT) 1165 return 0; 1166 1167 /* Unfrozen PE, nothing to do */ 1168 if (eeh_state_active(ret)) 1169 return 0; 1170 1171 /* Frozen PE, check if it needs PE level reset */ 1172 eeh_pe_for_each_dev(pe, edev, tmp) { 1173 pdev = eeh_dev_to_pci_dev(edev); 1174 if (!pdev) 1175 continue; 1176 1177 for (id = &eeh_reset_ids[0]; id->vendor != 0; id++) { 1178 if (id->vendor != PCI_ANY_ID && 1179 id->vendor != pdev->vendor) 1180 continue; 1181 if (id->device != PCI_ANY_ID && 1182 id->device != pdev->device) 1183 continue; 1184 if (id->subvendor != PCI_ANY_ID && 1185 id->subvendor != pdev->subsystem_vendor) 1186 continue; 1187 if (id->subdevice != PCI_ANY_ID && 1188 id->subdevice != pdev->subsystem_device) 1189 continue; 1190 1191 return eeh_pe_reset_and_recover(pe); 1192 } 1193 } 1194 1195 ret = eeh_unfreeze_pe(pe); 1196 if (!ret) 1197 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, true); 1198 return ret; 1199 } 1200 1201 /** 1202 * eeh_dev_open - Increase count of pass through devices for PE 1203 * @pdev: PCI device 1204 * 1205 * Increase count of passed through devices for the indicated 1206 * PE. In the result, the EEH errors detected on the PE won't be 1207 * reported. The PE owner will be responsible for detection 1208 * and recovery. 1209 */ 1210 int eeh_dev_open(struct pci_dev *pdev) 1211 { 1212 struct eeh_dev *edev; 1213 int ret = -ENODEV; 1214 1215 mutex_lock(&eeh_dev_mutex); 1216 1217 /* No PCI device ? */ 1218 if (!pdev) 1219 goto out; 1220 1221 /* No EEH device or PE ? */ 1222 edev = pci_dev_to_eeh_dev(pdev); 1223 if (!edev || !edev->pe) 1224 goto out; 1225 1226 /* 1227 * The PE might have been put into frozen state, but we 1228 * didn't detect that yet. The passed through PCI devices 1229 * in frozen PE won't work properly. Clear the frozen state 1230 * in advance. 1231 */ 1232 ret = eeh_pe_change_owner(edev->pe); 1233 if (ret) 1234 goto out; 1235 1236 /* Increase PE's pass through count */ 1237 atomic_inc(&edev->pe->pass_dev_cnt); 1238 mutex_unlock(&eeh_dev_mutex); 1239 1240 return 0; 1241 out: 1242 mutex_unlock(&eeh_dev_mutex); 1243 return ret; 1244 } 1245 EXPORT_SYMBOL_GPL(eeh_dev_open); 1246 1247 /** 1248 * eeh_dev_release - Decrease count of pass through devices for PE 1249 * @pdev: PCI device 1250 * 1251 * Decrease count of pass through devices for the indicated PE. If 1252 * there is no passed through device in PE, the EEH errors detected 1253 * on the PE will be reported and handled as usual. 1254 */ 1255 void eeh_dev_release(struct pci_dev *pdev) 1256 { 1257 struct eeh_dev *edev; 1258 1259 mutex_lock(&eeh_dev_mutex); 1260 1261 /* No PCI device ? */ 1262 if (!pdev) 1263 goto out; 1264 1265 /* No EEH device ? */ 1266 edev = pci_dev_to_eeh_dev(pdev); 1267 if (!edev || !edev->pe || !eeh_pe_passed(edev->pe)) 1268 goto out; 1269 1270 /* Decrease PE's pass through count */ 1271 WARN_ON(atomic_dec_if_positive(&edev->pe->pass_dev_cnt) < 0); 1272 eeh_pe_change_owner(edev->pe); 1273 out: 1274 mutex_unlock(&eeh_dev_mutex); 1275 } 1276 EXPORT_SYMBOL(eeh_dev_release); 1277 1278 #ifdef CONFIG_IOMMU_API 1279 1280 static int dev_has_iommu_table(struct device *dev, void *data) 1281 { 1282 struct pci_dev *pdev = to_pci_dev(dev); 1283 struct pci_dev **ppdev = data; 1284 1285 if (!dev) 1286 return 0; 1287 1288 if (device_iommu_mapped(dev)) { 1289 *ppdev = pdev; 1290 return 1; 1291 } 1292 1293 return 0; 1294 } 1295 1296 /** 1297 * eeh_iommu_group_to_pe - Convert IOMMU group to EEH PE 1298 * @group: IOMMU group 1299 * 1300 * The routine is called to convert IOMMU group to EEH PE. 1301 */ 1302 struct eeh_pe *eeh_iommu_group_to_pe(struct iommu_group *group) 1303 { 1304 struct pci_dev *pdev = NULL; 1305 struct eeh_dev *edev; 1306 int ret; 1307 1308 /* No IOMMU group ? */ 1309 if (!group) 1310 return NULL; 1311 1312 ret = iommu_group_for_each_dev(group, &pdev, dev_has_iommu_table); 1313 if (!ret || !pdev) 1314 return NULL; 1315 1316 /* No EEH device or PE ? */ 1317 edev = pci_dev_to_eeh_dev(pdev); 1318 if (!edev || !edev->pe) 1319 return NULL; 1320 1321 return edev->pe; 1322 } 1323 EXPORT_SYMBOL_GPL(eeh_iommu_group_to_pe); 1324 1325 #endif /* CONFIG_IOMMU_API */ 1326 1327 /** 1328 * eeh_pe_set_option - Set options for the indicated PE 1329 * @pe: EEH PE 1330 * @option: requested option 1331 * 1332 * The routine is called to enable or disable EEH functionality 1333 * on the indicated PE, to enable IO or DMA for the frozen PE. 1334 */ 1335 int eeh_pe_set_option(struct eeh_pe *pe, int option) 1336 { 1337 int ret = 0; 1338 1339 /* Invalid PE ? */ 1340 if (!pe) 1341 return -ENODEV; 1342 1343 /* 1344 * EEH functionality could possibly be disabled, just 1345 * return error for the case. And the EEH functinality 1346 * isn't expected to be disabled on one specific PE. 1347 */ 1348 switch (option) { 1349 case EEH_OPT_ENABLE: 1350 if (eeh_enabled()) { 1351 ret = eeh_pe_change_owner(pe); 1352 break; 1353 } 1354 ret = -EIO; 1355 break; 1356 case EEH_OPT_DISABLE: 1357 break; 1358 case EEH_OPT_THAW_MMIO: 1359 case EEH_OPT_THAW_DMA: 1360 case EEH_OPT_FREEZE_PE: 1361 if (!eeh_ops || !eeh_ops->set_option) { 1362 ret = -ENOENT; 1363 break; 1364 } 1365 1366 ret = eeh_pci_enable(pe, option); 1367 break; 1368 default: 1369 pr_debug("%s: Option %d out of range (%d, %d)\n", 1370 __func__, option, EEH_OPT_DISABLE, EEH_OPT_THAW_DMA); 1371 ret = -EINVAL; 1372 } 1373 1374 return ret; 1375 } 1376 EXPORT_SYMBOL_GPL(eeh_pe_set_option); 1377 1378 /** 1379 * eeh_pe_get_state - Retrieve PE's state 1380 * @pe: EEH PE 1381 * 1382 * Retrieve the PE's state, which includes 3 aspects: enabled 1383 * DMA, enabled IO and asserted reset. 1384 */ 1385 int eeh_pe_get_state(struct eeh_pe *pe) 1386 { 1387 int result, ret = 0; 1388 bool rst_active, dma_en, mmio_en; 1389 1390 /* Existing PE ? */ 1391 if (!pe) 1392 return -ENODEV; 1393 1394 if (!eeh_ops || !eeh_ops->get_state) 1395 return -ENOENT; 1396 1397 /* 1398 * If the parent PE is owned by the host kernel and is undergoing 1399 * error recovery, we should return the PE state as temporarily 1400 * unavailable so that the error recovery on the guest is suspended 1401 * until the recovery completes on the host. 1402 */ 1403 if (pe->parent && 1404 !(pe->state & EEH_PE_REMOVED) && 1405 (pe->parent->state & (EEH_PE_ISOLATED | EEH_PE_RECOVERING))) 1406 return EEH_PE_STATE_UNAVAIL; 1407 1408 result = eeh_ops->get_state(pe, NULL); 1409 rst_active = !!(result & EEH_STATE_RESET_ACTIVE); 1410 dma_en = !!(result & EEH_STATE_DMA_ENABLED); 1411 mmio_en = !!(result & EEH_STATE_MMIO_ENABLED); 1412 1413 if (rst_active) 1414 ret = EEH_PE_STATE_RESET; 1415 else if (dma_en && mmio_en) 1416 ret = EEH_PE_STATE_NORMAL; 1417 else if (!dma_en && !mmio_en) 1418 ret = EEH_PE_STATE_STOPPED_IO_DMA; 1419 else if (!dma_en && mmio_en) 1420 ret = EEH_PE_STATE_STOPPED_DMA; 1421 else 1422 ret = EEH_PE_STATE_UNAVAIL; 1423 1424 return ret; 1425 } 1426 EXPORT_SYMBOL_GPL(eeh_pe_get_state); 1427 1428 static int eeh_pe_reenable_devices(struct eeh_pe *pe, bool include_passed) 1429 { 1430 struct eeh_dev *edev, *tmp; 1431 struct pci_dev *pdev; 1432 int ret = 0; 1433 1434 eeh_pe_restore_bars(pe); 1435 1436 /* 1437 * Reenable PCI devices as the devices passed 1438 * through are always enabled before the reset. 1439 */ 1440 eeh_pe_for_each_dev(pe, edev, tmp) { 1441 pdev = eeh_dev_to_pci_dev(edev); 1442 if (!pdev) 1443 continue; 1444 1445 ret = pci_reenable_device(pdev); 1446 if (ret) { 1447 pr_warn("%s: Failure %d reenabling %s\n", 1448 __func__, ret, pci_name(pdev)); 1449 return ret; 1450 } 1451 } 1452 1453 /* The PE is still in frozen state */ 1454 if (include_passed || !eeh_pe_passed(pe)) { 1455 ret = eeh_unfreeze_pe(pe); 1456 } else 1457 pr_info("EEH: Note: Leaving passthrough PHB#%x-PE#%x frozen.\n", 1458 pe->phb->global_number, pe->addr); 1459 if (!ret) 1460 eeh_pe_state_clear(pe, EEH_PE_ISOLATED, include_passed); 1461 return ret; 1462 } 1463 1464 1465 /** 1466 * eeh_pe_reset - Issue PE reset according to specified type 1467 * @pe: EEH PE 1468 * @option: reset type 1469 * 1470 * The routine is called to reset the specified PE with the 1471 * indicated type, either fundamental reset or hot reset. 1472 * PE reset is the most important part for error recovery. 1473 */ 1474 int eeh_pe_reset(struct eeh_pe *pe, int option, bool include_passed) 1475 { 1476 int ret = 0; 1477 1478 /* Invalid PE ? */ 1479 if (!pe) 1480 return -ENODEV; 1481 1482 if (!eeh_ops || !eeh_ops->set_option || !eeh_ops->reset) 1483 return -ENOENT; 1484 1485 switch (option) { 1486 case EEH_RESET_DEACTIVATE: 1487 ret = eeh_ops->reset(pe, option); 1488 eeh_pe_state_clear(pe, EEH_PE_CFG_BLOCKED, include_passed); 1489 if (ret) 1490 break; 1491 1492 ret = eeh_pe_reenable_devices(pe, include_passed); 1493 break; 1494 case EEH_RESET_HOT: 1495 case EEH_RESET_FUNDAMENTAL: 1496 /* 1497 * Proactively freeze the PE to drop all MMIO access 1498 * during reset, which should be banned as it's always 1499 * cause recursive EEH error. 1500 */ 1501 eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE); 1502 1503 eeh_pe_state_mark(pe, EEH_PE_CFG_BLOCKED); 1504 ret = eeh_ops->reset(pe, option); 1505 break; 1506 default: 1507 pr_debug("%s: Unsupported option %d\n", 1508 __func__, option); 1509 ret = -EINVAL; 1510 } 1511 1512 return ret; 1513 } 1514 EXPORT_SYMBOL_GPL(eeh_pe_reset); 1515 1516 /** 1517 * eeh_pe_configure - Configure PCI bridges after PE reset 1518 * @pe: EEH PE 1519 * 1520 * The routine is called to restore the PCI config space for 1521 * those PCI devices, especially PCI bridges affected by PE 1522 * reset issued previously. 1523 */ 1524 int eeh_pe_configure(struct eeh_pe *pe) 1525 { 1526 int ret = 0; 1527 1528 /* Invalid PE ? */ 1529 if (!pe) 1530 return -ENODEV; 1531 1532 return ret; 1533 } 1534 EXPORT_SYMBOL_GPL(eeh_pe_configure); 1535 1536 /** 1537 * eeh_pe_inject_err - Injecting the specified PCI error to the indicated PE 1538 * @pe: the indicated PE 1539 * @type: error type 1540 * @function: error function 1541 * @addr: address 1542 * @mask: address mask 1543 * 1544 * The routine is called to inject the specified PCI error, which 1545 * is determined by @type and @function, to the indicated PE for 1546 * testing purpose. 1547 */ 1548 int eeh_pe_inject_err(struct eeh_pe *pe, int type, int func, 1549 unsigned long addr, unsigned long mask) 1550 { 1551 /* Invalid PE ? */ 1552 if (!pe) 1553 return -ENODEV; 1554 1555 /* Unsupported operation ? */ 1556 if (!eeh_ops || !eeh_ops->err_inject) 1557 return -ENOENT; 1558 1559 /* Check on PCI error type */ 1560 if (type != EEH_ERR_TYPE_32 && type != EEH_ERR_TYPE_64) 1561 return -EINVAL; 1562 1563 /* Check on PCI error function */ 1564 if (func < EEH_ERR_FUNC_MIN || func > EEH_ERR_FUNC_MAX) 1565 return -EINVAL; 1566 1567 return eeh_ops->err_inject(pe, type, func, addr, mask); 1568 } 1569 EXPORT_SYMBOL_GPL(eeh_pe_inject_err); 1570 1571 static int proc_eeh_show(struct seq_file *m, void *v) 1572 { 1573 if (!eeh_enabled()) { 1574 seq_printf(m, "EEH Subsystem is globally disabled\n"); 1575 seq_printf(m, "eeh_total_mmio_ffs=%llu\n", eeh_stats.total_mmio_ffs); 1576 } else { 1577 seq_printf(m, "EEH Subsystem is enabled\n"); 1578 seq_printf(m, 1579 "no device=%llu\n" 1580 "no device node=%llu\n" 1581 "no config address=%llu\n" 1582 "check not wanted=%llu\n" 1583 "eeh_total_mmio_ffs=%llu\n" 1584 "eeh_false_positives=%llu\n" 1585 "eeh_slot_resets=%llu\n", 1586 eeh_stats.no_device, 1587 eeh_stats.no_dn, 1588 eeh_stats.no_cfg_addr, 1589 eeh_stats.ignored_check, 1590 eeh_stats.total_mmio_ffs, 1591 eeh_stats.false_positives, 1592 eeh_stats.slot_resets); 1593 } 1594 1595 return 0; 1596 } 1597 1598 #ifdef CONFIG_DEBUG_FS 1599 1600 1601 static struct pci_dev *eeh_debug_lookup_pdev(struct file *filp, 1602 const char __user *user_buf, 1603 size_t count, loff_t *ppos) 1604 { 1605 uint32_t domain, bus, dev, fn; 1606 struct pci_dev *pdev; 1607 char buf[20]; 1608 int ret; 1609 1610 memset(buf, 0, sizeof(buf)); 1611 ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count); 1612 if (!ret) 1613 return ERR_PTR(-EFAULT); 1614 1615 ret = sscanf(buf, "%x:%x:%x.%x", &domain, &bus, &dev, &fn); 1616 if (ret != 4) { 1617 pr_err("%s: expected 4 args, got %d\n", __func__, ret); 1618 return ERR_PTR(-EINVAL); 1619 } 1620 1621 pdev = pci_get_domain_bus_and_slot(domain, bus, (dev << 3) | fn); 1622 if (!pdev) 1623 return ERR_PTR(-ENODEV); 1624 1625 return pdev; 1626 } 1627 1628 static int eeh_enable_dbgfs_set(void *data, u64 val) 1629 { 1630 if (val) 1631 eeh_clear_flag(EEH_FORCE_DISABLED); 1632 else 1633 eeh_add_flag(EEH_FORCE_DISABLED); 1634 1635 return 0; 1636 } 1637 1638 static int eeh_enable_dbgfs_get(void *data, u64 *val) 1639 { 1640 if (eeh_enabled()) 1641 *val = 0x1ul; 1642 else 1643 *val = 0x0ul; 1644 return 0; 1645 } 1646 1647 DEFINE_DEBUGFS_ATTRIBUTE(eeh_enable_dbgfs_ops, eeh_enable_dbgfs_get, 1648 eeh_enable_dbgfs_set, "0x%llx\n"); 1649 1650 static ssize_t eeh_force_recover_write(struct file *filp, 1651 const char __user *user_buf, 1652 size_t count, loff_t *ppos) 1653 { 1654 struct pci_controller *hose; 1655 uint32_t phbid, pe_no; 1656 struct eeh_pe *pe; 1657 char buf[20]; 1658 int ret; 1659 1660 ret = simple_write_to_buffer(buf, sizeof(buf), ppos, user_buf, count); 1661 if (!ret) 1662 return -EFAULT; 1663 1664 /* 1665 * When PE is NULL the event is a "special" event. Rather than 1666 * recovering a specific PE it forces the EEH core to scan for failed 1667 * PHBs and recovers each. This needs to be done before any device 1668 * recoveries can occur. 1669 */ 1670 if (!strncmp(buf, "hwcheck", 7)) { 1671 __eeh_send_failure_event(NULL); 1672 return count; 1673 } 1674 1675 ret = sscanf(buf, "%x:%x", &phbid, &pe_no); 1676 if (ret != 2) 1677 return -EINVAL; 1678 1679 hose = pci_find_controller_for_domain(phbid); 1680 if (!hose) 1681 return -ENODEV; 1682 1683 /* Retrieve PE */ 1684 pe = eeh_pe_get(hose, pe_no); 1685 if (!pe) 1686 return -ENODEV; 1687 1688 /* 1689 * We don't do any state checking here since the detection 1690 * process is async to the recovery process. The recovery 1691 * thread *should* not break even if we schedule a recovery 1692 * from an odd state (e.g. PE removed, or recovery of a 1693 * non-isolated PE) 1694 */ 1695 __eeh_send_failure_event(pe); 1696 1697 return ret < 0 ? ret : count; 1698 } 1699 1700 static const struct file_operations eeh_force_recover_fops = { 1701 .open = simple_open, 1702 .llseek = no_llseek, 1703 .write = eeh_force_recover_write, 1704 }; 1705 1706 static ssize_t eeh_debugfs_dev_usage(struct file *filp, 1707 char __user *user_buf, 1708 size_t count, loff_t *ppos) 1709 { 1710 static const char usage[] = "input format: <domain>:<bus>:<dev>.<fn>\n"; 1711 1712 return simple_read_from_buffer(user_buf, count, ppos, 1713 usage, sizeof(usage) - 1); 1714 } 1715 1716 static ssize_t eeh_dev_check_write(struct file *filp, 1717 const char __user *user_buf, 1718 size_t count, loff_t *ppos) 1719 { 1720 struct pci_dev *pdev; 1721 struct eeh_dev *edev; 1722 int ret; 1723 1724 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1725 if (IS_ERR(pdev)) 1726 return PTR_ERR(pdev); 1727 1728 edev = pci_dev_to_eeh_dev(pdev); 1729 if (!edev) { 1730 pci_err(pdev, "No eeh_dev for this device!\n"); 1731 pci_dev_put(pdev); 1732 return -ENODEV; 1733 } 1734 1735 ret = eeh_dev_check_failure(edev); 1736 pci_info(pdev, "eeh_dev_check_failure(%s) = %d\n", 1737 pci_name(pdev), ret); 1738 1739 pci_dev_put(pdev); 1740 1741 return count; 1742 } 1743 1744 static const struct file_operations eeh_dev_check_fops = { 1745 .open = simple_open, 1746 .llseek = no_llseek, 1747 .write = eeh_dev_check_write, 1748 .read = eeh_debugfs_dev_usage, 1749 }; 1750 1751 static int eeh_debugfs_break_device(struct pci_dev *pdev) 1752 { 1753 struct resource *bar = NULL; 1754 void __iomem *mapped; 1755 u16 old, bit; 1756 int i, pos; 1757 1758 /* Do we have an MMIO BAR to disable? */ 1759 for (i = 0; i <= PCI_STD_RESOURCE_END; i++) { 1760 struct resource *r = &pdev->resource[i]; 1761 1762 if (!r->flags || !r->start) 1763 continue; 1764 if (r->flags & IORESOURCE_IO) 1765 continue; 1766 if (r->flags & IORESOURCE_UNSET) 1767 continue; 1768 1769 bar = r; 1770 break; 1771 } 1772 1773 if (!bar) { 1774 pci_err(pdev, "Unable to find Memory BAR to cause EEH with\n"); 1775 return -ENXIO; 1776 } 1777 1778 pci_err(pdev, "Going to break: %pR\n", bar); 1779 1780 if (pdev->is_virtfn) { 1781 #ifndef CONFIG_PCI_IOV 1782 return -ENXIO; 1783 #else 1784 /* 1785 * VFs don't have a per-function COMMAND register, so the best 1786 * we can do is clear the Memory Space Enable bit in the PF's 1787 * SRIOV control reg. 1788 * 1789 * Unfortunately, this requires that we have a PF (i.e doesn't 1790 * work for a passed-through VF) and it has the potential side 1791 * effect of also causing an EEH on every other VF under the 1792 * PF. Oh well. 1793 */ 1794 pdev = pdev->physfn; 1795 if (!pdev) 1796 return -ENXIO; /* passed through VFs have no PF */ 1797 1798 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); 1799 pos += PCI_SRIOV_CTRL; 1800 bit = PCI_SRIOV_CTRL_MSE; 1801 #endif /* !CONFIG_PCI_IOV */ 1802 } else { 1803 bit = PCI_COMMAND_MEMORY; 1804 pos = PCI_COMMAND; 1805 } 1806 1807 /* 1808 * Process here is: 1809 * 1810 * 1. Disable Memory space. 1811 * 1812 * 2. Perform an MMIO to the device. This should result in an error 1813 * (CA / UR) being raised by the device which results in an EEH 1814 * PE freeze. Using the in_8() accessor skips the eeh detection hook 1815 * so the freeze hook so the EEH Detection machinery won't be 1816 * triggered here. This is to match the usual behaviour of EEH 1817 * where the HW will asyncronously freeze a PE and it's up to 1818 * the kernel to notice and deal with it. 1819 * 1820 * 3. Turn Memory space back on. This is more important for VFs 1821 * since recovery will probably fail if we don't. For normal 1822 * the COMMAND register is reset as a part of re-initialising 1823 * the device. 1824 * 1825 * Breaking stuff is the point so who cares if it's racy ;) 1826 */ 1827 pci_read_config_word(pdev, pos, &old); 1828 1829 mapped = ioremap(bar->start, PAGE_SIZE); 1830 if (!mapped) { 1831 pci_err(pdev, "Unable to map MMIO BAR %pR\n", bar); 1832 return -ENXIO; 1833 } 1834 1835 pci_write_config_word(pdev, pos, old & ~bit); 1836 in_8(mapped); 1837 pci_write_config_word(pdev, pos, old); 1838 1839 iounmap(mapped); 1840 1841 return 0; 1842 } 1843 1844 static ssize_t eeh_dev_break_write(struct file *filp, 1845 const char __user *user_buf, 1846 size_t count, loff_t *ppos) 1847 { 1848 struct pci_dev *pdev; 1849 int ret; 1850 1851 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1852 if (IS_ERR(pdev)) 1853 return PTR_ERR(pdev); 1854 1855 ret = eeh_debugfs_break_device(pdev); 1856 pci_dev_put(pdev); 1857 1858 if (ret < 0) 1859 return ret; 1860 1861 return count; 1862 } 1863 1864 static const struct file_operations eeh_dev_break_fops = { 1865 .open = simple_open, 1866 .llseek = no_llseek, 1867 .write = eeh_dev_break_write, 1868 .read = eeh_debugfs_dev_usage, 1869 }; 1870 1871 static ssize_t eeh_dev_can_recover(struct file *filp, 1872 const char __user *user_buf, 1873 size_t count, loff_t *ppos) 1874 { 1875 struct pci_driver *drv; 1876 struct pci_dev *pdev; 1877 size_t ret; 1878 1879 pdev = eeh_debug_lookup_pdev(filp, user_buf, count, ppos); 1880 if (IS_ERR(pdev)) 1881 return PTR_ERR(pdev); 1882 1883 /* 1884 * In order for error recovery to work the driver needs to implement 1885 * .error_detected(), so it can quiesce IO to the device, and 1886 * .slot_reset() so it can re-initialise the device after a reset. 1887 * 1888 * Ideally they'd implement .resume() too, but some drivers which 1889 * we need to support (notably IPR) don't so I guess we can tolerate 1890 * that. 1891 * 1892 * .mmio_enabled() is mostly there as a work-around for devices which 1893 * take forever to re-init after a hot reset. Implementing that is 1894 * strictly optional. 1895 */ 1896 drv = pci_dev_driver(pdev); 1897 if (drv && 1898 drv->err_handler && 1899 drv->err_handler->error_detected && 1900 drv->err_handler->slot_reset) { 1901 ret = count; 1902 } else { 1903 ret = -EOPNOTSUPP; 1904 } 1905 1906 pci_dev_put(pdev); 1907 1908 return ret; 1909 } 1910 1911 static const struct file_operations eeh_dev_can_recover_fops = { 1912 .open = simple_open, 1913 .llseek = no_llseek, 1914 .write = eeh_dev_can_recover, 1915 .read = eeh_debugfs_dev_usage, 1916 }; 1917 1918 #endif 1919 1920 static int __init eeh_init_proc(void) 1921 { 1922 if (machine_is(pseries) || machine_is(powernv)) { 1923 proc_create_single("powerpc/eeh", 0, NULL, proc_eeh_show); 1924 #ifdef CONFIG_DEBUG_FS 1925 debugfs_create_file_unsafe("eeh_enable", 0600, 1926 powerpc_debugfs_root, NULL, 1927 &eeh_enable_dbgfs_ops); 1928 debugfs_create_u32("eeh_max_freezes", 0600, 1929 powerpc_debugfs_root, &eeh_max_freezes); 1930 debugfs_create_bool("eeh_disable_recovery", 0600, 1931 powerpc_debugfs_root, 1932 &eeh_debugfs_no_recover); 1933 debugfs_create_file_unsafe("eeh_dev_check", 0600, 1934 powerpc_debugfs_root, NULL, 1935 &eeh_dev_check_fops); 1936 debugfs_create_file_unsafe("eeh_dev_break", 0600, 1937 powerpc_debugfs_root, NULL, 1938 &eeh_dev_break_fops); 1939 debugfs_create_file_unsafe("eeh_force_recover", 0600, 1940 powerpc_debugfs_root, NULL, 1941 &eeh_force_recover_fops); 1942 debugfs_create_file_unsafe("eeh_dev_can_recover", 0600, 1943 powerpc_debugfs_root, NULL, 1944 &eeh_dev_can_recover_fops); 1945 eeh_cache_debugfs_init(); 1946 #endif 1947 } 1948 1949 return 0; 1950 } 1951 __initcall(eeh_init_proc); 1952