1 // SPDX-License-Identifier: GPL-2.0-or-later
6  * Copyright 2001-2012 IBM Corporation.
32 #include <asm/ppc-pci.h>
33 #include <asm/rtas.h>
34 #include <asm/pte-walk.h>
40  *  usual PCI framework, except by check-stopping the CPU.  Systems
41  *  that are designed for high-availability/reliability cannot afford
43  *  An EEH-capable bridge operates by converting a detected error
44  *  into a "slot freeze", taking the PCI adapter off-line, making
47  *  ignored.  EEH slot isolation events can be triggered by parity
49  *  which in turn might be caused by low voltage on the bus, dust,
50  *  vibration, humidity, radioactivity or plain-old failed hardware.
54  *  or buggy device hardware.  This is because any attempt by the
55  *  device to bus-master data to a memory address that is not
57  *  is to prevent devices-gone-wild from corrupting system memory).
58  *  Buggy hardware/drivers will have a miserable time co-existing
62  *  event has occurred (e.g. by reading 0xff's), will then ask EEH
86  * from hardware. The flag is used to distinguish that.
119  * not dynamically alloced, so that it ends up in RMO where RTAS
126  * The struct is used to maintain the EEH global statistic
156 		pr_info("EEH: Recovery disabled by kernel parameter.\n");  in eeh_show_enabled()
166  * for RTAS error logging.
175 	n += scnprintf(buf+n, len-n, "%04x:%02x:%02x.%01x\n",  in eeh_dump_dev_log()
176 			edev->pe->phb->global_number, edev->bdfn >> 8,  in eeh_dump_dev_log()
177 			PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn));  in eeh_dump_dev_log()
179 		edev->pe->phb->global_number, edev->bdfn >> 8,  in eeh_dump_dev_log()
180 		PCI_SLOT(edev->bdfn), PCI_FUNC(edev->bdfn));  in eeh_dump_dev_log()
182 	eeh_ops->read_config(edev, PCI_VENDOR_ID, 4, &cfg);  in eeh_dump_dev_log()
183 	n += scnprintf(buf+n, len-n, "dev/vend:%08x\n", cfg);  in eeh_dump_dev_log()
186 	eeh_ops->read_config(edev, PCI_COMMAND, 4, &cfg);  in eeh_dump_dev_log()
187 	n += scnprintf(buf+n, len-n, "cmd/stat:%x\n", cfg);  in eeh_dump_dev_log()
190 	/* Gather bridge-specific registers */  in eeh_dump_dev_log()
191 	if (edev->mode & EEH_DEV_BRIDGE) {  in eeh_dump_dev_log()
192 		eeh_ops->read_config(edev, PCI_SEC_STATUS, 2, &cfg);  in eeh_dump_dev_log()
193 		n += scnprintf(buf+n, len-n, "sec stat:%x\n", cfg);  in eeh_dump_dev_log()
196 		eeh_ops->read_config(edev, PCI_BRIDGE_CONTROL, 2, &cfg);  in eeh_dump_dev_log()
197 		n += scnprintf(buf+n, len-n, "brdg ctl:%x\n", cfg);  in eeh_dump_dev_log()
201 	/* Dump out the PCI-X command and status regs */  in eeh_dump_dev_log()
202 	cap = edev->pcix_cap;  in eeh_dump_dev_log()
204 		eeh_ops->read_config(edev, cap, 4, &cfg);  in eeh_dump_dev_log()
205 		n += scnprintf(buf+n, len-n, "pcix-cmd:%x\n", cfg);  in eeh_dump_dev_log()
206 		pr_warn("EEH: PCI-X cmd: %08x\n", cfg);  in eeh_dump_dev_log()
208 		eeh_ops->read_config(edev, cap+4, 4, &cfg);  in eeh_dump_dev_log()
209 		n += scnprintf(buf+n, len-n, "pcix-stat:%x\n", cfg);  in eeh_dump_dev_log()
210 		pr_warn("EEH: PCI-X status: %08x\n", cfg);  in eeh_dump_dev_log()
213 	/* If PCI-E capable, dump PCI-E cap 10 */  in eeh_dump_dev_log()
214 	cap = edev->pcie_cap;  in eeh_dump_dev_log()
216 		n += scnprintf(buf+n, len-n, "pci-e cap10:\n");  in eeh_dump_dev_log()
217 		pr_warn("EEH: PCI-E capabilities and status follow:\n");  in eeh_dump_dev_log()
220 			eeh_ops->read_config(edev, cap+4*i, 4, &cfg);  in eeh_dump_dev_log()
221 			n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg);  in eeh_dump_dev_log()
228 					      "EEH: PCI-E %02x: %08x ",  in eeh_dump_dev_log()
231 				l += scnprintf(buffer+l, sizeof(buffer)-l,  in eeh_dump_dev_log()
241 	cap = edev->aer_cap;  in eeh_dump_dev_log()
243 		n += scnprintf(buf+n, len-n, "pci-e AER:\n");  in eeh_dump_dev_log()
244 		pr_warn("EEH: PCI-E AER capability register set follows:\n");  in eeh_dump_dev_log()
247 			eeh_ops->read_config(edev, cap+4*i, 4, &cfg);  in eeh_dump_dev_log()
248 			n += scnprintf(buf+n, len-n, "%02x:%x\n", 4*i, cfg);  in eeh_dump_dev_log()
255 					      "EEH: PCI-E AER %02x: %08x ",  in eeh_dump_dev_log()
258 				l += scnprintf(buffer+l, sizeof(buffer)-l,  in eeh_dump_dev_log()
276 					  EEH_PCI_REGS_LOG_LEN - *plen);  in eeh_dump_pe_log()
282  * eeh_slot_error_detail - Generate combined log including driver log and error log
311 	if (!(pe->type & EEH_PE_PHB)) {  in eeh_slot_error_detail()
328 		eeh_ops->configure_bridge(pe);  in eeh_slot_error_detail()
329 		if (!(pe->state & EEH_PE_CFG_BLOCKED)) {  in eeh_slot_error_detail()
337 	eeh_ops->get_log(pe, severity, pci_regs_buf, loglen);  in eeh_slot_error_detail()
341  * eeh_token_to_phys - Convert EEH address token to phys address
364 		return -EPERM;  in eeh_phb_check_failure()
367 	phb_pe = eeh_phb_pe_get(pe->phb);  in eeh_phb_check_failure()
370 			__func__, pe->phb->global_number);  in eeh_phb_check_failure()
371 		return -EEXIST;  in eeh_phb_check_failure()
376 	if (phb_pe->state & EEH_PE_ISOLATED) {  in eeh_phb_check_failure()
382 	ret = eeh_ops->get_state(phb_pe, NULL);  in eeh_phb_check_failure()
394 		phb_pe->phb->global_number, eeh_pe_loc_get(phb_pe));  in eeh_phb_check_failure()
405 		return dev_driver_string(&pdev->dev);  in eeh_driver_name()
411  * eeh_dev_check_failure - Check if all 1's data is due to EEH slot freeze
420  * a non-zero value and queues up a slot isolation event notification.
462 	 * If the PE isn't owned by us, we shouldn't check the  in eeh_dev_check_failure()
477 	if (pe->state & EEH_PE_ISOLATED) {  in eeh_dev_check_failure()
478 		pe->check_count++;  in eeh_dev_check_failure()
479 		if (pe->check_count == EEH_MAX_FAILS) {  in eeh_dev_check_failure()
482 				location = of_get_property(dn, "ibm,loc-code",  in eeh_dev_check_failure()
485 				pe->check_count,  in eeh_dev_check_failure()
499 	 * function zero of a multi-function device.  in eeh_dev_check_failure()
502 	ret = eeh_ops->get_state(pe, NULL);  in eeh_dev_check_failure()
504 	/* Note that config-io to empty slots may fail;  in eeh_dev_check_failure()
519 	     dev->error_state == pci_channel_io_perm_failure) ||  in eeh_dev_check_failure()
522 		pe->false_positives++;  in eeh_dev_check_failure()
532 	parent_pe = pe->parent;  in eeh_dev_check_failure()
535 		if (parent_pe->type & EEH_PE_PHB)  in eeh_dev_check_failure()
539 		ret = eeh_ops->get_state(parent_pe, NULL);  in eeh_dev_check_failure()
542 			pr_err("EEH: Failure of PHB#%x-PE#%x will be handled at parent PHB#%x-PE#%x.\n",  in eeh_dev_check_failure()
543 			       pe->phb->global_number, pe->addr,  in eeh_dev_check_failure()
544 			       pe->phb->global_number, parent_pe->addr);  in eeh_dev_check_failure()
548 		parent_pe = parent_pe->parent;  in eeh_dev_check_failure()
561 	 * a stack trace will help the device-driver authors figure  in eeh_dev_check_failure()
564 	pr_debug("EEH: %s: Frozen PHB#%x-PE#%x detected\n",  in eeh_dev_check_failure()
565 		__func__, pe->phb->global_number, pe->addr);  in eeh_dev_check_failure()
578  * eeh_check_failure - Check if all 1's data is due to EEH slot freeze
607  * eeh_pci_enable - Enable MMIO or DMA transfers for this slot
639 		return -EINVAL;  in eeh_pci_enable()
647 		rc = eeh_ops->get_state(pe, NULL);  in eeh_pci_enable()
662 	rc = eeh_ops->set_option(pe, function);  in eeh_pci_enable()
665 			"PHB#%x-PE#%x, err=%d\n",  in eeh_pci_enable()
666 			__func__, function, pe->phb->global_number,  in eeh_pci_enable()
667 			pe->addr, rc);  in eeh_pci_enable()
678 		return -EIO;  in eeh_pci_enable()
719 	if (eeh_ops->restore_config)  in eeh_restore_dev_state()
720 		eeh_ops->restore_config(edev);  in eeh_restore_dev_state()
728  * pcibios_set_pcie_reset_state - Set PCI-E reset state
743 		return -EINVAL;  in pcibios_set_pcie_reset_state()
748 		eeh_ops->reset(pe, EEH_RESET_DEACTIVATE);  in pcibios_set_pcie_reset_state()
750 		if (!(pe->type & EEH_PE_VF))  in pcibios_set_pcie_reset_state()
758 		eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);  in pcibios_set_pcie_reset_state()
760 		if (!(pe->type & EEH_PE_VF))  in pcibios_set_pcie_reset_state()
762 		eeh_ops->reset(pe, EEH_RESET_HOT);  in pcibios_set_pcie_reset_state()
767 		eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);  in pcibios_set_pcie_reset_state()
769 		if (!(pe->type & EEH_PE_VF))  in pcibios_set_pcie_reset_state()
771 		eeh_ops->reset(pe, EEH_RESET_FUNDAMENTAL);  in pcibios_set_pcie_reset_state()
775 		return -EINVAL;  in pcibios_set_pcie_reset_state()
782  * eeh_set_dev_freset - Check the required reset for the indicated device
787  * hot reset. The routine is used to collected the information for
798 		*freset |= dev->needs_freset;  in eeh_set_dev_freset()
808 			state = eeh_ops->get_state(pe, NULL);  in eeh_pe_refreeze_passed()
811 				pr_info("EEH: Passed-through PE PHB#%x-PE#%x was thawed by reset, re-freezing for safety.\n",  in eeh_pe_refreeze_passed()
812 					pe->phb->global_number, pe->addr);  in eeh_pe_refreeze_passed()
820  * eeh_pe_reset_full - Complete a full reset process on the indicated PE
822  * @include_passed: include passed-through devices?
826  * deactivating the reset status.  It is designed to be used within the EEH
840 	 * Determine the type of reset to perform - hot or fundamental.  in eeh_pe_reset_full()
859 			ret = -EIO;  in eeh_pe_reset_full()
860 			pr_warn("EEH: Failure %d resetting PHB#%x-PE#%x (attempt %d)\n\n",  in eeh_pe_reset_full()
861 				state, pe->phb->global_number, pe->addr, i + 1);  in eeh_pe_reset_full()
865 			pr_warn("EEH: PHB#%x-PE#%x: Successful reset (attempt %d)\n",  in eeh_pe_reset_full()
866 				pe->phb->global_number, pe->addr, i + 1);  in eeh_pe_reset_full()
871 			pr_warn("EEH: Unrecoverable slot failure on PHB#%x-PE#%x",  in eeh_pe_reset_full()
872 				pe->phb->global_number, pe->addr);  in eeh_pe_reset_full()
873 			ret = -ENOTRECOVERABLE;  in eeh_pe_reset_full()
879 			pr_warn("EEH: PHB#%x-PE#%x: Slot inactive after reset: 0x%x (attempt %d)\n",  in eeh_pe_reset_full()
880 				pe->phb->global_number, pe->addr, state, i + 1);  in eeh_pe_reset_full()
884 	 * (potentially) passed through to a guest, re-freeze them:  in eeh_pe_reset_full()
894  * eeh_save_bars - Save device bars
910 		eeh_ops->read_config(edev, i * 4, 4, &edev->config_space[i]);  in eeh_save_bars()
918 	if (edev->mode & EEH_DEV_BRIDGE)  in eeh_save_bars()
919 		edev->config_space[1] |= PCI_COMMAND_MASTER;  in eeh_save_bars()
958  * eeh_init - System wide EEH initialization
970 		return -EEXIST;  in eeh_init()
972 		return -ENOENT;  in eeh_init()
1001  * eeh_probe_device() - Perform EEH initialization for the indicated pci device
1004  * This routine must be used to complete EEH initialization for PCI
1022 	edev = eeh_ops->probe(dev);  in eeh_probe_device()
1037 	if (edev->pdev && edev->pdev != dev) {  in eeh_probe_device()
1039 		eeh_addr_cache_rmv_dev(edev->pdev);  in eeh_probe_device()
1040 		eeh_sysfs_remove_device(edev->pdev);  in eeh_probe_device()
1047 		edev->mode |= EEH_DEV_NO_HANDLER;  in eeh_probe_device()
1051 	edev->pdev = dev;  in eeh_probe_device()
1052 	dev->dev.archdata.edev = edev;  in eeh_probe_device()
1058  * eeh_remove_device - Undo EEH setup for the indicated pci device
1062  * a running system (e.g. by hotplug or dlpar).  It unregisters
1076 	dev_dbg(&dev->dev, "EEH: Removing device\n");  in eeh_remove_device()
1078 	if (!edev || !edev->pdev || !edev->pe) {  in eeh_remove_device()
1079 		dev_dbg(&dev->dev, "EEH: Device not referenced!\n");  in eeh_remove_device()
1089 	edev->pdev = NULL;  in eeh_remove_device()
1095 	if (edev->mode & EEH_DEV_SYSFS)  in eeh_remove_device()
1104 	edev->mode |= EEH_DEV_NO_HANDLER;  in eeh_remove_device()
1109 	 * The flag "in_error" is used to trace EEH devices for VFs  in eeh_remove_device()
1114 	edev->in_error = false;  in eeh_remove_device()
1115 	dev->dev.archdata.edev = NULL;  in eeh_remove_device()
1116 	if (!(edev->pe->state & EEH_PE_KEEP))  in eeh_remove_device()
1119 		edev->mode |= EEH_DEV_DISCONNECTED;  in eeh_remove_device()
1128 		pr_warn("%s: Failure %d enabling IO on PHB#%x-PE#%x\n",  in eeh_unfreeze_pe()
1129 			__func__, ret, pe->phb->global_number, pe->addr);  in eeh_unfreeze_pe()
1135 		pr_warn("%s: Failure %d enabling DMA on PHB#%x-PE#%x\n",  in eeh_unfreeze_pe()
1136 			__func__, ret, pe->phb->global_number, pe->addr);  in eeh_unfreeze_pe()
1159 	ret = eeh_ops->get_state(pe, NULL);  in eeh_pe_change_owner()
1173 		for (id = &eeh_reset_ids[0]; id->vendor != 0; id++) {  in eeh_pe_change_owner()
1174 			if (id->vendor != PCI_ANY_ID &&  in eeh_pe_change_owner()
1175 			    id->vendor != pdev->vendor)  in eeh_pe_change_owner()
1177 			if (id->device != PCI_ANY_ID &&  in eeh_pe_change_owner()
1178 			    id->device != pdev->device)  in eeh_pe_change_owner()
1180 			if (id->subvendor != PCI_ANY_ID &&  in eeh_pe_change_owner()
1181 			    id->subvendor != pdev->subsystem_vendor)  in eeh_pe_change_owner()
1183 			if (id->subdevice != PCI_ANY_ID &&  in eeh_pe_change_owner()
1184 			    id->subdevice != pdev->subsystem_device)  in eeh_pe_change_owner()
1198  * eeh_dev_open - Increase count of pass through devices for PE
1209 	int ret = -ENODEV;  in eeh_dev_open()
1219 	if (!edev || !edev->pe)  in eeh_dev_open()
1228 	ret = eeh_pe_change_owner(edev->pe);  in eeh_dev_open()
1233 	atomic_inc(&edev->pe->pass_dev_cnt);  in eeh_dev_open()
1244  * eeh_dev_release - Decrease count of pass through devices for PE
1263 	if (!edev || !edev->pe || !eeh_pe_passed(edev->pe))  in eeh_dev_release()
1267 	WARN_ON(atomic_dec_if_positive(&edev->pe->pass_dev_cnt) < 0);  in eeh_dev_release()
1268 	eeh_pe_change_owner(edev->pe);  in eeh_dev_release()
1277  * eeh_iommu_group_to_pe - Convert IOMMU group to EEH PE
1298 	if (!edev || !edev->pe)  in eeh_iommu_group_to_pe()
1301 	return edev->pe;  in eeh_iommu_group_to_pe()
1308  * eeh_pe_set_option - Set options for the indicated PE
1321 		return -ENODEV;  in eeh_pe_set_option()
1334 		ret = -EIO;  in eeh_pe_set_option()
1341 		if (!eeh_ops || !eeh_ops->set_option) {  in eeh_pe_set_option()
1342 			ret = -ENOENT;  in eeh_pe_set_option()
1351 		ret = -EINVAL;  in eeh_pe_set_option()
1359  * eeh_pe_get_state - Retrieve PE's state
1372 		return -ENODEV;  in eeh_pe_get_state()
1374 	if (!eeh_ops || !eeh_ops->get_state)  in eeh_pe_get_state()
1375 		return -ENOENT;  in eeh_pe_get_state()
1378 	 * If the parent PE is owned by the host kernel and is undergoing  in eeh_pe_get_state()
1383 	if (pe->parent &&  in eeh_pe_get_state()
1384 	    !(pe->state & EEH_PE_REMOVED) &&  in eeh_pe_get_state()
1385 	    (pe->parent->state & (EEH_PE_ISOLATED | EEH_PE_RECOVERING)))  in eeh_pe_get_state()
1388 	result = eeh_ops->get_state(pe, NULL);  in eeh_pe_get_state()
1437 		pr_info("EEH: Note: Leaving passthrough PHB#%x-PE#%x frozen.\n",  in eeh_pe_reenable_devices()
1438 			pe->phb->global_number, pe->addr);  in eeh_pe_reenable_devices()
1446  * eeh_pe_reset - Issue PE reset according to specified type
1449  * @include_passed: include passed-through devices?
1461 		return -ENODEV;  in eeh_pe_reset()
1463 	if (!eeh_ops || !eeh_ops->set_option || !eeh_ops->reset)  in eeh_pe_reset()
1464 		return -ENOENT;  in eeh_pe_reset()
1468 		ret = eeh_ops->reset(pe, option);  in eeh_pe_reset()
1482 		eeh_ops->set_option(pe, EEH_OPT_FREEZE_PE);  in eeh_pe_reset()
1485 		ret = eeh_ops->reset(pe, option);  in eeh_pe_reset()
1490 		ret = -EINVAL;  in eeh_pe_reset()
1498  * eeh_pe_configure - Configure PCI bridges after PE reset
1502  * those PCI devices, especially PCI bridges affected by PE
1511 		return -ENODEV;  in eeh_pe_configure()
1518  * eeh_pe_inject_err - Injecting the specified PCI error to the indicated PE
1526  * is determined by @type and @func, to the indicated PE for
1534 		return -ENODEV;  in eeh_pe_inject_err()
1537 	if (!eeh_ops || !eeh_ops->err_inject)  in eeh_pe_inject_err()
1538 		return -ENOENT;  in eeh_pe_inject_err()
1542 		return -EINVAL;  in eeh_pe_inject_err()
1544 	return eeh_ops->err_inject(pe, type, func, addr, mask);  in eeh_pe_inject_err()
1586 		struct resource *r = &pdev->resource[i];  in eeh_break_device()
1588 		if (!r->flags || !r->start)  in eeh_break_device()
1590 		if (r->flags & IORESOURCE_IO)  in eeh_break_device()
1592 		if (r->flags & IORESOURCE_UNSET)  in eeh_break_device()
1601 		return -ENXIO;  in eeh_break_device()
1606 	if (pdev->is_virtfn) {  in eeh_break_device()
1608 		return -ENXIO;  in eeh_break_device()
1611 		 * VFs don't have a per-function COMMAND register, so the best  in eeh_break_device()
1616 		 * work for a passed-through VF) and it has the potential side  in eeh_break_device()
1620 		pdev = pdev->physfn;  in eeh_break_device()
1622 			return -ENXIO; /* passed through VFs have no PF */  in eeh_break_device()
1639 	 *    (CA  / UR) being raised by the device which results in an EEH  in eeh_break_device()
1648 	 *    the COMMAND register is reset as a part of re-initialising  in eeh_break_device()
1655 	mapped = ioremap(bar->start, PAGE_SIZE);  in eeh_break_device()
1658 		return -ENXIO;  in eeh_break_device()
1688 	ret = simple_write_to_buffer(buf, sizeof(buf)-1, ppos, user_buf, count);  in eeh_debug_lookup_pdev()
1690 		return ERR_PTR(-EFAULT);  in eeh_debug_lookup_pdev()
1695 		return ERR_PTR(-EINVAL);  in eeh_debug_lookup_pdev()
1700 		return ERR_PTR(-ENODEV);  in eeh_debug_lookup_pdev()
1739 		return -EFAULT;  in eeh_force_recover_write()
1754 		return -EINVAL;  in eeh_force_recover_write()
1758 		return -ENODEV;  in eeh_force_recover_write()
1763 		return -ENODEV;  in eeh_force_recover_write()
1770 	 * non-isolated PE)  in eeh_force_recover_write()
1789 				       usage, sizeof(usage) - 1);  in eeh_debugfs_dev_usage()
1808 		return -ENODEV;  in eeh_dev_check_write()
1867 	 * .slot_reset() so it can re-initialise the device after a reset.  in eeh_dev_can_recover()
1873 	 * .mmio_enabled() is mostly there as a work-around for devices which  in eeh_dev_can_recover()
1874 	 * take forever to re-init after a hot reset. Implementing that is  in eeh_dev_can_recover()
1879 	    drv->err_handler &&  in eeh_dev_can_recover()
1880 	    drv->err_handler->error_detected &&  in eeh_dev_can_recover()
1881 	    drv->err_handler->slot_reset) {  in eeh_dev_can_recover()
1884 		ret = -EOPNOTSUPP;  in eeh_dev_can_recover()