1 // SPDX-License-Identifier: GPL-2.0-or-later
3  * PCI Error Recovery Driver for RPA-compliant PPC64 platform.
17 #include <asm/ppc-pci.h>
18 #include <asm/pci-bridge.h>
55 		return "need reset";  in pci_ers_result_name()
78 	return !edev || (edev->mode & EEH_DEV_REMOVED);  in eeh_dev_removed()
83 	if (!edev->pdev)  in eeh_edev_actionable()
85 	if (edev->pdev->error_state == pci_channel_io_perm_failure)  in eeh_edev_actionable()
89 	if (eeh_pe_passed(edev->pe))  in eeh_edev_actionable()
96  * eeh_pcid_get - Get the PCI device driver
101  * of the PCI device driver to prevent that being unloaded on
106 	if (!pdev || !pdev->dev.driver)  in eeh_pcid_get()
109 	if (!try_module_get(pdev->dev.driver->owner))  in eeh_pcid_get()
112 	return to_pci_driver(pdev->dev.driver);  in eeh_pcid_get()
116  * eeh_pcid_put - Dereference on the PCI device driver
119  * The function is called to do dereference on the PCI device
124 	if (!pdev || !pdev->dev.driver)  in eeh_pcid_put()
127 	module_put(pdev->dev.driver->owner);  in eeh_pcid_put()
131  * eeh_disable_irq - Disable interrupt for the recovering device
136  * device. If the device has enabled MSI or MSI-X interrupt, we needn't
138  * devices encountering EEH errors, which includes MSI or MSI-X.
142 	/* Don't disable MSI and MSI-X interrupts. They are  in eeh_disable_irq()
146 	if (edev->pdev->msi_enabled || edev->pdev->msix_enabled)  in eeh_disable_irq()
149 	if (!irq_has_action(edev->pdev->irq))  in eeh_disable_irq()
152 	edev->mode |= EEH_DEV_IRQ_DISABLED;  in eeh_disable_irq()
153 	disable_irq_nosync(edev->pdev->irq);  in eeh_disable_irq()
157  * eeh_enable_irq - Enable interrupt for the recovering device
165 	if ((edev->mode) & EEH_DEV_IRQ_DISABLED) {  in eeh_enable_irq()
166 		edev->mode &= ~EEH_DEV_IRQ_DISABLED;  in eeh_enable_irq()
187 		if (irqd_irq_disabled(irq_get_irq_data(edev->pdev->irq)))  in eeh_enable_irq()
188 			enable_irq(edev->pdev->irq);  in eeh_enable_irq()
200 	 * We cannot access the config space on some adapters.  in eeh_dev_save_state()
206 	if (edev->pe && (edev->pe->state & EEH_PE_CFG_RESTRICTED))  in eeh_dev_save_state()
224 				edev->pdev->error_state = s;  in eeh_set_channel_state()
237 			if (!eeh_pcid_get(edev->pdev))  in eeh_set_irq_state()
245 			eeh_pcid_put(edev->pdev);  in eeh_set_irq_state()
260 	pdev = edev->pdev;  in eeh_pe_report_edev()
262 		get_device(&pdev->dev);  in eeh_pe_report_edev()
268 	device_lock(&pdev->dev);  in eeh_pe_report_edev()
274 		else if (!driver->err_handler)  in eeh_pe_report_edev()
276 		else if (edev->mode & EEH_DEV_NO_HANDLER)  in eeh_pe_report_edev()
281 				      driver->name,  in eeh_pe_report_edev()
291 			      !eeh_dev_removed(edev), !eeh_pe_passed(edev->pe));  in eeh_pe_report_edev()
293 	device_unlock(&pdev->dev);  in eeh_pe_report_edev()
294 	if (edev->pdev != pdev)  in eeh_pe_report_edev()
296 	put_device(&pdev->dev);  in eeh_pe_report_edev()
317  * eeh_report_error - Report pci error to each device driver
329 	if (!driver->err_handler->error_detected)  in eeh_report_error()
332 	eeh_edev_info(edev, "Invoking %s->error_detected(IO frozen)",  in eeh_report_error()
333 		      driver->name);  in eeh_report_error()
334 	rc = driver->err_handler->error_detected(pdev, pci_channel_io_frozen);  in eeh_report_error()
336 	edev->in_error = true;  in eeh_report_error()
342  * eeh_report_mmio_enabled - Tell drivers that MMIO has been enabled
353 	if (!driver->err_handler->mmio_enabled)  in eeh_report_mmio_enabled()
355 	eeh_edev_info(edev, "Invoking %s->mmio_enabled()", driver->name);  in eeh_report_mmio_enabled()
356 	return driver->err_handler->mmio_enabled(pdev);  in eeh_report_mmio_enabled()
360  * eeh_report_reset - Tell device that slot has been reset
364  * This routine must be called while EEH tries to reset particular
366  * some actions, usually to save data the driver needs so that the
373 	if (!driver->err_handler->slot_reset || !edev->in_error)  in eeh_report_reset()
375 	eeh_edev_info(edev, "Invoking %s->slot_reset()", driver->name);  in eeh_report_reset()
376 	return driver->err_handler->slot_reset(pdev);  in eeh_report_reset()
390 	 * the blocked config space on some adapters. We have  in eeh_dev_restore_state()
394 	if (edev->pe && (edev->pe->state & EEH_PE_CFG_RESTRICTED)) {  in eeh_dev_restore_state()
395 		if (list_is_last(&edev->entry, &edev->pe->edevs))  in eeh_dev_restore_state()
396 			eeh_pe_restore_bars(edev->pe);  in eeh_dev_restore_state()
414  * eeh_report_resume - Tell device to resume normal operations
419  * could resume so that the device driver can do some initialization
426 	if (!driver->err_handler->resume || !edev->in_error)  in eeh_report_resume()
429 	eeh_edev_info(edev, "Invoking %s->resume()", driver->name);  in eeh_report_resume()
430 	driver->err_handler->resume(pdev);  in eeh_report_resume()
432 	pci_uevent_ers(edev->pdev, PCI_ERS_RESULT_RECOVERED);  in eeh_report_resume()
434 	if (eeh_ops->notify_resume)  in eeh_report_resume()
435 		eeh_ops->notify_resume(edev);  in eeh_report_resume()
441  * eeh_report_failure - Tell device driver that device is dead.
446  * dead, and that no further recovery attempts will be made on it.
454 	if (!driver->err_handler->error_detected)  in eeh_report_failure()
457 	eeh_edev_info(edev, "Invoking %s->error_detected(permanent failure)",  in eeh_report_failure()
458 		      driver->name);  in eeh_report_failure()
459 	rc = driver->err_handler->error_detected(pdev,  in eeh_report_failure()
471 	if (!(edev->physfn)) {  in eeh_add_virt_device()
478 		if (driver->err_handler) {  in eeh_add_virt_device()
486 	pci_iov_add_virtfn(edev->physfn, edev->vf_index);  in eeh_add_virt_device()
505 	    (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE))  in eeh_rmv_device()
511 			if (driver->err_handler &&  in eeh_rmv_device()
512 			    driver->err_handler->error_detected &&  in eeh_rmv_device()
513 			    driver->err_handler->slot_reset) {  in eeh_rmv_device()
524 	edev->mode |= EEH_DEV_DISCONNECTED;  in eeh_rmv_device()
526 		rmv_data->removed_dev_count++;  in eeh_rmv_device()
528 	if (edev->physfn) {  in eeh_rmv_device()
530 		pci_iov_remove_virtfn(edev->physfn, edev->vf_index);  in eeh_rmv_device()
531 		edev->pdev = NULL;  in eeh_rmv_device()
534 			list_add(&edev->rmv_entry, &rmv_data->removed_vf_list);  in eeh_rmv_device()
547 		if (!(edev->mode & EEH_DEV_DISCONNECTED))  in eeh_pe_detach_dev()
550 		edev->mode &= ~(EEH_DEV_DISCONNECTED | EEH_DEV_IRQ_DISABLED);  in eeh_pe_detach_dev()
561  * PE reset (for 3 times), we try to clear the frozen state
575 				return -EIO;  in eeh_clear_pe_frozen_state()
587 	if (pe->state & EEH_PE_RECOVERING)  in eeh_pe_reset_and_recover()
596 	/* Issue reset */  in eeh_pe_reset_and_recover()
620  * eeh_reset_device - Perform actual reset of a pci slot
626  * This routine must be called to do reset on the indicated PE.
627  * During the reset, udev might be invoked because those affected
644 	cnt = pe->freeze_count;  in eeh_reset_device()
645 	tstamp = pe->tstamp;  in eeh_reset_device()
654 	if (any_passed || driver_eeh_aware || (pe->type & EEH_PE_VF)) {  in eeh_reset_device()
661 	 * Reset the pci controller. (Asserts RST#; resets config space).  in eeh_reset_device()
663 	 * up if the reset failed for some reason.  in eeh_reset_device()
665 	 * During the reset, it's very dangerous to have uncontrolled PCI  in eeh_reset_device()
674 	eeh_ops->configure_bridge(pe);  in eeh_reset_device()
683 	/* Give the system 5 seconds to finish running the user-space  in eeh_reset_device()
689 	if (!driver_eeh_aware || rmv_data->removed_dev_count) {  in eeh_reset_device()
699 		edev = list_first_entry(&pe->edevs, struct eeh_dev, entry);  in eeh_reset_device()
701 		if (pe->type & EEH_PE_VF) {  in eeh_reset_device()
711 	pe->tstamp = tstamp;  in eeh_reset_device()
712 	pe->freeze_count = cnt;  in eeh_reset_device()
718  * to come back on line, in seconds.
725  * NB: This needs to be recursive to ensure the leaf PEs get removed
734 	list_for_each_entry_safe(child_pe, tmp, &pe->child_list, child)  in eeh_pe_cleanup()
737 	if (pe->state & EEH_PE_KEEP)  in eeh_pe_cleanup()
740 	if (!(pe->state & EEH_PE_INVALID))  in eeh_pe_cleanup()
743 	if (list_empty(&pe->edevs) && list_empty(&pe->child_list)) {  in eeh_pe_cleanup()
744 		list_del(&pe->child);  in eeh_pe_cleanup()
750  * eeh_check_slot_presence - Check if a device is still present in a slot
771 	if (pdev->error_state == pci_channel_io_perm_failure)  in eeh_slot_presence_check()
774 	slot = pdev->slot;  in eeh_slot_presence_check()
775 	if (!slot || !slot->hotplug)  in eeh_slot_presence_check()
778 	ops = slot->hotplug->ops;  in eeh_slot_presence_check()
779 	if (!ops || !ops->get_adapter_status)  in eeh_slot_presence_check()
783 	if (ops->set_attention_status)  in eeh_slot_presence_check()
784 		ops->set_attention_status(slot->hotplug, 1);  in eeh_slot_presence_check()
786 	rc = ops->get_adapter_status(slot->hotplug, &state);  in eeh_slot_presence_check()
801 	if (pdev->error_state == pci_channel_io_perm_failure)  in eeh_clear_slot_attention()
804 	slot = pdev->slot;  in eeh_clear_slot_attention()
805 	if (!slot || !slot->hotplug)  in eeh_clear_slot_attention()
808 	ops = slot->hotplug->ops;  in eeh_clear_slot_attention()
809 	if (!ops || !ops->set_attention_status)  in eeh_clear_slot_attention()
812 	ops->set_attention_status(slot->hotplug, 0);  in eeh_clear_slot_attention()
816  * eeh_handle_normal_event - Handle EEH events on a specific PE
817  * @pe: EEH PE - which should not be used after we return, as it may
823  * While PHB detects address or data parity errors on particular PCI
827  * #PERR or other misc PCI-related errors also can trigger EEH errors.
851 		pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n",  in eeh_handle_normal_event()
852 			__func__, pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
858 	 * When devices are hot-removed we might get an EEH due to  in eeh_handle_normal_event()
868 			if (eeh_slot_presence_check(edev->pdev))  in eeh_handle_normal_event()
872 		pr_warn("EEH: Frozen PHB#%x-PE#%x is empty!\n",  in eeh_handle_normal_event()
873 			pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
875 		 * The device is removed, tear down its state, on powernv  in eeh_handle_normal_event()
876 		 * hotplug driver would take care of it but not on pseries,  in eeh_handle_normal_event()
887 	if (pe->type & EEH_PE_PHB) {  in eeh_handle_normal_event()
889 			pe->phb->global_number, eeh_pe_loc_get(pe));  in eeh_handle_normal_event()
891 		struct eeh_pe *phb_pe = eeh_phb_pe_get(pe->phb);  in eeh_handle_normal_event()
893 		pr_err("EEH: Recovering PHB#%x-PE#%x\n",  in eeh_handle_normal_event()
894 		       pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
904 	if (pe->trace_entries) {  in eeh_handle_normal_event()
905 		void **ptrs = (void **) pe->stack_trace;  in eeh_handle_normal_event()
908 		pr_err("EEH: Frozen PHB#%x-PE#%x detected\n",  in eeh_handle_normal_event()
909 		       pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
913 		for (i = 0; i < pe->trace_entries; i++)  in eeh_handle_normal_event()
916 		pe->trace_entries = 0;  in eeh_handle_normal_event()
922 			edev->mode &= ~EEH_DEV_NO_HANDLER;  in eeh_handle_normal_event()
925 	pe->freeze_count++;  in eeh_handle_normal_event()
926 	if (pe->freeze_count > eeh_max_freezes) {  in eeh_handle_normal_event()
927 …pr_err("EEH: PHB#%x-PE#%x has failed %d times in the last hour and has been permanently disabled.\…  in eeh_handle_normal_event()
928 		       pe->phb->global_number, pe->addr,  in eeh_handle_normal_event()
929 		       pe->freeze_count);  in eeh_handle_normal_event()
935 	 * a reset sequence, giving each an opportunity to do what it needs  in eeh_handle_normal_event()
936 	 * to accomplish the reset.  Each child gets a report of the  in eeh_handle_normal_event()
937 	 * status ... if any child can't handle the reset, then the entire  in eeh_handle_normal_event()
940 	 * When the PHB is fenced, we have to issue a reset to recover from  in eeh_handle_normal_event()
945 		pe->freeze_count, eeh_max_freezes);  in eeh_handle_normal_event()
955 	 * Error logged on a PHB are always fences which need a full  in eeh_handle_normal_event()
956 	 * PHB reset to clear so force that to happen.  in eeh_handle_normal_event()
958 	if ((pe->type & EEH_PE_PHB) && result != PCI_ERS_RESULT_NONE)  in eeh_handle_normal_event()
977 	/* If all device drivers were EEH-unaware, then shut  in eeh_handle_normal_event()
982 		pr_info("EEH: Reset with hotplug activity\n");  in eeh_handle_normal_event()
985 			pr_warn("%s: Unable to reset, err=%d\n", __func__, rc);  in eeh_handle_normal_event()
990 	/* If all devices reported they can proceed, then re-enable MMIO */  in eeh_handle_normal_event()
1000 			pr_info("EEH: Notify device drivers to resume I/O\n");  in eeh_handle_normal_event()
1015 			 * We didn't do PE reset for the case. The PE  in eeh_handle_normal_event()
1024 	/* If any device called out for a reset, then reset the slot */  in eeh_handle_normal_event()
1026 		pr_info("EEH: Reset without hotplug activity\n");  in eeh_handle_normal_event()
1029 			pr_warn("%s: Cannot reset, err=%d\n", __func__, rc);  in eeh_handle_normal_event()
1049 			list_del(&edev->rmv_entry);  in eeh_handle_normal_event()
1052 		/* Tell all device drivers that they can resume operations */  in eeh_handle_normal_event()
1053 		pr_info("EEH: Notify device driver to resume\n");  in eeh_handle_normal_event()
1056 		eeh_pe_report("resume", pe, eeh_report_resume, NULL);  in eeh_handle_normal_event()
1059 				edev->mode &= ~EEH_DEV_NO_HANDLER;  in eeh_handle_normal_event()
1060 				edev->in_error = false;  in eeh_handle_normal_event()
1070 	 * About 90% of all real-life EEH failures in the field  in eeh_handle_normal_event()
1074 	pr_err("EEH: Unable to recover from failure from PHB#%x-PE#%x.\n"  in eeh_handle_normal_event()
1076 		pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
1094 	if (pe->type & EEH_PE_VF) {  in eeh_handle_normal_event()
1105 			pr_err("%s: PCI bus for PHB#%x-PE#%x disappeared\n",  in eeh_handle_normal_event()
1106 				__func__, pe->phb->global_number, pe->addr);  in eeh_handle_normal_event()
1123 			eeh_clear_slot_attention(edev->pdev);  in eeh_handle_normal_event()
1131  * eeh_handle_special_event - Handle EEH events without a specific failing PE
1149 		rc = eeh_ops->next_error(&pe);  in eeh_handle_special_event()
1209 					edev->mode &= ~EEH_DEV_NO_HANDLER;  in eeh_handle_special_event()
1221 				    !(phb_pe->state & EEH_PE_ISOLATED) ||  in eeh_handle_special_event()
1222 				    (phb_pe->state & EEH_PE_RECOVERING))  in eeh_handle_special_event()
1228 					       "PHB#%x-PE#%x\n",  in eeh_handle_special_event()
1230 					       pe->phb->global_number,  in eeh_handle_special_event()
1231 					       pe->addr);  in eeh_handle_special_event()