1 // SPDX-License-Identifier: GPL-2.0
3  * Implement the AER root port service driver. The driver registers an IRQ
4  * handler. When a root port triggers an AER interrupt, the IRQ handler
5  * collects Root Port status and schedules work.
11  * (C) Copyright 2009 Hewlett-Packard Development Company, L.P.
22 #include <linux/pci-acpi.h>
41 	dev_printk(level, &(pdev)->dev, fmt, ##arg)
54 	struct pci_dev *rpd;		/* Root Port device */
65 	 * at its link partner (e.g. Root Port) because the errors will be
84 	 * Fields for Root Ports & Root Complex Event Collectors only; these
86 	 * messages received by the Root Port / Event Collector, INCLUDING the
87 	 * ones that are generated internally (by the Root Port itself)
146 #define ECRC_POLICY_ON      2		/* ECRC on for data integrity */
157  * enable_ecrc_checking - enable PCIe ECRC checking for a device
164 	int aer = dev->aer_cap;  in enable_ecrc_checking()
168 		return -ENODEV;  in enable_ecrc_checking()
181  * disable_ecrc_checking - disable PCIe ECRC checking for a device
188 	int aer = dev->aer_cap;  in disable_ecrc_checking()
192 		return -ENODEV;  in disable_ecrc_checking()
202  * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based
226  * pcie_ecrc_get_policy - parse kernel command-line ecrc option
246 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);  in pcie_aer_is_native()
248 	if (!dev->aer_cap)  in pcie_aer_is_native()
251 	return pcie_ports_native || host->native_aer;  in pcie_aer_is_native()
260 		return -EIO;  in pci_enable_pcie_error_reporting()
268 	int aer = dev->aer_cap;  in pci_aer_clear_nonfatal_status()
272 		return -EIO;  in pci_aer_clear_nonfatal_status()
287 	int aer = dev->aer_cap;  in pci_aer_clear_fatal_status()
302  * pci_aer_raw_clear_status - Clear AER error registers.
312 	int aer = dev->aer_cap;  in pci_aer_raw_clear_status()
317 		return -EIO;  in pci_aer_raw_clear_status()
338 		return -EIO;  in pci_aer_clear_status()
345 	int aer = dev->aer_cap;  in pci_save_aer_state()
356 	cap = &save_state->cap.data[0];  in pci_save_aer_state()
367 	int aer = dev->aer_cap;  in pci_restore_aer_state()
378 	cap = &save_state->cap.data[0];  in pci_restore_aer_state()
391 	dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);  in pci_aer_init()
392 	if (!dev->aer_cap)  in pci_aer_init()
395 	dev->aer_info = kzalloc(sizeof(*dev->aer_info), GFP_KERNEL);  in pci_aer_init()
396 	if (!dev->aer_info) {  in pci_aer_init()
397 		dev->aer_cap = 0;  in pci_aer_init()
401 	ratelimit_state_init(&dev->aer_info->correctable_ratelimit,  in pci_aer_init()
403 	ratelimit_state_init(&dev->aer_info->nonfatal_ratelimit,  in pci_aer_init()
408 	 * PCI_ERR_COR_MASK, and PCI_ERR_CAP.  Root and Root Complex Event  in pci_aer_init()
409 	 * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r6.0, sec  in pci_aer_init()
425 	kfree(dev->aer_info);  in pci_aer_exit()
426 	dev->aer_info = NULL;  in pci_aer_exit()
431 #define AER_AGENT_COMPLETER		2
449 #define AER_TRANSACTION_LAYER_ERROR	2
468 	"Uncorrectable (Non-Fatal)",
564 	u64 *stats = pdev->aer_info->stats_array;			\
567 	for (i = 0; i < ARRAY_SIZE(pdev->aer_info->stats_array); i++) {	\
578 			     pdev->aer_info->total_field);		\
599 	return sysfs_emit(buf, "%llu\n", pdev->aer_info->field);	\
626 	if (!pdev->aer_info)  in aer_stats_attrs_are_visible()
636 	return a->mode;  in aer_stats_attrs_are_visible()
657 				  pdev->aer_info->ratelimit.interval);	\
668 			return -EPERM;					\
671 			return -EINVAL;					\
678 		pdev->aer_info->ratelimit.interval = interval;		\
692 				  pdev->aer_info->ratelimit.burst);	\
703 			return -EPERM;					\
706 			return -EINVAL;					\
708 		pdev->aer_info->ratelimit.burst = burst;		\
737 	if (!pdev->aer_info)  in aer_attrs_are_visible()
740 	return a->mode;  in aer_attrs_are_visible()
752 	unsigned long status = info->status & ~info->mask;  in pci_dev_aer_stats_incr()
753 	int i, max = -1;  in pci_dev_aer_stats_incr()
755 	struct aer_info *aer_info = pdev->aer_info;  in pci_dev_aer_stats_incr()
760 	switch (info->severity) {  in pci_dev_aer_stats_incr()
762 		aer_info->dev_total_cor_errs++;  in pci_dev_aer_stats_incr()
763 		counter = &aer_info->dev_cor_errs[0];  in pci_dev_aer_stats_incr()
767 		aer_info->dev_total_nonfatal_errs++;  in pci_dev_aer_stats_incr()
768 		counter = &aer_info->dev_nonfatal_errs[0];  in pci_dev_aer_stats_incr()
772 		aer_info->dev_total_fatal_errs++;  in pci_dev_aer_stats_incr()
773 		counter = &aer_info->dev_fatal_errs[0];  in pci_dev_aer_stats_incr()
785 	struct aer_info *aer_info = pdev->aer_info;  in pci_rootport_aer_stats_incr()
790 	if (e_src->status & PCI_ERR_ROOT_COR_RCV)  in pci_rootport_aer_stats_incr()
791 		aer_info->rootport_total_cor_errs++;  in pci_rootport_aer_stats_incr()
793 	if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {  in pci_rootport_aer_stats_incr()
794 		if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)  in pci_rootport_aer_stats_incr()
795 			aer_info->rootport_total_fatal_errs++;  in pci_rootport_aer_stats_incr()
797 			aer_info->rootport_total_nonfatal_errs++;  in pci_rootport_aer_stats_incr()
803 	if (!dev->aer_info)  in aer_ratelimit()
808 		return __ratelimit(&dev->aer_info->nonfatal_ratelimit);  in aer_ratelimit()
810 		return __ratelimit(&dev->aer_info->correctable_ratelimit);  in aer_ratelimit()
818 	/* Errors for which a header is always logged (PCIe r7.0 sec 6.2.7) */  in tlp_header_logged()
833 	unsigned long status = info->status & ~info->mask;  in __aer_print_error()
834 	const char *level = info->level;  in __aer_print_error()
838 	if (info->severity == AER_CORRECTABLE)  in __aer_print_error()
848 		aer_printk(level, dev, "   [%2d] %-22s%s\n", i, errmsg,  in __aer_print_error()
849 				info->first_error == i ? " (First)" : "");  in __aer_print_error()
856 	u16 source = info->id;  in aer_print_source()
859 		 info->multi_error_valid ? "Multiple " : "",  in aer_print_source()
860 		 aer_error_severity_string[info->severity],  in aer_print_source()
861 		 pci_domain_nr(dev->bus), PCI_BUS_NUM(source),  in aer_print_source()
870 	const char *level = info->level;  in aer_print_error()
875 	dev = info->dev[i];  in aer_print_error()
879 	trace_aer_event(pci_name(dev), (info->status & ~info->mask),  in aer_print_error()
880 			info->severity, info->tlp_header_valid, &info->tlp);  in aer_print_error()
882 	if (!info->ratelimit_print[i])  in aer_print_error()
885 	if (!info->status) {  in aer_print_error()
886 		pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",  in aer_print_error()
887 			aer_error_severity_string[info->severity]);  in aer_print_error()
891 	layer = AER_GET_LAYER_ERROR(info->severity, info->status);  in aer_print_error()
892 	agent = AER_GET_AGENT(info->severity, info->status);  in aer_print_error()
894 	aer_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",  in aer_print_error()
895 		   aer_error_severity_string[info->severity],  in aer_print_error()
899 		   dev->vendor, dev->device, info->status, info->mask);  in aer_print_error()
903 	if (info->tlp_header_valid)  in aer_print_error()
904 		pcie_print_tlp_log(dev, &info->tlp, level, dev_fmt("  "));  in aer_print_error()
907 	if (info->id && info->error_dev_num > 1 && info->id == id)  in aer_print_error()
933 		.first_error = PCI_ERR_CAP_FEP(aer->cap_control),  in pci_print_aer()
937 		status = aer->cor_status;  in pci_print_aer()
938 		mask = aer->cor_mask;  in pci_print_aer()
941 		status = aer->uncor_status;  in pci_print_aer()
942 		mask = aer->uncor_mask;  in pci_print_aer()
944 		tlp_header_valid = tlp_header_logged(status, aer->cap_control);  in pci_print_aer()
952 			aer_severity, tlp_header_valid, &aer->header_log);  in pci_print_aer()
968 			   aer->uncor_severity);  in pci_print_aer()
971 		pcie_print_tlp_log(dev, &aer->header_log, info.level,  in pci_print_aer()
977  * add_error_device - list device to be handled
983 	int i = e_info->error_dev_num;  in add_error_device()
986 		return -ENOSPC;  in add_error_device()
988 	e_info->dev[i] = pci_dev_get(dev);  in add_error_device()
989 	e_info->error_dev_num++;  in add_error_device()
993 	 * identified by the root's Error Source ID or it has an unmasked  in add_error_device()
995 	 * when "ratelimit_print[i]" is non-zero.  If we will print detail  in add_error_device()
997 	 * from the root as well.  in add_error_device()
999 	if (aer_ratelimit(dev, e_info->severity)) {  in add_error_device()
1000 		e_info->ratelimit_print[i] = 1;  in add_error_device()
1001 		e_info->root_ratelimit_print = 1;  in add_error_device()
1007  * is_error_source - check whether the device is source of reported error
1013 	int aer = dev->aer_cap;  in is_error_source()
1019 	 * reported by Root Port.  in is_error_source()
1021 	if ((PCI_BUS_NUM(e_info->id) != 0) &&  in is_error_source()
1022 	    !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) {  in is_error_source()
1024 		if (e_info->id == pci_dev_id(dev))  in is_error_source()
1028 		if (!e_info->multi_error_valid)  in is_error_source()
1036 	 *      2) bus flag PCI_BUS_FLAGS_NO_AERSID is set  in is_error_source()
1040 	if (atomic_read(&dev->enable_cnt) == 0)  in is_error_source()
1052 	if (e_info->severity == AER_CORRECTABLE) {  in is_error_source()
1079 		if (!e_info->multi_error_valid)  in find_device_iter()
1086  * find_source_device - search through device hierarchy for source device
1087  * @parent: pointer to Root Port pci_dev data structure
1092  * Invoked by DPC when error is detected at the Root Port.
1095  * e_info->error_dev_num and e_info->dev[], based on the given information.
1104 	e_info->error_dev_num = 0;  in find_source_device()
1106 	/* Is Root Port an agent that sends error message? */  in find_source_device()
1114 		pci_walk_bus(parent->subordinate, find_device_iter, e_info);  in find_source_device()
1116 	if (!e_info->error_dev_num)  in find_source_device()
1124  * pci_aer_unmask_internal_errors - unmask internal errors
1135 	int aer = dev->aer_cap;  in pci_aer_unmask_internal_errors()
1154 	if (dev->devfn != PCI_DEVFN(0, 0))  in is_cxl_mem_dev()
1161 	if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL)  in is_cxl_mem_dev()
1169 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);  in cxl_error_is_native()
1171 	return (pcie_ports_native || host->native_aer);  in cxl_error_is_native()
1176 	if (info->severity == AER_CORRECTABLE)  in is_internal_error()
1177 		return info->status & PCI_ERR_COR_INTERNAL;  in is_internal_error()
1179 	return info->status & PCI_ERR_UNC_INTN;  in is_internal_error()
1190 	/* Protect dev->driver */  in cxl_rch_handle_error_iter()
1191 	device_lock(&dev->dev);  in cxl_rch_handle_error_iter()
1193 	err_handler = dev->driver ? dev->driver->err_handler : NULL;  in cxl_rch_handle_error_iter()
1197 	if (info->severity == AER_CORRECTABLE) {  in cxl_rch_handle_error_iter()
1198 		if (err_handler->cor_error_detected)  in cxl_rch_handle_error_iter()
1199 			err_handler->cor_error_detected(dev);  in cxl_rch_handle_error_iter()
1200 	} else if (err_handler->error_detected) {  in cxl_rch_handle_error_iter()
1201 		if (info->severity == AER_NONFATAL)  in cxl_rch_handle_error_iter()
1202 			err_handler->error_detected(dev, pci_channel_io_normal);  in cxl_rch_handle_error_iter()
1203 		else if (info->severity == AER_FATAL)  in cxl_rch_handle_error_iter()
1204 			err_handler->error_detected(dev, pci_channel_io_frozen);  in cxl_rch_handle_error_iter()
1207 	device_unlock(&dev->dev);  in cxl_rch_handle_error_iter()
1215 	 * RCH's downstream port. Check and handle them in the CXL.mem  in cxl_rch_handle_error()
1230 	/* Non-zero terminates iteration */  in handles_cxl_error_iter()
1261  * pci_aer_handle_error - handle logging error into an event log
1265  * Invoked when an error being detected by Root Port.
1269 	int aer = dev->aer_cap;  in pci_aer_handle_error()
1271 	if (info->severity == AER_CORRECTABLE) {  in pci_aer_handle_error()
1278 					info->status);  in pci_aer_handle_error()
1280 			struct pci_driver *pdrv = dev->driver;  in pci_aer_handle_error()
1282 			if (pdrv && pdrv->err_handler &&  in pci_aer_handle_error()
1283 			    pdrv->err_handler->cor_error_detected)  in pci_aer_handle_error()
1284 				pdrv->err_handler->cor_error_detected(dev);  in pci_aer_handle_error()
1287 	} else if (info->severity == AER_NONFATAL)  in pci_aer_handle_error()
1289 	else if (info->severity == AER_FATAL)  in pci_aer_handle_error()
1382  * aer_get_device_error_info - read error status from dev and store it to info
1384  * @i: index into info->dev[]
1399 	dev = info->dev[i];  in aer_get_device_error_info()
1400 	aer = dev->aer_cap;  in aer_get_device_error_info()
1404 	info->status = 0;  in aer_get_device_error_info()
1405 	info->tlp_header_valid = 0;  in aer_get_device_error_info()
1411 	if (info->severity == AER_CORRECTABLE) {  in aer_get_device_error_info()
1413 			&info->status);  in aer_get_device_error_info()
1415 			&info->mask);  in aer_get_device_error_info()
1416 		if (!(info->status & ~info->mask))  in aer_get_device_error_info()
1421 		   info->severity == AER_NONFATAL) {  in aer_get_device_error_info()
1425 			&info->status);  in aer_get_device_error_info()
1427 			&info->mask);  in aer_get_device_error_info()
1428 		if (!(info->status & ~info->mask))  in aer_get_device_error_info()
1433 		info->first_error = PCI_ERR_CAP_FEP(aercc);  in aer_get_device_error_info()
1435 		if (tlp_header_logged(info->status, aercc)) {  in aer_get_device_error_info()
1436 			info->tlp_header_valid = 1;  in aer_get_device_error_info()
1441 					  &info->tlp);  in aer_get_device_error_info()
1453 	for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {  in aer_process_err_devices()
1457 	for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {  in aer_process_err_devices()
1459 			handle_error_source(e_info->dev[i], e_info);  in aer_process_err_devices()
1464  * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error
1465  *			    detected by Root Port or RCEC
1466  * @root: pointer to Root Port or RCEC that signaled AER interrupt
1469 static void aer_isr_one_error_type(struct pci_dev *root,  in aer_isr_one_error_type()  argument
1474 	found = find_source_device(root, info);  in aer_isr_one_error_type()
1478 	 * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to  in aer_isr_one_error_type()
1479 	 * non-zero (which enables printing) because this is either an  in aer_isr_one_error_type()
1484 	 * Requester ID from the ERR_* Message received by the Root Port or  in aer_isr_one_error_type()
1487 	if (info->root_ratelimit_print ||  in aer_isr_one_error_type()
1488 	    (!found && aer_ratelimit(root, info->severity)))  in aer_isr_one_error_type()
1489 		aer_print_source(root, info, found);  in aer_isr_one_error_type()
1496  * aer_isr_one_error - consume error(s) signaled by an AER interrupt from
1497  *		       Root Port or RCEC
1498  * @root: pointer to Root Port or RCEC that signaled AER interrupt
1501 static void aer_isr_one_error(struct pci_dev *root,  in aer_isr_one_error()  argument
1504 	u32 status = e_src->status;  in aer_isr_one_error()
1506 	pci_rootport_aer_stats_incr(root, e_src);  in aer_isr_one_error()
1515 			.id = ERR_COR_ID(e_src->id),  in aer_isr_one_error()
1521 		aer_isr_one_error_type(root, &e_info);  in aer_isr_one_error()
1528 			.id = ERR_UNCOR_ID(e_src->id),  in aer_isr_one_error()
1534 		aer_isr_one_error_type(root, &e_info);  in aer_isr_one_error()
1539  * aer_isr - consume errors detected by Root Port
1540  * @irq: IRQ assigned to Root Port
1541  * @context: pointer to Root Port data structure
1543  * Invoked, as DPC, when Root Port records new detected error
1551 	if (kfifo_is_empty(&rpc->aer_fifo))  in aer_isr()
1554 	while (kfifo_get(&rpc->aer_fifo, &e_src))  in aer_isr()
1555 		aer_isr_one_error(rpc->rpd, &e_src);  in aer_isr()
1560  * aer_irq - Root Port's ISR
1561  * @irq: IRQ assigned to Root Port
1562  * @context: pointer to Root Port data structure
1564  * Invoked when Root Port detects AER messages.
1570 	struct pci_dev *rp = rpc->rpd;  in aer_irq()
1571 	int aer = rp->aer_cap;  in aer_irq()
1581 	if (!kfifo_put(&rpc->aer_fifo, e_src))  in aer_irq()
1589 	int aer = pdev->aer_cap;  in aer_enable_irq()
1592 	/* Enable Root Port's interrupt in response to error messages */  in aer_enable_irq()
1600 	int aer = pdev->aer_cap;  in aer_disable_irq()
1603 	/* Disable Root Port's interrupt in response to error messages */  in aer_disable_irq()
1610  * aer_enable_rootport - enable Root Port's interrupts when receiving messages
1611  * @rpc: pointer to a Root Port data structure
1613  * Invoked when PCIe bus loads AER service driver.
1617 	struct pci_dev *pdev = rpc->rpd;  in aer_enable_rootport()
1618 	int aer = pdev->aer_cap;  in aer_enable_rootport()
1622 	/* Clear PCIe Capability's Device Status */  in aer_enable_rootport()
1642  * aer_disable_rootport - disable Root Port's interrupts when receiving messages
1643  * @rpc: pointer to a Root Port data structure
1645  * Invoked when PCIe bus unloads AER service driver.
1649 	struct pci_dev *pdev = rpc->rpd;  in aer_disable_rootport()
1650 	int aer = pdev->aer_cap;  in aer_disable_rootport()
1655 	/* Clear Root's error status reg */  in aer_disable_rootport()
1661  * aer_remove - clean up resources
1674  * aer_probe - initialize resources
1683 	struct device *device = &dev->device;  in aer_probe()
1684 	struct pci_dev *port = dev->port;  in aer_probe()  local
1691 	/* Limit to Root Ports or Root Complex Event Collectors */  in aer_probe()
1692 	if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) &&  in aer_probe()
1693 	    (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT))  in aer_probe()
1694 		return -ENODEV;  in aer_probe()
1698 		return -ENOMEM;  in aer_probe()
1700 	rpc->rpd = port;  in aer_probe()
1701 	INIT_KFIFO(rpc->aer_fifo);  in aer_probe()
1704 	status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr,  in aer_probe()
1707 		pci_err(port, "request AER IRQ %d failed\n", dev->irq);  in aer_probe()
1711 	cxl_rch_enable_rcec(port);  in aer_probe()
1713 	pci_info(port, "enabled with IRQ %d\n", dev->irq);  in aer_probe()
1734  * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP
1735  * @dev: pointer to Root Port, RCEC, or RCiEP
1737  * Invoked by Port Bus driver when performing reset.
1742 	struct pci_dev *root;  in aer_root_reset()  local
1744 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);  in aer_root_reset()
1749 	 * Only Root Ports and RCECs have AER Root Command and Root Status  in aer_root_reset()
1754 		root = dev->rcec;  in aer_root_reset()
1756 		root = pcie_find_root_port(dev);  in aer_root_reset()
1760 	 * an RCEC visible to us, so dev->rcec ("root") may be NULL.  In  in aer_root_reset()
1763 	aer = root ? root->aer_cap : 0;  in aer_root_reset()
1765 	if ((host->native_aer || pcie_ports_native) && aer)  in aer_root_reset()
1766 		aer_disable_irq(root);  in aer_root_reset()
1776 		pci_info(dev, "%s Port link has been reset (%d)\n",  in aer_root_reset()
1777 			pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc);  in aer_root_reset()
1780 	if ((host->native_aer || pcie_ports_native) && aer) {  in aer_root_reset()
1781 		/* Clear Root Error Status */  in aer_root_reset()
1782 		pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, &reg32);  in aer_root_reset()
1783 		pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32);  in aer_root_reset()
1785 		aer_enable_irq(root);  in aer_root_reset()
1803  * pcie_aer_init - register AER service driver
1810 		return -ENXIO;  in pcie_aer_init()