1 // SPDX-License-Identifier: GPL-2.0
3  * Implement the AER root port service driver. The driver registers an IRQ
4  * handler. When a root port triggers an AER interrupt, the IRQ handler
5  * collects Root Port status and schedules work.
11  * (C) Copyright 2009 Hewlett-Packard Development Company, L.P.
22 #include <linux/pci-acpi.h>
40 #define aer_printk(level, pdev, fmt, arg...) \  argument
41 	dev_printk(level, &(pdev)->dev, fmt, ##arg)
54 	struct pci_dev *rpd;		/* Root Port device */
65 	 * at its link partner (e.g. Root Port) because the errors will be
86 	 * messages received by the Root Port / Event Collector, INCLUDING the
87 	 * ones that are generated internally (by the Root Port itself)
147  * enable_ecrc_checking - enable PCIe ECRC checking for a device
154 	int aer = dev->aer_cap;  in enable_ecrc_checking()
158 		return -ENODEV;  in enable_ecrc_checking()
171  * disable_ecrc_checking - disable PCIe ECRC checking for a device
178 	int aer = dev->aer_cap;  in disable_ecrc_checking()
182 		return -ENODEV;  in disable_ecrc_checking()
192  * pcie_set_ecrc_checking - set/unset PCIe ECRC checking for a device based
216  * pcie_ecrc_get_policy - parse kernel command-line ecrc option
236 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);  in pcie_aer_is_native()
238 	if (!dev->aer_cap)  in pcie_aer_is_native()
241 	return pcie_ports_native || host->native_aer;  in pcie_aer_is_native()
250 		return -EIO;  in pci_enable_pcie_error_reporting()
258 	int aer = dev->aer_cap;  in pci_aer_clear_nonfatal_status()
262 		return -EIO;  in pci_aer_clear_nonfatal_status()
277 	int aer = dev->aer_cap;  in pci_aer_clear_fatal_status()
292  * pci_aer_raw_clear_status - Clear AER error registers.
302 	int aer = dev->aer_cap;  in pci_aer_raw_clear_status()
307 		return -EIO;  in pci_aer_raw_clear_status()
328 		return -EIO;  in pci_aer_clear_status()
335 	int aer = dev->aer_cap;  in pci_save_aer_state()
346 	cap = &save_state->cap.data[0];  in pci_save_aer_state()
357 	int aer = dev->aer_cap;  in pci_restore_aer_state()
368 	cap = &save_state->cap.data[0];  in pci_restore_aer_state()
381 	dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);  in pci_aer_init()
382 	if (!dev->aer_cap)  in pci_aer_init()
385 	dev->aer_info = kzalloc(sizeof(*dev->aer_info), GFP_KERNEL);  in pci_aer_init()
387 	ratelimit_state_init(&dev->aer_info->correctable_ratelimit,  in pci_aer_init()
389 	ratelimit_state_init(&dev->aer_info->nonfatal_ratelimit,  in pci_aer_init()
411 	kfree(dev->aer_info);  in pci_aer_exit()
412 	dev->aer_info = NULL;  in pci_aer_exit()
454 	"Uncorrectable (Non-Fatal)",
550 	u64 *stats = pdev->aer_info->stats_array;			\
553 	for (i = 0; i < ARRAY_SIZE(pdev->aer_info->stats_array); i++) {	\
564 			     pdev->aer_info->total_field);		\
585 	return sysfs_emit(buf, "%llu\n", pdev->aer_info->field);	\
612 	if (!pdev->aer_info)  in aer_stats_attrs_are_visible()
622 	return a->mode;  in aer_stats_attrs_are_visible()
643 				  pdev->aer_info->ratelimit.interval);	\
654 			return -EPERM;					\
657 			return -EINVAL;					\
664 		pdev->aer_info->ratelimit.interval = interval;		\
678 				  pdev->aer_info->ratelimit.burst);	\
689 			return -EPERM;					\
692 			return -EINVAL;					\
694 		pdev->aer_info->ratelimit.burst = burst;		\
723 	if (!pdev->aer_info)  in aer_attrs_are_visible()
726 	return a->mode;  in aer_attrs_are_visible()
738 	unsigned long status = info->status & ~info->mask;  in pci_dev_aer_stats_incr()
739 	int i, max = -1;  in pci_dev_aer_stats_incr()
741 	struct aer_info *aer_info = pdev->aer_info;  in pci_dev_aer_stats_incr()
746 	switch (info->severity) {  in pci_dev_aer_stats_incr()
748 		aer_info->dev_total_cor_errs++;  in pci_dev_aer_stats_incr()
749 		counter = &aer_info->dev_cor_errs[0];  in pci_dev_aer_stats_incr()
753 		aer_info->dev_total_nonfatal_errs++;  in pci_dev_aer_stats_incr()
754 		counter = &aer_info->dev_nonfatal_errs[0];  in pci_dev_aer_stats_incr()
758 		aer_info->dev_total_fatal_errs++;  in pci_dev_aer_stats_incr()
759 		counter = &aer_info->dev_fatal_errs[0];  in pci_dev_aer_stats_incr()
771 	struct aer_info *aer_info = pdev->aer_info;  in pci_rootport_aer_stats_incr()
776 	if (e_src->status & PCI_ERR_ROOT_COR_RCV)  in pci_rootport_aer_stats_incr()
777 		aer_info->rootport_total_cor_errs++;  in pci_rootport_aer_stats_incr()
779 	if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {  in pci_rootport_aer_stats_incr()
780 		if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)  in pci_rootport_aer_stats_incr()
781 			aer_info->rootport_total_fatal_errs++;  in pci_rootport_aer_stats_incr()
783 			aer_info->rootport_total_nonfatal_errs++;  in pci_rootport_aer_stats_incr()
791 		return __ratelimit(&dev->aer_info->nonfatal_ratelimit);  in aer_ratelimit()
793 		return __ratelimit(&dev->aer_info->correctable_ratelimit);  in aer_ratelimit()
802 	unsigned long status = info->status & ~info->mask;  in __aer_print_error()
803 	const char *level = info->level;  in __aer_print_error()  local
807 	if (info->severity == AER_CORRECTABLE)  in __aer_print_error()
817 		aer_printk(level, dev, "   [%2d] %-22s%s\n", i, errmsg,  in __aer_print_error()
818 				info->first_error == i ? " (First)" : "");  in __aer_print_error()
825 	u16 source = info->id;  in aer_print_source()
828 		 info->multi_error_valid ? "Multiple " : "",  in aer_print_source()
829 		 aer_error_severity_string[info->severity],  in aer_print_source()
830 		 pci_domain_nr(dev->bus), PCI_BUS_NUM(source),  in aer_print_source()
839 	const char *level = info->level;  in aer_print_error()  local
844 	dev = info->dev[i];  in aer_print_error()
848 	trace_aer_event(pci_name(dev), (info->status & ~info->mask),  in aer_print_error()
849 			info->severity, info->tlp_header_valid, &info->tlp);  in aer_print_error()
851 	if (!info->ratelimit_print[i])  in aer_print_error()
854 	if (!info->status) {  in aer_print_error()
856 			aer_error_severity_string[info->severity]);  in aer_print_error()
860 	layer = AER_GET_LAYER_ERROR(info->severity, info->status);  in aer_print_error()
861 	agent = AER_GET_AGENT(info->severity, info->status);  in aer_print_error()
863 	aer_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",  in aer_print_error()
864 		   aer_error_severity_string[info->severity],  in aer_print_error()
867 	aer_printk(level, dev, "  device [%04x:%04x] error status/mask=%08x/%08x\n",  in aer_print_error()
868 		   dev->vendor, dev->device, info->status, info->mask);  in aer_print_error()
872 	if (info->tlp_header_valid)  in aer_print_error()
873 		pcie_print_tlp_log(dev, &info->tlp, level, dev_fmt("  "));  in aer_print_error()
876 	if (info->id && info->error_dev_num > 1 && info->id == id)  in aer_print_error()
902 		.first_error = PCI_ERR_CAP_FEP(aer->cap_control),  in pci_print_aer()
906 		status = aer->cor_status;  in pci_print_aer()
907 		mask = aer->cor_mask;  in pci_print_aer()
908 		info.level = KERN_WARNING;  in pci_print_aer()
910 		status = aer->uncor_status;  in pci_print_aer()
911 		mask = aer->uncor_mask;  in pci_print_aer()
912 		info.level = KERN_ERR;  in pci_print_aer()
921 			aer_severity, tlp_header_valid, &aer->header_log);  in pci_print_aer()
929 	aer_printk(info.level, dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n",  in pci_print_aer()
932 	aer_printk(info.level, dev, "aer_layer=%s, aer_agent=%s\n",  in pci_print_aer()
936 		aer_printk(info.level, dev, "aer_uncor_severity: 0x%08x\n",  in pci_print_aer()
937 			   aer->uncor_severity);  in pci_print_aer()
940 		pcie_print_tlp_log(dev, &aer->header_log, info.level,  in pci_print_aer()
946  * add_error_device - list device to be handled
952 	int i = e_info->error_dev_num;  in add_error_device()
955 		return -ENOSPC;  in add_error_device()
957 	e_info->dev[i] = pci_dev_get(dev);  in add_error_device()
958 	e_info->error_dev_num++;  in add_error_device()
964 	 * when "ratelimit_print[i]" is non-zero.  If we will print detail  in add_error_device()
968 	if (aer_ratelimit(dev, e_info->severity)) {  in add_error_device()
969 		e_info->ratelimit_print[i] = 1;  in add_error_device()
970 		e_info->root_ratelimit_print = 1;  in add_error_device()
976  * is_error_source - check whether the device is source of reported error
982 	int aer = dev->aer_cap;  in is_error_source()
988 	 * reported by Root Port.  in is_error_source()
990 	if ((PCI_BUS_NUM(e_info->id) != 0) &&  in is_error_source()
991 	    !(dev->bus->bus_flags & PCI_BUS_FLAGS_NO_AERSID)) {  in is_error_source()
993 		if (e_info->id == pci_dev_id(dev))  in is_error_source()
997 		if (!e_info->multi_error_valid)  in is_error_source()
1009 	if (atomic_read(&dev->enable_cnt) == 0)  in is_error_source()
1021 	if (e_info->severity == AER_CORRECTABLE) {  in is_error_source()
1048 		if (!e_info->multi_error_valid)  in find_device_iter()
1055  * find_source_device - search through device hierarchy for source device
1056  * @parent: pointer to Root Port pci_dev data structure
1061  * Invoked by DPC when error is detected at the Root Port.
1064  * e_info->error_dev_num and e_info->dev[], based on the given information.
1073 	e_info->error_dev_num = 0;  in find_source_device()
1075 	/* Is Root Port an agent that sends error message? */  in find_source_device()
1083 		pci_walk_bus(parent->subordinate, find_device_iter, e_info);  in find_source_device()
1085 	if (!e_info->error_dev_num)  in find_source_device()
1093  * pci_aer_unmask_internal_errors - unmask internal errors
1104 	int aer = dev->aer_cap;  in pci_aer_unmask_internal_errors()
1123 	if (dev->devfn != PCI_DEVFN(0, 0))  in is_cxl_mem_dev()
1130 	if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL)  in is_cxl_mem_dev()
1138 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);  in cxl_error_is_native()
1140 	return (pcie_ports_native || host->native_aer);  in cxl_error_is_native()
1145 	if (info->severity == AER_CORRECTABLE)  in is_internal_error()
1146 		return info->status & PCI_ERR_COR_INTERNAL;  in is_internal_error()
1148 	return info->status & PCI_ERR_UNC_INTN;  in is_internal_error()
1159 	/* Protect dev->driver */  in cxl_rch_handle_error_iter()
1160 	device_lock(&dev->dev);  in cxl_rch_handle_error_iter()
1162 	err_handler = dev->driver ? dev->driver->err_handler : NULL;  in cxl_rch_handle_error_iter()
1166 	if (info->severity == AER_CORRECTABLE) {  in cxl_rch_handle_error_iter()
1167 		if (err_handler->cor_error_detected)  in cxl_rch_handle_error_iter()
1168 			err_handler->cor_error_detected(dev);  in cxl_rch_handle_error_iter()
1169 	} else if (err_handler->error_detected) {  in cxl_rch_handle_error_iter()
1170 		if (info->severity == AER_NONFATAL)  in cxl_rch_handle_error_iter()
1171 			err_handler->error_detected(dev, pci_channel_io_normal);  in cxl_rch_handle_error_iter()
1172 		else if (info->severity == AER_FATAL)  in cxl_rch_handle_error_iter()
1173 			err_handler->error_detected(dev, pci_channel_io_frozen);  in cxl_rch_handle_error_iter()
1176 	device_unlock(&dev->dev);  in cxl_rch_handle_error_iter()
1184 	 * RCH's downstream port. Check and handle them in the CXL.mem  in cxl_rch_handle_error()
1199 	/* Non-zero terminates iteration */  in handles_cxl_error_iter()
1230  * pci_aer_handle_error - handle logging error into an event log
1234  * Invoked when an error being detected by Root Port.
1238 	int aer = dev->aer_cap;  in pci_aer_handle_error()
1240 	if (info->severity == AER_CORRECTABLE) {  in pci_aer_handle_error()
1247 					info->status);  in pci_aer_handle_error()
1249 			struct pci_driver *pdrv = dev->driver;  in pci_aer_handle_error()
1251 			if (pdrv && pdrv->err_handler &&  in pci_aer_handle_error()
1252 			    pdrv->err_handler->cor_error_detected)  in pci_aer_handle_error()
1253 				pdrv->err_handler->cor_error_detected(dev);  in pci_aer_handle_error()
1256 	} else if (info->severity == AER_NONFATAL)  in pci_aer_handle_error()
1258 	else if (info->severity == AER_FATAL)  in pci_aer_handle_error()
1351  * aer_get_device_error_info - read error status from dev and store it to info
1353  * @i: index into info->dev[]
1368 	dev = info->dev[i];  in aer_get_device_error_info()
1369 	aer = dev->aer_cap;  in aer_get_device_error_info()
1373 	info->status = 0;  in aer_get_device_error_info()
1374 	info->tlp_header_valid = 0;  in aer_get_device_error_info()
1380 	if (info->severity == AER_CORRECTABLE) {  in aer_get_device_error_info()
1382 			&info->status);  in aer_get_device_error_info()
1384 			&info->mask);  in aer_get_device_error_info()
1385 		if (!(info->status & ~info->mask))  in aer_get_device_error_info()
1390 		   info->severity == AER_NONFATAL) {  in aer_get_device_error_info()
1394 			&info->status);  in aer_get_device_error_info()
1396 			&info->mask);  in aer_get_device_error_info()
1397 		if (!(info->status & ~info->mask))  in aer_get_device_error_info()
1402 		info->first_error = PCI_ERR_CAP_FEP(aercc);  in aer_get_device_error_info()
1404 		if (info->status & AER_LOG_TLP_MASKS) {  in aer_get_device_error_info()
1405 			info->tlp_header_valid = 1;  in aer_get_device_error_info()
1410 					  &info->tlp);  in aer_get_device_error_info()
1422 	for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {  in aer_process_err_devices()
1426 	for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) {  in aer_process_err_devices()
1428 			handle_error_source(e_info->dev[i], e_info);  in aer_process_err_devices()
1433  * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error
1434  *			    detected by Root Port or RCEC
1435  * @root: pointer to Root Port or RCEC that signaled AER interrupt
1447 	 * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to  in aer_isr_one_error_type()
1448 	 * non-zero (which enables printing) because this is either an  in aer_isr_one_error_type()
1453 	 * Requester ID from the ERR_* Message received by the Root Port or  in aer_isr_one_error_type()
1456 	if (info->root_ratelimit_print ||  in aer_isr_one_error_type()
1457 	    (!found && aer_ratelimit(root, info->severity)))  in aer_isr_one_error_type()
1465  * aer_isr_one_error - consume error(s) signaled by an AER interrupt from
1466  *		       Root Port or RCEC
1467  * @root: pointer to Root Port or RCEC that signaled AER interrupt
1473 	u32 status = e_src->status;  in aer_isr_one_error()
1484 			.id = ERR_COR_ID(e_src->id),  in aer_isr_one_error()
1486 			.level = KERN_WARNING,  in aer_isr_one_error()
1497 			.id = ERR_UNCOR_ID(e_src->id),  in aer_isr_one_error()
1499 			.level = KERN_ERR,  in aer_isr_one_error()
1508  * aer_isr - consume errors detected by Root Port
1509  * @irq: IRQ assigned to Root Port
1510  * @context: pointer to Root Port data structure
1512  * Invoked, as DPC, when Root Port records new detected error
1520 	if (kfifo_is_empty(&rpc->aer_fifo))  in aer_isr()
1523 	while (kfifo_get(&rpc->aer_fifo, &e_src))  in aer_isr()
1524 		aer_isr_one_error(rpc->rpd, &e_src);  in aer_isr()
1529  * aer_irq - Root Port's ISR
1530  * @irq: IRQ assigned to Root Port
1531  * @context: pointer to Root Port data structure
1533  * Invoked when Root Port detects AER messages.
1539 	struct pci_dev *rp = rpc->rpd;  in aer_irq()
1540 	int aer = rp->aer_cap;  in aer_irq()
1550 	if (!kfifo_put(&rpc->aer_fifo, e_src))  in aer_irq()
1558 	int aer = pdev->aer_cap;  in aer_enable_irq()
1561 	/* Enable Root Port's interrupt in response to error messages */  in aer_enable_irq()
1569 	int aer = pdev->aer_cap;  in aer_disable_irq()
1572 	/* Disable Root Port's interrupt in response to error messages */  in aer_disable_irq()
1579  * aer_enable_rootport - enable Root Port's interrupts when receiving messages
1580  * @rpc: pointer to a Root Port data structure
1586 	struct pci_dev *pdev = rpc->rpd;  in aer_enable_rootport()
1587 	int aer = pdev->aer_cap;  in aer_enable_rootport()
1611  * aer_disable_rootport - disable Root Port's interrupts when receiving messages
1612  * @rpc: pointer to a Root Port data structure
1618 	struct pci_dev *pdev = rpc->rpd;  in aer_disable_rootport()
1619 	int aer = pdev->aer_cap;  in aer_disable_rootport()
1630  * aer_remove - clean up resources
1643  * aer_probe - initialize resources
1652 	struct device *device = &dev->device;  in aer_probe()
1653 	struct pci_dev *port = dev->port;  in aer_probe()  local
1661 	if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) &&  in aer_probe()
1662 	    (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT))  in aer_probe()
1663 		return -ENODEV;  in aer_probe()
1667 		return -ENOMEM;  in aer_probe()
1669 	rpc->rpd = port;  in aer_probe()
1670 	INIT_KFIFO(rpc->aer_fifo);  in aer_probe()
1673 	status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr,  in aer_probe()
1676 		pci_err(port, "request AER IRQ %d failed\n", dev->irq);  in aer_probe()
1680 	cxl_rch_enable_rcec(port);  in aer_probe()
1682 	pci_info(port, "enabled with IRQ %d\n", dev->irq);  in aer_probe()
1703  * aer_root_reset - reset Root Port hierarchy, RCEC, or RCiEP
1704  * @dev: pointer to Root Port, RCEC, or RCiEP
1706  * Invoked by Port Bus driver when performing reset.
1713 	struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);  in aer_root_reset()
1723 		root = dev->rcec;  in aer_root_reset()
1729 	 * an RCEC visible to us, so dev->rcec ("root") may be NULL.  In  in aer_root_reset()
1732 	aer = root ? root->aer_cap : 0;  in aer_root_reset()
1734 	if ((host->native_aer || pcie_ports_native) && aer)  in aer_root_reset()
1745 		pci_info(dev, "%s Port link has been reset (%d)\n",  in aer_root_reset()
1746 			pci_is_root_bus(dev->bus) ? "Root" : "Downstream", rc);  in aer_root_reset()
1749 	if ((host->native_aer || pcie_ports_native) && aer) {  in aer_root_reset()
1772  * pcie_aer_init - register AER service driver
1779 		return -ENXIO;  in pcie_aer_init()