1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2025 AMD Corporation. All rights reserved. */ 3 4 #include <linux/pci.h> 5 #include <linux/aer.h> 6 #include <cxl/event.h> 7 #include <cxlmem.h> 8 #include <cxlpci.h> 9 #include "trace.h" 10 11 /* Check that UCE header definition is maintained to keep ABI intact */ 12 static_assert(CXL_HEADERLOG_TRACE_SIZE_U32 == 128, 13 "rasdaemon ABI requires exactly 128 u32s"); 14 15 static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev, 16 struct cxl_ras_capability_regs ras_cap) 17 { 18 u32 status = ras_cap.cor_status & ~ras_cap.cor_mask; 19 20 trace_cxl_port_aer_correctable_error(&pdev->dev, status); 21 } 22 23 static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, 24 struct cxl_ras_capability_regs ras_cap) 25 { 26 u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; 27 u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; 28 u32 fe; 29 30 if (hweight32(status) > 1) 31 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, 32 ras_cap.cap_control)); 33 else 34 fe = status; 35 36 memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); 37 trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, hl); 38 } 39 40 static void cxl_cper_trace_corr_prot_err(struct cxl_memdev *cxlmd, 41 struct cxl_ras_capability_regs ras_cap) 42 { 43 u32 status = ras_cap.cor_status & ~ras_cap.cor_mask; 44 45 trace_cxl_aer_correctable_error(cxlmd, status); 46 } 47 48 static void 49 cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, 50 struct cxl_ras_capability_regs ras_cap) 51 { 52 u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; 53 u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; 54 u32 fe; 55 56 if (hweight32(status) > 1) 57 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, 58 ras_cap.cap_control)); 59 else 60 fe = status; 61 62 /* 63 * ras_cap.header_log[] holds CXL_HEADERLOG_SIZE_U32 (16) hardware 64 * dwords. Copy them into the front of a zero-filled 65 * CXL_HEADERLOG_TRACE_SIZE_U32 (128) u32 staging buffer so the trace 66 * event memcpy sees a full 512-byte source and the userspace ABI 67 * (rasdaemon) is preserved. 68 */ 69 memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); 70 trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, hl); 71 } 72 73 static int match_memdev_by_parent(struct device *dev, const void *uport) 74 { 75 if (is_cxl_memdev(dev) && dev->parent == uport) 76 return 1; 77 return 0; 78 } 79 80 void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data) 81 { 82 unsigned int devfn = PCI_DEVFN(data->prot_err.agent_addr.device, 83 data->prot_err.agent_addr.function); 84 struct pci_dev *pdev __free(pci_dev_put) = 85 pci_get_domain_bus_and_slot(data->prot_err.agent_addr.segment, 86 data->prot_err.agent_addr.bus, 87 devfn); 88 struct cxl_memdev *cxlmd; 89 int port_type; 90 91 if (!pdev) 92 return; 93 94 port_type = pci_pcie_type(pdev); 95 if (port_type == PCI_EXP_TYPE_ROOT_PORT || 96 port_type == PCI_EXP_TYPE_DOWNSTREAM || 97 port_type == PCI_EXP_TYPE_UPSTREAM) { 98 if (data->severity == AER_CORRECTABLE) 99 cxl_cper_trace_corr_port_prot_err(pdev, data->ras_cap); 100 else 101 cxl_cper_trace_uncorr_port_prot_err(pdev, data->ras_cap); 102 103 return; 104 } 105 106 guard(device)(&pdev->dev); 107 if (!pdev->dev.driver) 108 return; 109 110 struct device *mem_dev __free(put_device) = bus_find_device( 111 &cxl_bus_type, NULL, pdev, match_memdev_by_parent); 112 if (!mem_dev) 113 return; 114 115 cxlmd = to_cxl_memdev(mem_dev); 116 if (data->severity == AER_CORRECTABLE) 117 cxl_cper_trace_corr_prot_err(cxlmd, data->ras_cap); 118 else 119 cxl_cper_trace_uncorr_prot_err(cxlmd, data->ras_cap); 120 } 121 EXPORT_SYMBOL_GPL(cxl_cper_handle_prot_err); 122 123 static void cxl_cper_prot_err_work_fn(struct work_struct *work) 124 { 125 struct cxl_cper_prot_err_work_data wd; 126 127 while (cxl_cper_prot_err_kfifo_get(&wd)) 128 cxl_cper_handle_prot_err(&wd); 129 } 130 static DECLARE_WORK(cxl_cper_prot_err_work, cxl_cper_prot_err_work_fn); 131 132 void cxl_ras_init(void) 133 { 134 cxl_cper_register_prot_err_work(&cxl_cper_prot_err_work); 135 } 136 137 void cxl_ras_exit(void) 138 { 139 cxl_cper_unregister_prot_err_work(); 140 } 141 142 static void cxl_dport_map_ras(struct cxl_dport *dport) 143 { 144 struct cxl_register_map *map = &dport->reg_map; 145 struct device *dev = dport->dport_dev; 146 147 if (!map->component_map.ras.valid) 148 dev_dbg(dev, "RAS registers not found\n"); 149 else if (cxl_map_component_regs(map, &dport->regs.component, 150 BIT(CXL_CM_CAP_CAP_ID_RAS))) 151 dev_dbg(dev, "Failed to map RAS capability.\n"); 152 } 153 154 /** 155 * devm_cxl_dport_ras_setup - Setup CXL RAS report on this dport 156 * @dport: the cxl_dport that needs to be initialized 157 */ 158 void devm_cxl_dport_ras_setup(struct cxl_dport *dport) 159 { 160 dport->reg_map.host = dport_to_host(dport); 161 cxl_dport_map_ras(dport); 162 } 163 164 void devm_cxl_dport_rch_ras_setup(struct cxl_dport *dport) 165 { 166 struct pci_host_bridge *host_bridge; 167 168 if (!dev_is_pci(dport->dport_dev)) 169 return; 170 171 devm_cxl_dport_ras_setup(dport); 172 173 host_bridge = to_pci_host_bridge(dport->dport_dev); 174 if (!host_bridge->native_aer) 175 return; 176 177 cxl_dport_map_rch_aer(dport); 178 cxl_disable_rch_root_ints(dport); 179 } 180 EXPORT_SYMBOL_NS_GPL(devm_cxl_dport_rch_ras_setup, "CXL"); 181 182 void devm_cxl_port_ras_setup(struct cxl_port *port) 183 { 184 struct cxl_register_map *map = &port->reg_map; 185 186 if (!map->component_map.ras.valid) { 187 dev_dbg(&port->dev, "RAS registers not found\n"); 188 return; 189 } 190 191 map->host = &port->dev; 192 if (cxl_map_component_regs(map, &port->regs, 193 BIT(CXL_CM_CAP_CAP_ID_RAS))) 194 dev_dbg(&port->dev, "Failed to map RAS capability\n"); 195 } 196 EXPORT_SYMBOL_NS_GPL(devm_cxl_port_ras_setup, "CXL"); 197 198 void cxl_handle_cor_ras(struct device *dev, void __iomem *ras_base) 199 { 200 void __iomem *addr; 201 u32 status; 202 203 if (!ras_base) 204 return; 205 206 addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET; 207 status = readl(addr); 208 if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) { 209 writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr); 210 trace_cxl_aer_correctable_error(to_cxl_memdev(dev), status); 211 } 212 } 213 214 /* CXL spec rev3.0 8.2.4.16.1 */ 215 static void header_log_copy(void __iomem *ras_base, u32 *log) 216 { 217 void __iomem *addr; 218 u32 *log_addr; 219 int i; 220 221 addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET; 222 log_addr = log; 223 224 for (i = 0; i < CXL_HEADERLOG_SIZE_U32; i++) { 225 *log_addr = readl(addr); 226 log_addr++; 227 addr += sizeof(u32); 228 } 229 } 230 231 /* 232 * Log the state of the RAS status registers and prepare them to log the 233 * next error status. Return 1 if reset needed. 234 */ 235 bool cxl_handle_ras(struct device *dev, void __iomem *ras_base) 236 { 237 u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; 238 void __iomem *addr; 239 u32 status; 240 u32 fe; 241 242 if (!ras_base) 243 return false; 244 245 addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; 246 status = readl(addr); 247 if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK)) 248 return false; 249 250 /* If multiple errors, log header points to first error from ctrl reg */ 251 if (hweight32(status) > 1) { 252 void __iomem *rcc_addr = 253 ras_base + CXL_RAS_CAP_CONTROL_OFFSET; 254 255 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, 256 readl(rcc_addr))); 257 } else { 258 fe = status; 259 } 260 261 header_log_copy(ras_base, hl); 262 trace_cxl_aer_uncorrectable_error(to_cxl_memdev(dev), status, fe, hl); 263 writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr); 264 265 return true; 266 } 267 268 void cxl_cor_error_detected(struct pci_dev *pdev) 269 { 270 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 271 struct cxl_memdev *cxlmd = cxlds->cxlmd; 272 struct device *dev = &cxlds->cxlmd->dev; 273 274 scoped_guard(device, dev) { 275 if (!dev->driver) { 276 dev_warn(&pdev->dev, 277 "%s: memdev disabled, abort error handling\n", 278 dev_name(dev)); 279 return; 280 } 281 282 if (cxlds->rcd) 283 cxl_handle_rdport_errors(cxlds); 284 285 cxl_handle_cor_ras(&cxlds->cxlmd->dev, cxlmd->endpoint->regs.ras); 286 } 287 } 288 EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, "CXL"); 289 290 pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, 291 pci_channel_state_t state) 292 { 293 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 294 struct cxl_memdev *cxlmd = cxlds->cxlmd; 295 struct device *dev = &cxlmd->dev; 296 bool ue; 297 298 scoped_guard(device, dev) { 299 if (!dev->driver) { 300 dev_warn(&pdev->dev, 301 "%s: memdev disabled, abort error handling\n", 302 dev_name(dev)); 303 return PCI_ERS_RESULT_DISCONNECT; 304 } 305 306 if (cxlds->rcd) 307 cxl_handle_rdport_errors(cxlds); 308 /* 309 * A frozen channel indicates an impending reset which is fatal to 310 * CXL.mem operation, and will likely crash the system. On the off 311 * chance the situation is recoverable dump the status of the RAS 312 * capability registers and bounce the active state of the memdev. 313 */ 314 ue = cxl_handle_ras(&cxlds->cxlmd->dev, cxlmd->endpoint->regs.ras); 315 } 316 317 switch (state) { 318 case pci_channel_io_normal: 319 if (ue) { 320 device_release_driver(dev); 321 return PCI_ERS_RESULT_NEED_RESET; 322 } 323 return PCI_ERS_RESULT_CAN_RECOVER; 324 case pci_channel_io_frozen: 325 dev_warn(&pdev->dev, 326 "%s: frozen state error detected, disable CXL.mem\n", 327 dev_name(dev)); 328 device_release_driver(dev); 329 return PCI_ERS_RESULT_NEED_RESET; 330 case pci_channel_io_perm_failure: 331 dev_warn(&pdev->dev, 332 "failure state error detected, request disconnect\n"); 333 return PCI_ERS_RESULT_DISCONNECT; 334 } 335 return PCI_ERS_RESULT_NEED_RESET; 336 } 337 EXPORT_SYMBOL_NS_GPL(cxl_error_detected, "CXL"); 338