1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2025 AMD Corporation. All rights reserved. */ 3 4 #include <linux/pci.h> 5 #include <linux/aer.h> 6 #include <cxl/event.h> 7 #include <cxlmem.h> 8 #include <cxlpci.h> 9 #include "trace.h" 10 11 /* Check that UCE header definition is maintained to keep ABI intact */ 12 static_assert(CXL_HEADERLOG_TRACE_SIZE_U32 == 128, 13 "rasdaemon ABI requires exactly 128 u32s"); 14 15 static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev, 16 struct cxl_ras_capability_regs ras_cap) 17 { 18 u32 status = ras_cap.cor_status & ~ras_cap.cor_mask; 19 20 trace_cxl_port_aer_correctable_error(&pdev->dev, status); 21 } 22 23 static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev, 24 struct cxl_ras_capability_regs ras_cap) 25 { 26 u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; 27 u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; 28 u32 fe; 29 30 if (hweight32(status) > 1) 31 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, 32 ras_cap.cap_control)); 33 else 34 fe = status; 35 36 memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); 37 trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, hl); 38 } 39 40 static void cxl_cper_trace_corr_prot_err(struct cxl_memdev *cxlmd, 41 struct cxl_ras_capability_regs ras_cap) 42 { 43 u32 status = ras_cap.cor_status & ~ras_cap.cor_mask; 44 45 trace_cxl_aer_correctable_error(cxlmd, status); 46 } 47 48 static void 49 cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, 50 struct cxl_ras_capability_regs ras_cap) 51 { 52 u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; 53 u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask; 54 u32 fe; 55 56 if (hweight32(status) > 1) 57 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, 58 ras_cap.cap_control)); 59 else 60 fe = status; 61 62 /* 63 * ras_cap.header_log[] holds CXL_HEADERLOG_SIZE_U32 (16) hardware 64 * dwords. Copy them into the front of a zero-filled 65 * CXL_HEADERLOG_TRACE_SIZE_U32 (128) u32 staging buffer so the trace 66 * event memcpy sees a full 512-byte source and the userspace ABI 67 * (rasdaemon) is preserved. 68 */ 69 memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE); 70 trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, hl); 71 } 72 73 static int match_memdev_by_parent(struct device *dev, const void *uport) 74 { 75 if (is_cxl_memdev(dev) && dev->parent == uport) 76 return 1; 77 return 0; 78 } 79 80 void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data) 81 { 82 unsigned int devfn = PCI_DEVFN(data->prot_err.agent_addr.device, 83 data->prot_err.agent_addr.function); 84 struct pci_dev *pdev __free(pci_dev_put) = 85 pci_get_domain_bus_and_slot(data->prot_err.agent_addr.segment, 86 data->prot_err.agent_addr.bus, 87 devfn); 88 struct cxl_memdev *cxlmd; 89 int port_type; 90 91 if (!pdev) 92 return; 93 94 port_type = pci_pcie_type(pdev); 95 if (port_type == PCI_EXP_TYPE_ROOT_PORT || 96 port_type == PCI_EXP_TYPE_DOWNSTREAM || 97 port_type == PCI_EXP_TYPE_UPSTREAM) { 98 if (data->severity == AER_CORRECTABLE) 99 cxl_cper_trace_corr_port_prot_err(pdev, data->ras_cap); 100 else 101 cxl_cper_trace_uncorr_port_prot_err(pdev, data->ras_cap); 102 103 return; 104 } 105 106 guard(device)(&pdev->dev); 107 if (!pdev->dev.driver) 108 return; 109 110 struct device *mem_dev __free(put_device) = bus_find_device( 111 &cxl_bus_type, NULL, pdev, match_memdev_by_parent); 112 if (!mem_dev) 113 return; 114 115 cxlmd = to_cxl_memdev(mem_dev); 116 if (data->severity == AER_CORRECTABLE) 117 cxl_cper_trace_corr_prot_err(cxlmd, data->ras_cap); 118 else 119 cxl_cper_trace_uncorr_prot_err(cxlmd, data->ras_cap); 120 } 121 EXPORT_SYMBOL_GPL(cxl_cper_handle_prot_err); 122 123 static void cxl_cper_prot_err_work_fn(struct work_struct *work) 124 { 125 struct cxl_cper_prot_err_work_data wd; 126 127 while (cxl_cper_prot_err_kfifo_get(&wd)) 128 cxl_cper_handle_prot_err(&wd); 129 } 130 static DECLARE_WORK(cxl_cper_prot_err_work, cxl_cper_prot_err_work_fn); 131 132 int cxl_ras_init(void) 133 { 134 return cxl_cper_register_prot_err_work(&cxl_cper_prot_err_work); 135 } 136 137 void cxl_ras_exit(void) 138 { 139 cxl_cper_unregister_prot_err_work(&cxl_cper_prot_err_work); 140 cancel_work_sync(&cxl_cper_prot_err_work); 141 } 142 143 static void cxl_dport_map_ras(struct cxl_dport *dport) 144 { 145 struct cxl_register_map *map = &dport->reg_map; 146 struct device *dev = dport->dport_dev; 147 148 if (!map->component_map.ras.valid) 149 dev_dbg(dev, "RAS registers not found\n"); 150 else if (cxl_map_component_regs(map, &dport->regs.component, 151 BIT(CXL_CM_CAP_CAP_ID_RAS))) 152 dev_dbg(dev, "Failed to map RAS capability.\n"); 153 } 154 155 /** 156 * devm_cxl_dport_ras_setup - Setup CXL RAS report on this dport 157 * @dport: the cxl_dport that needs to be initialized 158 */ 159 void devm_cxl_dport_ras_setup(struct cxl_dport *dport) 160 { 161 dport->reg_map.host = dport_to_host(dport); 162 cxl_dport_map_ras(dport); 163 } 164 165 void devm_cxl_dport_rch_ras_setup(struct cxl_dport *dport) 166 { 167 struct pci_host_bridge *host_bridge; 168 169 if (!dev_is_pci(dport->dport_dev)) 170 return; 171 172 devm_cxl_dport_ras_setup(dport); 173 174 host_bridge = to_pci_host_bridge(dport->dport_dev); 175 if (!host_bridge->native_aer) 176 return; 177 178 cxl_dport_map_rch_aer(dport); 179 cxl_disable_rch_root_ints(dport); 180 } 181 EXPORT_SYMBOL_NS_GPL(devm_cxl_dport_rch_ras_setup, "CXL"); 182 183 void devm_cxl_port_ras_setup(struct cxl_port *port) 184 { 185 struct cxl_register_map *map = &port->reg_map; 186 187 if (!map->component_map.ras.valid) { 188 dev_dbg(&port->dev, "RAS registers not found\n"); 189 return; 190 } 191 192 map->host = &port->dev; 193 if (cxl_map_component_regs(map, &port->regs, 194 BIT(CXL_CM_CAP_CAP_ID_RAS))) 195 dev_dbg(&port->dev, "Failed to map RAS capability\n"); 196 } 197 EXPORT_SYMBOL_NS_GPL(devm_cxl_port_ras_setup, "CXL"); 198 199 void cxl_handle_cor_ras(struct device *dev, void __iomem *ras_base) 200 { 201 void __iomem *addr; 202 u32 status; 203 204 if (!ras_base) 205 return; 206 207 addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET; 208 status = readl(addr); 209 if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) { 210 writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr); 211 trace_cxl_aer_correctable_error(to_cxl_memdev(dev), status); 212 } 213 } 214 215 /* CXL spec rev3.0 8.2.4.16.1 */ 216 static void header_log_copy(void __iomem *ras_base, u32 *log) 217 { 218 void __iomem *addr; 219 u32 *log_addr; 220 int i; 221 222 addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET; 223 log_addr = log; 224 225 for (i = 0; i < CXL_HEADERLOG_SIZE_U32; i++) { 226 *log_addr = readl(addr); 227 log_addr++; 228 addr += sizeof(u32); 229 } 230 } 231 232 /* 233 * Log the state of the RAS status registers and prepare them to log the 234 * next error status. Return 1 if reset needed. 235 */ 236 bool cxl_handle_ras(struct device *dev, void __iomem *ras_base) 237 { 238 u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {}; 239 void __iomem *addr; 240 u32 status; 241 u32 fe; 242 243 if (!ras_base) 244 return false; 245 246 addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET; 247 status = readl(addr); 248 if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK)) 249 return false; 250 251 /* If multiple errors, log header points to first error from ctrl reg */ 252 if (hweight32(status) > 1) { 253 void __iomem *rcc_addr = 254 ras_base + CXL_RAS_CAP_CONTROL_OFFSET; 255 256 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK, 257 readl(rcc_addr))); 258 } else { 259 fe = status; 260 } 261 262 header_log_copy(ras_base, hl); 263 trace_cxl_aer_uncorrectable_error(to_cxl_memdev(dev), status, fe, hl); 264 writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr); 265 266 return true; 267 } 268 269 void cxl_cor_error_detected(struct pci_dev *pdev) 270 { 271 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 272 struct cxl_memdev *cxlmd = cxlds->cxlmd; 273 struct device *dev = &cxlds->cxlmd->dev; 274 275 scoped_guard(device, dev) { 276 if (!dev->driver) { 277 dev_warn(&pdev->dev, 278 "%s: memdev disabled, abort error handling\n", 279 dev_name(dev)); 280 return; 281 } 282 283 if (cxlds->rcd) 284 cxl_handle_rdport_errors(cxlds); 285 286 cxl_handle_cor_ras(&cxlds->cxlmd->dev, cxlmd->endpoint->regs.ras); 287 } 288 } 289 EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, "CXL"); 290 291 pci_ers_result_t cxl_error_detected(struct pci_dev *pdev, 292 pci_channel_state_t state) 293 { 294 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev); 295 struct cxl_memdev *cxlmd = cxlds->cxlmd; 296 struct device *dev = &cxlmd->dev; 297 bool ue; 298 299 scoped_guard(device, dev) { 300 if (!dev->driver) { 301 dev_warn(&pdev->dev, 302 "%s: memdev disabled, abort error handling\n", 303 dev_name(dev)); 304 return PCI_ERS_RESULT_DISCONNECT; 305 } 306 307 if (cxlds->rcd) 308 cxl_handle_rdport_errors(cxlds); 309 /* 310 * A frozen channel indicates an impending reset which is fatal to 311 * CXL.mem operation, and will likely crash the system. On the off 312 * chance the situation is recoverable dump the status of the RAS 313 * capability registers and bounce the active state of the memdev. 314 */ 315 ue = cxl_handle_ras(&cxlds->cxlmd->dev, cxlmd->endpoint->regs.ras); 316 } 317 318 switch (state) { 319 case pci_channel_io_normal: 320 if (ue) { 321 device_release_driver(dev); 322 return PCI_ERS_RESULT_NEED_RESET; 323 } 324 return PCI_ERS_RESULT_CAN_RECOVER; 325 case pci_channel_io_frozen: 326 dev_warn(&pdev->dev, 327 "%s: frozen state error detected, disable CXL.mem\n", 328 dev_name(dev)); 329 device_release_driver(dev); 330 return PCI_ERS_RESULT_NEED_RESET; 331 case pci_channel_io_perm_failure: 332 dev_warn(&pdev->dev, 333 "failure state error detected, request disconnect\n"); 334 return PCI_ERS_RESULT_DISCONNECT; 335 } 336 return PCI_ERS_RESULT_NEED_RESET; 337 } 338 EXPORT_SYMBOL_NS_GPL(cxl_error_detected, "CXL"); 339