1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2025 AMD Corporation. All rights reserved. */
3
4 #include <linux/pci.h>
5 #include <linux/aer.h>
6 #include <cxl/event.h>
7 #include <cxlmem.h>
8 #include <cxlpci.h>
9 #include "trace.h"
10
11 /* Check that UCE header definition is maintained to keep ABI intact */
12 static_assert(CXL_HEADERLOG_TRACE_SIZE_U32 == 128,
13 "rasdaemon ABI requires exactly 128 u32s");
14
cxl_cper_trace_corr_port_prot_err(struct pci_dev * pdev,struct cxl_ras_capability_regs ras_cap)15 static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev,
16 struct cxl_ras_capability_regs ras_cap)
17 {
18 u32 status = ras_cap.cor_status & ~ras_cap.cor_mask;
19
20 trace_cxl_port_aer_correctable_error(&pdev->dev, status);
21 }
22
cxl_cper_trace_uncorr_port_prot_err(struct pci_dev * pdev,struct cxl_ras_capability_regs ras_cap)23 static void cxl_cper_trace_uncorr_port_prot_err(struct pci_dev *pdev,
24 struct cxl_ras_capability_regs ras_cap)
25 {
26 u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {};
27 u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask;
28 u32 fe;
29
30 if (hweight32(status) > 1)
31 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK,
32 ras_cap.cap_control));
33 else
34 fe = status;
35
36 memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE);
37 trace_cxl_port_aer_uncorrectable_error(&pdev->dev, status, fe, hl);
38 }
39
cxl_cper_trace_corr_prot_err(struct cxl_memdev * cxlmd,struct cxl_ras_capability_regs ras_cap)40 static void cxl_cper_trace_corr_prot_err(struct cxl_memdev *cxlmd,
41 struct cxl_ras_capability_regs ras_cap)
42 {
43 u32 status = ras_cap.cor_status & ~ras_cap.cor_mask;
44
45 trace_cxl_aer_correctable_error(cxlmd, status);
46 }
47
48 static void
cxl_cper_trace_uncorr_prot_err(struct cxl_memdev * cxlmd,struct cxl_ras_capability_regs ras_cap)49 cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd,
50 struct cxl_ras_capability_regs ras_cap)
51 {
52 u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {};
53 u32 status = ras_cap.uncor_status & ~ras_cap.uncor_mask;
54 u32 fe;
55
56 if (hweight32(status) > 1)
57 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK,
58 ras_cap.cap_control));
59 else
60 fe = status;
61
62 /*
63 * ras_cap.header_log[] holds CXL_HEADERLOG_SIZE_U32 (16) hardware
64 * dwords. Copy them into the front of a zero-filled
65 * CXL_HEADERLOG_TRACE_SIZE_U32 (128) u32 staging buffer so the trace
66 * event memcpy sees a full 512-byte source and the userspace ABI
67 * (rasdaemon) is preserved.
68 */
69 memcpy(hl, ras_cap.header_log, CXL_HEADERLOG_SIZE);
70 trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, hl);
71 }
72
match_memdev_by_parent(struct device * dev,const void * uport)73 static int match_memdev_by_parent(struct device *dev, const void *uport)
74 {
75 if (is_cxl_memdev(dev) && dev->parent == uport)
76 return 1;
77 return 0;
78 }
79
cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data * data)80 void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data)
81 {
82 unsigned int devfn = PCI_DEVFN(data->prot_err.agent_addr.device,
83 data->prot_err.agent_addr.function);
84 struct pci_dev *pdev __free(pci_dev_put) =
85 pci_get_domain_bus_and_slot(data->prot_err.agent_addr.segment,
86 data->prot_err.agent_addr.bus,
87 devfn);
88 struct cxl_memdev *cxlmd;
89 int port_type;
90
91 if (!pdev)
92 return;
93
94 port_type = pci_pcie_type(pdev);
95 if (port_type == PCI_EXP_TYPE_ROOT_PORT ||
96 port_type == PCI_EXP_TYPE_DOWNSTREAM ||
97 port_type == PCI_EXP_TYPE_UPSTREAM) {
98 if (data->severity == AER_CORRECTABLE)
99 cxl_cper_trace_corr_port_prot_err(pdev, data->ras_cap);
100 else
101 cxl_cper_trace_uncorr_port_prot_err(pdev, data->ras_cap);
102
103 return;
104 }
105
106 guard(device)(&pdev->dev);
107 if (!pdev->dev.driver)
108 return;
109
110 struct device *mem_dev __free(put_device) = bus_find_device(
111 &cxl_bus_type, NULL, pdev, match_memdev_by_parent);
112 if (!mem_dev)
113 return;
114
115 cxlmd = to_cxl_memdev(mem_dev);
116 if (data->severity == AER_CORRECTABLE)
117 cxl_cper_trace_corr_prot_err(cxlmd, data->ras_cap);
118 else
119 cxl_cper_trace_uncorr_prot_err(cxlmd, data->ras_cap);
120 }
121 EXPORT_SYMBOL_GPL(cxl_cper_handle_prot_err);
122
cxl_cper_prot_err_work_fn(struct work_struct * work)123 static void cxl_cper_prot_err_work_fn(struct work_struct *work)
124 {
125 struct cxl_cper_prot_err_work_data wd;
126
127 while (cxl_cper_prot_err_kfifo_get(&wd))
128 cxl_cper_handle_prot_err(&wd);
129 }
130 static DECLARE_WORK(cxl_cper_prot_err_work, cxl_cper_prot_err_work_fn);
131
cxl_ras_init(void)132 void cxl_ras_init(void)
133 {
134 cxl_cper_register_prot_err_work(&cxl_cper_prot_err_work);
135 }
136
cxl_ras_exit(void)137 void cxl_ras_exit(void)
138 {
139 cxl_cper_unregister_prot_err_work();
140 }
141
cxl_dport_map_ras(struct cxl_dport * dport)142 static void cxl_dport_map_ras(struct cxl_dport *dport)
143 {
144 struct cxl_register_map *map = &dport->reg_map;
145 struct device *dev = dport->dport_dev;
146
147 if (!map->component_map.ras.valid)
148 dev_dbg(dev, "RAS registers not found\n");
149 else if (cxl_map_component_regs(map, &dport->regs.component,
150 BIT(CXL_CM_CAP_CAP_ID_RAS)))
151 dev_dbg(dev, "Failed to map RAS capability.\n");
152 }
153
154 /**
155 * devm_cxl_dport_ras_setup - Setup CXL RAS report on this dport
156 * @dport: the cxl_dport that needs to be initialized
157 */
devm_cxl_dport_ras_setup(struct cxl_dport * dport)158 void devm_cxl_dport_ras_setup(struct cxl_dport *dport)
159 {
160 dport->reg_map.host = dport_to_host(dport);
161 cxl_dport_map_ras(dport);
162 }
163
devm_cxl_dport_rch_ras_setup(struct cxl_dport * dport)164 void devm_cxl_dport_rch_ras_setup(struct cxl_dport *dport)
165 {
166 struct pci_host_bridge *host_bridge;
167
168 if (!dev_is_pci(dport->dport_dev))
169 return;
170
171 devm_cxl_dport_ras_setup(dport);
172
173 host_bridge = to_pci_host_bridge(dport->dport_dev);
174 if (!host_bridge->native_aer)
175 return;
176
177 cxl_dport_map_rch_aer(dport);
178 cxl_disable_rch_root_ints(dport);
179 }
180 EXPORT_SYMBOL_NS_GPL(devm_cxl_dport_rch_ras_setup, "CXL");
181
devm_cxl_port_ras_setup(struct cxl_port * port)182 void devm_cxl_port_ras_setup(struct cxl_port *port)
183 {
184 struct cxl_register_map *map = &port->reg_map;
185
186 if (!map->component_map.ras.valid) {
187 dev_dbg(&port->dev, "RAS registers not found\n");
188 return;
189 }
190
191 map->host = &port->dev;
192 if (cxl_map_component_regs(map, &port->regs,
193 BIT(CXL_CM_CAP_CAP_ID_RAS)))
194 dev_dbg(&port->dev, "Failed to map RAS capability\n");
195 }
196 EXPORT_SYMBOL_NS_GPL(devm_cxl_port_ras_setup, "CXL");
197
cxl_handle_cor_ras(struct device * dev,void __iomem * ras_base)198 void cxl_handle_cor_ras(struct device *dev, void __iomem *ras_base)
199 {
200 void __iomem *addr;
201 u32 status;
202
203 if (!ras_base)
204 return;
205
206 addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET;
207 status = readl(addr);
208 if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) {
209 writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr);
210 trace_cxl_aer_correctable_error(to_cxl_memdev(dev), status);
211 }
212 }
213
214 /* CXL spec rev3.0 8.2.4.16.1 */
header_log_copy(void __iomem * ras_base,u32 * log)215 static void header_log_copy(void __iomem *ras_base, u32 *log)
216 {
217 void __iomem *addr;
218 u32 *log_addr;
219 int i;
220
221 addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET;
222 log_addr = log;
223
224 for (i = 0; i < CXL_HEADERLOG_SIZE_U32; i++) {
225 *log_addr = readl(addr);
226 log_addr++;
227 addr += sizeof(u32);
228 }
229 }
230
231 /*
232 * Log the state of the RAS status registers and prepare them to log the
233 * next error status. Return 1 if reset needed.
234 */
cxl_handle_ras(struct device * dev,void __iomem * ras_base)235 bool cxl_handle_ras(struct device *dev, void __iomem *ras_base)
236 {
237 u32 hl[CXL_HEADERLOG_TRACE_SIZE_U32] = {};
238 void __iomem *addr;
239 u32 status;
240 u32 fe;
241
242 if (!ras_base)
243 return false;
244
245 addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET;
246 status = readl(addr);
247 if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK))
248 return false;
249
250 /* If multiple errors, log header points to first error from ctrl reg */
251 if (hweight32(status) > 1) {
252 void __iomem *rcc_addr =
253 ras_base + CXL_RAS_CAP_CONTROL_OFFSET;
254
255 fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK,
256 readl(rcc_addr)));
257 } else {
258 fe = status;
259 }
260
261 header_log_copy(ras_base, hl);
262 trace_cxl_aer_uncorrectable_error(to_cxl_memdev(dev), status, fe, hl);
263 writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr);
264
265 return true;
266 }
267
cxl_cor_error_detected(struct pci_dev * pdev)268 void cxl_cor_error_detected(struct pci_dev *pdev)
269 {
270 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
271 struct cxl_memdev *cxlmd = cxlds->cxlmd;
272 struct device *dev = &cxlds->cxlmd->dev;
273
274 scoped_guard(device, dev) {
275 if (!dev->driver) {
276 dev_warn(&pdev->dev,
277 "%s: memdev disabled, abort error handling\n",
278 dev_name(dev));
279 return;
280 }
281
282 if (cxlds->rcd)
283 cxl_handle_rdport_errors(cxlds);
284
285 cxl_handle_cor_ras(&cxlds->cxlmd->dev, cxlmd->endpoint->regs.ras);
286 }
287 }
288 EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, "CXL");
289
cxl_error_detected(struct pci_dev * pdev,pci_channel_state_t state)290 pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
291 pci_channel_state_t state)
292 {
293 struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
294 struct cxl_memdev *cxlmd = cxlds->cxlmd;
295 struct device *dev = &cxlmd->dev;
296 bool ue;
297
298 scoped_guard(device, dev) {
299 if (!dev->driver) {
300 dev_warn(&pdev->dev,
301 "%s: memdev disabled, abort error handling\n",
302 dev_name(dev));
303 return PCI_ERS_RESULT_DISCONNECT;
304 }
305
306 if (cxlds->rcd)
307 cxl_handle_rdport_errors(cxlds);
308 /*
309 * A frozen channel indicates an impending reset which is fatal to
310 * CXL.mem operation, and will likely crash the system. On the off
311 * chance the situation is recoverable dump the status of the RAS
312 * capability registers and bounce the active state of the memdev.
313 */
314 ue = cxl_handle_ras(&cxlds->cxlmd->dev, cxlmd->endpoint->regs.ras);
315 }
316
317 switch (state) {
318 case pci_channel_io_normal:
319 if (ue) {
320 device_release_driver(dev);
321 return PCI_ERS_RESULT_NEED_RESET;
322 }
323 return PCI_ERS_RESULT_CAN_RECOVER;
324 case pci_channel_io_frozen:
325 dev_warn(&pdev->dev,
326 "%s: frozen state error detected, disable CXL.mem\n",
327 dev_name(dev));
328 device_release_driver(dev);
329 return PCI_ERS_RESULT_NEED_RESET;
330 case pci_channel_io_perm_failure:
331 dev_warn(&pdev->dev,
332 "failure state error detected, request disconnect\n");
333 return PCI_ERS_RESULT_DISCONNECT;
334 }
335 return PCI_ERS_RESULT_NEED_RESET;
336 }
337 EXPORT_SYMBOL_NS_GPL(cxl_error_detected, "CXL");
338