1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright IBM Corp. 2012
4 *
5 * Author(s):
6 * Jan Glauber <jang@linux.vnet.ibm.com>
7 */
8
9 #define pr_fmt(fmt) "zpci: " fmt
10
11 #include <linux/kernel.h>
12 #include <linux/pci.h>
13 #include <asm/pci_debug.h>
14 #include <asm/pci_dma.h>
15 #include <asm/sclp.h>
16
17 #include "pci_bus.h"
18 #include "pci_report.h"
19
20 /* Content Code Description for PCI Function Availability */
21 struct zpci_ccdf_avail {
22 u32 reserved1;
23 u32 fh; /* function handle */
24 u32 fid; /* function id */
25 u32 reserved2;
26 u32 reserved3;
27 u32 reserved4;
28 u32 reserved5;
29 u16 reserved6;
30 u16 pec; /* PCI event code */
31 } __packed;
32
ers_result_indicates_abort(pci_ers_result_t ers_res)33 static inline bool ers_result_indicates_abort(pci_ers_result_t ers_res)
34 {
35 switch (ers_res) {
36 case PCI_ERS_RESULT_CAN_RECOVER:
37 case PCI_ERS_RESULT_RECOVERED:
38 case PCI_ERS_RESULT_NEED_RESET:
39 case PCI_ERS_RESULT_NONE:
40 return false;
41 default:
42 return true;
43 }
44 }
45
is_driver_supported(struct pci_driver * driver)46 static bool is_driver_supported(struct pci_driver *driver)
47 {
48 if (!driver || !driver->err_handler)
49 return false;
50 if (!driver->err_handler->error_detected)
51 return false;
52 return true;
53 }
54
zpci_store_pci_error(struct pci_dev * pdev,struct zpci_ccdf_err * ccdf)55 static int zpci_store_pci_error(struct pci_dev *pdev,
56 struct zpci_ccdf_err *ccdf)
57 {
58 struct zpci_dev *zdev = to_zpci(pdev);
59 int i;
60
61 guard(mutex)(&zdev->pending_errs_lock);
62 if (!zdev->pending_errs.mediated_recovery)
63 return -EINVAL;
64
65 if (zdev->pending_errs.count >= ZPCI_ERR_PENDING_MAX) {
66 dev_warn_ratelimited(&pdev->dev,
67 "%s: Maximum number (%d) of pending error events queued\n",
68 pci_name(pdev),
69 ZPCI_ERR_PENDING_MAX);
70 return -ENOMEM;
71 }
72
73 i = zdev->pending_errs.tail % ZPCI_ERR_PENDING_MAX;
74 memcpy(&zdev->pending_errs.err[i], ccdf, sizeof(struct zpci_ccdf_err));
75 zdev->pending_errs.tail++;
76 zdev->pending_errs.count++;
77 return 0;
78 }
79
zpci_get_pending_error(struct zpci_dev * zdev,struct zpci_ccdf_err * ccdf)80 int zpci_get_pending_error(struct zpci_dev *zdev,
81 struct zpci_ccdf_err *ccdf)
82 {
83 int head;
84
85 guard(mutex)(&zdev->pending_errs_lock);
86
87 if (!zdev->pending_errs.count)
88 return -ENOMSG;
89
90 head = zdev->pending_errs.head % ZPCI_ERR_PENDING_MAX;
91 memcpy(ccdf, &zdev->pending_errs.err[head],
92 sizeof(struct zpci_ccdf_err));
93 zdev->pending_errs.head++;
94 zdev->pending_errs.count--;
95 return 0;
96 }
97 EXPORT_SYMBOL_GPL(zpci_get_pending_error);
98
zpci_start_mediated_recovery(struct zpci_dev * zdev)99 void zpci_start_mediated_recovery(struct zpci_dev *zdev)
100 {
101 guard(mutex)(&zdev->pending_errs_lock);
102 zdev->pending_errs.mediated_recovery = true;
103 }
104 EXPORT_SYMBOL_GPL(zpci_start_mediated_recovery);
105
zpci_stop_mediated_recovery(struct zpci_dev * zdev)106 void zpci_stop_mediated_recovery(struct zpci_dev *zdev)
107 {
108 guard(mutex)(&zdev->pending_errs_lock);
109 zdev->pending_errs.mediated_recovery = false;
110 if (zdev->pending_errs.count)
111 pr_info("Unhandled PCI error events count=%d for PCI function 0x%x\n",
112 zdev->pending_errs.count, zdev->fid);
113 memset(&zdev->pending_errs, 0, sizeof(struct zpci_ccdf_pending));
114 }
115 EXPORT_SYMBOL_GPL(zpci_stop_mediated_recovery);
116
zpci_event_notify_error_detected(struct pci_dev * pdev,struct pci_driver * driver)117 static pci_ers_result_t zpci_event_notify_error_detected(struct pci_dev *pdev,
118 struct pci_driver *driver)
119 {
120 pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT;
121
122 ers_res = driver->err_handler->error_detected(pdev, pdev->error_state);
123 pci_uevent_ers(pdev, ers_res);
124 if (ers_result_indicates_abort(ers_res))
125 pr_info("%s: Automatic recovery failed after initial reporting\n", pci_name(pdev));
126 else if (ers_res == PCI_ERS_RESULT_NEED_RESET)
127 pr_debug("%s: Driver needs reset to recover\n", pci_name(pdev));
128
129 return ers_res;
130 }
131
zpci_event_do_error_state_clear(struct pci_dev * pdev,struct pci_driver * driver)132 static pci_ers_result_t zpci_event_do_error_state_clear(struct pci_dev *pdev,
133 struct pci_driver *driver)
134 {
135 pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT;
136 struct zpci_dev *zdev = to_zpci(pdev);
137 int rc;
138
139 /* The underlying device may have been disabled by the event */
140 if (!zdev_enabled(zdev))
141 return PCI_ERS_RESULT_NEED_RESET;
142
143 pr_info("%s: Unblocking device access for examination\n", pci_name(pdev));
144 rc = zpci_reset_load_store_blocked(zdev);
145 if (rc) {
146 pr_err("%s: Unblocking device access failed\n", pci_name(pdev));
147 /* Let's try a full reset instead */
148 return PCI_ERS_RESULT_NEED_RESET;
149 }
150
151 if (driver->err_handler->mmio_enabled)
152 ers_res = driver->err_handler->mmio_enabled(pdev);
153 else
154 ers_res = PCI_ERS_RESULT_NONE;
155
156 if (ers_result_indicates_abort(ers_res)) {
157 pr_info("%s: Automatic recovery failed after MMIO re-enable\n",
158 pci_name(pdev));
159 return ers_res;
160 } else if (ers_res == PCI_ERS_RESULT_NEED_RESET) {
161 pr_debug("%s: Driver needs reset to recover\n", pci_name(pdev));
162 return ers_res;
163 }
164
165 pr_debug("%s: Unblocking DMA\n", pci_name(pdev));
166 rc = zpci_clear_error_state(zdev);
167 if (!rc) {
168 pdev->error_state = pci_channel_io_normal;
169 } else {
170 pr_err("%s: Unblocking DMA failed\n", pci_name(pdev));
171 /* Let's try a full reset instead */
172 return PCI_ERS_RESULT_NEED_RESET;
173 }
174
175 return ers_res;
176 }
177
zpci_event_do_reset(struct pci_dev * pdev,struct pci_driver * driver)178 static pci_ers_result_t zpci_event_do_reset(struct pci_dev *pdev,
179 struct pci_driver *driver)
180 {
181 pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT;
182
183 pr_info("%s: Initiating reset\n", pci_name(pdev));
184 if (zpci_hot_reset_device(to_zpci(pdev))) {
185 pr_err("%s: The reset request failed\n", pci_name(pdev));
186 return ers_res;
187 }
188 pdev->error_state = pci_channel_io_normal;
189
190 if (driver->err_handler->slot_reset)
191 ers_res = driver->err_handler->slot_reset(pdev);
192 else
193 ers_res = PCI_ERS_RESULT_NONE;
194
195 if (ers_result_indicates_abort(ers_res)) {
196 pr_info("%s: Automatic recovery failed after slot reset\n", pci_name(pdev));
197 return ers_res;
198 }
199
200 return ers_res;
201 }
202
203 /* zpci_event_attempt_error_recovery - Try to recover the given PCI function
204 * @pdev: PCI function to recover currently in the error state
205 *
206 * We follow the scheme outlined in Documentation/PCI/pci-error-recovery.rst.
207 * With the simplification that recovery always happens per function
208 * and the platform determines which functions are affected for
209 * multi-function devices.
210 */
zpci_event_attempt_error_recovery(struct pci_dev * pdev,struct zpci_ccdf_err * ccdf)211 static pci_ers_result_t zpci_event_attempt_error_recovery(struct pci_dev *pdev,
212 struct zpci_ccdf_err *ccdf)
213 {
214 pci_ers_result_t ers_res = PCI_ERS_RESULT_DISCONNECT;
215 struct zpci_dev *zdev = to_zpci(pdev);
216 bool mediated_recovery = false;
217 char *status_str = "success";
218 struct pci_driver *driver;
219 int rc;
220
221 /*
222 * Ensure that the PCI function is not removed concurrently, no driver
223 * is unbound or probed and that userspace can't access its
224 * configuration space while we perform recovery.
225 */
226 device_lock(&pdev->dev);
227 if (pdev->error_state == pci_channel_io_perm_failure) {
228 ers_res = PCI_ERS_RESULT_DISCONNECT;
229 status_str = "skipped (permanent failure)";
230 goto out_unlock;
231 }
232 pdev->error_state = pci_channel_io_frozen;
233
234 driver = to_pci_driver(pdev->dev.driver);
235 if (!is_driver_supported(driver)) {
236 if (!driver) {
237 pr_info("%s: Cannot be recovered because no driver is bound to the device\n",
238 pci_name(pdev));
239 status_str = "failed (no driver)";
240 } else {
241 pr_info("%s: The %s driver bound to the device does not support error recovery\n",
242 pci_name(pdev),
243 driver->name);
244 status_str = "failed (no driver support)";
245 }
246 goto out_unlock;
247 }
248
249 rc = zpci_store_pci_error(pdev, ccdf);
250 if (!rc || rc == -ENOMEM)
251 mediated_recovery = true;
252
253 ers_res = zpci_event_notify_error_detected(pdev, driver);
254 if (ers_result_indicates_abort(ers_res)) {
255 status_str = "failed (abort on detection)";
256 goto out_unlock;
257 }
258
259 if (mediated_recovery) {
260 pr_info("%s: Leaving recovery of pass-through device to user-space\n",
261 pci_name(pdev));
262 ers_res = PCI_ERS_RESULT_RECOVERED;
263 status_str = "in progress";
264 goto out_unlock;
265 }
266
267 if (ers_res != PCI_ERS_RESULT_NEED_RESET) {
268 ers_res = zpci_event_do_error_state_clear(pdev, driver);
269 if (ers_result_indicates_abort(ers_res)) {
270 status_str = "failed (abort on MMIO enable)";
271 goto out_unlock;
272 }
273 }
274
275 if (ers_res == PCI_ERS_RESULT_NEED_RESET)
276 ers_res = zpci_event_do_reset(pdev, driver);
277
278 /*
279 * ers_res can be PCI_ERS_RESULT_NONE either because the driver
280 * decided to return it, indicating that it abstains from voting
281 * on how to recover, or because it didn't implement the callback.
282 * Both cases assume, that if there is nothing else causing a
283 * disconnect, we recovered successfully.
284 */
285 if (ers_res == PCI_ERS_RESULT_NONE)
286 ers_res = PCI_ERS_RESULT_RECOVERED;
287
288 if (ers_res != PCI_ERS_RESULT_RECOVERED) {
289 pci_uevent_ers(pdev, PCI_ERS_RESULT_DISCONNECT);
290 pr_err("%s: Automatic recovery failed; operator intervention is required\n",
291 pci_name(pdev));
292 status_str = "failed (driver can't recover)";
293 goto out_unlock;
294 }
295
296 pr_info("%s: The device is ready to resume operations\n", pci_name(pdev));
297 if (driver->err_handler->resume)
298 driver->err_handler->resume(pdev);
299 pci_uevent_ers(pdev, PCI_ERS_RESULT_RECOVERED);
300 out_unlock:
301 zpci_report_status(zdev, pdev, "recovery", status_str);
302 device_unlock(&pdev->dev);
303
304 return ers_res;
305 }
306
307 /* zpci_event_io_failure - Report PCI channel failure state to driver
308 * @pdev: PCI function for which to report
309 * @es: PCI channel failure state to report
310 */
zpci_event_io_failure(struct pci_dev * pdev,pci_channel_state_t es,struct zpci_ccdf_err * ccdf)311 static void zpci_event_io_failure(struct pci_dev *pdev, pci_channel_state_t es,
312 struct zpci_ccdf_err *ccdf)
313 {
314 struct pci_driver *driver;
315
316 pci_dev_lock(pdev);
317 pdev->error_state = es;
318
319 zpci_store_pci_error(pdev, ccdf);
320 driver = to_pci_driver(pdev->dev.driver);
321 if (driver && driver->err_handler && driver->err_handler->error_detected)
322 driver->err_handler->error_detected(pdev, pdev->error_state);
323
324 pci_dev_unlock(pdev);
325 }
326
__zpci_event_print_error(struct pci_dev * pdev,struct zpci_ccdf_err * ccdf)327 static void __zpci_event_print_error(struct pci_dev *pdev, struct zpci_ccdf_err *ccdf)
328 {
329 pr_err("%s: Event 0x%x reports an error for PCI function 0x%x\n",
330 pdev ? pci_name(pdev) : "n/a", ccdf->pec, ccdf->fid);
331 }
332
__zpci_event_error(struct zpci_ccdf_err * ccdf)333 static void __zpci_event_error(struct zpci_ccdf_err *ccdf)
334 {
335 struct zpci_dev *zdev = get_zdev_by_fid(ccdf->fid);
336 struct pci_dev *pdev = NULL;
337 pci_ers_result_t ers_res;
338 u32 fh = 0;
339 int rc;
340
341 zpci_dbg(3, "err fid:%x, fh:%x, pec:%x\n",
342 ccdf->fid, ccdf->fh, ccdf->pec);
343 zpci_err("error CCDF:\n");
344 zpci_err_hex(ccdf, sizeof(*ccdf));
345
346 if (!zdev)
347 return __zpci_event_print_error(NULL, ccdf);
348
349 mutex_lock(&zdev->state_lock);
350 rc = clp_refresh_fh(zdev->fid, &fh);
351 if (rc)
352 goto no_pdev;
353 if (!fh || ccdf->fh != fh) {
354 /* Ignore events with stale handles */
355 zpci_dbg(3, "err fid:%x, fh:%x (stale %x)\n",
356 ccdf->fid, fh, ccdf->fh);
357 goto no_pdev;
358 }
359 zpci_update_fh(zdev, ccdf->fh);
360 if (zdev->zbus->bus)
361 pdev = pci_get_slot(zdev->zbus->bus, zdev->devfn);
362
363 __zpci_event_print_error(pdev, ccdf);
364
365 if (!pdev) {
366 zpci_report_status(zdev, NULL, "error event", "no pdev bound");
367 goto no_pdev;
368 }
369
370 switch (ccdf->pec) {
371 case 0x002a: /* Error event concerns FMB */
372 case 0x002b:
373 case 0x002c:
374 break;
375 case 0x0040: /* Service Action or Error Recovery Failed */
376 case 0x003b:
377 zpci_event_io_failure(pdev, pci_channel_io_perm_failure, ccdf);
378 break;
379 default: /* PCI function left in the error state attempt to recover */
380 ers_res = zpci_event_attempt_error_recovery(pdev, ccdf);
381 if (ers_res != PCI_ERS_RESULT_RECOVERED)
382 zpci_event_io_failure(pdev, pci_channel_io_perm_failure, ccdf);
383 break;
384 }
385 pci_dev_put(pdev);
386 no_pdev:
387 mutex_unlock(&zdev->state_lock);
388 zpci_zdev_put(zdev);
389 }
390
zpci_event_error(void * data)391 void zpci_event_error(void *data)
392 {
393 if (zpci_is_enabled())
394 __zpci_event_error(data);
395 }
396
zpci_event_hard_deconfigured(struct zpci_dev * zdev,u32 fh)397 static void zpci_event_hard_deconfigured(struct zpci_dev *zdev, u32 fh)
398 {
399 zpci_update_fh(zdev, fh);
400 /* Give the driver a hint that the function is
401 * already unusable.
402 */
403 zpci_bus_remove_device(zdev, true);
404 /* Even though the device is already gone we still
405 * need to free zPCI resources as part of the disable.
406 */
407 if (zdev_enabled(zdev))
408 zpci_disable_device(zdev);
409 zdev->state = ZPCI_FN_STATE_STANDBY;
410 }
411
zpci_event_reappear(struct zpci_dev * zdev)412 static void zpci_event_reappear(struct zpci_dev *zdev)
413 {
414 lockdep_assert_held(&zdev->state_lock);
415 /*
416 * The zdev is in the reserved state. This means that it was presumed to
417 * go away but there are still undropped references. Now, the platform
418 * announced its availability again. Bring back the lingering zdev
419 * to standby. This is safe because we hold a temporary reference
420 * now so that it won't go away. Account for the re-appearance of the
421 * underlying device by incrementing the reference count.
422 */
423 zdev->state = ZPCI_FN_STATE_STANDBY;
424 zpci_zdev_get(zdev);
425 zpci_dbg(1, "rea fid:%x, fh:%x\n", zdev->fid, zdev->fh);
426 }
427
zpci_event_avail_any_device(struct zpci_ccdf_avail * ccdf)428 static bool zpci_event_avail_any_device(struct zpci_ccdf_avail *ccdf)
429 {
430 /* 0x0306 - No handle or fid stored */
431 if (ccdf->pec != 0x0306)
432 return false;
433 /* 0x308 or 0x302 for multiple devices */
434 zpci_remove_reserved_devices();
435 zpci_scan_devices();
436 return true;
437 }
438
zpci_event_avail_new_device(struct zpci_ccdf_avail * ccdf)439 static void zpci_event_avail_new_device(struct zpci_ccdf_avail *ccdf)
440 {
441 struct zpci_dev *zdev;
442
443 switch (ccdf->pec) {
444 case 0x0301: /* Reserved|Standby -> Configured */
445 zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_CONFIGURED);
446 if (IS_ERR(zdev))
447 break;
448 if (zpci_add_device(zdev)) {
449 kfree(zdev);
450 break;
451 }
452 zpci_scan_configured_device(zdev, ccdf->fh);
453 break;
454 case 0x0302: /* Reserved -> Standby */
455 zdev = zpci_create_device(ccdf->fid, ccdf->fh, ZPCI_FN_STATE_STANDBY);
456 if (IS_ERR(zdev))
457 break;
458 if (zpci_add_device(zdev)) {
459 kfree(zdev);
460 break;
461 }
462 break;
463 }
464 }
465
zpci_event_avail_existing_device(struct zpci_dev * zdev,struct zpci_ccdf_avail * ccdf)466 static void zpci_event_avail_existing_device(struct zpci_dev *zdev, struct zpci_ccdf_avail *ccdf)
467 {
468 enum zpci_state state;
469
470 switch (ccdf->pec) {
471 case 0x0301: /* Reserved|Standby -> Configured */
472 if (zdev->state == ZPCI_FN_STATE_RESERVED)
473 zpci_event_reappear(zdev);
474 /* the configuration request may be stale */
475 else if (zdev->state != ZPCI_FN_STATE_STANDBY)
476 break;
477 zdev->state = ZPCI_FN_STATE_CONFIGURED;
478 zpci_scan_configured_device(zdev, ccdf->fh);
479 break;
480 case 0x0302: /* Reserved -> Standby */
481 if (zdev->state == ZPCI_FN_STATE_RESERVED)
482 zpci_event_reappear(zdev);
483 zpci_update_fh(zdev, ccdf->fh);
484 break;
485 case 0x0303: /* Deconfiguration requested */
486 /* The event may have been queued before we configured
487 * the device.
488 */
489 if (zdev->state != ZPCI_FN_STATE_CONFIGURED)
490 break;
491 zpci_update_fh(zdev, ccdf->fh);
492 zpci_deconfigure_device(zdev);
493 break;
494 case 0x0304: /* Configured -> Standby|Reserved */
495 /* The event may have been queued before we configured
496 * the device.:
497 */
498 if (zdev->state == ZPCI_FN_STATE_CONFIGURED)
499 zpci_event_hard_deconfigured(zdev, ccdf->fh);
500 /* The 0x0304 event may immediately reserve the device */
501 if (!clp_get_state(zdev->fid, &state) &&
502 state == ZPCI_FN_STATE_RESERVED) {
503 zpci_device_reserved(zdev);
504 }
505 break;
506 case 0x0308: /* Standby -> Reserved */
507 zpci_device_reserved(zdev);
508 break;
509 }
510 }
511
zpci_event_availability(void * data)512 void zpci_event_availability(void *data)
513 {
514 struct zpci_ccdf_avail *ccdf = data;
515 struct zpci_dev *zdev;
516
517 if (!zpci_is_enabled())
518 return;
519 zpci_dbg(3, "avl fid:%x, fh:%x, pec:%x\n",
520 ccdf->fid, ccdf->fh, ccdf->pec);
521 if (zpci_event_avail_any_device(ccdf))
522 return;
523 zdev = get_zdev_by_fid(ccdf->fid);
524 if (!zdev)
525 return zpci_event_avail_new_device(ccdf);
526 mutex_lock(&zdev->state_lock);
527 zpci_event_avail_existing_device(zdev, ccdf);
528 mutex_unlock(&zdev->state_lock);
529 zpci_zdev_put(zdev);
530 }
531