xref: /linux/drivers/misc/genwqe/card_base.c (revision 505b08777d78868e6c47051a4e7f011718a3aba1)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /**
3  * IBM Accelerator Family 'GenWQE'
4  *
5  * (C) Copyright IBM Corp. 2013
6  *
7  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
8  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
9  * Author: Michael Jung <mijung@gmx.net>
10  * Author: Michael Ruettger <michael@ibmra.de>
11  */
12 
13 /*
14  * Module initialization and PCIe setup. Card health monitoring and
15  * recovery functionality. Character device creation and deletion are
16  * controlled from here.
17  */
18 
19 #include <linux/types.h>
20 #include <linux/pci.h>
21 #include <linux/err.h>
22 #include <linux/aer.h>
23 #include <linux/string.h>
24 #include <linux/sched.h>
25 #include <linux/wait.h>
26 #include <linux/delay.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/module.h>
29 #include <linux/notifier.h>
30 #include <linux/device.h>
31 #include <linux/log2.h>
32 
33 #include "card_base.h"
34 #include "card_ddcb.h"
35 
36 MODULE_AUTHOR("Frank Haverkamp <haver@linux.vnet.ibm.com>");
37 MODULE_AUTHOR("Michael Ruettger <michael@ibmra.de>");
38 MODULE_AUTHOR("Joerg-Stephan Vogt <jsvogt@de.ibm.com>");
39 MODULE_AUTHOR("Michael Jung <mijung@gmx.net>");
40 
41 MODULE_DESCRIPTION("GenWQE Card");
42 MODULE_VERSION(DRV_VERSION);
43 MODULE_LICENSE("GPL");
44 
45 static char genwqe_driver_name[] = GENWQE_DEVNAME;
46 static struct class *class_genwqe;
47 static struct dentry *debugfs_genwqe;
48 static struct genwqe_dev *genwqe_devices[GENWQE_CARD_NO_MAX];
49 
50 /* PCI structure for identifying device by PCI vendor and device ID */
51 static const struct pci_device_id genwqe_device_table[] = {
52 	{ .vendor      = PCI_VENDOR_ID_IBM,
53 	  .device      = PCI_DEVICE_GENWQE,
54 	  .subvendor   = PCI_SUBVENDOR_ID_IBM,
55 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
56 	  .class       = (PCI_CLASSCODE_GENWQE5 << 8),
57 	  .class_mask  = ~0,
58 	  .driver_data = 0 },
59 
60 	/* Initial SR-IOV bring-up image */
61 	{ .vendor      = PCI_VENDOR_ID_IBM,
62 	  .device      = PCI_DEVICE_GENWQE,
63 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
64 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,
65 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
66 	  .class_mask  = ~0,
67 	  .driver_data = 0 },
68 
69 	{ .vendor      = PCI_VENDOR_ID_IBM,  /* VF Vendor ID */
70 	  .device      = 0x0000,  /* VF Device ID */
71 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
72 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,
73 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
74 	  .class_mask  = ~0,
75 	  .driver_data = 0 },
76 
77 	/* Fixed up image */
78 	{ .vendor      = PCI_VENDOR_ID_IBM,
79 	  .device      = PCI_DEVICE_GENWQE,
80 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
81 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
82 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
83 	  .class_mask  = ~0,
84 	  .driver_data = 0 },
85 
86 	{ .vendor      = PCI_VENDOR_ID_IBM,  /* VF Vendor ID */
87 	  .device      = 0x0000,  /* VF Device ID */
88 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
89 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
90 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
91 	  .class_mask  = ~0,
92 	  .driver_data = 0 },
93 
94 	/* Even one more ... */
95 	{ .vendor      = PCI_VENDOR_ID_IBM,
96 	  .device      = PCI_DEVICE_GENWQE,
97 	  .subvendor   = PCI_SUBVENDOR_ID_IBM,
98 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_NEW,
99 	  .class       = (PCI_CLASSCODE_GENWQE5 << 8),
100 	  .class_mask  = ~0,
101 	  .driver_data = 0 },
102 
103 	{ 0, }			/* 0 terminated list. */
104 };
105 
106 MODULE_DEVICE_TABLE(pci, genwqe_device_table);
107 
108 /**
109  * genwqe_dev_alloc() - Create and prepare a new card descriptor
110  *
111  * Return: Pointer to card descriptor, or ERR_PTR(err) on error
112  */
113 static struct genwqe_dev *genwqe_dev_alloc(void)
114 {
115 	unsigned int i = 0, j;
116 	struct genwqe_dev *cd;
117 
118 	for (i = 0; i < GENWQE_CARD_NO_MAX; i++) {
119 		if (genwqe_devices[i] == NULL)
120 			break;
121 	}
122 	if (i >= GENWQE_CARD_NO_MAX)
123 		return ERR_PTR(-ENODEV);
124 
125 	cd = kzalloc(sizeof(struct genwqe_dev), GFP_KERNEL);
126 	if (!cd)
127 		return ERR_PTR(-ENOMEM);
128 
129 	cd->card_idx = i;
130 	cd->class_genwqe = class_genwqe;
131 	cd->debugfs_genwqe = debugfs_genwqe;
132 
133 	/*
134 	 * This comes from kernel config option and can be overritten via
135 	 * debugfs.
136 	 */
137 	cd->use_platform_recovery = CONFIG_GENWQE_PLATFORM_ERROR_RECOVERY;
138 
139 	init_waitqueue_head(&cd->queue_waitq);
140 
141 	spin_lock_init(&cd->file_lock);
142 	INIT_LIST_HEAD(&cd->file_list);
143 
144 	cd->card_state = GENWQE_CARD_UNUSED;
145 	spin_lock_init(&cd->print_lock);
146 
147 	cd->ddcb_software_timeout = GENWQE_DDCB_SOFTWARE_TIMEOUT;
148 	cd->kill_timeout = GENWQE_KILL_TIMEOUT;
149 
150 	for (j = 0; j < GENWQE_MAX_VFS; j++)
151 		cd->vf_jobtimeout_msec[j] = GENWQE_VF_JOBTIMEOUT_MSEC;
152 
153 	genwqe_devices[i] = cd;
154 	return cd;
155 }
156 
157 static void genwqe_dev_free(struct genwqe_dev *cd)
158 {
159 	if (!cd)
160 		return;
161 
162 	genwqe_devices[cd->card_idx] = NULL;
163 	kfree(cd);
164 }
165 
166 /**
167  * genwqe_bus_reset() - Card recovery
168  * @cd: GenWQE device information
169  *
170  * pci_reset_function() will recover the device and ensure that the
171  * registers are accessible again when it completes with success. If
172  * not, the card will stay dead and registers will be unaccessible
173  * still.
174  */
175 static int genwqe_bus_reset(struct genwqe_dev *cd)
176 {
177 	int rc = 0;
178 	struct pci_dev *pci_dev = cd->pci_dev;
179 	void __iomem *mmio;
180 
181 	if (cd->err_inject & GENWQE_INJECT_BUS_RESET_FAILURE)
182 		return -EIO;
183 
184 	mmio = cd->mmio;
185 	cd->mmio = NULL;
186 	pci_iounmap(pci_dev, mmio);
187 
188 	pci_release_mem_regions(pci_dev);
189 
190 	/*
191 	 * Firmware/BIOS might change memory mapping during bus reset.
192 	 * Settings like enable bus-mastering, ... are backuped and
193 	 * restored by the pci_reset_function().
194 	 */
195 	dev_dbg(&pci_dev->dev, "[%s] pci_reset function ...\n", __func__);
196 	rc = pci_reset_function(pci_dev);
197 	if (rc) {
198 		dev_err(&pci_dev->dev,
199 			"[%s] err: failed reset func (rc %d)\n", __func__, rc);
200 		return rc;
201 	}
202 	dev_dbg(&pci_dev->dev, "[%s] done with rc=%d\n", __func__, rc);
203 
204 	/*
205 	 * Here is the right spot to clear the register read
206 	 * failure. pci_bus_reset() does this job in real systems.
207 	 */
208 	cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |
209 			    GENWQE_INJECT_GFIR_FATAL |
210 			    GENWQE_INJECT_GFIR_INFO);
211 
212 	rc = pci_request_mem_regions(pci_dev, genwqe_driver_name);
213 	if (rc) {
214 		dev_err(&pci_dev->dev,
215 			"[%s] err: request bars failed (%d)\n", __func__, rc);
216 		return -EIO;
217 	}
218 
219 	cd->mmio = pci_iomap(pci_dev, 0, 0);
220 	if (cd->mmio == NULL) {
221 		dev_err(&pci_dev->dev,
222 			"[%s] err: mapping BAR0 failed\n", __func__);
223 		return -ENOMEM;
224 	}
225 	return 0;
226 }
227 
228 /*
229  * Hardware circumvention section. Certain bitstreams in our test-lab
230  * had different kinds of problems. Here is where we adjust those
231  * bitstreams to function will with this version of our device driver.
232  *
233  * Thise circumventions are applied to the physical function only.
234  * The magical numbers below are identifying development/manufacturing
235  * versions of the bitstream used on the card.
236  *
237  * Turn off error reporting for old/manufacturing images.
238  */
239 
240 bool genwqe_need_err_masking(struct genwqe_dev *cd)
241 {
242 	return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;
243 }
244 
245 static void genwqe_tweak_hardware(struct genwqe_dev *cd)
246 {
247 	struct pci_dev *pci_dev = cd->pci_dev;
248 
249 	/* Mask FIRs for development images */
250 	if (((cd->slu_unitcfg & 0xFFFF0ull) >= 0x32000ull) &&
251 	    ((cd->slu_unitcfg & 0xFFFF0ull) <= 0x33250ull)) {
252 		dev_warn(&pci_dev->dev,
253 			 "FIRs masked due to bitstream %016llx.%016llx\n",
254 			 cd->slu_unitcfg, cd->app_unitcfg);
255 
256 		__genwqe_writeq(cd, IO_APP_SEC_LEM_DEBUG_OVR,
257 				0xFFFFFFFFFFFFFFFFull);
258 
259 		__genwqe_writeq(cd, IO_APP_ERR_ACT_MASK,
260 				0x0000000000000000ull);
261 	}
262 }
263 
264 /**
265  * genwqe_recovery_on_fatal_gfir_required() - Version depended actions
266  * @cd: GenWQE device information
267  *
268  * Bitstreams older than 2013-02-17 have a bug where fatal GFIRs must
269  * be ignored. This is e.g. true for the bitstream we gave to the card
270  * manufacturer, but also for some old bitstreams we released to our
271  * test-lab.
272  */
273 int genwqe_recovery_on_fatal_gfir_required(struct genwqe_dev *cd)
274 {
275 	return (cd->slu_unitcfg & 0xFFFF0ull) >= 0x32170ull;
276 }
277 
278 int genwqe_flash_readback_fails(struct genwqe_dev *cd)
279 {
280 	return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;
281 }
282 
283 /**
284  * genwqe_T_psec() - Calculate PF/VF timeout register content
285  * @cd: GenWQE device information
286  *
287  * Note: From a design perspective it turned out to be a bad idea to
288  * use codes here to specifiy the frequency/speed values. An old
289  * driver cannot understand new codes and is therefore always a
290  * problem. Better is to measure out the value or put the
291  * speed/frequency directly into a register which is always a valid
292  * value for old as well as for new software.
293  */
294 /* T = 1/f */
295 static int genwqe_T_psec(struct genwqe_dev *cd)
296 {
297 	u16 speed;	/* 1/f -> 250,  200,  166,  175 */
298 	static const int T[] = { 4000, 5000, 6000, 5714 };
299 
300 	speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);
301 	if (speed >= ARRAY_SIZE(T))
302 		return -1;	/* illegal value */
303 
304 	return T[speed];
305 }
306 
307 /**
308  * genwqe_setup_pf_jtimer() - Setup PF hardware timeouts for DDCB execution
309  * @cd: GenWQE device information
310  *
311  * Do this _after_ card_reset() is called. Otherwise the values will
312  * vanish. The settings need to be done when the queues are inactive.
313  *
314  * The max. timeout value is 2^(10+x) * T (6ns for 166MHz) * 15/16.
315  * The min. timeout value is 2^(10+x) * T (6ns for 166MHz) * 14/16.
316  */
317 static bool genwqe_setup_pf_jtimer(struct genwqe_dev *cd)
318 {
319 	u32 T = genwqe_T_psec(cd);
320 	u64 x;
321 
322 	if (GENWQE_PF_JOBTIMEOUT_MSEC == 0)
323 		return false;
324 
325 	/* PF: large value needed, flash update 2sec per block */
326 	x = ilog2(GENWQE_PF_JOBTIMEOUT_MSEC *
327 		  16000000000uL/(T * 15)) - 10;
328 
329 	genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
330 			  0xff00 | (x & 0xff), 0);
331 	return true;
332 }
333 
334 /**
335  * genwqe_setup_vf_jtimer() - Setup VF hardware timeouts for DDCB execution
336  * @cd: GenWQE device information
337  */
338 static bool genwqe_setup_vf_jtimer(struct genwqe_dev *cd)
339 {
340 	struct pci_dev *pci_dev = cd->pci_dev;
341 	unsigned int vf;
342 	u32 T = genwqe_T_psec(cd);
343 	u64 x;
344 	int totalvfs;
345 
346 	totalvfs = pci_sriov_get_totalvfs(pci_dev);
347 	if (totalvfs <= 0)
348 		return false;
349 
350 	for (vf = 0; vf < totalvfs; vf++) {
351 
352 		if (cd->vf_jobtimeout_msec[vf] == 0)
353 			continue;
354 
355 		x = ilog2(cd->vf_jobtimeout_msec[vf] *
356 			  16000000000uL/(T * 15)) - 10;
357 
358 		genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
359 				  0xff00 | (x & 0xff), vf + 1);
360 	}
361 	return true;
362 }
363 
364 static int genwqe_ffdc_buffs_alloc(struct genwqe_dev *cd)
365 {
366 	unsigned int type, e = 0;
367 
368 	for (type = 0; type < GENWQE_DBG_UNITS; type++) {
369 		switch (type) {
370 		case GENWQE_DBG_UNIT0:
371 			e = genwqe_ffdc_buff_size(cd, 0);
372 			break;
373 		case GENWQE_DBG_UNIT1:
374 			e = genwqe_ffdc_buff_size(cd, 1);
375 			break;
376 		case GENWQE_DBG_UNIT2:
377 			e = genwqe_ffdc_buff_size(cd, 2);
378 			break;
379 		case GENWQE_DBG_REGS:
380 			e = GENWQE_FFDC_REGS;
381 			break;
382 		}
383 
384 		/* currently support only the debug units mentioned here */
385 		cd->ffdc[type].entries = e;
386 		cd->ffdc[type].regs =
387 			kmalloc_array(e, sizeof(struct genwqe_reg),
388 				      GFP_KERNEL);
389 		/*
390 		 * regs == NULL is ok, the using code treats this as no regs,
391 		 * Printing warning is ok in this case.
392 		 */
393 	}
394 	return 0;
395 }
396 
397 static void genwqe_ffdc_buffs_free(struct genwqe_dev *cd)
398 {
399 	unsigned int type;
400 
401 	for (type = 0; type < GENWQE_DBG_UNITS; type++) {
402 		kfree(cd->ffdc[type].regs);
403 		cd->ffdc[type].regs = NULL;
404 	}
405 }
406 
407 static int genwqe_read_ids(struct genwqe_dev *cd)
408 {
409 	int err = 0;
410 	int slu_id;
411 	struct pci_dev *pci_dev = cd->pci_dev;
412 
413 	cd->slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);
414 	if (cd->slu_unitcfg == IO_ILLEGAL_VALUE) {
415 		dev_err(&pci_dev->dev,
416 			"err: SLUID=%016llx\n", cd->slu_unitcfg);
417 		err = -EIO;
418 		goto out_err;
419 	}
420 
421 	slu_id = genwqe_get_slu_id(cd);
422 	if (slu_id < GENWQE_SLU_ARCH_REQ || slu_id == 0xff) {
423 		dev_err(&pci_dev->dev,
424 			"err: incompatible SLU Architecture %u\n", slu_id);
425 		err = -ENOENT;
426 		goto out_err;
427 	}
428 
429 	cd->app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);
430 	if (cd->app_unitcfg == IO_ILLEGAL_VALUE) {
431 		dev_err(&pci_dev->dev,
432 			"err: APPID=%016llx\n", cd->app_unitcfg);
433 		err = -EIO;
434 		goto out_err;
435 	}
436 	genwqe_read_app_id(cd, cd->app_name, sizeof(cd->app_name));
437 
438 	/*
439 	 * Is access to all registers possible? If we are a VF the
440 	 * answer is obvious. If we run fully virtualized, we need to
441 	 * check if we can access all registers. If we do not have
442 	 * full access we will cause an UR and some informational FIRs
443 	 * in the PF, but that should not harm.
444 	 */
445 	if (pci_dev->is_virtfn)
446 		cd->is_privileged = 0;
447 	else
448 		cd->is_privileged = (__genwqe_readq(cd, IO_SLU_BITSTREAM)
449 				     != IO_ILLEGAL_VALUE);
450 
451  out_err:
452 	return err;
453 }
454 
455 static int genwqe_start(struct genwqe_dev *cd)
456 {
457 	int err;
458 	struct pci_dev *pci_dev = cd->pci_dev;
459 
460 	err = genwqe_read_ids(cd);
461 	if (err)
462 		return err;
463 
464 	if (genwqe_is_privileged(cd)) {
465 		/* do this after the tweaks. alloc fail is acceptable */
466 		genwqe_ffdc_buffs_alloc(cd);
467 		genwqe_stop_traps(cd);
468 
469 		/* Collect registers e.g. FIRs, UNITIDs, traces ... */
470 		genwqe_read_ffdc_regs(cd, cd->ffdc[GENWQE_DBG_REGS].regs,
471 				      cd->ffdc[GENWQE_DBG_REGS].entries, 0);
472 
473 		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT0,
474 				      cd->ffdc[GENWQE_DBG_UNIT0].regs,
475 				      cd->ffdc[GENWQE_DBG_UNIT0].entries);
476 
477 		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT1,
478 				      cd->ffdc[GENWQE_DBG_UNIT1].regs,
479 				      cd->ffdc[GENWQE_DBG_UNIT1].entries);
480 
481 		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT2,
482 				      cd->ffdc[GENWQE_DBG_UNIT2].regs,
483 				      cd->ffdc[GENWQE_DBG_UNIT2].entries);
484 
485 		genwqe_start_traps(cd);
486 
487 		if (cd->card_state == GENWQE_CARD_FATAL_ERROR) {
488 			dev_warn(&pci_dev->dev,
489 				 "[%s] chip reload/recovery!\n", __func__);
490 
491 			/*
492 			 * Stealth Mode: Reload chip on either hot
493 			 * reset or PERST.
494 			 */
495 			cd->softreset = 0x7Cull;
496 			__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,
497 				       cd->softreset);
498 
499 			err = genwqe_bus_reset(cd);
500 			if (err != 0) {
501 				dev_err(&pci_dev->dev,
502 					"[%s] err: bus reset failed!\n",
503 					__func__);
504 				goto out;
505 			}
506 
507 			/*
508 			 * Re-read the IDs because
509 			 * it could happen that the bitstream load
510 			 * failed!
511 			 */
512 			err = genwqe_read_ids(cd);
513 			if (err)
514 				goto out;
515 		}
516 	}
517 
518 	err = genwqe_setup_service_layer(cd);  /* does a reset to the card */
519 	if (err != 0) {
520 		dev_err(&pci_dev->dev,
521 			"[%s] err: could not setup servicelayer!\n", __func__);
522 		err = -ENODEV;
523 		goto out;
524 	}
525 
526 	if (genwqe_is_privileged(cd)) {	 /* code is running _after_ reset */
527 		genwqe_tweak_hardware(cd);
528 
529 		genwqe_setup_pf_jtimer(cd);
530 		genwqe_setup_vf_jtimer(cd);
531 	}
532 
533 	err = genwqe_device_create(cd);
534 	if (err < 0) {
535 		dev_err(&pci_dev->dev,
536 			"err: chdev init failed! (err=%d)\n", err);
537 		goto out_release_service_layer;
538 	}
539 	return 0;
540 
541  out_release_service_layer:
542 	genwqe_release_service_layer(cd);
543  out:
544 	if (genwqe_is_privileged(cd))
545 		genwqe_ffdc_buffs_free(cd);
546 	return -EIO;
547 }
548 
549 /**
550  * genwqe_stop() - Stop card operation
551  * @cd: GenWQE device information
552  *
553  * Recovery notes:
554  *   As long as genwqe_thread runs we might access registers during
555  *   error data capture. Same is with the genwqe_health_thread.
556  *   When genwqe_bus_reset() fails this function might called two times:
557  *   first by the genwqe_health_thread() and later by genwqe_remove() to
558  *   unbind the device. We must be able to survive that.
559  *
560  * This function must be robust enough to be called twice.
561  */
562 static int genwqe_stop(struct genwqe_dev *cd)
563 {
564 	genwqe_finish_queue(cd);	    /* no register access */
565 	genwqe_device_remove(cd);	    /* device removed, procs killed */
566 	genwqe_release_service_layer(cd);   /* here genwqe_thread is stopped */
567 
568 	if (genwqe_is_privileged(cd)) {
569 		pci_disable_sriov(cd->pci_dev);	/* access pci config space */
570 		genwqe_ffdc_buffs_free(cd);
571 	}
572 
573 	return 0;
574 }
575 
576 /**
577  * genwqe_recover_card() - Try to recover the card if it is possible
578  * @cd: GenWQE device information
579  * @fatal_err: Indicate whether to attempt soft reset
580  *
581  * If fatal_err is set no register access is possible anymore. It is
582  * likely that genwqe_start fails in that situation. Proper error
583  * handling is required in this case.
584  *
585  * genwqe_bus_reset() will cause the pci code to call genwqe_remove()
586  * and later genwqe_probe() for all virtual functions.
587  */
588 static int genwqe_recover_card(struct genwqe_dev *cd, int fatal_err)
589 {
590 	int rc;
591 	struct pci_dev *pci_dev = cd->pci_dev;
592 
593 	genwqe_stop(cd);
594 
595 	/*
596 	 * Make sure chip is not reloaded to maintain FFDC. Write SLU
597 	 * Reset Register, CPLDReset field to 0.
598 	 */
599 	if (!fatal_err) {
600 		cd->softreset = 0x70ull;
601 		__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, cd->softreset);
602 	}
603 
604 	rc = genwqe_bus_reset(cd);
605 	if (rc != 0) {
606 		dev_err(&pci_dev->dev,
607 			"[%s] err: card recovery impossible!\n", __func__);
608 		return rc;
609 	}
610 
611 	rc = genwqe_start(cd);
612 	if (rc < 0) {
613 		dev_err(&pci_dev->dev,
614 			"[%s] err: failed to launch device!\n", __func__);
615 		return rc;
616 	}
617 	return 0;
618 }
619 
620 static int genwqe_health_check_cond(struct genwqe_dev *cd, u64 *gfir)
621 {
622 	*gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
623 	return (*gfir & GFIR_ERR_TRIGGER) &&
624 		genwqe_recovery_on_fatal_gfir_required(cd);
625 }
626 
627 /**
628  * genwqe_fir_checking() - Check the fault isolation registers of the card
629  * @cd: GenWQE device information
630  *
631  * If this code works ok, can be tried out with help of the genwqe_poke tool:
632  *   sudo ./tools/genwqe_poke 0x8 0xfefefefefef
633  *
634  * Now the relevant FIRs/sFIRs should be printed out and the driver should
635  * invoke recovery (devices are removed and readded).
636  */
637 static u64 genwqe_fir_checking(struct genwqe_dev *cd)
638 {
639 	int j, iterations = 0;
640 	u64 mask, fir, fec, uid, gfir, gfir_masked, sfir, sfec;
641 	u32 fir_addr, fir_clr_addr, fec_addr, sfir_addr, sfec_addr;
642 	struct pci_dev *pci_dev = cd->pci_dev;
643 
644  healthMonitor:
645 	iterations++;
646 	if (iterations > 16) {
647 		dev_err(&pci_dev->dev, "* exit looping after %d times\n",
648 			iterations);
649 		goto fatal_error;
650 	}
651 
652 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
653 	if (gfir != 0x0)
654 		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n",
655 				    IO_SLC_CFGREG_GFIR, gfir);
656 	if (gfir == IO_ILLEGAL_VALUE)
657 		goto fatal_error;
658 
659 	/*
660 	 * Avoid printing when to GFIR bit is on prevents contignous
661 	 * printout e.g. for the following bug:
662 	 *   FIR set without a 2ndary FIR/FIR cannot be cleared
663 	 * Comment out the following if to get the prints:
664 	 */
665 	if (gfir == 0)
666 		return 0;
667 
668 	gfir_masked = gfir & GFIR_ERR_TRIGGER;  /* fatal errors */
669 
670 	for (uid = 0; uid < GENWQE_MAX_UNITS; uid++) { /* 0..2 in zEDC */
671 
672 		/* read the primary FIR (pfir) */
673 		fir_addr = (uid << 24) + 0x08;
674 		fir = __genwqe_readq(cd, fir_addr);
675 		if (fir == 0x0)
676 			continue;  /* no error in this unit */
677 
678 		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fir_addr, fir);
679 		if (fir == IO_ILLEGAL_VALUE)
680 			goto fatal_error;
681 
682 		/* read primary FEC */
683 		fec_addr = (uid << 24) + 0x18;
684 		fec = __genwqe_readq(cd, fec_addr);
685 
686 		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fec_addr, fec);
687 		if (fec == IO_ILLEGAL_VALUE)
688 			goto fatal_error;
689 
690 		for (j = 0, mask = 1ULL; j < 64; j++, mask <<= 1) {
691 
692 			/* secondary fir empty, skip it */
693 			if ((fir & mask) == 0x0)
694 				continue;
695 
696 			sfir_addr = (uid << 24) + 0x100 + 0x08 * j;
697 			sfir = __genwqe_readq(cd, sfir_addr);
698 
699 			if (sfir == IO_ILLEGAL_VALUE)
700 				goto fatal_error;
701 			dev_err(&pci_dev->dev,
702 				"* 0x%08x 0x%016llx\n", sfir_addr, sfir);
703 
704 			sfec_addr = (uid << 24) + 0x300 + 0x08 * j;
705 			sfec = __genwqe_readq(cd, sfec_addr);
706 
707 			if (sfec == IO_ILLEGAL_VALUE)
708 				goto fatal_error;
709 			dev_err(&pci_dev->dev,
710 				"* 0x%08x 0x%016llx\n", sfec_addr, sfec);
711 
712 			gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
713 			if (gfir == IO_ILLEGAL_VALUE)
714 				goto fatal_error;
715 
716 			/* gfir turned on during routine! get out and
717 			   start over. */
718 			if ((gfir_masked == 0x0) &&
719 			    (gfir & GFIR_ERR_TRIGGER)) {
720 				goto healthMonitor;
721 			}
722 
723 			/* do not clear if we entered with a fatal gfir */
724 			if (gfir_masked == 0x0) {
725 
726 				/* NEW clear by mask the logged bits */
727 				sfir_addr = (uid << 24) + 0x100 + 0x08 * j;
728 				__genwqe_writeq(cd, sfir_addr, sfir);
729 
730 				dev_dbg(&pci_dev->dev,
731 					"[HM] Clearing  2ndary FIR 0x%08x with 0x%016llx\n",
732 					sfir_addr, sfir);
733 
734 				/*
735 				 * note, these cannot be error-Firs
736 				 * since gfir_masked is 0 after sfir
737 				 * was read. Also, it is safe to do
738 				 * this write if sfir=0. Still need to
739 				 * clear the primary. This just means
740 				 * there is no secondary FIR.
741 				 */
742 
743 				/* clear by mask the logged bit. */
744 				fir_clr_addr = (uid << 24) + 0x10;
745 				__genwqe_writeq(cd, fir_clr_addr, mask);
746 
747 				dev_dbg(&pci_dev->dev,
748 					"[HM] Clearing primary FIR 0x%08x with 0x%016llx\n",
749 					fir_clr_addr, mask);
750 			}
751 		}
752 	}
753 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
754 	if (gfir == IO_ILLEGAL_VALUE)
755 		goto fatal_error;
756 
757 	if ((gfir_masked == 0x0) && (gfir & GFIR_ERR_TRIGGER)) {
758 		/*
759 		 * Check once more that it didn't go on after all the
760 		 * FIRS were cleared.
761 		 */
762 		dev_dbg(&pci_dev->dev, "ACK! Another FIR! Recursing %d!\n",
763 			iterations);
764 		goto healthMonitor;
765 	}
766 	return gfir_masked;
767 
768  fatal_error:
769 	return IO_ILLEGAL_VALUE;
770 }
771 
772 /**
773  * genwqe_pci_fundamental_reset() - trigger a PCIe fundamental reset on the slot
774  * @pci_dev:	PCI device information struct
775  *
776  * Note: pci_set_pcie_reset_state() is not implemented on all archs, so this
777  * reset method will not work in all cases.
778  *
779  * Return: 0 on success or error code from pci_set_pcie_reset_state()
780  */
781 static int genwqe_pci_fundamental_reset(struct pci_dev *pci_dev)
782 {
783 	int rc;
784 
785 	/*
786 	 * lock pci config space access from userspace,
787 	 * save state and issue PCIe fundamental reset
788 	 */
789 	pci_cfg_access_lock(pci_dev);
790 	pci_save_state(pci_dev);
791 	rc = pci_set_pcie_reset_state(pci_dev, pcie_warm_reset);
792 	if (!rc) {
793 		/* keep PCIe reset asserted for 250ms */
794 		msleep(250);
795 		pci_set_pcie_reset_state(pci_dev, pcie_deassert_reset);
796 		/* Wait for 2s to reload flash and train the link */
797 		msleep(2000);
798 	}
799 	pci_restore_state(pci_dev);
800 	pci_cfg_access_unlock(pci_dev);
801 	return rc;
802 }
803 
804 
805 static int genwqe_platform_recovery(struct genwqe_dev *cd)
806 {
807 	struct pci_dev *pci_dev = cd->pci_dev;
808 	int rc;
809 
810 	dev_info(&pci_dev->dev,
811 		 "[%s] resetting card for error recovery\n", __func__);
812 
813 	/* Clear out error injection flags */
814 	cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |
815 			    GENWQE_INJECT_GFIR_FATAL |
816 			    GENWQE_INJECT_GFIR_INFO);
817 
818 	genwqe_stop(cd);
819 
820 	/* Try recoverying the card with fundamental reset */
821 	rc = genwqe_pci_fundamental_reset(pci_dev);
822 	if (!rc) {
823 		rc = genwqe_start(cd);
824 		if (!rc)
825 			dev_info(&pci_dev->dev,
826 				 "[%s] card recovered\n", __func__);
827 		else
828 			dev_err(&pci_dev->dev,
829 				"[%s] err: cannot start card services! (err=%d)\n",
830 				__func__, rc);
831 	} else {
832 		dev_err(&pci_dev->dev,
833 			"[%s] card reset failed\n", __func__);
834 	}
835 
836 	return rc;
837 }
838 
839 /**
840  * genwqe_reload_bistream() - reload card bitstream
841  * @cd: GenWQE device information
842  *
843  * Set the appropriate register and call fundamental reset to reaload the card
844  * bitstream.
845  *
846  * Return: 0 on success, error code otherwise
847  */
848 static int genwqe_reload_bistream(struct genwqe_dev *cd)
849 {
850 	struct pci_dev *pci_dev = cd->pci_dev;
851 	int rc;
852 
853 	dev_info(&pci_dev->dev,
854 		 "[%s] resetting card for bitstream reload\n",
855 		 __func__);
856 
857 	genwqe_stop(cd);
858 
859 	/*
860 	 * Cause a CPLD reprogram with the 'next_bitstream'
861 	 * partition on PCIe hot or fundamental reset
862 	 */
863 	__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,
864 			(cd->softreset & 0xcull) | 0x70ull);
865 
866 	rc = genwqe_pci_fundamental_reset(pci_dev);
867 	if (rc) {
868 		/*
869 		 * A fundamental reset failure can be caused
870 		 * by lack of support on the arch, so we just
871 		 * log the error and try to start the card
872 		 * again.
873 		 */
874 		dev_err(&pci_dev->dev,
875 			"[%s] err: failed to reset card for bitstream reload\n",
876 			__func__);
877 	}
878 
879 	rc = genwqe_start(cd);
880 	if (rc) {
881 		dev_err(&pci_dev->dev,
882 			"[%s] err: cannot start card services! (err=%d)\n",
883 			__func__, rc);
884 		return rc;
885 	}
886 	dev_info(&pci_dev->dev,
887 		 "[%s] card reloaded\n", __func__);
888 	return 0;
889 }
890 
891 
892 /**
893  * genwqe_health_thread() - Health checking thread
894  * @data: GenWQE device information
895  *
896  * This thread is only started for the PF of the card.
897  *
898  * This thread monitors the health of the card. A critical situation
899  * is when we read registers which contain -1 (IO_ILLEGAL_VALUE). In
900  * this case we need to be recovered from outside. Writing to
901  * registers will very likely not work either.
902  *
903  * This thread must only exit if kthread_should_stop() becomes true.
904  *
905  * Condition for the health-thread to trigger:
906  *   a) when a kthread_stop() request comes in or
907  *   b) a critical GFIR occured
908  *
909  * Informational GFIRs are checked and potentially printed in
910  * GENWQE_HEALTH_CHECK_INTERVAL seconds.
911  */
912 static int genwqe_health_thread(void *data)
913 {
914 	int rc, should_stop = 0;
915 	struct genwqe_dev *cd = data;
916 	struct pci_dev *pci_dev = cd->pci_dev;
917 	u64 gfir, gfir_masked, slu_unitcfg, app_unitcfg;
918 
919  health_thread_begin:
920 	while (!kthread_should_stop()) {
921 		rc = wait_event_interruptible_timeout(cd->health_waitq,
922 			 (genwqe_health_check_cond(cd, &gfir) ||
923 			  (should_stop = kthread_should_stop())),
924 				GENWQE_HEALTH_CHECK_INTERVAL * HZ);
925 
926 		if (should_stop)
927 			break;
928 
929 		if (gfir == IO_ILLEGAL_VALUE) {
930 			dev_err(&pci_dev->dev,
931 				"[%s] GFIR=%016llx\n", __func__, gfir);
932 			goto fatal_error;
933 		}
934 
935 		slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);
936 		if (slu_unitcfg == IO_ILLEGAL_VALUE) {
937 			dev_err(&pci_dev->dev,
938 				"[%s] SLU_UNITCFG=%016llx\n",
939 				__func__, slu_unitcfg);
940 			goto fatal_error;
941 		}
942 
943 		app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);
944 		if (app_unitcfg == IO_ILLEGAL_VALUE) {
945 			dev_err(&pci_dev->dev,
946 				"[%s] APP_UNITCFG=%016llx\n",
947 				__func__, app_unitcfg);
948 			goto fatal_error;
949 		}
950 
951 		gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
952 		if (gfir == IO_ILLEGAL_VALUE) {
953 			dev_err(&pci_dev->dev,
954 				"[%s] %s: GFIR=%016llx\n", __func__,
955 				(gfir & GFIR_ERR_TRIGGER) ? "err" : "info",
956 				gfir);
957 			goto fatal_error;
958 		}
959 
960 		gfir_masked = genwqe_fir_checking(cd);
961 		if (gfir_masked == IO_ILLEGAL_VALUE)
962 			goto fatal_error;
963 
964 		/*
965 		 * GFIR ErrorTrigger bits set => reset the card!
966 		 * Never do this for old/manufacturing images!
967 		 */
968 		if ((gfir_masked) && !cd->skip_recovery &&
969 		    genwqe_recovery_on_fatal_gfir_required(cd)) {
970 
971 			cd->card_state = GENWQE_CARD_FATAL_ERROR;
972 
973 			rc = genwqe_recover_card(cd, 0);
974 			if (rc < 0) {
975 				/* FIXME Card is unusable and needs unbind! */
976 				goto fatal_error;
977 			}
978 		}
979 
980 		if (cd->card_state == GENWQE_CARD_RELOAD_BITSTREAM) {
981 			/* Userspace requested card bitstream reload */
982 			rc = genwqe_reload_bistream(cd);
983 			if (rc)
984 				goto fatal_error;
985 		}
986 
987 		cd->last_gfir = gfir;
988 		cond_resched();
989 	}
990 
991 	return 0;
992 
993  fatal_error:
994 	if (cd->use_platform_recovery) {
995 		/*
996 		 * Since we use raw accessors, EEH errors won't be detected
997 		 * by the platform until we do a non-raw MMIO or config space
998 		 * read
999 		 */
1000 		readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1001 
1002 		/* We do nothing if the card is going over PCI recovery */
1003 		if (pci_channel_offline(pci_dev))
1004 			return -EIO;
1005 
1006 		/*
1007 		 * If it's supported by the platform, we try a fundamental reset
1008 		 * to recover from a fatal error. Otherwise, we continue to wait
1009 		 * for an external recovery procedure to take care of it.
1010 		 */
1011 		rc = genwqe_platform_recovery(cd);
1012 		if (!rc)
1013 			goto health_thread_begin;
1014 	}
1015 
1016 	dev_err(&pci_dev->dev,
1017 		"[%s] card unusable. Please trigger unbind!\n", __func__);
1018 
1019 	/* Bring down logical devices to inform user space via udev remove. */
1020 	cd->card_state = GENWQE_CARD_FATAL_ERROR;
1021 	genwqe_stop(cd);
1022 
1023 	/* genwqe_bus_reset failed(). Now wait for genwqe_remove(). */
1024 	while (!kthread_should_stop())
1025 		cond_resched();
1026 
1027 	return -EIO;
1028 }
1029 
1030 static int genwqe_health_check_start(struct genwqe_dev *cd)
1031 {
1032 	int rc;
1033 
1034 	if (GENWQE_HEALTH_CHECK_INTERVAL <= 0)
1035 		return 0;	/* valid for disabling the service */
1036 
1037 	/* moved before request_irq() */
1038 	/* init_waitqueue_head(&cd->health_waitq); */
1039 
1040 	cd->health_thread = kthread_run(genwqe_health_thread, cd,
1041 					GENWQE_DEVNAME "%d_health",
1042 					cd->card_idx);
1043 	if (IS_ERR(cd->health_thread)) {
1044 		rc = PTR_ERR(cd->health_thread);
1045 		cd->health_thread = NULL;
1046 		return rc;
1047 	}
1048 	return 0;
1049 }
1050 
1051 static int genwqe_health_thread_running(struct genwqe_dev *cd)
1052 {
1053 	return cd->health_thread != NULL;
1054 }
1055 
1056 static int genwqe_health_check_stop(struct genwqe_dev *cd)
1057 {
1058 	if (!genwqe_health_thread_running(cd))
1059 		return -EIO;
1060 
1061 	kthread_stop(cd->health_thread);
1062 	cd->health_thread = NULL;
1063 	return 0;
1064 }
1065 
1066 /**
1067  * genwqe_pci_setup() - Allocate PCIe related resources for our card
1068  * @cd: GenWQE device information
1069  */
1070 static int genwqe_pci_setup(struct genwqe_dev *cd)
1071 {
1072 	int err;
1073 	struct pci_dev *pci_dev = cd->pci_dev;
1074 
1075 	err = pci_enable_device_mem(pci_dev);
1076 	if (err) {
1077 		dev_err(&pci_dev->dev,
1078 			"err: failed to enable pci memory (err=%d)\n", err);
1079 		goto err_out;
1080 	}
1081 
1082 	/* Reserve PCI I/O and memory resources */
1083 	err = pci_request_mem_regions(pci_dev, genwqe_driver_name);
1084 	if (err) {
1085 		dev_err(&pci_dev->dev,
1086 			"[%s] err: request bars failed (%d)\n", __func__, err);
1087 		err = -EIO;
1088 		goto err_disable_device;
1089 	}
1090 
1091 	/* check for 64-bit DMA address supported (DAC) */
1092 	/* check for 32-bit DMA address supported (SAC) */
1093 	if (dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)) ||
1094 	    dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32))) {
1095 		dev_err(&pci_dev->dev,
1096 			"err: neither DMA32 nor DMA64 supported\n");
1097 		err = -EIO;
1098 		goto out_release_resources;
1099 	}
1100 
1101 	pci_set_master(pci_dev);
1102 	pci_enable_pcie_error_reporting(pci_dev);
1103 
1104 	/* EEH recovery requires PCIe fundamental reset */
1105 	pci_dev->needs_freset = 1;
1106 
1107 	/* request complete BAR-0 space (length = 0) */
1108 	cd->mmio_len = pci_resource_len(pci_dev, 0);
1109 	cd->mmio = pci_iomap(pci_dev, 0, 0);
1110 	if (cd->mmio == NULL) {
1111 		dev_err(&pci_dev->dev,
1112 			"[%s] err: mapping BAR0 failed\n", __func__);
1113 		err = -ENOMEM;
1114 		goto out_release_resources;
1115 	}
1116 
1117 	cd->num_vfs = pci_sriov_get_totalvfs(pci_dev);
1118 	if (cd->num_vfs < 0)
1119 		cd->num_vfs = 0;
1120 
1121 	err = genwqe_read_ids(cd);
1122 	if (err)
1123 		goto out_iounmap;
1124 
1125 	return 0;
1126 
1127  out_iounmap:
1128 	pci_iounmap(pci_dev, cd->mmio);
1129  out_release_resources:
1130 	pci_release_mem_regions(pci_dev);
1131  err_disable_device:
1132 	pci_disable_device(pci_dev);
1133  err_out:
1134 	return err;
1135 }
1136 
1137 /**
1138  * genwqe_pci_remove() - Free PCIe related resources for our card
1139  * @cd: GenWQE device information
1140  */
1141 static void genwqe_pci_remove(struct genwqe_dev *cd)
1142 {
1143 	struct pci_dev *pci_dev = cd->pci_dev;
1144 
1145 	if (cd->mmio)
1146 		pci_iounmap(pci_dev, cd->mmio);
1147 
1148 	pci_release_mem_regions(pci_dev);
1149 	pci_disable_device(pci_dev);
1150 }
1151 
1152 /**
1153  * genwqe_probe() - Device initialization
1154  * @pci_dev:	PCI device information struct
1155  * @id:		PCI device ID
1156  *
1157  * Callable for multiple cards. This function is called on bind.
1158  *
1159  * Return: 0 if succeeded, < 0 when failed
1160  */
1161 static int genwqe_probe(struct pci_dev *pci_dev,
1162 			const struct pci_device_id *id)
1163 {
1164 	int err;
1165 	struct genwqe_dev *cd;
1166 
1167 	genwqe_init_crc32();
1168 
1169 	cd = genwqe_dev_alloc();
1170 	if (IS_ERR(cd)) {
1171 		dev_err(&pci_dev->dev, "err: could not alloc mem (err=%d)!\n",
1172 			(int)PTR_ERR(cd));
1173 		return PTR_ERR(cd);
1174 	}
1175 
1176 	dev_set_drvdata(&pci_dev->dev, cd);
1177 	cd->pci_dev = pci_dev;
1178 
1179 	err = genwqe_pci_setup(cd);
1180 	if (err < 0) {
1181 		dev_err(&pci_dev->dev,
1182 			"err: problems with PCI setup (err=%d)\n", err);
1183 		goto out_free_dev;
1184 	}
1185 
1186 	err = genwqe_start(cd);
1187 	if (err < 0) {
1188 		dev_err(&pci_dev->dev,
1189 			"err: cannot start card services! (err=%d)\n", err);
1190 		goto out_pci_remove;
1191 	}
1192 
1193 	if (genwqe_is_privileged(cd)) {
1194 		err = genwqe_health_check_start(cd);
1195 		if (err < 0) {
1196 			dev_err(&pci_dev->dev,
1197 				"err: cannot start health checking! (err=%d)\n",
1198 				err);
1199 			goto out_stop_services;
1200 		}
1201 	}
1202 	return 0;
1203 
1204  out_stop_services:
1205 	genwqe_stop(cd);
1206  out_pci_remove:
1207 	genwqe_pci_remove(cd);
1208  out_free_dev:
1209 	genwqe_dev_free(cd);
1210 	return err;
1211 }
1212 
1213 /**
1214  * genwqe_remove() - Called when device is removed (hot-plugable)
1215  * @pci_dev:	PCI device information struct
1216  *
1217  * Or when driver is unloaded respecitively when unbind is done.
1218  */
1219 static void genwqe_remove(struct pci_dev *pci_dev)
1220 {
1221 	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1222 
1223 	genwqe_health_check_stop(cd);
1224 
1225 	/*
1226 	 * genwqe_stop() must survive if it is called twice
1227 	 * sequentially. This happens when the health thread calls it
1228 	 * and fails on genwqe_bus_reset().
1229 	 */
1230 	genwqe_stop(cd);
1231 	genwqe_pci_remove(cd);
1232 	genwqe_dev_free(cd);
1233 }
1234 
1235 /**
1236  * genwqe_err_error_detected() - Error detection callback
1237  * @pci_dev:	PCI device information struct
1238  * @state:	PCI channel state
1239  *
1240  * This callback is called by the PCI subsystem whenever a PCI bus
1241  * error is detected.
1242  */
1243 static pci_ers_result_t genwqe_err_error_detected(struct pci_dev *pci_dev,
1244 						 pci_channel_state_t state)
1245 {
1246 	struct genwqe_dev *cd;
1247 
1248 	dev_err(&pci_dev->dev, "[%s] state=%d\n", __func__, state);
1249 
1250 	cd = dev_get_drvdata(&pci_dev->dev);
1251 	if (cd == NULL)
1252 		return PCI_ERS_RESULT_DISCONNECT;
1253 
1254 	/* Stop the card */
1255 	genwqe_health_check_stop(cd);
1256 	genwqe_stop(cd);
1257 
1258 	/*
1259 	 * On permanent failure, the PCI code will call device remove
1260 	 * after the return of this function.
1261 	 * genwqe_stop() can be called twice.
1262 	 */
1263 	if (state == pci_channel_io_perm_failure) {
1264 		return PCI_ERS_RESULT_DISCONNECT;
1265 	} else {
1266 		genwqe_pci_remove(cd);
1267 		return PCI_ERS_RESULT_NEED_RESET;
1268 	}
1269 }
1270 
1271 static pci_ers_result_t genwqe_err_slot_reset(struct pci_dev *pci_dev)
1272 {
1273 	int rc;
1274 	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1275 
1276 	rc = genwqe_pci_setup(cd);
1277 	if (!rc) {
1278 		return PCI_ERS_RESULT_RECOVERED;
1279 	} else {
1280 		dev_err(&pci_dev->dev,
1281 			"err: problems with PCI setup (err=%d)\n", rc);
1282 		return PCI_ERS_RESULT_DISCONNECT;
1283 	}
1284 }
1285 
1286 static pci_ers_result_t genwqe_err_result_none(struct pci_dev *dev)
1287 {
1288 	return PCI_ERS_RESULT_NONE;
1289 }
1290 
1291 static void genwqe_err_resume(struct pci_dev *pci_dev)
1292 {
1293 	int rc;
1294 	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1295 
1296 	rc = genwqe_start(cd);
1297 	if (!rc) {
1298 		rc = genwqe_health_check_start(cd);
1299 		if (rc)
1300 			dev_err(&pci_dev->dev,
1301 				"err: cannot start health checking! (err=%d)\n",
1302 				rc);
1303 	} else {
1304 		dev_err(&pci_dev->dev,
1305 			"err: cannot start card services! (err=%d)\n", rc);
1306 	}
1307 }
1308 
1309 static int genwqe_sriov_configure(struct pci_dev *dev, int numvfs)
1310 {
1311 	int rc;
1312 	struct genwqe_dev *cd = dev_get_drvdata(&dev->dev);
1313 
1314 	if (numvfs > 0) {
1315 		genwqe_setup_vf_jtimer(cd);
1316 		rc = pci_enable_sriov(dev, numvfs);
1317 		if (rc < 0)
1318 			return rc;
1319 		return numvfs;
1320 	}
1321 	if (numvfs == 0) {
1322 		pci_disable_sriov(dev);
1323 		return 0;
1324 	}
1325 	return 0;
1326 }
1327 
1328 static const struct pci_error_handlers genwqe_err_handler = {
1329 	.error_detected = genwqe_err_error_detected,
1330 	.mmio_enabled	= genwqe_err_result_none,
1331 	.slot_reset	= genwqe_err_slot_reset,
1332 	.resume		= genwqe_err_resume,
1333 };
1334 
1335 static struct pci_driver genwqe_driver = {
1336 	.name	  = genwqe_driver_name,
1337 	.id_table = genwqe_device_table,
1338 	.probe	  = genwqe_probe,
1339 	.remove	  = genwqe_remove,
1340 	.sriov_configure = genwqe_sriov_configure,
1341 	.err_handler = &genwqe_err_handler,
1342 };
1343 
1344 /**
1345  * genwqe_devnode() - Set default access mode for genwqe devices.
1346  * @dev:	Pointer to device (unused)
1347  * @mode:	Carrier to pass-back given mode (permissions)
1348  *
1349  * Default mode should be rw for everybody. Do not change default
1350  * device name.
1351  */
1352 static char *genwqe_devnode(struct device *dev, umode_t *mode)
1353 {
1354 	if (mode)
1355 		*mode = 0666;
1356 	return NULL;
1357 }
1358 
1359 /**
1360  * genwqe_init_module() - Driver registration and initialization
1361  */
1362 static int __init genwqe_init_module(void)
1363 {
1364 	int rc;
1365 
1366 	class_genwqe = class_create(THIS_MODULE, GENWQE_DEVNAME);
1367 	if (IS_ERR(class_genwqe)) {
1368 		pr_err("[%s] create class failed\n", __func__);
1369 		return -ENOMEM;
1370 	}
1371 
1372 	class_genwqe->devnode = genwqe_devnode;
1373 
1374 	debugfs_genwqe = debugfs_create_dir(GENWQE_DEVNAME, NULL);
1375 
1376 	rc = pci_register_driver(&genwqe_driver);
1377 	if (rc != 0) {
1378 		pr_err("[%s] pci_reg_driver (rc=%d)\n", __func__, rc);
1379 		goto err_out0;
1380 	}
1381 
1382 	return rc;
1383 
1384  err_out0:
1385 	debugfs_remove(debugfs_genwqe);
1386 	class_destroy(class_genwqe);
1387 	return rc;
1388 }
1389 
1390 /**
1391  * genwqe_exit_module() - Driver exit
1392  */
1393 static void __exit genwqe_exit_module(void)
1394 {
1395 	pci_unregister_driver(&genwqe_driver);
1396 	debugfs_remove(debugfs_genwqe);
1397 	class_destroy(class_genwqe);
1398 }
1399 
1400 module_init(genwqe_init_module);
1401 module_exit(genwqe_exit_module);
1402