xref: /linux/drivers/misc/genwqe/card_base.c (revision a1c613ae4c322ddd58d5a8539dbfba2a0380a8c0)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * IBM Accelerator Family 'GenWQE'
4  *
5  * (C) Copyright IBM Corp. 2013
6  *
7  * Author: Frank Haverkamp <haver@linux.vnet.ibm.com>
8  * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com>
9  * Author: Michael Jung <mijung@gmx.net>
10  * Author: Michael Ruettger <michael@ibmra.de>
11  */
12 
13 /*
14  * Module initialization and PCIe setup. Card health monitoring and
15  * recovery functionality. Character device creation and deletion are
16  * controlled from here.
17  */
18 
19 #include <linux/types.h>
20 #include <linux/pci.h>
21 #include <linux/err.h>
22 #include <linux/string.h>
23 #include <linux/sched.h>
24 #include <linux/wait.h>
25 #include <linux/delay.h>
26 #include <linux/dma-mapping.h>
27 #include <linux/module.h>
28 #include <linux/notifier.h>
29 #include <linux/device.h>
30 #include <linux/log2.h>
31 
32 #include "card_base.h"
33 #include "card_ddcb.h"
34 
35 MODULE_AUTHOR("Frank Haverkamp <haver@linux.vnet.ibm.com>");
36 MODULE_AUTHOR("Michael Ruettger <michael@ibmra.de>");
37 MODULE_AUTHOR("Joerg-Stephan Vogt <jsvogt@de.ibm.com>");
38 MODULE_AUTHOR("Michael Jung <mijung@gmx.net>");
39 
40 MODULE_DESCRIPTION("GenWQE Card");
41 MODULE_VERSION(DRV_VERSION);
42 MODULE_LICENSE("GPL");
43 
44 static char genwqe_driver_name[] = GENWQE_DEVNAME;
45 
46 static struct dentry *debugfs_genwqe;
47 static struct genwqe_dev *genwqe_devices[GENWQE_CARD_NO_MAX];
48 
49 /* PCI structure for identifying device by PCI vendor and device ID */
50 static const struct pci_device_id genwqe_device_table[] = {
51 	{ .vendor      = PCI_VENDOR_ID_IBM,
52 	  .device      = PCI_DEVICE_GENWQE,
53 	  .subvendor   = PCI_SUBVENDOR_ID_IBM,
54 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
55 	  .class       = (PCI_CLASSCODE_GENWQE5 << 8),
56 	  .class_mask  = ~0,
57 	  .driver_data = 0 },
58 
59 	/* Initial SR-IOV bring-up image */
60 	{ .vendor      = PCI_VENDOR_ID_IBM,
61 	  .device      = PCI_DEVICE_GENWQE,
62 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
63 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,
64 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
65 	  .class_mask  = ~0,
66 	  .driver_data = 0 },
67 
68 	{ .vendor      = PCI_VENDOR_ID_IBM,  /* VF Vendor ID */
69 	  .device      = 0x0000,  /* VF Device ID */
70 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
71 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_SRIOV,
72 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
73 	  .class_mask  = ~0,
74 	  .driver_data = 0 },
75 
76 	/* Fixed up image */
77 	{ .vendor      = PCI_VENDOR_ID_IBM,
78 	  .device      = PCI_DEVICE_GENWQE,
79 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
80 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
81 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
82 	  .class_mask  = ~0,
83 	  .driver_data = 0 },
84 
85 	{ .vendor      = PCI_VENDOR_ID_IBM,  /* VF Vendor ID */
86 	  .device      = 0x0000,  /* VF Device ID */
87 	  .subvendor   = PCI_SUBVENDOR_ID_IBM_SRIOV,
88 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5,
89 	  .class       = (PCI_CLASSCODE_GENWQE5_SRIOV << 8),
90 	  .class_mask  = ~0,
91 	  .driver_data = 0 },
92 
93 	/* Even one more ... */
94 	{ .vendor      = PCI_VENDOR_ID_IBM,
95 	  .device      = PCI_DEVICE_GENWQE,
96 	  .subvendor   = PCI_SUBVENDOR_ID_IBM,
97 	  .subdevice   = PCI_SUBSYSTEM_ID_GENWQE5_NEW,
98 	  .class       = (PCI_CLASSCODE_GENWQE5 << 8),
99 	  .class_mask  = ~0,
100 	  .driver_data = 0 },
101 
102 	{ 0, }			/* 0 terminated list. */
103 };
104 
105 MODULE_DEVICE_TABLE(pci, genwqe_device_table);
106 
107 /**
108  * genwqe_devnode() - Set default access mode for genwqe devices.
109  * @dev:	Pointer to device (unused)
110  * @mode:	Carrier to pass-back given mode (permissions)
111  *
112  * Default mode should be rw for everybody. Do not change default
113  * device name.
114  */
genwqe_devnode(const struct device * dev,umode_t * mode)115 static char *genwqe_devnode(const struct device *dev, umode_t *mode)
116 {
117 	if (mode)
118 		*mode = 0666;
119 	return NULL;
120 }
121 
122 static const struct class class_genwqe = {
123 	.name = GENWQE_DEVNAME,
124 	.devnode = genwqe_devnode,
125 };
126 
127 /**
128  * genwqe_dev_alloc() - Create and prepare a new card descriptor
129  *
130  * Return: Pointer to card descriptor, or ERR_PTR(err) on error
131  */
genwqe_dev_alloc(void)132 static struct genwqe_dev *genwqe_dev_alloc(void)
133 {
134 	unsigned int i = 0, j;
135 	struct genwqe_dev *cd;
136 
137 	for (i = 0; i < GENWQE_CARD_NO_MAX; i++) {
138 		if (genwqe_devices[i] == NULL)
139 			break;
140 	}
141 	if (i >= GENWQE_CARD_NO_MAX)
142 		return ERR_PTR(-ENODEV);
143 
144 	cd = kzalloc(sizeof(struct genwqe_dev), GFP_KERNEL);
145 	if (!cd)
146 		return ERR_PTR(-ENOMEM);
147 
148 	cd->card_idx = i;
149 	cd->class_genwqe = &class_genwqe;
150 	cd->debugfs_genwqe = debugfs_genwqe;
151 
152 	/*
153 	 * This comes from kernel config option and can be overritten via
154 	 * debugfs.
155 	 */
156 	cd->use_platform_recovery = CONFIG_GENWQE_PLATFORM_ERROR_RECOVERY;
157 
158 	init_waitqueue_head(&cd->queue_waitq);
159 
160 	spin_lock_init(&cd->file_lock);
161 	INIT_LIST_HEAD(&cd->file_list);
162 
163 	cd->card_state = GENWQE_CARD_UNUSED;
164 	spin_lock_init(&cd->print_lock);
165 
166 	cd->ddcb_software_timeout = GENWQE_DDCB_SOFTWARE_TIMEOUT;
167 	cd->kill_timeout = GENWQE_KILL_TIMEOUT;
168 
169 	for (j = 0; j < GENWQE_MAX_VFS; j++)
170 		cd->vf_jobtimeout_msec[j] = GENWQE_VF_JOBTIMEOUT_MSEC;
171 
172 	genwqe_devices[i] = cd;
173 	return cd;
174 }
175 
genwqe_dev_free(struct genwqe_dev * cd)176 static void genwqe_dev_free(struct genwqe_dev *cd)
177 {
178 	if (!cd)
179 		return;
180 
181 	genwqe_devices[cd->card_idx] = NULL;
182 	kfree(cd);
183 }
184 
185 /**
186  * genwqe_bus_reset() - Card recovery
187  * @cd: GenWQE device information
188  *
189  * pci_reset_function() will recover the device and ensure that the
190  * registers are accessible again when it completes with success. If
191  * not, the card will stay dead and registers will be unaccessible
192  * still.
193  */
genwqe_bus_reset(struct genwqe_dev * cd)194 static int genwqe_bus_reset(struct genwqe_dev *cd)
195 {
196 	int rc = 0;
197 	struct pci_dev *pci_dev = cd->pci_dev;
198 	void __iomem *mmio;
199 
200 	if (cd->err_inject & GENWQE_INJECT_BUS_RESET_FAILURE)
201 		return -EIO;
202 
203 	mmio = cd->mmio;
204 	cd->mmio = NULL;
205 	pci_iounmap(pci_dev, mmio);
206 
207 	pci_release_mem_regions(pci_dev);
208 
209 	/*
210 	 * Firmware/BIOS might change memory mapping during bus reset.
211 	 * Settings like enable bus-mastering, ... are backuped and
212 	 * restored by the pci_reset_function().
213 	 */
214 	dev_dbg(&pci_dev->dev, "[%s] pci_reset function ...\n", __func__);
215 	rc = pci_reset_function(pci_dev);
216 	if (rc) {
217 		dev_err(&pci_dev->dev,
218 			"[%s] err: failed reset func (rc %d)\n", __func__, rc);
219 		return rc;
220 	}
221 	dev_dbg(&pci_dev->dev, "[%s] done with rc=%d\n", __func__, rc);
222 
223 	/*
224 	 * Here is the right spot to clear the register read
225 	 * failure. pci_bus_reset() does this job in real systems.
226 	 */
227 	cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |
228 			    GENWQE_INJECT_GFIR_FATAL |
229 			    GENWQE_INJECT_GFIR_INFO);
230 
231 	rc = pci_request_mem_regions(pci_dev, genwqe_driver_name);
232 	if (rc) {
233 		dev_err(&pci_dev->dev,
234 			"[%s] err: request bars failed (%d)\n", __func__, rc);
235 		return -EIO;
236 	}
237 
238 	cd->mmio = pci_iomap(pci_dev, 0, 0);
239 	if (cd->mmio == NULL) {
240 		dev_err(&pci_dev->dev,
241 			"[%s] err: mapping BAR0 failed\n", __func__);
242 		return -ENOMEM;
243 	}
244 	return 0;
245 }
246 
247 /*
248  * Hardware circumvention section. Certain bitstreams in our test-lab
249  * had different kinds of problems. Here is where we adjust those
250  * bitstreams to function will with this version of our device driver.
251  *
252  * Thise circumventions are applied to the physical function only.
253  * The magical numbers below are identifying development/manufacturing
254  * versions of the bitstream used on the card.
255  *
256  * Turn off error reporting for old/manufacturing images.
257  */
258 
genwqe_need_err_masking(struct genwqe_dev * cd)259 bool genwqe_need_err_masking(struct genwqe_dev *cd)
260 {
261 	return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;
262 }
263 
genwqe_tweak_hardware(struct genwqe_dev * cd)264 static void genwqe_tweak_hardware(struct genwqe_dev *cd)
265 {
266 	struct pci_dev *pci_dev = cd->pci_dev;
267 
268 	/* Mask FIRs for development images */
269 	if (((cd->slu_unitcfg & 0xFFFF0ull) >= 0x32000ull) &&
270 	    ((cd->slu_unitcfg & 0xFFFF0ull) <= 0x33250ull)) {
271 		dev_warn(&pci_dev->dev,
272 			 "FIRs masked due to bitstream %016llx.%016llx\n",
273 			 cd->slu_unitcfg, cd->app_unitcfg);
274 
275 		__genwqe_writeq(cd, IO_APP_SEC_LEM_DEBUG_OVR,
276 				0xFFFFFFFFFFFFFFFFull);
277 
278 		__genwqe_writeq(cd, IO_APP_ERR_ACT_MASK,
279 				0x0000000000000000ull);
280 	}
281 }
282 
283 /**
284  * genwqe_recovery_on_fatal_gfir_required() - Version depended actions
285  * @cd: GenWQE device information
286  *
287  * Bitstreams older than 2013-02-17 have a bug where fatal GFIRs must
288  * be ignored. This is e.g. true for the bitstream we gave to the card
289  * manufacturer, but also for some old bitstreams we released to our
290  * test-lab.
291  */
genwqe_recovery_on_fatal_gfir_required(struct genwqe_dev * cd)292 int genwqe_recovery_on_fatal_gfir_required(struct genwqe_dev *cd)
293 {
294 	return (cd->slu_unitcfg & 0xFFFF0ull) >= 0x32170ull;
295 }
296 
genwqe_flash_readback_fails(struct genwqe_dev * cd)297 int genwqe_flash_readback_fails(struct genwqe_dev *cd)
298 {
299 	return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull;
300 }
301 
302 /**
303  * genwqe_T_psec() - Calculate PF/VF timeout register content
304  * @cd: GenWQE device information
305  *
306  * Note: From a design perspective it turned out to be a bad idea to
307  * use codes here to specifiy the frequency/speed values. An old
308  * driver cannot understand new codes and is therefore always a
309  * problem. Better is to measure out the value or put the
310  * speed/frequency directly into a register which is always a valid
311  * value for old as well as for new software.
312  */
313 /* T = 1/f */
genwqe_T_psec(struct genwqe_dev * cd)314 static int genwqe_T_psec(struct genwqe_dev *cd)
315 {
316 	u16 speed;	/* 1/f -> 250,  200,  166,  175 */
317 	static const int T[] = { 4000, 5000, 6000, 5714 };
318 
319 	speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full);
320 	if (speed >= ARRAY_SIZE(T))
321 		return -1;	/* illegal value */
322 
323 	return T[speed];
324 }
325 
326 /**
327  * genwqe_setup_pf_jtimer() - Setup PF hardware timeouts for DDCB execution
328  * @cd: GenWQE device information
329  *
330  * Do this _after_ card_reset() is called. Otherwise the values will
331  * vanish. The settings need to be done when the queues are inactive.
332  *
333  * The max. timeout value is 2^(10+x) * T (6ns for 166MHz) * 15/16.
334  * The min. timeout value is 2^(10+x) * T (6ns for 166MHz) * 14/16.
335  */
genwqe_setup_pf_jtimer(struct genwqe_dev * cd)336 static bool genwqe_setup_pf_jtimer(struct genwqe_dev *cd)
337 {
338 	u32 T = genwqe_T_psec(cd);
339 	u64 x;
340 
341 	if (GENWQE_PF_JOBTIMEOUT_MSEC == 0)
342 		return false;
343 
344 	/* PF: large value needed, flash update 2sec per block */
345 	x = ilog2(GENWQE_PF_JOBTIMEOUT_MSEC *
346 		  16000000000uL/(T * 15)) - 10;
347 
348 	genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
349 			  0xff00 | (x & 0xff), 0);
350 	return true;
351 }
352 
353 /**
354  * genwqe_setup_vf_jtimer() - Setup VF hardware timeouts for DDCB execution
355  * @cd: GenWQE device information
356  */
genwqe_setup_vf_jtimer(struct genwqe_dev * cd)357 static bool genwqe_setup_vf_jtimer(struct genwqe_dev *cd)
358 {
359 	struct pci_dev *pci_dev = cd->pci_dev;
360 	unsigned int vf;
361 	u32 T = genwqe_T_psec(cd);
362 	u64 x;
363 	int totalvfs;
364 
365 	totalvfs = pci_sriov_get_totalvfs(pci_dev);
366 	if (totalvfs <= 0)
367 		return false;
368 
369 	for (vf = 0; vf < totalvfs; vf++) {
370 
371 		if (cd->vf_jobtimeout_msec[vf] == 0)
372 			continue;
373 
374 		x = ilog2(cd->vf_jobtimeout_msec[vf] *
375 			  16000000000uL/(T * 15)) - 10;
376 
377 		genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT,
378 				  0xff00 | (x & 0xff), vf + 1);
379 	}
380 	return true;
381 }
382 
genwqe_ffdc_buffs_alloc(struct genwqe_dev * cd)383 static int genwqe_ffdc_buffs_alloc(struct genwqe_dev *cd)
384 {
385 	unsigned int type, e = 0;
386 
387 	for (type = 0; type < GENWQE_DBG_UNITS; type++) {
388 		switch (type) {
389 		case GENWQE_DBG_UNIT0:
390 			e = genwqe_ffdc_buff_size(cd, 0);
391 			break;
392 		case GENWQE_DBG_UNIT1:
393 			e = genwqe_ffdc_buff_size(cd, 1);
394 			break;
395 		case GENWQE_DBG_UNIT2:
396 			e = genwqe_ffdc_buff_size(cd, 2);
397 			break;
398 		case GENWQE_DBG_REGS:
399 			e = GENWQE_FFDC_REGS;
400 			break;
401 		}
402 
403 		/* currently support only the debug units mentioned here */
404 		cd->ffdc[type].entries = e;
405 		cd->ffdc[type].regs =
406 			kmalloc_array(e, sizeof(struct genwqe_reg),
407 				      GFP_KERNEL);
408 		/*
409 		 * regs == NULL is ok, the using code treats this as no regs,
410 		 * Printing warning is ok in this case.
411 		 */
412 	}
413 	return 0;
414 }
415 
genwqe_ffdc_buffs_free(struct genwqe_dev * cd)416 static void genwqe_ffdc_buffs_free(struct genwqe_dev *cd)
417 {
418 	unsigned int type;
419 
420 	for (type = 0; type < GENWQE_DBG_UNITS; type++) {
421 		kfree(cd->ffdc[type].regs);
422 		cd->ffdc[type].regs = NULL;
423 	}
424 }
425 
genwqe_read_ids(struct genwqe_dev * cd)426 static int genwqe_read_ids(struct genwqe_dev *cd)
427 {
428 	int err = 0;
429 	int slu_id;
430 	struct pci_dev *pci_dev = cd->pci_dev;
431 
432 	cd->slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);
433 	if (cd->slu_unitcfg == IO_ILLEGAL_VALUE) {
434 		dev_err(&pci_dev->dev,
435 			"err: SLUID=%016llx\n", cd->slu_unitcfg);
436 		err = -EIO;
437 		goto out_err;
438 	}
439 
440 	slu_id = genwqe_get_slu_id(cd);
441 	if (slu_id < GENWQE_SLU_ARCH_REQ || slu_id == 0xff) {
442 		dev_err(&pci_dev->dev,
443 			"err: incompatible SLU Architecture %u\n", slu_id);
444 		err = -ENOENT;
445 		goto out_err;
446 	}
447 
448 	cd->app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);
449 	if (cd->app_unitcfg == IO_ILLEGAL_VALUE) {
450 		dev_err(&pci_dev->dev,
451 			"err: APPID=%016llx\n", cd->app_unitcfg);
452 		err = -EIO;
453 		goto out_err;
454 	}
455 	genwqe_read_app_id(cd, cd->app_name, sizeof(cd->app_name));
456 
457 	/*
458 	 * Is access to all registers possible? If we are a VF the
459 	 * answer is obvious. If we run fully virtualized, we need to
460 	 * check if we can access all registers. If we do not have
461 	 * full access we will cause an UR and some informational FIRs
462 	 * in the PF, but that should not harm.
463 	 */
464 	if (pci_dev->is_virtfn)
465 		cd->is_privileged = 0;
466 	else
467 		cd->is_privileged = (__genwqe_readq(cd, IO_SLU_BITSTREAM)
468 				     != IO_ILLEGAL_VALUE);
469 
470  out_err:
471 	return err;
472 }
473 
genwqe_start(struct genwqe_dev * cd)474 static int genwqe_start(struct genwqe_dev *cd)
475 {
476 	int err;
477 	struct pci_dev *pci_dev = cd->pci_dev;
478 
479 	err = genwqe_read_ids(cd);
480 	if (err)
481 		return err;
482 
483 	if (genwqe_is_privileged(cd)) {
484 		/* do this after the tweaks. alloc fail is acceptable */
485 		genwqe_ffdc_buffs_alloc(cd);
486 		genwqe_stop_traps(cd);
487 
488 		/* Collect registers e.g. FIRs, UNITIDs, traces ... */
489 		genwqe_read_ffdc_regs(cd, cd->ffdc[GENWQE_DBG_REGS].regs,
490 				      cd->ffdc[GENWQE_DBG_REGS].entries, 0);
491 
492 		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT0,
493 				      cd->ffdc[GENWQE_DBG_UNIT0].regs,
494 				      cd->ffdc[GENWQE_DBG_UNIT0].entries);
495 
496 		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT1,
497 				      cd->ffdc[GENWQE_DBG_UNIT1].regs,
498 				      cd->ffdc[GENWQE_DBG_UNIT1].entries);
499 
500 		genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT2,
501 				      cd->ffdc[GENWQE_DBG_UNIT2].regs,
502 				      cd->ffdc[GENWQE_DBG_UNIT2].entries);
503 
504 		genwqe_start_traps(cd);
505 
506 		if (cd->card_state == GENWQE_CARD_FATAL_ERROR) {
507 			dev_warn(&pci_dev->dev,
508 				 "[%s] chip reload/recovery!\n", __func__);
509 
510 			/*
511 			 * Stealth Mode: Reload chip on either hot
512 			 * reset or PERST.
513 			 */
514 			cd->softreset = 0x7Cull;
515 			__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,
516 				       cd->softreset);
517 
518 			err = genwqe_bus_reset(cd);
519 			if (err != 0) {
520 				dev_err(&pci_dev->dev,
521 					"[%s] err: bus reset failed!\n",
522 					__func__);
523 				goto out;
524 			}
525 
526 			/*
527 			 * Re-read the IDs because
528 			 * it could happen that the bitstream load
529 			 * failed!
530 			 */
531 			err = genwqe_read_ids(cd);
532 			if (err)
533 				goto out;
534 		}
535 	}
536 
537 	err = genwqe_setup_service_layer(cd);  /* does a reset to the card */
538 	if (err != 0) {
539 		dev_err(&pci_dev->dev,
540 			"[%s] err: could not setup servicelayer!\n", __func__);
541 		err = -ENODEV;
542 		goto out;
543 	}
544 
545 	if (genwqe_is_privileged(cd)) {	 /* code is running _after_ reset */
546 		genwqe_tweak_hardware(cd);
547 
548 		genwqe_setup_pf_jtimer(cd);
549 		genwqe_setup_vf_jtimer(cd);
550 	}
551 
552 	err = genwqe_device_create(cd);
553 	if (err < 0) {
554 		dev_err(&pci_dev->dev,
555 			"err: chdev init failed! (err=%d)\n", err);
556 		goto out_release_service_layer;
557 	}
558 	return 0;
559 
560  out_release_service_layer:
561 	genwqe_release_service_layer(cd);
562  out:
563 	if (genwqe_is_privileged(cd))
564 		genwqe_ffdc_buffs_free(cd);
565 	return -EIO;
566 }
567 
568 /**
569  * genwqe_stop() - Stop card operation
570  * @cd: GenWQE device information
571  *
572  * Recovery notes:
573  *   As long as genwqe_thread runs we might access registers during
574  *   error data capture. Same is with the genwqe_health_thread.
575  *   When genwqe_bus_reset() fails this function might called two times:
576  *   first by the genwqe_health_thread() and later by genwqe_remove() to
577  *   unbind the device. We must be able to survive that.
578  *
579  * This function must be robust enough to be called twice.
580  */
genwqe_stop(struct genwqe_dev * cd)581 static int genwqe_stop(struct genwqe_dev *cd)
582 {
583 	genwqe_finish_queue(cd);	    /* no register access */
584 	genwqe_device_remove(cd);	    /* device removed, procs killed */
585 	genwqe_release_service_layer(cd);   /* here genwqe_thread is stopped */
586 
587 	if (genwqe_is_privileged(cd)) {
588 		pci_disable_sriov(cd->pci_dev);	/* access pci config space */
589 		genwqe_ffdc_buffs_free(cd);
590 	}
591 
592 	return 0;
593 }
594 
595 /**
596  * genwqe_recover_card() - Try to recover the card if it is possible
597  * @cd: GenWQE device information
598  * @fatal_err: Indicate whether to attempt soft reset
599  *
600  * If fatal_err is set no register access is possible anymore. It is
601  * likely that genwqe_start fails in that situation. Proper error
602  * handling is required in this case.
603  *
604  * genwqe_bus_reset() will cause the pci code to call genwqe_remove()
605  * and later genwqe_probe() for all virtual functions.
606  */
genwqe_recover_card(struct genwqe_dev * cd,int fatal_err)607 static int genwqe_recover_card(struct genwqe_dev *cd, int fatal_err)
608 {
609 	int rc;
610 	struct pci_dev *pci_dev = cd->pci_dev;
611 
612 	genwqe_stop(cd);
613 
614 	/*
615 	 * Make sure chip is not reloaded to maintain FFDC. Write SLU
616 	 * Reset Register, CPLDReset field to 0.
617 	 */
618 	if (!fatal_err) {
619 		cd->softreset = 0x70ull;
620 		__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, cd->softreset);
621 	}
622 
623 	rc = genwqe_bus_reset(cd);
624 	if (rc != 0) {
625 		dev_err(&pci_dev->dev,
626 			"[%s] err: card recovery impossible!\n", __func__);
627 		return rc;
628 	}
629 
630 	rc = genwqe_start(cd);
631 	if (rc < 0) {
632 		dev_err(&pci_dev->dev,
633 			"[%s] err: failed to launch device!\n", __func__);
634 		return rc;
635 	}
636 	return 0;
637 }
638 
genwqe_health_check_cond(struct genwqe_dev * cd,u64 * gfir)639 static int genwqe_health_check_cond(struct genwqe_dev *cd, u64 *gfir)
640 {
641 	*gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
642 	return (*gfir & GFIR_ERR_TRIGGER) &&
643 		genwqe_recovery_on_fatal_gfir_required(cd);
644 }
645 
646 /**
647  * genwqe_fir_checking() - Check the fault isolation registers of the card
648  * @cd: GenWQE device information
649  *
650  * If this code works ok, can be tried out with help of the genwqe_poke tool:
651  *   sudo ./tools/genwqe_poke 0x8 0xfefefefefef
652  *
653  * Now the relevant FIRs/sFIRs should be printed out and the driver should
654  * invoke recovery (devices are removed and readded).
655  */
genwqe_fir_checking(struct genwqe_dev * cd)656 static u64 genwqe_fir_checking(struct genwqe_dev *cd)
657 {
658 	int j, iterations = 0;
659 	u64 mask, fir, fec, uid, gfir, gfir_masked, sfir, sfec;
660 	u32 fir_addr, fir_clr_addr, fec_addr, sfir_addr, sfec_addr;
661 	struct pci_dev *pci_dev = cd->pci_dev;
662 
663  healthMonitor:
664 	iterations++;
665 	if (iterations > 16) {
666 		dev_err(&pci_dev->dev, "* exit looping after %d times\n",
667 			iterations);
668 		goto fatal_error;
669 	}
670 
671 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
672 	if (gfir != 0x0)
673 		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n",
674 				    IO_SLC_CFGREG_GFIR, gfir);
675 	if (gfir == IO_ILLEGAL_VALUE)
676 		goto fatal_error;
677 
678 	/*
679 	 * Avoid printing when to GFIR bit is on prevents contignous
680 	 * printout e.g. for the following bug:
681 	 *   FIR set without a 2ndary FIR/FIR cannot be cleared
682 	 * Comment out the following if to get the prints:
683 	 */
684 	if (gfir == 0)
685 		return 0;
686 
687 	gfir_masked = gfir & GFIR_ERR_TRIGGER;  /* fatal errors */
688 
689 	for (uid = 0; uid < GENWQE_MAX_UNITS; uid++) { /* 0..2 in zEDC */
690 
691 		/* read the primary FIR (pfir) */
692 		fir_addr = (uid << 24) + 0x08;
693 		fir = __genwqe_readq(cd, fir_addr);
694 		if (fir == 0x0)
695 			continue;  /* no error in this unit */
696 
697 		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fir_addr, fir);
698 		if (fir == IO_ILLEGAL_VALUE)
699 			goto fatal_error;
700 
701 		/* read primary FEC */
702 		fec_addr = (uid << 24) + 0x18;
703 		fec = __genwqe_readq(cd, fec_addr);
704 
705 		dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fec_addr, fec);
706 		if (fec == IO_ILLEGAL_VALUE)
707 			goto fatal_error;
708 
709 		for (j = 0, mask = 1ULL; j < 64; j++, mask <<= 1) {
710 
711 			/* secondary fir empty, skip it */
712 			if ((fir & mask) == 0x0)
713 				continue;
714 
715 			sfir_addr = (uid << 24) + 0x100 + 0x08 * j;
716 			sfir = __genwqe_readq(cd, sfir_addr);
717 
718 			if (sfir == IO_ILLEGAL_VALUE)
719 				goto fatal_error;
720 			dev_err(&pci_dev->dev,
721 				"* 0x%08x 0x%016llx\n", sfir_addr, sfir);
722 
723 			sfec_addr = (uid << 24) + 0x300 + 0x08 * j;
724 			sfec = __genwqe_readq(cd, sfec_addr);
725 
726 			if (sfec == IO_ILLEGAL_VALUE)
727 				goto fatal_error;
728 			dev_err(&pci_dev->dev,
729 				"* 0x%08x 0x%016llx\n", sfec_addr, sfec);
730 
731 			gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
732 			if (gfir == IO_ILLEGAL_VALUE)
733 				goto fatal_error;
734 
735 			/* gfir turned on during routine! get out and
736 			   start over. */
737 			if ((gfir_masked == 0x0) &&
738 			    (gfir & GFIR_ERR_TRIGGER)) {
739 				goto healthMonitor;
740 			}
741 
742 			/* do not clear if we entered with a fatal gfir */
743 			if (gfir_masked == 0x0) {
744 
745 				/* NEW clear by mask the logged bits */
746 				sfir_addr = (uid << 24) + 0x100 + 0x08 * j;
747 				__genwqe_writeq(cd, sfir_addr, sfir);
748 
749 				dev_dbg(&pci_dev->dev,
750 					"[HM] Clearing  2ndary FIR 0x%08x with 0x%016llx\n",
751 					sfir_addr, sfir);
752 
753 				/*
754 				 * note, these cannot be error-Firs
755 				 * since gfir_masked is 0 after sfir
756 				 * was read. Also, it is safe to do
757 				 * this write if sfir=0. Still need to
758 				 * clear the primary. This just means
759 				 * there is no secondary FIR.
760 				 */
761 
762 				/* clear by mask the logged bit. */
763 				fir_clr_addr = (uid << 24) + 0x10;
764 				__genwqe_writeq(cd, fir_clr_addr, mask);
765 
766 				dev_dbg(&pci_dev->dev,
767 					"[HM] Clearing primary FIR 0x%08x with 0x%016llx\n",
768 					fir_clr_addr, mask);
769 			}
770 		}
771 	}
772 	gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
773 	if (gfir == IO_ILLEGAL_VALUE)
774 		goto fatal_error;
775 
776 	if ((gfir_masked == 0x0) && (gfir & GFIR_ERR_TRIGGER)) {
777 		/*
778 		 * Check once more that it didn't go on after all the
779 		 * FIRS were cleared.
780 		 */
781 		dev_dbg(&pci_dev->dev, "ACK! Another FIR! Recursing %d!\n",
782 			iterations);
783 		goto healthMonitor;
784 	}
785 	return gfir_masked;
786 
787  fatal_error:
788 	return IO_ILLEGAL_VALUE;
789 }
790 
791 /**
792  * genwqe_pci_fundamental_reset() - trigger a PCIe fundamental reset on the slot
793  * @pci_dev:	PCI device information struct
794  *
795  * Note: pci_set_pcie_reset_state() is not implemented on all archs, so this
796  * reset method will not work in all cases.
797  *
798  * Return: 0 on success or error code from pci_set_pcie_reset_state()
799  */
genwqe_pci_fundamental_reset(struct pci_dev * pci_dev)800 static int genwqe_pci_fundamental_reset(struct pci_dev *pci_dev)
801 {
802 	int rc;
803 
804 	/*
805 	 * lock pci config space access from userspace,
806 	 * save state and issue PCIe fundamental reset
807 	 */
808 	pci_cfg_access_lock(pci_dev);
809 	pci_save_state(pci_dev);
810 	rc = pci_set_pcie_reset_state(pci_dev, pcie_warm_reset);
811 	if (!rc) {
812 		/* keep PCIe reset asserted for 250ms */
813 		msleep(250);
814 		pci_set_pcie_reset_state(pci_dev, pcie_deassert_reset);
815 		/* Wait for 2s to reload flash and train the link */
816 		msleep(2000);
817 	}
818 	pci_restore_state(pci_dev);
819 	pci_cfg_access_unlock(pci_dev);
820 	return rc;
821 }
822 
823 
genwqe_platform_recovery(struct genwqe_dev * cd)824 static int genwqe_platform_recovery(struct genwqe_dev *cd)
825 {
826 	struct pci_dev *pci_dev = cd->pci_dev;
827 	int rc;
828 
829 	dev_info(&pci_dev->dev,
830 		 "[%s] resetting card for error recovery\n", __func__);
831 
832 	/* Clear out error injection flags */
833 	cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE |
834 			    GENWQE_INJECT_GFIR_FATAL |
835 			    GENWQE_INJECT_GFIR_INFO);
836 
837 	genwqe_stop(cd);
838 
839 	/* Try recoverying the card with fundamental reset */
840 	rc = genwqe_pci_fundamental_reset(pci_dev);
841 	if (!rc) {
842 		rc = genwqe_start(cd);
843 		if (!rc)
844 			dev_info(&pci_dev->dev,
845 				 "[%s] card recovered\n", __func__);
846 		else
847 			dev_err(&pci_dev->dev,
848 				"[%s] err: cannot start card services! (err=%d)\n",
849 				__func__, rc);
850 	} else {
851 		dev_err(&pci_dev->dev,
852 			"[%s] card reset failed\n", __func__);
853 	}
854 
855 	return rc;
856 }
857 
858 /**
859  * genwqe_reload_bistream() - reload card bitstream
860  * @cd: GenWQE device information
861  *
862  * Set the appropriate register and call fundamental reset to reaload the card
863  * bitstream.
864  *
865  * Return: 0 on success, error code otherwise
866  */
genwqe_reload_bistream(struct genwqe_dev * cd)867 static int genwqe_reload_bistream(struct genwqe_dev *cd)
868 {
869 	struct pci_dev *pci_dev = cd->pci_dev;
870 	int rc;
871 
872 	dev_info(&pci_dev->dev,
873 		 "[%s] resetting card for bitstream reload\n",
874 		 __func__);
875 
876 	genwqe_stop(cd);
877 
878 	/*
879 	 * Cause a CPLD reprogram with the 'next_bitstream'
880 	 * partition on PCIe hot or fundamental reset
881 	 */
882 	__genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET,
883 			(cd->softreset & 0xcull) | 0x70ull);
884 
885 	rc = genwqe_pci_fundamental_reset(pci_dev);
886 	if (rc) {
887 		/*
888 		 * A fundamental reset failure can be caused
889 		 * by lack of support on the arch, so we just
890 		 * log the error and try to start the card
891 		 * again.
892 		 */
893 		dev_err(&pci_dev->dev,
894 			"[%s] err: failed to reset card for bitstream reload\n",
895 			__func__);
896 	}
897 
898 	rc = genwqe_start(cd);
899 	if (rc) {
900 		dev_err(&pci_dev->dev,
901 			"[%s] err: cannot start card services! (err=%d)\n",
902 			__func__, rc);
903 		return rc;
904 	}
905 	dev_info(&pci_dev->dev,
906 		 "[%s] card reloaded\n", __func__);
907 	return 0;
908 }
909 
910 
911 /**
912  * genwqe_health_thread() - Health checking thread
913  * @data: GenWQE device information
914  *
915  * This thread is only started for the PF of the card.
916  *
917  * This thread monitors the health of the card. A critical situation
918  * is when we read registers which contain -1 (IO_ILLEGAL_VALUE). In
919  * this case we need to be recovered from outside. Writing to
920  * registers will very likely not work either.
921  *
922  * This thread must only exit if kthread_should_stop() becomes true.
923  *
924  * Condition for the health-thread to trigger:
925  *   a) when a kthread_stop() request comes in or
926  *   b) a critical GFIR occured
927  *
928  * Informational GFIRs are checked and potentially printed in
929  * GENWQE_HEALTH_CHECK_INTERVAL seconds.
930  */
genwqe_health_thread(void * data)931 static int genwqe_health_thread(void *data)
932 {
933 	int rc, should_stop = 0;
934 	struct genwqe_dev *cd = data;
935 	struct pci_dev *pci_dev = cd->pci_dev;
936 	u64 gfir, gfir_masked, slu_unitcfg, app_unitcfg;
937 
938  health_thread_begin:
939 	while (!kthread_should_stop()) {
940 		rc = wait_event_interruptible_timeout(cd->health_waitq,
941 			 (genwqe_health_check_cond(cd, &gfir) ||
942 			  (should_stop = kthread_should_stop())),
943 				GENWQE_HEALTH_CHECK_INTERVAL * HZ);
944 
945 		if (should_stop)
946 			break;
947 
948 		if (gfir == IO_ILLEGAL_VALUE) {
949 			dev_err(&pci_dev->dev,
950 				"[%s] GFIR=%016llx\n", __func__, gfir);
951 			goto fatal_error;
952 		}
953 
954 		slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG);
955 		if (slu_unitcfg == IO_ILLEGAL_VALUE) {
956 			dev_err(&pci_dev->dev,
957 				"[%s] SLU_UNITCFG=%016llx\n",
958 				__func__, slu_unitcfg);
959 			goto fatal_error;
960 		}
961 
962 		app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG);
963 		if (app_unitcfg == IO_ILLEGAL_VALUE) {
964 			dev_err(&pci_dev->dev,
965 				"[%s] APP_UNITCFG=%016llx\n",
966 				__func__, app_unitcfg);
967 			goto fatal_error;
968 		}
969 
970 		gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR);
971 		if (gfir == IO_ILLEGAL_VALUE) {
972 			dev_err(&pci_dev->dev,
973 				"[%s] %s: GFIR=%016llx\n", __func__,
974 				(gfir & GFIR_ERR_TRIGGER) ? "err" : "info",
975 				gfir);
976 			goto fatal_error;
977 		}
978 
979 		gfir_masked = genwqe_fir_checking(cd);
980 		if (gfir_masked == IO_ILLEGAL_VALUE)
981 			goto fatal_error;
982 
983 		/*
984 		 * GFIR ErrorTrigger bits set => reset the card!
985 		 * Never do this for old/manufacturing images!
986 		 */
987 		if ((gfir_masked) && !cd->skip_recovery &&
988 		    genwqe_recovery_on_fatal_gfir_required(cd)) {
989 
990 			cd->card_state = GENWQE_CARD_FATAL_ERROR;
991 
992 			rc = genwqe_recover_card(cd, 0);
993 			if (rc < 0) {
994 				/* FIXME Card is unusable and needs unbind! */
995 				goto fatal_error;
996 			}
997 		}
998 
999 		if (cd->card_state == GENWQE_CARD_RELOAD_BITSTREAM) {
1000 			/* Userspace requested card bitstream reload */
1001 			rc = genwqe_reload_bistream(cd);
1002 			if (rc)
1003 				goto fatal_error;
1004 		}
1005 
1006 		cd->last_gfir = gfir;
1007 		cond_resched();
1008 	}
1009 
1010 	return 0;
1011 
1012  fatal_error:
1013 	if (cd->use_platform_recovery) {
1014 		/*
1015 		 * Since we use raw accessors, EEH errors won't be detected
1016 		 * by the platform until we do a non-raw MMIO or config space
1017 		 * read
1018 		 */
1019 		readq(cd->mmio + IO_SLC_CFGREG_GFIR);
1020 
1021 		/* We do nothing if the card is going over PCI recovery */
1022 		if (pci_channel_offline(pci_dev))
1023 			return -EIO;
1024 
1025 		/*
1026 		 * If it's supported by the platform, we try a fundamental reset
1027 		 * to recover from a fatal error. Otherwise, we continue to wait
1028 		 * for an external recovery procedure to take care of it.
1029 		 */
1030 		rc = genwqe_platform_recovery(cd);
1031 		if (!rc)
1032 			goto health_thread_begin;
1033 	}
1034 
1035 	dev_err(&pci_dev->dev,
1036 		"[%s] card unusable. Please trigger unbind!\n", __func__);
1037 
1038 	/* Bring down logical devices to inform user space via udev remove. */
1039 	cd->card_state = GENWQE_CARD_FATAL_ERROR;
1040 	genwqe_stop(cd);
1041 
1042 	/* genwqe_bus_reset failed(). Now wait for genwqe_remove(). */
1043 	while (!kthread_should_stop())
1044 		cond_resched();
1045 
1046 	return -EIO;
1047 }
1048 
genwqe_health_check_start(struct genwqe_dev * cd)1049 static int genwqe_health_check_start(struct genwqe_dev *cd)
1050 {
1051 	int rc;
1052 
1053 	if (GENWQE_HEALTH_CHECK_INTERVAL <= 0)
1054 		return 0;	/* valid for disabling the service */
1055 
1056 	/* moved before request_irq() */
1057 	/* init_waitqueue_head(&cd->health_waitq); */
1058 
1059 	cd->health_thread = kthread_run(genwqe_health_thread, cd,
1060 					GENWQE_DEVNAME "%d_health",
1061 					cd->card_idx);
1062 	if (IS_ERR(cd->health_thread)) {
1063 		rc = PTR_ERR(cd->health_thread);
1064 		cd->health_thread = NULL;
1065 		return rc;
1066 	}
1067 	return 0;
1068 }
1069 
genwqe_health_thread_running(struct genwqe_dev * cd)1070 static int genwqe_health_thread_running(struct genwqe_dev *cd)
1071 {
1072 	return cd->health_thread != NULL;
1073 }
1074 
genwqe_health_check_stop(struct genwqe_dev * cd)1075 static int genwqe_health_check_stop(struct genwqe_dev *cd)
1076 {
1077 	if (!genwqe_health_thread_running(cd))
1078 		return -EIO;
1079 
1080 	kthread_stop(cd->health_thread);
1081 	cd->health_thread = NULL;
1082 	return 0;
1083 }
1084 
1085 /**
1086  * genwqe_pci_setup() - Allocate PCIe related resources for our card
1087  * @cd: GenWQE device information
1088  */
genwqe_pci_setup(struct genwqe_dev * cd)1089 static int genwqe_pci_setup(struct genwqe_dev *cd)
1090 {
1091 	int err;
1092 	struct pci_dev *pci_dev = cd->pci_dev;
1093 
1094 	err = pci_enable_device_mem(pci_dev);
1095 	if (err) {
1096 		dev_err(&pci_dev->dev,
1097 			"err: failed to enable pci memory (err=%d)\n", err);
1098 		goto err_out;
1099 	}
1100 
1101 	/* Reserve PCI I/O and memory resources */
1102 	err = pci_request_mem_regions(pci_dev, genwqe_driver_name);
1103 	if (err) {
1104 		dev_err(&pci_dev->dev,
1105 			"[%s] err: request bars failed (%d)\n", __func__, err);
1106 		err = -EIO;
1107 		goto err_disable_device;
1108 	}
1109 
1110 	/* check for 64-bit DMA address supported (DAC) */
1111 	/* check for 32-bit DMA address supported (SAC) */
1112 	if (dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)) &&
1113 	    dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32))) {
1114 		dev_err(&pci_dev->dev,
1115 			"err: neither DMA32 nor DMA64 supported\n");
1116 		err = -EIO;
1117 		goto out_release_resources;
1118 	}
1119 
1120 	pci_set_master(pci_dev);
1121 
1122 	/* EEH recovery requires PCIe fundamental reset */
1123 	pci_dev->needs_freset = 1;
1124 
1125 	/* request complete BAR-0 space (length = 0) */
1126 	cd->mmio_len = pci_resource_len(pci_dev, 0);
1127 	cd->mmio = pci_iomap(pci_dev, 0, 0);
1128 	if (cd->mmio == NULL) {
1129 		dev_err(&pci_dev->dev,
1130 			"[%s] err: mapping BAR0 failed\n", __func__);
1131 		err = -ENOMEM;
1132 		goto out_release_resources;
1133 	}
1134 
1135 	cd->num_vfs = pci_sriov_get_totalvfs(pci_dev);
1136 	if (cd->num_vfs < 0)
1137 		cd->num_vfs = 0;
1138 
1139 	err = genwqe_read_ids(cd);
1140 	if (err)
1141 		goto out_iounmap;
1142 
1143 	return 0;
1144 
1145  out_iounmap:
1146 	pci_iounmap(pci_dev, cd->mmio);
1147  out_release_resources:
1148 	pci_release_mem_regions(pci_dev);
1149  err_disable_device:
1150 	pci_disable_device(pci_dev);
1151  err_out:
1152 	return err;
1153 }
1154 
1155 /**
1156  * genwqe_pci_remove() - Free PCIe related resources for our card
1157  * @cd: GenWQE device information
1158  */
genwqe_pci_remove(struct genwqe_dev * cd)1159 static void genwqe_pci_remove(struct genwqe_dev *cd)
1160 {
1161 	struct pci_dev *pci_dev = cd->pci_dev;
1162 
1163 	if (cd->mmio)
1164 		pci_iounmap(pci_dev, cd->mmio);
1165 
1166 	pci_release_mem_regions(pci_dev);
1167 	pci_disable_device(pci_dev);
1168 }
1169 
1170 /**
1171  * genwqe_probe() - Device initialization
1172  * @pci_dev:	PCI device information struct
1173  * @id:		PCI device ID
1174  *
1175  * Callable for multiple cards. This function is called on bind.
1176  *
1177  * Return: 0 if succeeded, < 0 when failed
1178  */
genwqe_probe(struct pci_dev * pci_dev,const struct pci_device_id * id)1179 static int genwqe_probe(struct pci_dev *pci_dev,
1180 			const struct pci_device_id *id)
1181 {
1182 	int err;
1183 	struct genwqe_dev *cd;
1184 
1185 	genwqe_init_crc32();
1186 
1187 	cd = genwqe_dev_alloc();
1188 	if (IS_ERR(cd)) {
1189 		dev_err(&pci_dev->dev, "err: could not alloc mem (err=%d)!\n",
1190 			(int)PTR_ERR(cd));
1191 		return PTR_ERR(cd);
1192 	}
1193 
1194 	dev_set_drvdata(&pci_dev->dev, cd);
1195 	cd->pci_dev = pci_dev;
1196 
1197 	err = genwqe_pci_setup(cd);
1198 	if (err < 0) {
1199 		dev_err(&pci_dev->dev,
1200 			"err: problems with PCI setup (err=%d)\n", err);
1201 		goto out_free_dev;
1202 	}
1203 
1204 	err = genwqe_start(cd);
1205 	if (err < 0) {
1206 		dev_err(&pci_dev->dev,
1207 			"err: cannot start card services! (err=%d)\n", err);
1208 		goto out_pci_remove;
1209 	}
1210 
1211 	if (genwqe_is_privileged(cd)) {
1212 		err = genwqe_health_check_start(cd);
1213 		if (err < 0) {
1214 			dev_err(&pci_dev->dev,
1215 				"err: cannot start health checking! (err=%d)\n",
1216 				err);
1217 			goto out_stop_services;
1218 		}
1219 	}
1220 	return 0;
1221 
1222  out_stop_services:
1223 	genwqe_stop(cd);
1224  out_pci_remove:
1225 	genwqe_pci_remove(cd);
1226  out_free_dev:
1227 	genwqe_dev_free(cd);
1228 	return err;
1229 }
1230 
1231 /**
1232  * genwqe_remove() - Called when device is removed (hot-plugable)
1233  * @pci_dev:	PCI device information struct
1234  *
1235  * Or when driver is unloaded respecitively when unbind is done.
1236  */
genwqe_remove(struct pci_dev * pci_dev)1237 static void genwqe_remove(struct pci_dev *pci_dev)
1238 {
1239 	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1240 
1241 	genwqe_health_check_stop(cd);
1242 
1243 	/*
1244 	 * genwqe_stop() must survive if it is called twice
1245 	 * sequentially. This happens when the health thread calls it
1246 	 * and fails on genwqe_bus_reset().
1247 	 */
1248 	genwqe_stop(cd);
1249 	genwqe_pci_remove(cd);
1250 	genwqe_dev_free(cd);
1251 }
1252 
1253 /**
1254  * genwqe_err_error_detected() - Error detection callback
1255  * @pci_dev:	PCI device information struct
1256  * @state:	PCI channel state
1257  *
1258  * This callback is called by the PCI subsystem whenever a PCI bus
1259  * error is detected.
1260  */
genwqe_err_error_detected(struct pci_dev * pci_dev,pci_channel_state_t state)1261 static pci_ers_result_t genwqe_err_error_detected(struct pci_dev *pci_dev,
1262 						 pci_channel_state_t state)
1263 {
1264 	struct genwqe_dev *cd;
1265 
1266 	dev_err(&pci_dev->dev, "[%s] state=%d\n", __func__, state);
1267 
1268 	cd = dev_get_drvdata(&pci_dev->dev);
1269 	if (cd == NULL)
1270 		return PCI_ERS_RESULT_DISCONNECT;
1271 
1272 	/* Stop the card */
1273 	genwqe_health_check_stop(cd);
1274 	genwqe_stop(cd);
1275 
1276 	/*
1277 	 * On permanent failure, the PCI code will call device remove
1278 	 * after the return of this function.
1279 	 * genwqe_stop() can be called twice.
1280 	 */
1281 	if (state == pci_channel_io_perm_failure) {
1282 		return PCI_ERS_RESULT_DISCONNECT;
1283 	} else {
1284 		genwqe_pci_remove(cd);
1285 		return PCI_ERS_RESULT_NEED_RESET;
1286 	}
1287 }
1288 
genwqe_err_slot_reset(struct pci_dev * pci_dev)1289 static pci_ers_result_t genwqe_err_slot_reset(struct pci_dev *pci_dev)
1290 {
1291 	int rc;
1292 	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1293 
1294 	rc = genwqe_pci_setup(cd);
1295 	if (!rc) {
1296 		return PCI_ERS_RESULT_RECOVERED;
1297 	} else {
1298 		dev_err(&pci_dev->dev,
1299 			"err: problems with PCI setup (err=%d)\n", rc);
1300 		return PCI_ERS_RESULT_DISCONNECT;
1301 	}
1302 }
1303 
genwqe_err_result_none(struct pci_dev * dev)1304 static pci_ers_result_t genwqe_err_result_none(struct pci_dev *dev)
1305 {
1306 	return PCI_ERS_RESULT_NONE;
1307 }
1308 
genwqe_err_resume(struct pci_dev * pci_dev)1309 static void genwqe_err_resume(struct pci_dev *pci_dev)
1310 {
1311 	int rc;
1312 	struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev);
1313 
1314 	rc = genwqe_start(cd);
1315 	if (!rc) {
1316 		rc = genwqe_health_check_start(cd);
1317 		if (rc)
1318 			dev_err(&pci_dev->dev,
1319 				"err: cannot start health checking! (err=%d)\n",
1320 				rc);
1321 	} else {
1322 		dev_err(&pci_dev->dev,
1323 			"err: cannot start card services! (err=%d)\n", rc);
1324 	}
1325 }
1326 
genwqe_sriov_configure(struct pci_dev * dev,int numvfs)1327 static int genwqe_sriov_configure(struct pci_dev *dev, int numvfs)
1328 {
1329 	int rc;
1330 	struct genwqe_dev *cd = dev_get_drvdata(&dev->dev);
1331 
1332 	if (numvfs > 0) {
1333 		genwqe_setup_vf_jtimer(cd);
1334 		rc = pci_enable_sriov(dev, numvfs);
1335 		if (rc < 0)
1336 			return rc;
1337 		return numvfs;
1338 	}
1339 	if (numvfs == 0) {
1340 		pci_disable_sriov(dev);
1341 		return 0;
1342 	}
1343 	return 0;
1344 }
1345 
1346 static const struct pci_error_handlers genwqe_err_handler = {
1347 	.error_detected = genwqe_err_error_detected,
1348 	.mmio_enabled	= genwqe_err_result_none,
1349 	.slot_reset	= genwqe_err_slot_reset,
1350 	.resume		= genwqe_err_resume,
1351 };
1352 
1353 static struct pci_driver genwqe_driver = {
1354 	.name	  = genwqe_driver_name,
1355 	.id_table = genwqe_device_table,
1356 	.probe	  = genwqe_probe,
1357 	.remove	  = genwqe_remove,
1358 	.sriov_configure = genwqe_sriov_configure,
1359 	.err_handler = &genwqe_err_handler,
1360 };
1361 
1362 /**
1363  * genwqe_init_module() - Driver registration and initialization
1364  */
genwqe_init_module(void)1365 static int __init genwqe_init_module(void)
1366 {
1367 	int rc;
1368 
1369 	rc = class_register(&class_genwqe);
1370 	if (rc) {
1371 		pr_err("[%s] create class failed\n", __func__);
1372 		return -ENOMEM;
1373 	}
1374 
1375 	debugfs_genwqe = debugfs_create_dir(GENWQE_DEVNAME, NULL);
1376 
1377 	rc = pci_register_driver(&genwqe_driver);
1378 	if (rc != 0) {
1379 		pr_err("[%s] pci_reg_driver (rc=%d)\n", __func__, rc);
1380 		goto err_out0;
1381 	}
1382 
1383 	return rc;
1384 
1385  err_out0:
1386 	debugfs_remove(debugfs_genwqe);
1387 	class_unregister(&class_genwqe);
1388 	return rc;
1389 }
1390 
1391 /**
1392  * genwqe_exit_module() - Driver exit
1393  */
genwqe_exit_module(void)1394 static void __exit genwqe_exit_module(void)
1395 {
1396 	pci_unregister_driver(&genwqe_driver);
1397 	debugfs_remove(debugfs_genwqe);
1398 	class_unregister(&class_genwqe);
1399 }
1400 
1401 module_init(genwqe_init_module);
1402 module_exit(genwqe_exit_module);
1403