1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * IBM Accelerator Family 'GenWQE' 4 * 5 * (C) Copyright IBM Corp. 2013 6 * 7 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> 8 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> 9 * Author: Michael Jung <mijung@gmx.net> 10 * Author: Michael Ruettger <michael@ibmra.de> 11 */ 12 13 /* 14 * Module initialization and PCIe setup. Card health monitoring and 15 * recovery functionality. Character device creation and deletion are 16 * controlled from here. 17 */ 18 19 #include <linux/types.h> 20 #include <linux/pci.h> 21 #include <linux/err.h> 22 #include <linux/string.h> 23 #include <linux/sched.h> 24 #include <linux/wait.h> 25 #include <linux/delay.h> 26 #include <linux/dma-mapping.h> 27 #include <linux/module.h> 28 #include <linux/notifier.h> 29 #include <linux/device.h> 30 #include <linux/log2.h> 31 32 #include "card_base.h" 33 #include "card_ddcb.h" 34 35 MODULE_AUTHOR("Frank Haverkamp <haver@linux.vnet.ibm.com>"); 36 MODULE_AUTHOR("Michael Ruettger <michael@ibmra.de>"); 37 MODULE_AUTHOR("Joerg-Stephan Vogt <jsvogt@de.ibm.com>"); 38 MODULE_AUTHOR("Michael Jung <mijung@gmx.net>"); 39 40 MODULE_DESCRIPTION("GenWQE Card"); 41 MODULE_VERSION(DRV_VERSION); 42 MODULE_LICENSE("GPL"); 43 44 static char genwqe_driver_name[] = GENWQE_DEVNAME; 45 46 static struct dentry *debugfs_genwqe; 47 static struct genwqe_dev *genwqe_devices[GENWQE_CARD_NO_MAX]; 48 49 /* PCI structure for identifying device by PCI vendor and device ID */ 50 static const struct pci_device_id genwqe_device_table[] = { 51 { 52 PCI_DEVICE_SUB(PCI_VENDOR_ID_IBM, PCI_DEVICE_GENWQE, 53 PCI_SUBVENDOR_ID_IBM, PCI_SUBSYSTEM_ID_GENWQE5), 54 .class = PCI_CLASSCODE_GENWQE5 << 8, 55 .class_mask = ~0, 56 }, { 57 /* Initial SR-IOV bring-up image */ 58 PCI_DEVICE_SUB(PCI_VENDOR_ID_IBM, PCI_VENDOR_ID_IBM, PCI_DEVICE_GENWQE, 59 PCI_SUBVENDOR_ID_IBM_SRIOV), 60 .class = PCI_CLASSCODE_GENWQE5_SRIOV << 8, 61 .class_mask = ~0, 62 }, { 63 PCI_DEVICE_SUB(PCI_VENDOR_ID_IBM, 0x0000, 64 PCI_SUBVENDOR_ID_IBM_SRIOV, PCI_SUBSYSTEM_ID_GENWQE5_SRIOV), 65 .class = PCI_CLASSCODE_GENWQE5_SRIOV << 8, 66 .class_mask = ~0, 67 }, { 68 /* Fixed up image */ 69 PCI_DEVICE_SUB(PCI_VENDOR_ID_IBM, PCI_DEVICE_GENWQE, 70 PCI_SUBVENDOR_ID_IBM_SRIOV, PCI_SUBSYSTEM_ID_GENWQE5), 71 .class = PCI_CLASSCODE_GENWQE5_SRIOV << 8, 72 .class_mask = ~0, 73 }, { 74 PCI_DEVICE_SUB(PCI_VENDOR_ID_IBM, 0x0000, 75 PCI_SUBVENDOR_ID_IBM_SRIOV, PCI_SUBSYSTEM_ID_GENWQE5), 76 .class = PCI_CLASSCODE_GENWQE5_SRIOV << 8, 77 .class_mask = ~0, 78 }, { 79 /* Even one more ... */ 80 PCI_DEVICE_SUB(PCI_VENDOR_ID_IBM, PCI_DEVICE_GENWQE, 81 PCI_SUBVENDOR_ID_IBM, PCI_SUBSYSTEM_ID_GENWQE5_NEW), 82 .class = (PCI_CLASSCODE_GENWQE5 << 8), 83 .class_mask = ~0, 84 }, 85 { } /* 0 terminated list. */ 86 }; 87 MODULE_DEVICE_TABLE(pci, genwqe_device_table); 88 89 /** 90 * genwqe_devnode() - Set default access mode for genwqe devices. 91 * @dev: Pointer to device (unused) 92 * @mode: Carrier to pass-back given mode (permissions) 93 * 94 * Default mode should be rw for everybody. Do not change default 95 * device name. 96 */ 97 static char *genwqe_devnode(const struct device *dev, umode_t *mode) 98 { 99 if (mode) 100 *mode = 0666; 101 return NULL; 102 } 103 104 static const struct class class_genwqe = { 105 .name = GENWQE_DEVNAME, 106 .devnode = genwqe_devnode, 107 }; 108 109 /** 110 * genwqe_dev_alloc() - Create and prepare a new card descriptor 111 * 112 * Return: Pointer to card descriptor, or ERR_PTR(err) on error 113 */ 114 static struct genwqe_dev *genwqe_dev_alloc(void) 115 { 116 unsigned int i = 0, j; 117 struct genwqe_dev *cd; 118 119 for (i = 0; i < GENWQE_CARD_NO_MAX; i++) { 120 if (genwqe_devices[i] == NULL) 121 break; 122 } 123 if (i >= GENWQE_CARD_NO_MAX) 124 return ERR_PTR(-ENODEV); 125 126 cd = kzalloc_obj(struct genwqe_dev); 127 if (!cd) 128 return ERR_PTR(-ENOMEM); 129 130 cd->card_idx = i; 131 cd->class_genwqe = &class_genwqe; 132 cd->debugfs_genwqe = debugfs_genwqe; 133 134 /* 135 * This comes from kernel config option and can be overritten via 136 * debugfs. 137 */ 138 cd->use_platform_recovery = CONFIG_GENWQE_PLATFORM_ERROR_RECOVERY; 139 140 init_waitqueue_head(&cd->queue_waitq); 141 142 spin_lock_init(&cd->file_lock); 143 INIT_LIST_HEAD(&cd->file_list); 144 145 cd->card_state = GENWQE_CARD_UNUSED; 146 spin_lock_init(&cd->print_lock); 147 148 cd->ddcb_software_timeout = GENWQE_DDCB_SOFTWARE_TIMEOUT; 149 cd->kill_timeout = GENWQE_KILL_TIMEOUT; 150 151 for (j = 0; j < GENWQE_MAX_VFS; j++) 152 cd->vf_jobtimeout_msec[j] = GENWQE_VF_JOBTIMEOUT_MSEC; 153 154 genwqe_devices[i] = cd; 155 return cd; 156 } 157 158 static void genwqe_dev_free(struct genwqe_dev *cd) 159 { 160 if (!cd) 161 return; 162 163 genwqe_devices[cd->card_idx] = NULL; 164 kfree(cd); 165 } 166 167 /** 168 * genwqe_bus_reset() - Card recovery 169 * @cd: GenWQE device information 170 * 171 * pci_reset_function() will recover the device and ensure that the 172 * registers are accessible again when it completes with success. If 173 * not, the card will stay dead and registers will be unaccessible 174 * still. 175 */ 176 static int genwqe_bus_reset(struct genwqe_dev *cd) 177 { 178 int rc = 0; 179 struct pci_dev *pci_dev = cd->pci_dev; 180 void __iomem *mmio; 181 182 if (cd->err_inject & GENWQE_INJECT_BUS_RESET_FAILURE) 183 return -EIO; 184 185 mmio = cd->mmio; 186 cd->mmio = NULL; 187 pci_iounmap(pci_dev, mmio); 188 189 pci_release_mem_regions(pci_dev); 190 191 /* 192 * Firmware/BIOS might change memory mapping during bus reset. 193 * Settings like enable bus-mastering, ... are backuped and 194 * restored by the pci_reset_function(). 195 */ 196 dev_dbg(&pci_dev->dev, "[%s] pci_reset function ...\n", __func__); 197 rc = pci_reset_function(pci_dev); 198 if (rc) { 199 dev_err(&pci_dev->dev, 200 "[%s] err: failed reset func (rc %d)\n", __func__, rc); 201 return rc; 202 } 203 dev_dbg(&pci_dev->dev, "[%s] done with rc=%d\n", __func__, rc); 204 205 /* 206 * Here is the right spot to clear the register read 207 * failure. pci_bus_reset() does this job in real systems. 208 */ 209 cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE | 210 GENWQE_INJECT_GFIR_FATAL | 211 GENWQE_INJECT_GFIR_INFO); 212 213 rc = pci_request_mem_regions(pci_dev, genwqe_driver_name); 214 if (rc) { 215 dev_err(&pci_dev->dev, 216 "[%s] err: request bars failed (%d)\n", __func__, rc); 217 return -EIO; 218 } 219 220 cd->mmio = pci_iomap(pci_dev, 0, 0); 221 if (cd->mmio == NULL) { 222 dev_err(&pci_dev->dev, 223 "[%s] err: mapping BAR0 failed\n", __func__); 224 return -ENOMEM; 225 } 226 return 0; 227 } 228 229 /* 230 * Hardware circumvention section. Certain bitstreams in our test-lab 231 * had different kinds of problems. Here is where we adjust those 232 * bitstreams to function will with this version of our device driver. 233 * 234 * Thise circumventions are applied to the physical function only. 235 * The magical numbers below are identifying development/manufacturing 236 * versions of the bitstream used on the card. 237 * 238 * Turn off error reporting for old/manufacturing images. 239 */ 240 241 bool genwqe_need_err_masking(struct genwqe_dev *cd) 242 { 243 return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull; 244 } 245 246 static void genwqe_tweak_hardware(struct genwqe_dev *cd) 247 { 248 struct pci_dev *pci_dev = cd->pci_dev; 249 250 /* Mask FIRs for development images */ 251 if (((cd->slu_unitcfg & 0xFFFF0ull) >= 0x32000ull) && 252 ((cd->slu_unitcfg & 0xFFFF0ull) <= 0x33250ull)) { 253 dev_warn(&pci_dev->dev, 254 "FIRs masked due to bitstream %016llx.%016llx\n", 255 cd->slu_unitcfg, cd->app_unitcfg); 256 257 __genwqe_writeq(cd, IO_APP_SEC_LEM_DEBUG_OVR, 258 0xFFFFFFFFFFFFFFFFull); 259 260 __genwqe_writeq(cd, IO_APP_ERR_ACT_MASK, 261 0x0000000000000000ull); 262 } 263 } 264 265 /** 266 * genwqe_recovery_on_fatal_gfir_required() - Version depended actions 267 * @cd: GenWQE device information 268 * 269 * Bitstreams older than 2013-02-17 have a bug where fatal GFIRs must 270 * be ignored. This is e.g. true for the bitstream we gave to the card 271 * manufacturer, but also for some old bitstreams we released to our 272 * test-lab. 273 */ 274 int genwqe_recovery_on_fatal_gfir_required(struct genwqe_dev *cd) 275 { 276 return (cd->slu_unitcfg & 0xFFFF0ull) >= 0x32170ull; 277 } 278 279 int genwqe_flash_readback_fails(struct genwqe_dev *cd) 280 { 281 return (cd->slu_unitcfg & 0xFFFF0ull) < 0x32170ull; 282 } 283 284 /** 285 * genwqe_T_psec() - Calculate PF/VF timeout register content 286 * @cd: GenWQE device information 287 * 288 * Note: From a design perspective it turned out to be a bad idea to 289 * use codes here to specifiy the frequency/speed values. An old 290 * driver cannot understand new codes and is therefore always a 291 * problem. Better is to measure out the value or put the 292 * speed/frequency directly into a register which is always a valid 293 * value for old as well as for new software. 294 */ 295 /* T = 1/f */ 296 static int genwqe_T_psec(struct genwqe_dev *cd) 297 { 298 u16 speed; /* 1/f -> 250, 200, 166, 175 */ 299 static const int T[] = { 4000, 5000, 6000, 5714 }; 300 301 speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full); 302 if (speed >= ARRAY_SIZE(T)) 303 return -1; /* illegal value */ 304 305 return T[speed]; 306 } 307 308 /** 309 * genwqe_setup_pf_jtimer() - Setup PF hardware timeouts for DDCB execution 310 * @cd: GenWQE device information 311 * 312 * Do this _after_ card_reset() is called. Otherwise the values will 313 * vanish. The settings need to be done when the queues are inactive. 314 * 315 * The max. timeout value is 2^(10+x) * T (6ns for 166MHz) * 15/16. 316 * The min. timeout value is 2^(10+x) * T (6ns for 166MHz) * 14/16. 317 */ 318 static bool genwqe_setup_pf_jtimer(struct genwqe_dev *cd) 319 { 320 u32 T = genwqe_T_psec(cd); 321 u64 x; 322 323 if (GENWQE_PF_JOBTIMEOUT_MSEC == 0) 324 return false; 325 326 /* PF: large value needed, flash update 2sec per block */ 327 x = ilog2(GENWQE_PF_JOBTIMEOUT_MSEC * 328 16000000000uL/(T * 15)) - 10; 329 330 genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT, 331 0xff00 | (x & 0xff), 0); 332 return true; 333 } 334 335 /** 336 * genwqe_setup_vf_jtimer() - Setup VF hardware timeouts for DDCB execution 337 * @cd: GenWQE device information 338 */ 339 static bool genwqe_setup_vf_jtimer(struct genwqe_dev *cd) 340 { 341 struct pci_dev *pci_dev = cd->pci_dev; 342 unsigned int vf; 343 u32 T = genwqe_T_psec(cd); 344 u64 x; 345 int totalvfs; 346 347 totalvfs = pci_sriov_get_totalvfs(pci_dev); 348 if (totalvfs <= 0) 349 return false; 350 351 for (vf = 0; vf < totalvfs; vf++) { 352 353 if (cd->vf_jobtimeout_msec[vf] == 0) 354 continue; 355 356 x = ilog2(cd->vf_jobtimeout_msec[vf] * 357 16000000000uL/(T * 15)) - 10; 358 359 genwqe_write_vreg(cd, IO_SLC_VF_APPJOB_TIMEOUT, 360 0xff00 | (x & 0xff), vf + 1); 361 } 362 return true; 363 } 364 365 static int genwqe_ffdc_buffs_alloc(struct genwqe_dev *cd) 366 { 367 unsigned int type, e = 0; 368 369 for (type = 0; type < GENWQE_DBG_UNITS; type++) { 370 switch (type) { 371 case GENWQE_DBG_UNIT0: 372 e = genwqe_ffdc_buff_size(cd, 0); 373 break; 374 case GENWQE_DBG_UNIT1: 375 e = genwqe_ffdc_buff_size(cd, 1); 376 break; 377 case GENWQE_DBG_UNIT2: 378 e = genwqe_ffdc_buff_size(cd, 2); 379 break; 380 case GENWQE_DBG_REGS: 381 e = GENWQE_FFDC_REGS; 382 break; 383 } 384 385 /* currently support only the debug units mentioned here */ 386 cd->ffdc[type].entries = e; 387 cd->ffdc[type].regs = 388 kmalloc_objs(struct genwqe_reg, e); 389 /* 390 * regs == NULL is ok, the using code treats this as no regs, 391 * Printing warning is ok in this case. 392 */ 393 } 394 return 0; 395 } 396 397 static void genwqe_ffdc_buffs_free(struct genwqe_dev *cd) 398 { 399 unsigned int type; 400 401 for (type = 0; type < GENWQE_DBG_UNITS; type++) { 402 kfree(cd->ffdc[type].regs); 403 cd->ffdc[type].regs = NULL; 404 } 405 } 406 407 static int genwqe_read_ids(struct genwqe_dev *cd) 408 { 409 int err = 0; 410 int slu_id; 411 struct pci_dev *pci_dev = cd->pci_dev; 412 413 cd->slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG); 414 if (cd->slu_unitcfg == IO_ILLEGAL_VALUE) { 415 dev_err(&pci_dev->dev, 416 "err: SLUID=%016llx\n", cd->slu_unitcfg); 417 err = -EIO; 418 goto out_err; 419 } 420 421 slu_id = genwqe_get_slu_id(cd); 422 if (slu_id < GENWQE_SLU_ARCH_REQ || slu_id == 0xff) { 423 dev_err(&pci_dev->dev, 424 "err: incompatible SLU Architecture %u\n", slu_id); 425 err = -ENOENT; 426 goto out_err; 427 } 428 429 cd->app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG); 430 if (cd->app_unitcfg == IO_ILLEGAL_VALUE) { 431 dev_err(&pci_dev->dev, 432 "err: APPID=%016llx\n", cd->app_unitcfg); 433 err = -EIO; 434 goto out_err; 435 } 436 genwqe_read_app_id(cd, cd->app_name, sizeof(cd->app_name)); 437 438 /* 439 * Is access to all registers possible? If we are a VF the 440 * answer is obvious. If we run fully virtualized, we need to 441 * check if we can access all registers. If we do not have 442 * full access we will cause an UR and some informational FIRs 443 * in the PF, but that should not harm. 444 */ 445 if (pci_dev->is_virtfn) 446 cd->is_privileged = 0; 447 else 448 cd->is_privileged = (__genwqe_readq(cd, IO_SLU_BITSTREAM) 449 != IO_ILLEGAL_VALUE); 450 451 out_err: 452 return err; 453 } 454 455 static int genwqe_start(struct genwqe_dev *cd) 456 { 457 int err; 458 struct pci_dev *pci_dev = cd->pci_dev; 459 460 err = genwqe_read_ids(cd); 461 if (err) 462 return err; 463 464 if (genwqe_is_privileged(cd)) { 465 /* do this after the tweaks. alloc fail is acceptable */ 466 genwqe_ffdc_buffs_alloc(cd); 467 genwqe_stop_traps(cd); 468 469 /* Collect registers e.g. FIRs, UNITIDs, traces ... */ 470 genwqe_read_ffdc_regs(cd, cd->ffdc[GENWQE_DBG_REGS].regs, 471 cd->ffdc[GENWQE_DBG_REGS].entries, 0); 472 473 genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT0, 474 cd->ffdc[GENWQE_DBG_UNIT0].regs, 475 cd->ffdc[GENWQE_DBG_UNIT0].entries); 476 477 genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT1, 478 cd->ffdc[GENWQE_DBG_UNIT1].regs, 479 cd->ffdc[GENWQE_DBG_UNIT1].entries); 480 481 genwqe_ffdc_buff_read(cd, GENWQE_DBG_UNIT2, 482 cd->ffdc[GENWQE_DBG_UNIT2].regs, 483 cd->ffdc[GENWQE_DBG_UNIT2].entries); 484 485 genwqe_start_traps(cd); 486 487 if (cd->card_state == GENWQE_CARD_FATAL_ERROR) { 488 dev_warn(&pci_dev->dev, 489 "[%s] chip reload/recovery!\n", __func__); 490 491 /* 492 * Stealth Mode: Reload chip on either hot 493 * reset or PERST. 494 */ 495 cd->softreset = 0x7Cull; 496 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, 497 cd->softreset); 498 499 err = genwqe_bus_reset(cd); 500 if (err != 0) { 501 dev_err(&pci_dev->dev, 502 "[%s] err: bus reset failed!\n", 503 __func__); 504 goto out; 505 } 506 507 /* 508 * Re-read the IDs because 509 * it could happen that the bitstream load 510 * failed! 511 */ 512 err = genwqe_read_ids(cd); 513 if (err) 514 goto out; 515 } 516 } 517 518 err = genwqe_setup_service_layer(cd); /* does a reset to the card */ 519 if (err != 0) { 520 dev_err(&pci_dev->dev, 521 "[%s] err: could not setup servicelayer!\n", __func__); 522 err = -ENODEV; 523 goto out; 524 } 525 526 if (genwqe_is_privileged(cd)) { /* code is running _after_ reset */ 527 genwqe_tweak_hardware(cd); 528 529 genwqe_setup_pf_jtimer(cd); 530 genwqe_setup_vf_jtimer(cd); 531 } 532 533 err = genwqe_device_create(cd); 534 if (err < 0) { 535 dev_err(&pci_dev->dev, 536 "err: chdev init failed! (err=%d)\n", err); 537 goto out_release_service_layer; 538 } 539 return 0; 540 541 out_release_service_layer: 542 genwqe_release_service_layer(cd); 543 out: 544 if (genwqe_is_privileged(cd)) 545 genwqe_ffdc_buffs_free(cd); 546 return -EIO; 547 } 548 549 /** 550 * genwqe_stop() - Stop card operation 551 * @cd: GenWQE device information 552 * 553 * Recovery notes: 554 * As long as genwqe_thread runs we might access registers during 555 * error data capture. Same is with the genwqe_health_thread. 556 * When genwqe_bus_reset() fails this function might called two times: 557 * first by the genwqe_health_thread() and later by genwqe_remove() to 558 * unbind the device. We must be able to survive that. 559 * 560 * This function must be robust enough to be called twice. 561 */ 562 static int genwqe_stop(struct genwqe_dev *cd) 563 { 564 genwqe_finish_queue(cd); /* no register access */ 565 genwqe_device_remove(cd); /* device removed, procs killed */ 566 genwqe_release_service_layer(cd); /* here genwqe_thread is stopped */ 567 568 if (genwqe_is_privileged(cd)) { 569 pci_disable_sriov(cd->pci_dev); /* access pci config space */ 570 genwqe_ffdc_buffs_free(cd); 571 } 572 573 return 0; 574 } 575 576 /** 577 * genwqe_recover_card() - Try to recover the card if it is possible 578 * @cd: GenWQE device information 579 * @fatal_err: Indicate whether to attempt soft reset 580 * 581 * If fatal_err is set no register access is possible anymore. It is 582 * likely that genwqe_start fails in that situation. Proper error 583 * handling is required in this case. 584 * 585 * genwqe_bus_reset() will cause the pci code to call genwqe_remove() 586 * and later genwqe_probe() for all virtual functions. 587 */ 588 static int genwqe_recover_card(struct genwqe_dev *cd, int fatal_err) 589 { 590 int rc; 591 struct pci_dev *pci_dev = cd->pci_dev; 592 593 genwqe_stop(cd); 594 595 /* 596 * Make sure chip is not reloaded to maintain FFDC. Write SLU 597 * Reset Register, CPLDReset field to 0. 598 */ 599 if (!fatal_err) { 600 cd->softreset = 0x70ull; 601 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, cd->softreset); 602 } 603 604 rc = genwqe_bus_reset(cd); 605 if (rc != 0) { 606 dev_err(&pci_dev->dev, 607 "[%s] err: card recovery impossible!\n", __func__); 608 return rc; 609 } 610 611 rc = genwqe_start(cd); 612 if (rc < 0) { 613 dev_err(&pci_dev->dev, 614 "[%s] err: failed to launch device!\n", __func__); 615 return rc; 616 } 617 return 0; 618 } 619 620 static int genwqe_health_check_cond(struct genwqe_dev *cd, u64 *gfir) 621 { 622 *gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 623 return (*gfir & GFIR_ERR_TRIGGER) && 624 genwqe_recovery_on_fatal_gfir_required(cd); 625 } 626 627 /** 628 * genwqe_fir_checking() - Check the fault isolation registers of the card 629 * @cd: GenWQE device information 630 * 631 * If this code works ok, can be tried out with help of the genwqe_poke tool: 632 * sudo ./tools/genwqe_poke 0x8 0xfefefefefef 633 * 634 * Now the relevant FIRs/sFIRs should be printed out and the driver should 635 * invoke recovery (devices are removed and readded). 636 */ 637 static u64 genwqe_fir_checking(struct genwqe_dev *cd) 638 { 639 int j, iterations = 0; 640 u64 mask, fir, fec, uid, gfir, gfir_masked, sfir, sfec; 641 u32 fir_addr, fir_clr_addr, fec_addr, sfir_addr, sfec_addr; 642 struct pci_dev *pci_dev = cd->pci_dev; 643 644 healthMonitor: 645 iterations++; 646 if (iterations > 16) { 647 dev_err(&pci_dev->dev, "* exit looping after %d times\n", 648 iterations); 649 goto fatal_error; 650 } 651 652 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 653 if (gfir != 0x0) 654 dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", 655 IO_SLC_CFGREG_GFIR, gfir); 656 if (gfir == IO_ILLEGAL_VALUE) 657 goto fatal_error; 658 659 /* 660 * Avoid printing when to GFIR bit is on prevents contignous 661 * printout e.g. for the following bug: 662 * FIR set without a 2ndary FIR/FIR cannot be cleared 663 * Comment out the following if to get the prints: 664 */ 665 if (gfir == 0) 666 return 0; 667 668 gfir_masked = gfir & GFIR_ERR_TRIGGER; /* fatal errors */ 669 670 for (uid = 0; uid < GENWQE_MAX_UNITS; uid++) { /* 0..2 in zEDC */ 671 672 /* read the primary FIR (pfir) */ 673 fir_addr = (uid << 24) + 0x08; 674 fir = __genwqe_readq(cd, fir_addr); 675 if (fir == 0x0) 676 continue; /* no error in this unit */ 677 678 dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fir_addr, fir); 679 if (fir == IO_ILLEGAL_VALUE) 680 goto fatal_error; 681 682 /* read primary FEC */ 683 fec_addr = (uid << 24) + 0x18; 684 fec = __genwqe_readq(cd, fec_addr); 685 686 dev_err(&pci_dev->dev, "* 0x%08x 0x%016llx\n", fec_addr, fec); 687 if (fec == IO_ILLEGAL_VALUE) 688 goto fatal_error; 689 690 for (j = 0, mask = 1ULL; j < 64; j++, mask <<= 1) { 691 692 /* secondary fir empty, skip it */ 693 if ((fir & mask) == 0x0) 694 continue; 695 696 sfir_addr = (uid << 24) + 0x100 + 0x08 * j; 697 sfir = __genwqe_readq(cd, sfir_addr); 698 699 if (sfir == IO_ILLEGAL_VALUE) 700 goto fatal_error; 701 dev_err(&pci_dev->dev, 702 "* 0x%08x 0x%016llx\n", sfir_addr, sfir); 703 704 sfec_addr = (uid << 24) + 0x300 + 0x08 * j; 705 sfec = __genwqe_readq(cd, sfec_addr); 706 707 if (sfec == IO_ILLEGAL_VALUE) 708 goto fatal_error; 709 dev_err(&pci_dev->dev, 710 "* 0x%08x 0x%016llx\n", sfec_addr, sfec); 711 712 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 713 if (gfir == IO_ILLEGAL_VALUE) 714 goto fatal_error; 715 716 /* gfir turned on during routine! get out and 717 start over. */ 718 if ((gfir_masked == 0x0) && 719 (gfir & GFIR_ERR_TRIGGER)) { 720 goto healthMonitor; 721 } 722 723 /* do not clear if we entered with a fatal gfir */ 724 if (gfir_masked == 0x0) { 725 726 /* NEW clear by mask the logged bits */ 727 sfir_addr = (uid << 24) + 0x100 + 0x08 * j; 728 __genwqe_writeq(cd, sfir_addr, sfir); 729 730 dev_dbg(&pci_dev->dev, 731 "[HM] Clearing 2ndary FIR 0x%08x with 0x%016llx\n", 732 sfir_addr, sfir); 733 734 /* 735 * note, these cannot be error-Firs 736 * since gfir_masked is 0 after sfir 737 * was read. Also, it is safe to do 738 * this write if sfir=0. Still need to 739 * clear the primary. This just means 740 * there is no secondary FIR. 741 */ 742 743 /* clear by mask the logged bit. */ 744 fir_clr_addr = (uid << 24) + 0x10; 745 __genwqe_writeq(cd, fir_clr_addr, mask); 746 747 dev_dbg(&pci_dev->dev, 748 "[HM] Clearing primary FIR 0x%08x with 0x%016llx\n", 749 fir_clr_addr, mask); 750 } 751 } 752 } 753 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 754 if (gfir == IO_ILLEGAL_VALUE) 755 goto fatal_error; 756 757 if ((gfir_masked == 0x0) && (gfir & GFIR_ERR_TRIGGER)) { 758 /* 759 * Check once more that it didn't go on after all the 760 * FIRS were cleared. 761 */ 762 dev_dbg(&pci_dev->dev, "ACK! Another FIR! Recursing %d!\n", 763 iterations); 764 goto healthMonitor; 765 } 766 return gfir_masked; 767 768 fatal_error: 769 return IO_ILLEGAL_VALUE; 770 } 771 772 /** 773 * genwqe_pci_fundamental_reset() - trigger a PCIe fundamental reset on the slot 774 * @pci_dev: PCI device information struct 775 * 776 * Note: pci_set_pcie_reset_state() is not implemented on all archs, so this 777 * reset method will not work in all cases. 778 * 779 * Return: 0 on success or error code from pci_set_pcie_reset_state() 780 */ 781 static int genwqe_pci_fundamental_reset(struct pci_dev *pci_dev) 782 { 783 int rc; 784 785 /* 786 * lock pci config space access from userspace, 787 * save state and issue PCIe fundamental reset 788 */ 789 pci_cfg_access_lock(pci_dev); 790 pci_save_state(pci_dev); 791 rc = pci_set_pcie_reset_state(pci_dev, pcie_warm_reset); 792 if (!rc) { 793 /* keep PCIe reset asserted for 250ms */ 794 msleep(250); 795 pci_set_pcie_reset_state(pci_dev, pcie_deassert_reset); 796 /* Wait for 2s to reload flash and train the link */ 797 msleep(2000); 798 } 799 pci_restore_state(pci_dev); 800 pci_cfg_access_unlock(pci_dev); 801 return rc; 802 } 803 804 805 static int genwqe_platform_recovery(struct genwqe_dev *cd) 806 { 807 struct pci_dev *pci_dev = cd->pci_dev; 808 int rc; 809 810 dev_info(&pci_dev->dev, 811 "[%s] resetting card for error recovery\n", __func__); 812 813 /* Clear out error injection flags */ 814 cd->err_inject &= ~(GENWQE_INJECT_HARDWARE_FAILURE | 815 GENWQE_INJECT_GFIR_FATAL | 816 GENWQE_INJECT_GFIR_INFO); 817 818 genwqe_stop(cd); 819 820 /* Try recoverying the card with fundamental reset */ 821 rc = genwqe_pci_fundamental_reset(pci_dev); 822 if (!rc) { 823 rc = genwqe_start(cd); 824 if (!rc) 825 dev_info(&pci_dev->dev, 826 "[%s] card recovered\n", __func__); 827 else 828 dev_err(&pci_dev->dev, 829 "[%s] err: cannot start card services! (err=%d)\n", 830 __func__, rc); 831 } else { 832 dev_err(&pci_dev->dev, 833 "[%s] card reset failed\n", __func__); 834 } 835 836 return rc; 837 } 838 839 /** 840 * genwqe_reload_bistream() - reload card bitstream 841 * @cd: GenWQE device information 842 * 843 * Set the appropriate register and call fundamental reset to reaload the card 844 * bitstream. 845 * 846 * Return: 0 on success, error code otherwise 847 */ 848 static int genwqe_reload_bistream(struct genwqe_dev *cd) 849 { 850 struct pci_dev *pci_dev = cd->pci_dev; 851 int rc; 852 853 dev_info(&pci_dev->dev, 854 "[%s] resetting card for bitstream reload\n", 855 __func__); 856 857 genwqe_stop(cd); 858 859 /* 860 * Cause a CPLD reprogram with the 'next_bitstream' 861 * partition on PCIe hot or fundamental reset 862 */ 863 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, 864 (cd->softreset & 0xcull) | 0x70ull); 865 866 rc = genwqe_pci_fundamental_reset(pci_dev); 867 if (rc) { 868 /* 869 * A fundamental reset failure can be caused 870 * by lack of support on the arch, so we just 871 * log the error and try to start the card 872 * again. 873 */ 874 dev_err(&pci_dev->dev, 875 "[%s] err: failed to reset card for bitstream reload\n", 876 __func__); 877 } 878 879 rc = genwqe_start(cd); 880 if (rc) { 881 dev_err(&pci_dev->dev, 882 "[%s] err: cannot start card services! (err=%d)\n", 883 __func__, rc); 884 return rc; 885 } 886 dev_info(&pci_dev->dev, 887 "[%s] card reloaded\n", __func__); 888 return 0; 889 } 890 891 892 /** 893 * genwqe_health_thread() - Health checking thread 894 * @data: GenWQE device information 895 * 896 * This thread is only started for the PF of the card. 897 * 898 * This thread monitors the health of the card. A critical situation 899 * is when we read registers which contain -1 (IO_ILLEGAL_VALUE). In 900 * this case we need to be recovered from outside. Writing to 901 * registers will very likely not work either. 902 * 903 * This thread must only exit if kthread_should_stop() becomes true. 904 * 905 * Condition for the health-thread to trigger: 906 * a) when a kthread_stop() request comes in or 907 * b) a critical GFIR occured 908 * 909 * Informational GFIRs are checked and potentially printed in 910 * GENWQE_HEALTH_CHECK_INTERVAL seconds. 911 */ 912 static int genwqe_health_thread(void *data) 913 { 914 int rc, should_stop = 0; 915 struct genwqe_dev *cd = data; 916 struct pci_dev *pci_dev = cd->pci_dev; 917 u64 gfir, gfir_masked, slu_unitcfg, app_unitcfg; 918 919 health_thread_begin: 920 while (!kthread_should_stop()) { 921 rc = wait_event_interruptible_timeout(cd->health_waitq, 922 (genwqe_health_check_cond(cd, &gfir) || 923 (should_stop = kthread_should_stop())), 924 GENWQE_HEALTH_CHECK_INTERVAL * HZ); 925 926 if (should_stop) 927 break; 928 929 if (gfir == IO_ILLEGAL_VALUE) { 930 dev_err(&pci_dev->dev, 931 "[%s] GFIR=%016llx\n", __func__, gfir); 932 goto fatal_error; 933 } 934 935 slu_unitcfg = __genwqe_readq(cd, IO_SLU_UNITCFG); 936 if (slu_unitcfg == IO_ILLEGAL_VALUE) { 937 dev_err(&pci_dev->dev, 938 "[%s] SLU_UNITCFG=%016llx\n", 939 __func__, slu_unitcfg); 940 goto fatal_error; 941 } 942 943 app_unitcfg = __genwqe_readq(cd, IO_APP_UNITCFG); 944 if (app_unitcfg == IO_ILLEGAL_VALUE) { 945 dev_err(&pci_dev->dev, 946 "[%s] APP_UNITCFG=%016llx\n", 947 __func__, app_unitcfg); 948 goto fatal_error; 949 } 950 951 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 952 if (gfir == IO_ILLEGAL_VALUE) { 953 dev_err(&pci_dev->dev, 954 "[%s] %s: GFIR=%016llx\n", __func__, 955 (gfir & GFIR_ERR_TRIGGER) ? "err" : "info", 956 gfir); 957 goto fatal_error; 958 } 959 960 gfir_masked = genwqe_fir_checking(cd); 961 if (gfir_masked == IO_ILLEGAL_VALUE) 962 goto fatal_error; 963 964 /* 965 * GFIR ErrorTrigger bits set => reset the card! 966 * Never do this for old/manufacturing images! 967 */ 968 if ((gfir_masked) && !cd->skip_recovery && 969 genwqe_recovery_on_fatal_gfir_required(cd)) { 970 971 cd->card_state = GENWQE_CARD_FATAL_ERROR; 972 973 rc = genwqe_recover_card(cd, 0); 974 if (rc < 0) { 975 /* FIXME Card is unusable and needs unbind! */ 976 goto fatal_error; 977 } 978 } 979 980 if (cd->card_state == GENWQE_CARD_RELOAD_BITSTREAM) { 981 /* Userspace requested card bitstream reload */ 982 rc = genwqe_reload_bistream(cd); 983 if (rc) 984 goto fatal_error; 985 } 986 987 cd->last_gfir = gfir; 988 cond_resched(); 989 } 990 991 return 0; 992 993 fatal_error: 994 if (cd->use_platform_recovery) { 995 /* 996 * Since we use raw accessors, EEH errors won't be detected 997 * by the platform until we do a non-raw MMIO or config space 998 * read 999 */ 1000 readq(cd->mmio + IO_SLC_CFGREG_GFIR); 1001 1002 /* We do nothing if the card is going over PCI recovery */ 1003 if (pci_channel_offline(pci_dev)) 1004 return -EIO; 1005 1006 /* 1007 * If it's supported by the platform, we try a fundamental reset 1008 * to recover from a fatal error. Otherwise, we continue to wait 1009 * for an external recovery procedure to take care of it. 1010 */ 1011 rc = genwqe_platform_recovery(cd); 1012 if (!rc) 1013 goto health_thread_begin; 1014 } 1015 1016 dev_err(&pci_dev->dev, 1017 "[%s] card unusable. Please trigger unbind!\n", __func__); 1018 1019 /* Bring down logical devices to inform user space via udev remove. */ 1020 cd->card_state = GENWQE_CARD_FATAL_ERROR; 1021 genwqe_stop(cd); 1022 1023 /* genwqe_bus_reset failed(). Now wait for genwqe_remove(). */ 1024 while (!kthread_should_stop()) 1025 cond_resched(); 1026 1027 return -EIO; 1028 } 1029 1030 static int genwqe_health_check_start(struct genwqe_dev *cd) 1031 { 1032 int rc; 1033 1034 if (GENWQE_HEALTH_CHECK_INTERVAL <= 0) 1035 return 0; /* valid for disabling the service */ 1036 1037 /* moved before request_irq() */ 1038 /* init_waitqueue_head(&cd->health_waitq); */ 1039 1040 cd->health_thread = kthread_run(genwqe_health_thread, cd, 1041 GENWQE_DEVNAME "%d_health", 1042 cd->card_idx); 1043 if (IS_ERR(cd->health_thread)) { 1044 rc = PTR_ERR(cd->health_thread); 1045 cd->health_thread = NULL; 1046 return rc; 1047 } 1048 return 0; 1049 } 1050 1051 static int genwqe_health_thread_running(struct genwqe_dev *cd) 1052 { 1053 return cd->health_thread != NULL; 1054 } 1055 1056 static int genwqe_health_check_stop(struct genwqe_dev *cd) 1057 { 1058 if (!genwqe_health_thread_running(cd)) 1059 return -EIO; 1060 1061 kthread_stop(cd->health_thread); 1062 cd->health_thread = NULL; 1063 return 0; 1064 } 1065 1066 /** 1067 * genwqe_pci_setup() - Allocate PCIe related resources for our card 1068 * @cd: GenWQE device information 1069 */ 1070 static int genwqe_pci_setup(struct genwqe_dev *cd) 1071 { 1072 int err; 1073 struct pci_dev *pci_dev = cd->pci_dev; 1074 1075 err = pci_enable_device_mem(pci_dev); 1076 if (err) { 1077 dev_err(&pci_dev->dev, 1078 "err: failed to enable pci memory (err=%d)\n", err); 1079 goto err_out; 1080 } 1081 1082 /* Reserve PCI I/O and memory resources */ 1083 err = pci_request_mem_regions(pci_dev, genwqe_driver_name); 1084 if (err) { 1085 dev_err(&pci_dev->dev, 1086 "[%s] err: request bars failed (%d)\n", __func__, err); 1087 err = -EIO; 1088 goto err_disable_device; 1089 } 1090 1091 /* check for 64-bit DMA address supported (DAC) */ 1092 /* check for 32-bit DMA address supported (SAC) */ 1093 if (dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)) && 1094 dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(32))) { 1095 dev_err(&pci_dev->dev, 1096 "err: neither DMA32 nor DMA64 supported\n"); 1097 err = -EIO; 1098 goto out_release_resources; 1099 } 1100 1101 pci_set_master(pci_dev); 1102 1103 /* EEH recovery requires PCIe fundamental reset */ 1104 pci_dev->needs_freset = 1; 1105 1106 /* request complete BAR-0 space (length = 0) */ 1107 cd->mmio_len = pci_resource_len(pci_dev, 0); 1108 cd->mmio = pci_iomap(pci_dev, 0, 0); 1109 if (cd->mmio == NULL) { 1110 dev_err(&pci_dev->dev, 1111 "[%s] err: mapping BAR0 failed\n", __func__); 1112 err = -ENOMEM; 1113 goto out_release_resources; 1114 } 1115 1116 cd->num_vfs = pci_sriov_get_totalvfs(pci_dev); 1117 if (cd->num_vfs < 0) 1118 cd->num_vfs = 0; 1119 1120 err = genwqe_read_ids(cd); 1121 if (err) 1122 goto out_iounmap; 1123 1124 return 0; 1125 1126 out_iounmap: 1127 pci_iounmap(pci_dev, cd->mmio); 1128 out_release_resources: 1129 pci_release_mem_regions(pci_dev); 1130 err_disable_device: 1131 pci_disable_device(pci_dev); 1132 err_out: 1133 return err; 1134 } 1135 1136 /** 1137 * genwqe_pci_remove() - Free PCIe related resources for our card 1138 * @cd: GenWQE device information 1139 */ 1140 static void genwqe_pci_remove(struct genwqe_dev *cd) 1141 { 1142 struct pci_dev *pci_dev = cd->pci_dev; 1143 1144 if (cd->mmio) 1145 pci_iounmap(pci_dev, cd->mmio); 1146 1147 pci_release_mem_regions(pci_dev); 1148 pci_disable_device(pci_dev); 1149 } 1150 1151 /** 1152 * genwqe_probe() - Device initialization 1153 * @pci_dev: PCI device information struct 1154 * @id: PCI device ID 1155 * 1156 * Callable for multiple cards. This function is called on bind. 1157 * 1158 * Return: 0 if succeeded, < 0 when failed 1159 */ 1160 static int genwqe_probe(struct pci_dev *pci_dev, 1161 const struct pci_device_id *id) 1162 { 1163 int err; 1164 struct genwqe_dev *cd; 1165 1166 genwqe_init_crc32(); 1167 1168 cd = genwqe_dev_alloc(); 1169 if (IS_ERR(cd)) { 1170 dev_err(&pci_dev->dev, "err: could not alloc mem (err=%d)!\n", 1171 (int)PTR_ERR(cd)); 1172 return PTR_ERR(cd); 1173 } 1174 1175 dev_set_drvdata(&pci_dev->dev, cd); 1176 cd->pci_dev = pci_dev; 1177 1178 err = genwqe_pci_setup(cd); 1179 if (err < 0) { 1180 dev_err(&pci_dev->dev, 1181 "err: problems with PCI setup (err=%d)\n", err); 1182 goto out_free_dev; 1183 } 1184 1185 err = genwqe_start(cd); 1186 if (err < 0) { 1187 dev_err(&pci_dev->dev, 1188 "err: cannot start card services! (err=%d)\n", err); 1189 goto out_pci_remove; 1190 } 1191 1192 if (genwqe_is_privileged(cd)) { 1193 err = genwqe_health_check_start(cd); 1194 if (err < 0) { 1195 dev_err(&pci_dev->dev, 1196 "err: cannot start health checking! (err=%d)\n", 1197 err); 1198 goto out_stop_services; 1199 } 1200 } 1201 return 0; 1202 1203 out_stop_services: 1204 genwqe_stop(cd); 1205 out_pci_remove: 1206 genwqe_pci_remove(cd); 1207 out_free_dev: 1208 genwqe_dev_free(cd); 1209 return err; 1210 } 1211 1212 /** 1213 * genwqe_remove() - Called when device is removed (hot-plugable) 1214 * @pci_dev: PCI device information struct 1215 * 1216 * Or when driver is unloaded respecitively when unbind is done. 1217 */ 1218 static void genwqe_remove(struct pci_dev *pci_dev) 1219 { 1220 struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev); 1221 1222 genwqe_health_check_stop(cd); 1223 1224 /* 1225 * genwqe_stop() must survive if it is called twice 1226 * sequentially. This happens when the health thread calls it 1227 * and fails on genwqe_bus_reset(). 1228 */ 1229 genwqe_stop(cd); 1230 genwqe_pci_remove(cd); 1231 genwqe_dev_free(cd); 1232 } 1233 1234 /** 1235 * genwqe_err_error_detected() - Error detection callback 1236 * @pci_dev: PCI device information struct 1237 * @state: PCI channel state 1238 * 1239 * This callback is called by the PCI subsystem whenever a PCI bus 1240 * error is detected. 1241 */ 1242 static pci_ers_result_t genwqe_err_error_detected(struct pci_dev *pci_dev, 1243 pci_channel_state_t state) 1244 { 1245 struct genwqe_dev *cd; 1246 1247 dev_err(&pci_dev->dev, "[%s] state=%d\n", __func__, state); 1248 1249 cd = dev_get_drvdata(&pci_dev->dev); 1250 if (cd == NULL) 1251 return PCI_ERS_RESULT_DISCONNECT; 1252 1253 /* Stop the card */ 1254 genwqe_health_check_stop(cd); 1255 genwqe_stop(cd); 1256 1257 /* 1258 * On permanent failure, the PCI code will call device remove 1259 * after the return of this function. 1260 * genwqe_stop() can be called twice. 1261 */ 1262 if (state == pci_channel_io_perm_failure) { 1263 return PCI_ERS_RESULT_DISCONNECT; 1264 } else { 1265 genwqe_pci_remove(cd); 1266 return PCI_ERS_RESULT_NEED_RESET; 1267 } 1268 } 1269 1270 static pci_ers_result_t genwqe_err_slot_reset(struct pci_dev *pci_dev) 1271 { 1272 int rc; 1273 struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev); 1274 1275 rc = genwqe_pci_setup(cd); 1276 if (!rc) { 1277 return PCI_ERS_RESULT_RECOVERED; 1278 } else { 1279 dev_err(&pci_dev->dev, 1280 "err: problems with PCI setup (err=%d)\n", rc); 1281 return PCI_ERS_RESULT_DISCONNECT; 1282 } 1283 } 1284 1285 static pci_ers_result_t genwqe_err_result_none(struct pci_dev *dev) 1286 { 1287 return PCI_ERS_RESULT_NONE; 1288 } 1289 1290 static void genwqe_err_resume(struct pci_dev *pci_dev) 1291 { 1292 int rc; 1293 struct genwqe_dev *cd = dev_get_drvdata(&pci_dev->dev); 1294 1295 rc = genwqe_start(cd); 1296 if (!rc) { 1297 rc = genwqe_health_check_start(cd); 1298 if (rc) 1299 dev_err(&pci_dev->dev, 1300 "err: cannot start health checking! (err=%d)\n", 1301 rc); 1302 } else { 1303 dev_err(&pci_dev->dev, 1304 "err: cannot start card services! (err=%d)\n", rc); 1305 } 1306 } 1307 1308 static int genwqe_sriov_configure(struct pci_dev *dev, int numvfs) 1309 { 1310 int rc; 1311 struct genwqe_dev *cd = dev_get_drvdata(&dev->dev); 1312 1313 if (numvfs > 0) { 1314 genwqe_setup_vf_jtimer(cd); 1315 rc = pci_enable_sriov(dev, numvfs); 1316 if (rc < 0) 1317 return rc; 1318 return numvfs; 1319 } 1320 if (numvfs == 0) { 1321 pci_disable_sriov(dev); 1322 return 0; 1323 } 1324 return 0; 1325 } 1326 1327 static const struct pci_error_handlers genwqe_err_handler = { 1328 .error_detected = genwqe_err_error_detected, 1329 .mmio_enabled = genwqe_err_result_none, 1330 .slot_reset = genwqe_err_slot_reset, 1331 .resume = genwqe_err_resume, 1332 }; 1333 1334 static struct pci_driver genwqe_driver = { 1335 .name = genwqe_driver_name, 1336 .id_table = genwqe_device_table, 1337 .probe = genwqe_probe, 1338 .remove = genwqe_remove, 1339 .sriov_configure = genwqe_sriov_configure, 1340 .err_handler = &genwqe_err_handler, 1341 }; 1342 1343 /** 1344 * genwqe_init_module() - Driver registration and initialization 1345 */ 1346 static int __init genwqe_init_module(void) 1347 { 1348 int rc; 1349 1350 rc = class_register(&class_genwqe); 1351 if (rc) { 1352 pr_err("[%s] create class failed\n", __func__); 1353 return -ENOMEM; 1354 } 1355 1356 debugfs_genwqe = debugfs_create_dir(GENWQE_DEVNAME, NULL); 1357 1358 rc = pci_register_driver(&genwqe_driver); 1359 if (rc != 0) { 1360 pr_err("[%s] pci_reg_driver (rc=%d)\n", __func__, rc); 1361 goto err_out0; 1362 } 1363 1364 return rc; 1365 1366 err_out0: 1367 debugfs_remove(debugfs_genwqe); 1368 class_unregister(&class_genwqe); 1369 return rc; 1370 } 1371 1372 /** 1373 * genwqe_exit_module() - Driver exit 1374 */ 1375 static void __exit genwqe_exit_module(void) 1376 { 1377 pci_unregister_driver(&genwqe_driver); 1378 debugfs_remove(debugfs_genwqe); 1379 class_unregister(&class_genwqe); 1380 } 1381 1382 module_init(genwqe_init_module); 1383 module_exit(genwqe_exit_module); 1384