xref: /linux/drivers/accel/habanalabs/common/habanalabs_drv.c (revision e3610441d1fb47b1f00e4c38bdf333176e824729) 
1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright 2016-2021 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  *
7  */
8 
9 #define pr_fmt(fmt)		"habanalabs: " fmt
10 
11 #include "habanalabs.h"
12 #include "../include/hw_ip/pci/pci_general.h"
13 
14 #include <linux/pci.h>
15 #include <linux/module.h>
16 #include <linux/vmalloc.h>
17 #include <linux/version.h>
18 
19 #include <drm/drm_accel.h>
20 #include <drm/drm_drv.h>
21 #include <drm/drm_ioctl.h>
22 
23 #define CREATE_TRACE_POINTS
24 #include <trace/events/habanalabs.h>
25 
26 #define HL_DRIVER_AUTHOR	"HabanaLabs Kernel Driver Team"
27 
28 #define HL_DRIVER_DESC		"Driver for HabanaLabs's AI Accelerators"
29 
30 MODULE_AUTHOR(HL_DRIVER_AUTHOR);
31 MODULE_DESCRIPTION(HL_DRIVER_DESC);
32 MODULE_LICENSE("GPL v2");
33 
34 static int hl_major;
35 static DEFINE_IDR(hl_devs_idr);
36 static DEFINE_MUTEX(hl_devs_idr_lock);
37 
38 #define HL_DEFAULT_TIMEOUT_LOCKED	30	/* 30 seconds */
39 #define GAUDI_DEFAULT_TIMEOUT_LOCKED	600	/* 10 minutes */
40 
41 static int timeout_locked = HL_DEFAULT_TIMEOUT_LOCKED;
42 static int reset_on_lockup = 1;
43 static int memory_scrub;
44 static ulong boot_error_status_mask = ULONG_MAX;
45 
46 module_param(timeout_locked, int, 0444);
47 MODULE_PARM_DESC(timeout_locked,
48 	"Device lockup timeout in seconds (0 = disabled, default 30s)");
49 
50 module_param(reset_on_lockup, int, 0444);
51 MODULE_PARM_DESC(reset_on_lockup,
52 	"Do device reset on lockup (0 = no, 1 = yes, default yes)");
53 
54 module_param(memory_scrub, int, 0444);
55 MODULE_PARM_DESC(memory_scrub,
56 	"Scrub device memory in various states (0 = no, 1 = yes, default no)");
57 
58 module_param(boot_error_status_mask, ulong, 0444);
59 MODULE_PARM_DESC(boot_error_status_mask,
60 	"Mask of the error status during device CPU boot (If bitX is cleared then error X is masked. Default all 1's)");
61 
62 #define PCI_IDS_GOYA			0x0001
63 #define PCI_IDS_GAUDI			0x1000
64 #define PCI_IDS_GAUDI_SEC		0x1010
65 
66 #define PCI_IDS_GAUDI2			0x1020
67 
68 static const struct pci_device_id ids[] = {
69 	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GOYA), },
70 	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI), },
71 	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI_SEC), },
72 	{ PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI2), },
73 	{ 0, }
74 };
75 MODULE_DEVICE_TABLE(pci, ids);
76 
77 static const struct drm_ioctl_desc hl_drm_ioctls[] = {
78 	DRM_IOCTL_DEF_DRV(HL_INFO, hl_info_ioctl, 0),
79 	DRM_IOCTL_DEF_DRV(HL_CB, hl_cb_ioctl, 0),
80 	DRM_IOCTL_DEF_DRV(HL_CS, hl_cs_ioctl, 0),
81 	DRM_IOCTL_DEF_DRV(HL_WAIT_CS, hl_wait_ioctl, 0),
82 	DRM_IOCTL_DEF_DRV(HL_MEMORY, hl_mem_ioctl, 0),
83 	DRM_IOCTL_DEF_DRV(HL_DEBUG, hl_debug_ioctl, 0),
84 };
85 
86 static const struct file_operations hl_fops = {
87 	.owner = THIS_MODULE,
88 	.open = accel_open,
89 	.release = drm_release,
90 	.unlocked_ioctl = drm_ioctl,
91 	.compat_ioctl = drm_compat_ioctl,
92 	.llseek = noop_llseek,
93 	.mmap = hl_mmap
94 };
95 
96 static const struct drm_driver hl_driver = {
97 	.driver_features = DRIVER_COMPUTE_ACCEL,
98 
99 	.name = HL_NAME,
100 	.desc = HL_DRIVER_DESC,
101 	.major = LINUX_VERSION_MAJOR,
102 	.minor = LINUX_VERSION_PATCHLEVEL,
103 	.patchlevel = LINUX_VERSION_SUBLEVEL,
104 
105 	.fops = &hl_fops,
106 	.open = hl_device_open,
107 	.postclose = hl_device_release,
108 	.ioctls = hl_drm_ioctls,
109 	.num_ioctls = ARRAY_SIZE(hl_drm_ioctls)
110 };
111 
112 /*
113  * get_asic_type - translate device id to asic type
114  *
115  * @hdev: pointer to habanalabs device structure.
116  *
117  * Translate device id and revision id to asic type.
118  * In case of unidentified device, return -1
119  */
120 static enum hl_asic_type get_asic_type(struct hl_device *hdev)
121 {
122 	struct pci_dev *pdev = hdev->pdev;
123 	enum hl_asic_type asic_type = ASIC_INVALID;
124 
125 	switch (pdev->device) {
126 	case PCI_IDS_GOYA:
127 		asic_type = ASIC_GOYA;
128 		break;
129 	case PCI_IDS_GAUDI:
130 		asic_type = ASIC_GAUDI;
131 		break;
132 	case PCI_IDS_GAUDI_SEC:
133 		asic_type = ASIC_GAUDI_SEC;
134 		break;
135 	case PCI_IDS_GAUDI2:
136 		switch (pdev->revision) {
137 		case REV_ID_A:
138 			asic_type = ASIC_GAUDI2;
139 			break;
140 		case REV_ID_B:
141 			asic_type = ASIC_GAUDI2B;
142 			break;
143 		case REV_ID_C:
144 			asic_type = ASIC_GAUDI2C;
145 			break;
146 		case REV_ID_D:
147 			asic_type = ASIC_GAUDI2D;
148 			break;
149 		default:
150 			break;
151 		}
152 		break;
153 	default:
154 		break;
155 	}
156 
157 	return asic_type;
158 }
159 
160 static bool is_asic_secured(enum hl_asic_type asic_type)
161 {
162 	switch (asic_type) {
163 	case ASIC_GAUDI_SEC:
164 		return true;
165 	default:
166 		return false;
167 	}
168 }
169 
170 /*
171  * hl_device_open() - open function for habanalabs device.
172  * @ddev: pointer to DRM device structure.
173  * @file: pointer to DRM file private data structure.
174  *
175  * Called when process opens an habanalabs device.
176  */
177 int hl_device_open(struct drm_device *ddev, struct drm_file *file_priv)
178 {
179 	struct hl_device *hdev = to_hl_device(ddev);
180 	enum hl_device_status status;
181 	struct hl_fpriv *hpriv;
182 	int rc;
183 
184 	hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
185 	if (!hpriv)
186 		return -ENOMEM;
187 
188 	hpriv->hdev = hdev;
189 	mutex_init(&hpriv->notifier_event.lock);
190 	mutex_init(&hpriv->restore_phase_mutex);
191 	mutex_init(&hpriv->ctx_lock);
192 	kref_init(&hpriv->refcount);
193 
194 	hl_ctx_mgr_init(&hpriv->ctx_mgr);
195 	hl_mem_mgr_init(hpriv->hdev->dev, &hpriv->mem_mgr);
196 
197 	hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);
198 
199 	mutex_lock(&hdev->fpriv_list_lock);
200 
201 	if (!hl_device_operational(hdev, &status)) {
202 		dev_dbg_ratelimited(hdev->dev,
203 			"Can't open %s because it is %s\n",
204 			dev_name(hdev->dev), hdev->status[status]);
205 
206 		if (status == HL_DEVICE_STATUS_IN_RESET ||
207 					status == HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE)
208 			rc = -EAGAIN;
209 		else
210 			rc = -EPERM;
211 
212 		goto out_err;
213 	}
214 
215 	if (hdev->is_in_dram_scrub) {
216 		dev_dbg_ratelimited(hdev->dev,
217 			"Can't open %s during dram scrub\n",
218 			dev_name(hdev->dev));
219 		rc = -EAGAIN;
220 		goto out_err;
221 	}
222 
223 	if (hdev->compute_ctx_in_release) {
224 		dev_dbg_ratelimited(hdev->dev,
225 			"Can't open %s because another user is still releasing it\n",
226 			dev_name(hdev->dev));
227 		rc = -EAGAIN;
228 		goto out_err;
229 	}
230 
231 	if (hdev->is_compute_ctx_active) {
232 		dev_dbg_ratelimited(hdev->dev,
233 			"Can't open %s because another user is working on it\n",
234 			dev_name(hdev->dev));
235 		rc = -EBUSY;
236 		goto out_err;
237 	}
238 
239 	rc = hl_ctx_create(hdev, hpriv);
240 	if (rc) {
241 		dev_err(hdev->dev, "Failed to create context %d\n", rc);
242 		goto out_err;
243 	}
244 
245 	list_add(&hpriv->dev_node, &hdev->fpriv_list);
246 	mutex_unlock(&hdev->fpriv_list_lock);
247 
248 	hdev->asic_funcs->send_device_activity(hdev, true);
249 
250 	hl_debugfs_add_file(hpriv);
251 
252 	hl_enable_err_info_capture(&hdev->captured_err_info);
253 
254 	hdev->open_counter++;
255 	hdev->last_successful_open_jif = jiffies;
256 	hdev->last_successful_open_ktime = ktime_get();
257 
258 	file_priv->driver_priv = hpriv;
259 	hpriv->file_priv = file_priv;
260 
261 	return 0;
262 
263 out_err:
264 	mutex_unlock(&hdev->fpriv_list_lock);
265 	hl_mem_mgr_fini(&hpriv->mem_mgr, NULL);
266 	hl_mem_mgr_idr_destroy(&hpriv->mem_mgr);
267 	hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
268 	mutex_destroy(&hpriv->ctx_lock);
269 	mutex_destroy(&hpriv->restore_phase_mutex);
270 	mutex_destroy(&hpriv->notifier_event.lock);
271 	put_pid(hpriv->taskpid);
272 
273 	kfree(hpriv);
274 
275 	return rc;
276 }
277 
278 int hl_device_open_ctrl(struct inode *inode, struct file *filp)
279 {
280 	struct hl_device *hdev;
281 	struct hl_fpriv *hpriv;
282 	int rc;
283 
284 	mutex_lock(&hl_devs_idr_lock);
285 	hdev = idr_find(&hl_devs_idr, iminor(inode));
286 	mutex_unlock(&hl_devs_idr_lock);
287 
288 	if (!hdev) {
289 		pr_err("Couldn't find device %d:%d\n",
290 			imajor(inode), iminor(inode));
291 		return -ENXIO;
292 	}
293 
294 	hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
295 	if (!hpriv)
296 		return -ENOMEM;
297 
298 	/* Prevent other routines from reading partial hpriv data by
299 	 * initializing hpriv fields before inserting it to the list
300 	 */
301 	hpriv->hdev = hdev;
302 	filp->private_data = hpriv;
303 
304 	nonseekable_open(inode, filp);
305 
306 	hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);
307 
308 	mutex_lock(&hdev->fpriv_ctrl_list_lock);
309 
310 	if (!hl_ctrl_device_operational(hdev, NULL)) {
311 		dev_dbg_ratelimited(hdev->dev_ctrl,
312 			"Can't open %s because it is disabled\n",
313 			dev_name(hdev->dev_ctrl));
314 		rc = -EPERM;
315 		goto out_err;
316 	}
317 
318 	list_add(&hpriv->dev_node, &hdev->fpriv_ctrl_list);
319 	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
320 
321 	return 0;
322 
323 out_err:
324 	mutex_unlock(&hdev->fpriv_ctrl_list_lock);
325 	filp->private_data = NULL;
326 	put_pid(hpriv->taskpid);
327 
328 	kfree(hpriv);
329 
330 	return rc;
331 }
332 
333 static void set_driver_behavior_per_device(struct hl_device *hdev)
334 {
335 	hdev->nic_ports_mask = 0;
336 	hdev->fw_components = FW_TYPE_ALL_TYPES;
337 	hdev->cpu_queues_enable = 1;
338 	hdev->pldm = 0;
339 	hdev->hard_reset_on_fw_events = 1;
340 	hdev->bmc_enable = 1;
341 	hdev->reset_on_preboot_fail = 1;
342 	hdev->heartbeat = 1;
343 }
344 
345 static void copy_kernel_module_params_to_device(struct hl_device *hdev)
346 {
347 	hdev->asic_prop.fw_security_enabled = is_asic_secured(hdev->asic_type);
348 
349 	hdev->major = hl_major;
350 	hdev->memory_scrub = memory_scrub;
351 	hdev->reset_on_lockup = reset_on_lockup;
352 	hdev->boot_error_status_mask = boot_error_status_mask;
353 }
354 
355 static void fixup_device_params_per_asic(struct hl_device *hdev, int timeout)
356 {
357 	switch (hdev->asic_type) {
358 	case ASIC_GAUDI:
359 	case ASIC_GAUDI_SEC:
360 		/* If user didn't request a different timeout than the default one, we have
361 		 * a different default timeout for Gaudi
362 		 */
363 		if (timeout == HL_DEFAULT_TIMEOUT_LOCKED)
364 			hdev->timeout_jiffies = msecs_to_jiffies(GAUDI_DEFAULT_TIMEOUT_LOCKED *
365 										MSEC_PER_SEC);
366 
367 		hdev->reset_upon_device_release = 0;
368 		break;
369 
370 	case ASIC_GOYA:
371 		hdev->reset_upon_device_release = 0;
372 		break;
373 
374 	default:
375 		hdev->reset_upon_device_release = 1;
376 		break;
377 	}
378 }
379 
380 static int fixup_device_params(struct hl_device *hdev)
381 {
382 	int tmp_timeout;
383 
384 	tmp_timeout = timeout_locked;
385 
386 	hdev->fw_poll_interval_usec = HL_FW_STATUS_POLL_INTERVAL_USEC;
387 	hdev->fw_comms_poll_interval_usec = HL_FW_STATUS_POLL_INTERVAL_USEC;
388 
389 	if (tmp_timeout)
390 		hdev->timeout_jiffies = msecs_to_jiffies(tmp_timeout * MSEC_PER_SEC);
391 	else
392 		hdev->timeout_jiffies = MAX_SCHEDULE_TIMEOUT;
393 
394 	hdev->stop_on_err = true;
395 	hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
396 	hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
397 
398 	/* Enable only after the initialization of the device */
399 	hdev->disabled = true;
400 
401 	if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU) &&
402 			(hdev->fw_components & ~FW_TYPE_PREBOOT_CPU)) {
403 		pr_err("Preboot must be set along with other components");
404 		return -EINVAL;
405 	}
406 
407 	/* If CPU queues not enabled, no way to do heartbeat */
408 	if (!hdev->cpu_queues_enable)
409 		hdev->heartbeat = 0;
410 	fixup_device_params_per_asic(hdev, tmp_timeout);
411 
412 	return 0;
413 }
414 
415 static int allocate_device_id(struct hl_device *hdev)
416 {
417 	int id;
418 
419 	mutex_lock(&hl_devs_idr_lock);
420 	id = idr_alloc(&hl_devs_idr, hdev, 0, HL_MAX_MINORS, GFP_KERNEL);
421 	mutex_unlock(&hl_devs_idr_lock);
422 
423 	if (id < 0) {
424 		if (id == -ENOSPC)
425 			pr_err("too many devices in the system\n");
426 		return -EBUSY;
427 	}
428 
429 	hdev->id = id;
430 
431 	/*
432 	 * Firstly initialized with the internal device ID.
433 	 * Will be updated later after the DRM device registration to hold the minor ID.
434 	 */
435 	hdev->cdev_idx = hdev->id;
436 
437 	return 0;
438 }
439 
440 /**
441  * create_hdev - create habanalabs device instance
442  *
443  * @dev: will hold the pointer to the new habanalabs device structure
444  * @pdev: pointer to the pci device
445  *
446  * Allocate memory for habanalabs device and initialize basic fields
447  * Identify the ASIC type
448  * Allocate ID (minor) for the device (only for real devices)
449  */
450 static int create_hdev(struct hl_device **dev, struct pci_dev *pdev)
451 {
452 	struct hl_device *hdev;
453 	int rc;
454 
455 	*dev = NULL;
456 
457 	hdev = devm_drm_dev_alloc(&pdev->dev, &hl_driver, struct hl_device, drm);
458 	if (IS_ERR(hdev))
459 		return PTR_ERR(hdev);
460 
461 	hdev->dev = hdev->drm.dev;
462 
463 	/* Will be NULL in case of simulator device */
464 	hdev->pdev = pdev;
465 
466 	/* Assign status description string */
467 	strscpy(hdev->status[HL_DEVICE_STATUS_OPERATIONAL], "operational", HL_STR_MAX);
468 	strscpy(hdev->status[HL_DEVICE_STATUS_IN_RESET], "in reset", HL_STR_MAX);
469 	strscpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION], "disabled", HL_STR_MAX);
470 	strscpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET], "needs reset", HL_STR_MAX);
471 	strscpy(hdev->status[HL_DEVICE_STATUS_IN_DEVICE_CREATION],
472 				"in device creation", HL_STR_MAX);
473 	strscpy(hdev->status[HL_DEVICE_STATUS_IN_RESET_AFTER_DEVICE_RELEASE],
474 				"in reset after device release", HL_STR_MAX);
475 
476 
477 	/* First, we must find out which ASIC are we handling. This is needed
478 	 * to configure the behavior of the driver (kernel parameters)
479 	 */
480 	hdev->asic_type = get_asic_type(hdev);
481 	if (hdev->asic_type == ASIC_INVALID) {
482 		dev_err(&pdev->dev, "Unsupported ASIC\n");
483 		rc = -ENODEV;
484 		goto out_err;
485 	}
486 
487 	copy_kernel_module_params_to_device(hdev);
488 
489 	set_driver_behavior_per_device(hdev);
490 
491 	fixup_device_params(hdev);
492 
493 	rc = allocate_device_id(hdev);
494 	if (rc)
495 		goto out_err;
496 
497 	*dev = hdev;
498 
499 	return 0;
500 
501 out_err:
502 	return rc;
503 }
504 
505 /*
506  * destroy_hdev - destroy habanalabs device instance
507  *
508  * @dev: pointer to the habanalabs device structure
509  *
510  */
511 static void destroy_hdev(struct hl_device *hdev)
512 {
513 	/* Remove device from the device list */
514 	mutex_lock(&hl_devs_idr_lock);
515 	idr_remove(&hl_devs_idr, hdev->id);
516 	mutex_unlock(&hl_devs_idr_lock);
517 
518 }
519 
520 static int hl_pmops_suspend(struct device *dev)
521 {
522 	struct hl_device *hdev = dev_get_drvdata(dev);
523 
524 	pr_debug("Going to suspend PCI device\n");
525 
526 	if (!hdev) {
527 		pr_err("device pointer is NULL in suspend\n");
528 		return 0;
529 	}
530 
531 	return hl_device_suspend(hdev);
532 }
533 
534 static int hl_pmops_resume(struct device *dev)
535 {
536 	struct hl_device *hdev = dev_get_drvdata(dev);
537 
538 	pr_debug("Going to resume PCI device\n");
539 
540 	if (!hdev) {
541 		pr_err("device pointer is NULL in resume\n");
542 		return 0;
543 	}
544 
545 	return hl_device_resume(hdev);
546 }
547 
548 /**
549  * hl_pci_probe - probe PCI habanalabs devices
550  *
551  * @pdev: pointer to pci device
552  * @id: pointer to pci device id structure
553  *
554  * Standard PCI probe function for habanalabs device.
555  * Create a new habanalabs device and initialize it according to the
556  * device's type
557  */
558 static int hl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
559 {
560 	struct hl_device *hdev;
561 	int rc;
562 
563 	dev_info(&pdev->dev, HL_NAME
564 		 " device found [%04x:%04x] (rev %x)\n",
565 		 (int)pdev->vendor, (int)pdev->device, (int)pdev->revision);
566 
567 	rc = create_hdev(&hdev, pdev);
568 	if (rc)
569 		return rc;
570 
571 	pci_set_drvdata(pdev, hdev);
572 
573 	rc = hl_device_init(hdev);
574 	if (rc) {
575 		dev_err(&pdev->dev, "Fatal error during habanalabs device init\n");
576 		rc = -ENODEV;
577 		goto disable_device;
578 	}
579 
580 	return 0;
581 
582 disable_device:
583 	pci_set_drvdata(pdev, NULL);
584 	destroy_hdev(hdev);
585 
586 	return rc;
587 }
588 
589 /*
590  * hl_pci_remove - remove PCI habanalabs devices
591  *
592  * @pdev: pointer to pci device
593  *
594  * Standard PCI remove function for habanalabs device
595  */
596 static void hl_pci_remove(struct pci_dev *pdev)
597 {
598 	struct hl_device *hdev;
599 
600 	hdev = pci_get_drvdata(pdev);
601 	if (!hdev)
602 		return;
603 
604 	hl_device_fini(hdev);
605 	pci_set_drvdata(pdev, NULL);
606 	destroy_hdev(hdev);
607 }
608 
609 /**
610  * hl_pci_err_detected - a PCI bus error detected on this device
611  *
612  * @pdev: pointer to pci device
613  * @state: PCI error type
614  *
615  * Called by the PCI subsystem whenever a non-correctable
616  * PCI bus error is detected
617  */
618 static pci_ers_result_t
619 hl_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state)
620 {
621 	struct hl_device *hdev = pci_get_drvdata(pdev);
622 	enum pci_ers_result result;
623 
624 	switch (state) {
625 	case pci_channel_io_normal:
626 		dev_warn(hdev->dev, "PCI normal state error detected\n");
627 		return PCI_ERS_RESULT_CAN_RECOVER;
628 
629 	case pci_channel_io_frozen:
630 		dev_warn(hdev->dev, "PCI frozen state error detected\n");
631 		result = PCI_ERS_RESULT_NEED_RESET;
632 		break;
633 
634 	case pci_channel_io_perm_failure:
635 		dev_warn(hdev->dev, "PCI failure state error detected\n");
636 		result = PCI_ERS_RESULT_DISCONNECT;
637 		break;
638 
639 	default:
640 		result = PCI_ERS_RESULT_NONE;
641 	}
642 
643 	hdev->asic_funcs->halt_engines(hdev, true, false);
644 
645 	return result;
646 }
647 
648 /**
649  * hl_pci_err_resume - resume after a PCI slot reset
650  *
651  * @pdev: pointer to pci device
652  *
653  */
654 static void hl_pci_err_resume(struct pci_dev *pdev)
655 {
656 	struct hl_device *hdev = pci_get_drvdata(pdev);
657 
658 	dev_warn(hdev->dev, "Resuming device after PCI slot reset\n");
659 	hl_device_resume(hdev);
660 }
661 
662 /**
663  * hl_pci_err_slot_reset - a PCI slot reset has just happened
664  *
665  * @pdev: pointer to pci device
666  *
667  * Determine if the driver can recover from the PCI slot reset
668  */
669 static pci_ers_result_t hl_pci_err_slot_reset(struct pci_dev *pdev)
670 {
671 	struct hl_device *hdev = pci_get_drvdata(pdev);
672 
673 	dev_warn(hdev->dev, "PCI slot reset detected\n");
674 
675 	return PCI_ERS_RESULT_RECOVERED;
676 }
677 
678 static void hl_pci_reset_prepare(struct pci_dev *pdev)
679 {
680 	struct hl_device *hdev;
681 
682 	hdev = pci_get_drvdata(pdev);
683 	if (!hdev)
684 		return;
685 
686 	hdev->disabled = true;
687 }
688 
689 static void hl_pci_reset_done(struct pci_dev *pdev)
690 {
691 	struct hl_device *hdev;
692 	u32 flags;
693 
694 	hdev = pci_get_drvdata(pdev);
695 	if (!hdev)
696 		return;
697 
698 	/*
699 	 * Schedule a thread to trigger hard reset.
700 	 * The reason for this handler, is for rare cases where the driver is up
701 	 * and FLR occurs. This is valid only when working with no VM, so FW handles FLR
702 	 * and resets the device. FW will go back preboot stage, so driver needs to perform
703 	 * hard reset in order to load FW fit again.
704 	 */
705 	flags = HL_DRV_RESET_HARD | HL_DRV_RESET_BYPASS_REQ_TO_FW;
706 
707 	hl_device_reset(hdev, flags);
708 }
709 
710 static const struct dev_pm_ops hl_pm_ops = {
711 	.suspend = hl_pmops_suspend,
712 	.resume = hl_pmops_resume,
713 };
714 
715 static const struct pci_error_handlers hl_pci_err_handler = {
716 	.error_detected = hl_pci_err_detected,
717 	.slot_reset = hl_pci_err_slot_reset,
718 	.resume = hl_pci_err_resume,
719 	.reset_prepare = hl_pci_reset_prepare,
720 	.reset_done = hl_pci_reset_done,
721 };
722 
723 static struct pci_driver hl_pci_driver = {
724 	.name = HL_NAME,
725 	.id_table = ids,
726 	.probe = hl_pci_probe,
727 	.remove = hl_pci_remove,
728 	.shutdown = hl_pci_remove,
729 	.driver = {
730 		.name = HL_NAME,
731 		.pm = &hl_pm_ops,
732 		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
733 	},
734 	.err_handler = &hl_pci_err_handler,
735 };
736 
737 /*
738  * hl_init - Initialize the habanalabs kernel driver
739  */
740 static int __init hl_init(void)
741 {
742 	int rc;
743 	dev_t dev;
744 
745 	pr_info("loading driver\n");
746 
747 	rc = alloc_chrdev_region(&dev, 0, HL_MAX_MINORS, HL_NAME);
748 	if (rc < 0) {
749 		pr_err("unable to get major\n");
750 		return rc;
751 	}
752 
753 	hl_major = MAJOR(dev);
754 
755 	rc = pci_register_driver(&hl_pci_driver);
756 	if (rc) {
757 		pr_err("failed to register pci device\n");
758 		goto remove_major;
759 	}
760 
761 	pr_debug("driver loaded\n");
762 
763 	return 0;
764 
765 remove_major:
766 	unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
767 	return rc;
768 }
769 
770 /*
771  * hl_exit - Release all resources of the habanalabs kernel driver
772  */
773 static void __exit hl_exit(void)
774 {
775 	pci_unregister_driver(&hl_pci_driver);
776 
777 	unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
778 
779 	idr_destroy(&hl_devs_idr);
780 
781 	pr_debug("driver removed\n");
782 }
783 
784 module_init(hl_init);
785 module_exit(hl_exit);
786