1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020-2024 Intel Corporation
4 */
5
6 #include <linux/firmware.h>
7 #include <linux/module.h>
8 #include <linux/pci.h>
9 #include <linux/pm_runtime.h>
10 #include <generated/utsrelease.h>
11
12 #include <drm/drm_accel.h>
13 #include <drm/drm_file.h>
14 #include <drm/drm_gem.h>
15 #include <drm/drm_ioctl.h>
16 #include <drm/drm_prime.h>
17
18 #include "ivpu_coredump.h"
19 #include "ivpu_debugfs.h"
20 #include "ivpu_drv.h"
21 #include "ivpu_fw.h"
22 #include "ivpu_fw_log.h"
23 #include "ivpu_gem.h"
24 #include "ivpu_hw.h"
25 #include "ivpu_ipc.h"
26 #include "ivpu_job.h"
27 #include "ivpu_jsm_msg.h"
28 #include "ivpu_mmu.h"
29 #include "ivpu_mmu_context.h"
30 #include "ivpu_ms.h"
31 #include "ivpu_pm.h"
32 #include "ivpu_sysfs.h"
33 #include "vpu_boot_api.h"
34
35 #ifndef DRIVER_VERSION_STR
36 #define DRIVER_VERSION_STR "1.0.0 " UTS_RELEASE
37 #endif
38
39 static struct lock_class_key submitted_jobs_xa_lock_class_key;
40
41 int ivpu_dbg_mask;
42 module_param_named(dbg_mask, ivpu_dbg_mask, int, 0644);
43 MODULE_PARM_DESC(dbg_mask, "Driver debug mask. See IVPU_DBG_* macros.");
44
45 int ivpu_test_mode;
46 #if IS_ENABLED(CONFIG_DRM_ACCEL_IVPU_DEBUG)
47 module_param_named_unsafe(test_mode, ivpu_test_mode, int, 0644);
48 MODULE_PARM_DESC(test_mode, "Test mode mask. See IVPU_TEST_MODE_* macros.");
49 #endif
50
51 u8 ivpu_pll_min_ratio;
52 module_param_named(pll_min_ratio, ivpu_pll_min_ratio, byte, 0644);
53 MODULE_PARM_DESC(pll_min_ratio, "Minimum PLL ratio used to set NPU frequency");
54
55 u8 ivpu_pll_max_ratio = U8_MAX;
56 module_param_named(pll_max_ratio, ivpu_pll_max_ratio, byte, 0644);
57 MODULE_PARM_DESC(pll_max_ratio, "Maximum PLL ratio used to set NPU frequency");
58
59 int ivpu_sched_mode = IVPU_SCHED_MODE_AUTO;
60 module_param_named(sched_mode, ivpu_sched_mode, int, 0444);
61 MODULE_PARM_DESC(sched_mode, "Scheduler mode: -1 - Use default scheduler, 0 - Use OS scheduler, 1 - Use HW scheduler");
62
63 bool ivpu_disable_mmu_cont_pages;
64 module_param_named(disable_mmu_cont_pages, ivpu_disable_mmu_cont_pages, bool, 0444);
65 MODULE_PARM_DESC(disable_mmu_cont_pages, "Disable MMU contiguous pages optimization");
66
67 bool ivpu_force_snoop;
68 module_param_named(force_snoop, ivpu_force_snoop, bool, 0444);
69 MODULE_PARM_DESC(force_snoop, "Force snooping for NPU host memory access");
70
ivpu_file_priv_get(struct ivpu_file_priv * file_priv)71 struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv)
72 {
73 struct ivpu_device *vdev = file_priv->vdev;
74
75 kref_get(&file_priv->ref);
76
77 ivpu_dbg(vdev, KREF, "file_priv get: ctx %u refcount %u\n",
78 file_priv->ctx.id, kref_read(&file_priv->ref));
79
80 return file_priv;
81 }
82
file_priv_unbind(struct ivpu_device * vdev,struct ivpu_file_priv * file_priv)83 static void file_priv_unbind(struct ivpu_device *vdev, struct ivpu_file_priv *file_priv)
84 {
85 mutex_lock(&file_priv->lock);
86 if (file_priv->bound) {
87 ivpu_dbg(vdev, FILE, "file_priv unbind: ctx %u\n", file_priv->ctx.id);
88
89 ivpu_cmdq_release_all_locked(file_priv);
90 ivpu_bo_unbind_all_bos_from_context(vdev, &file_priv->ctx);
91 ivpu_mmu_context_fini(vdev, &file_priv->ctx);
92 file_priv->bound = false;
93 drm_WARN_ON(&vdev->drm, !xa_erase_irq(&vdev->context_xa, file_priv->ctx.id));
94 }
95 mutex_unlock(&file_priv->lock);
96 }
97
file_priv_release(struct kref * ref)98 static void file_priv_release(struct kref *ref)
99 {
100 struct ivpu_file_priv *file_priv = container_of(ref, struct ivpu_file_priv, ref);
101 struct ivpu_device *vdev = file_priv->vdev;
102
103 ivpu_dbg(vdev, FILE, "file_priv release: ctx %u bound %d\n",
104 file_priv->ctx.id, (bool)file_priv->bound);
105
106 pm_runtime_get_sync(vdev->drm.dev);
107 mutex_lock(&vdev->context_list_lock);
108 file_priv_unbind(vdev, file_priv);
109 drm_WARN_ON(&vdev->drm, !xa_empty(&file_priv->cmdq_xa));
110 xa_destroy(&file_priv->cmdq_xa);
111 mutex_unlock(&vdev->context_list_lock);
112 pm_runtime_put_autosuspend(vdev->drm.dev);
113
114 mutex_destroy(&file_priv->ms_lock);
115 mutex_destroy(&file_priv->lock);
116 kfree(file_priv);
117 }
118
ivpu_file_priv_put(struct ivpu_file_priv ** link)119 void ivpu_file_priv_put(struct ivpu_file_priv **link)
120 {
121 struct ivpu_file_priv *file_priv = *link;
122 struct ivpu_device *vdev = file_priv->vdev;
123
124 ivpu_dbg(vdev, KREF, "file_priv put: ctx %u refcount %u\n",
125 file_priv->ctx.id, kref_read(&file_priv->ref));
126
127 *link = NULL;
128 kref_put(&file_priv->ref, file_priv_release);
129 }
130
ivpu_get_capabilities(struct ivpu_device * vdev,struct drm_ivpu_param * args)131 static int ivpu_get_capabilities(struct ivpu_device *vdev, struct drm_ivpu_param *args)
132 {
133 switch (args->index) {
134 case DRM_IVPU_CAP_METRIC_STREAMER:
135 args->value = 1;
136 break;
137 case DRM_IVPU_CAP_DMA_MEMORY_RANGE:
138 args->value = 1;
139 break;
140 default:
141 return -EINVAL;
142 }
143
144 return 0;
145 }
146
ivpu_get_param_ioctl(struct drm_device * dev,void * data,struct drm_file * file)147 static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
148 {
149 struct ivpu_file_priv *file_priv = file->driver_priv;
150 struct ivpu_device *vdev = file_priv->vdev;
151 struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
152 struct drm_ivpu_param *args = data;
153 int ret = 0;
154 int idx;
155
156 if (!drm_dev_enter(dev, &idx))
157 return -ENODEV;
158
159 switch (args->param) {
160 case DRM_IVPU_PARAM_DEVICE_ID:
161 args->value = pdev->device;
162 break;
163 case DRM_IVPU_PARAM_DEVICE_REVISION:
164 args->value = pdev->revision;
165 break;
166 case DRM_IVPU_PARAM_PLATFORM_TYPE:
167 args->value = vdev->platform;
168 break;
169 case DRM_IVPU_PARAM_CORE_CLOCK_RATE:
170 args->value = ivpu_hw_ratio_to_freq(vdev, vdev->hw->pll.max_ratio);
171 break;
172 case DRM_IVPU_PARAM_NUM_CONTEXTS:
173 args->value = ivpu_get_context_count(vdev);
174 break;
175 case DRM_IVPU_PARAM_CONTEXT_BASE_ADDRESS:
176 args->value = vdev->hw->ranges.user.start;
177 break;
178 case DRM_IVPU_PARAM_CONTEXT_ID:
179 args->value = file_priv->ctx.id;
180 break;
181 case DRM_IVPU_PARAM_FW_API_VERSION:
182 if (args->index < VPU_FW_API_VER_NUM) {
183 struct vpu_firmware_header *fw_hdr;
184
185 fw_hdr = (struct vpu_firmware_header *)vdev->fw->file->data;
186 args->value = fw_hdr->api_version[args->index];
187 } else {
188 ret = -EINVAL;
189 }
190 break;
191 case DRM_IVPU_PARAM_ENGINE_HEARTBEAT:
192 ret = ivpu_jsm_get_heartbeat(vdev, args->index, &args->value);
193 break;
194 case DRM_IVPU_PARAM_UNIQUE_INFERENCE_ID:
195 args->value = (u64)atomic64_inc_return(&vdev->unique_id_counter);
196 break;
197 case DRM_IVPU_PARAM_TILE_CONFIG:
198 args->value = vdev->hw->tile_fuse;
199 break;
200 case DRM_IVPU_PARAM_SKU:
201 args->value = vdev->hw->sku;
202 break;
203 case DRM_IVPU_PARAM_CAPABILITIES:
204 ret = ivpu_get_capabilities(vdev, args);
205 break;
206 default:
207 ret = -EINVAL;
208 break;
209 }
210
211 drm_dev_exit(idx);
212 return ret;
213 }
214
ivpu_set_param_ioctl(struct drm_device * dev,void * data,struct drm_file * file)215 static int ivpu_set_param_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
216 {
217 struct drm_ivpu_param *args = data;
218 int ret = 0;
219
220 switch (args->param) {
221 default:
222 ret = -EINVAL;
223 }
224
225 return ret;
226 }
227
ivpu_open(struct drm_device * dev,struct drm_file * file)228 static int ivpu_open(struct drm_device *dev, struct drm_file *file)
229 {
230 struct ivpu_device *vdev = to_ivpu_device(dev);
231 struct ivpu_file_priv *file_priv;
232 u32 ctx_id;
233 int idx, ret;
234
235 if (!drm_dev_enter(dev, &idx))
236 return -ENODEV;
237
238 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
239 if (!file_priv) {
240 ret = -ENOMEM;
241 goto err_dev_exit;
242 }
243
244 INIT_LIST_HEAD(&file_priv->ms_instance_list);
245
246 file_priv->vdev = vdev;
247 file_priv->bound = true;
248 kref_init(&file_priv->ref);
249 mutex_init(&file_priv->lock);
250 mutex_init(&file_priv->ms_lock);
251
252 mutex_lock(&vdev->context_list_lock);
253
254 ret = xa_alloc_irq(&vdev->context_xa, &ctx_id, file_priv,
255 vdev->context_xa_limit, GFP_KERNEL);
256 if (ret) {
257 ivpu_err(vdev, "Failed to allocate context id: %d\n", ret);
258 goto err_unlock;
259 }
260
261 ivpu_mmu_context_init(vdev, &file_priv->ctx, ctx_id);
262
263 file_priv->job_limit.min = FIELD_PREP(IVPU_JOB_ID_CONTEXT_MASK, (file_priv->ctx.id - 1));
264 file_priv->job_limit.max = file_priv->job_limit.min | IVPU_JOB_ID_JOB_MASK;
265
266 xa_init_flags(&file_priv->cmdq_xa, XA_FLAGS_ALLOC1);
267 file_priv->cmdq_limit.min = IVPU_CMDQ_MIN_ID;
268 file_priv->cmdq_limit.max = IVPU_CMDQ_MAX_ID;
269
270 mutex_unlock(&vdev->context_list_lock);
271 drm_dev_exit(idx);
272
273 file->driver_priv = file_priv;
274
275 ivpu_dbg(vdev, FILE, "file_priv create: ctx %u process %s pid %d\n",
276 ctx_id, current->comm, task_pid_nr(current));
277
278 return 0;
279
280 err_unlock:
281 mutex_unlock(&vdev->context_list_lock);
282 mutex_destroy(&file_priv->ms_lock);
283 mutex_destroy(&file_priv->lock);
284 kfree(file_priv);
285 err_dev_exit:
286 drm_dev_exit(idx);
287 return ret;
288 }
289
ivpu_postclose(struct drm_device * dev,struct drm_file * file)290 static void ivpu_postclose(struct drm_device *dev, struct drm_file *file)
291 {
292 struct ivpu_file_priv *file_priv = file->driver_priv;
293 struct ivpu_device *vdev = to_ivpu_device(dev);
294
295 ivpu_dbg(vdev, FILE, "file_priv close: ctx %u process %s pid %d\n",
296 file_priv->ctx.id, current->comm, task_pid_nr(current));
297
298 ivpu_ms_cleanup(file_priv);
299 ivpu_file_priv_put(&file_priv);
300 }
301
302 static const struct drm_ioctl_desc ivpu_drm_ioctls[] = {
303 DRM_IOCTL_DEF_DRV(IVPU_GET_PARAM, ivpu_get_param_ioctl, 0),
304 DRM_IOCTL_DEF_DRV(IVPU_SET_PARAM, ivpu_set_param_ioctl, 0),
305 DRM_IOCTL_DEF_DRV(IVPU_BO_CREATE, ivpu_bo_create_ioctl, 0),
306 DRM_IOCTL_DEF_DRV(IVPU_BO_INFO, ivpu_bo_info_ioctl, 0),
307 DRM_IOCTL_DEF_DRV(IVPU_SUBMIT, ivpu_submit_ioctl, 0),
308 DRM_IOCTL_DEF_DRV(IVPU_BO_WAIT, ivpu_bo_wait_ioctl, 0),
309 DRM_IOCTL_DEF_DRV(IVPU_METRIC_STREAMER_START, ivpu_ms_start_ioctl, 0),
310 DRM_IOCTL_DEF_DRV(IVPU_METRIC_STREAMER_GET_DATA, ivpu_ms_get_data_ioctl, 0),
311 DRM_IOCTL_DEF_DRV(IVPU_METRIC_STREAMER_STOP, ivpu_ms_stop_ioctl, 0),
312 DRM_IOCTL_DEF_DRV(IVPU_METRIC_STREAMER_GET_INFO, ivpu_ms_get_info_ioctl, 0),
313 };
314
ivpu_wait_for_ready(struct ivpu_device * vdev)315 static int ivpu_wait_for_ready(struct ivpu_device *vdev)
316 {
317 struct ivpu_ipc_consumer cons;
318 struct ivpu_ipc_hdr ipc_hdr;
319 unsigned long timeout;
320 int ret;
321
322 if (ivpu_test_mode & IVPU_TEST_MODE_FW_TEST)
323 return 0;
324
325 ivpu_ipc_consumer_add(vdev, &cons, IVPU_IPC_CHAN_BOOT_MSG, NULL);
326
327 timeout = jiffies + msecs_to_jiffies(vdev->timeout.boot);
328 while (1) {
329 ivpu_ipc_irq_handler(vdev);
330 ret = ivpu_ipc_receive(vdev, &cons, &ipc_hdr, NULL, 0);
331 if (ret != -ETIMEDOUT || time_after_eq(jiffies, timeout))
332 break;
333
334 cond_resched();
335 }
336
337 ivpu_ipc_consumer_del(vdev, &cons);
338
339 if (!ret && ipc_hdr.data_addr != IVPU_IPC_BOOT_MSG_DATA_ADDR) {
340 ivpu_err(vdev, "Invalid NPU ready message: 0x%x\n",
341 ipc_hdr.data_addr);
342 return -EIO;
343 }
344
345 if (!ret)
346 ivpu_dbg(vdev, PM, "NPU ready message received successfully\n");
347
348 return ret;
349 }
350
ivpu_hw_sched_init(struct ivpu_device * vdev)351 static int ivpu_hw_sched_init(struct ivpu_device *vdev)
352 {
353 int ret = 0;
354
355 if (vdev->fw->sched_mode == VPU_SCHEDULING_MODE_HW) {
356 ret = ivpu_jsm_hws_setup_priority_bands(vdev);
357 if (ret) {
358 ivpu_err(vdev, "Failed to enable hw scheduler: %d", ret);
359 return ret;
360 }
361 }
362
363 return ret;
364 }
365
366 /**
367 * ivpu_boot() - Start VPU firmware
368 * @vdev: VPU device
369 *
370 * This function is paired with ivpu_shutdown() but it doesn't power up the
371 * VPU because power up has to be called very early in ivpu_probe().
372 */
ivpu_boot(struct ivpu_device * vdev)373 int ivpu_boot(struct ivpu_device *vdev)
374 {
375 int ret;
376
377 /* Update boot params located at first 4KB of FW memory */
378 ivpu_fw_boot_params_setup(vdev, ivpu_bo_vaddr(vdev->fw->mem));
379
380 ret = ivpu_hw_boot_fw(vdev);
381 if (ret) {
382 ivpu_err(vdev, "Failed to start the firmware: %d\n", ret);
383 return ret;
384 }
385
386 ret = ivpu_wait_for_ready(vdev);
387 if (ret) {
388 ivpu_err(vdev, "Failed to boot the firmware: %d\n", ret);
389 goto err_diagnose_failure;
390 }
391
392 ivpu_hw_irq_clear(vdev);
393 enable_irq(vdev->irq);
394 ivpu_hw_irq_enable(vdev);
395 ivpu_ipc_enable(vdev);
396
397 if (ivpu_fw_is_cold_boot(vdev)) {
398 ret = ivpu_pm_dct_init(vdev);
399 if (ret)
400 goto err_disable_ipc;
401
402 ret = ivpu_hw_sched_init(vdev);
403 if (ret)
404 goto err_disable_ipc;
405 }
406
407 return 0;
408
409 err_disable_ipc:
410 ivpu_ipc_disable(vdev);
411 ivpu_hw_irq_disable(vdev);
412 disable_irq(vdev->irq);
413 err_diagnose_failure:
414 ivpu_hw_diagnose_failure(vdev);
415 ivpu_mmu_evtq_dump(vdev);
416 ivpu_dev_coredump(vdev);
417 return ret;
418 }
419
ivpu_prepare_for_reset(struct ivpu_device * vdev)420 void ivpu_prepare_for_reset(struct ivpu_device *vdev)
421 {
422 ivpu_hw_irq_disable(vdev);
423 disable_irq(vdev->irq);
424 ivpu_ipc_disable(vdev);
425 ivpu_mmu_disable(vdev);
426 }
427
ivpu_shutdown(struct ivpu_device * vdev)428 int ivpu_shutdown(struct ivpu_device *vdev)
429 {
430 int ret;
431
432 /* Save PCI state before powering down as it sometimes gets corrupted if NPU hangs */
433 pci_save_state(to_pci_dev(vdev->drm.dev));
434
435 ret = ivpu_hw_power_down(vdev);
436 if (ret)
437 ivpu_warn(vdev, "Failed to power down HW: %d\n", ret);
438
439 pci_set_power_state(to_pci_dev(vdev->drm.dev), PCI_D3hot);
440
441 return ret;
442 }
443
444 static const struct file_operations ivpu_fops = {
445 .owner = THIS_MODULE,
446 DRM_ACCEL_FOPS,
447 };
448
449 static const struct drm_driver driver = {
450 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
451
452 .open = ivpu_open,
453 .postclose = ivpu_postclose,
454
455 .gem_create_object = ivpu_gem_create_object,
456 .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table,
457
458 .ioctls = ivpu_drm_ioctls,
459 .num_ioctls = ARRAY_SIZE(ivpu_drm_ioctls),
460 .fops = &ivpu_fops,
461
462 .name = DRIVER_NAME,
463 .desc = DRIVER_DESC,
464
465 .major = 1,
466 };
467
ivpu_context_abort_invalid(struct ivpu_device * vdev)468 static void ivpu_context_abort_invalid(struct ivpu_device *vdev)
469 {
470 struct ivpu_file_priv *file_priv;
471 unsigned long ctx_id;
472
473 mutex_lock(&vdev->context_list_lock);
474
475 xa_for_each(&vdev->context_xa, ctx_id, file_priv) {
476 if (!file_priv->has_mmu_faults || file_priv->aborted)
477 continue;
478
479 mutex_lock(&file_priv->lock);
480 ivpu_context_abort_locked(file_priv);
481 file_priv->aborted = true;
482 mutex_unlock(&file_priv->lock);
483 }
484
485 mutex_unlock(&vdev->context_list_lock);
486 }
487
ivpu_irq_thread_handler(int irq,void * arg)488 static irqreturn_t ivpu_irq_thread_handler(int irq, void *arg)
489 {
490 struct ivpu_device *vdev = arg;
491 u8 irq_src;
492
493 if (kfifo_is_empty(&vdev->hw->irq.fifo))
494 return IRQ_NONE;
495
496 while (kfifo_get(&vdev->hw->irq.fifo, &irq_src)) {
497 switch (irq_src) {
498 case IVPU_HW_IRQ_SRC_IPC:
499 ivpu_ipc_irq_thread_handler(vdev);
500 break;
501 case IVPU_HW_IRQ_SRC_MMU_EVTQ:
502 ivpu_context_abort_invalid(vdev);
503 break;
504 case IVPU_HW_IRQ_SRC_DCT:
505 ivpu_pm_dct_irq_thread_handler(vdev);
506 break;
507 default:
508 ivpu_err_ratelimited(vdev, "Unknown IRQ source: %u\n", irq_src);
509 break;
510 }
511 }
512
513 return IRQ_HANDLED;
514 }
515
ivpu_irq_init(struct ivpu_device * vdev)516 static int ivpu_irq_init(struct ivpu_device *vdev)
517 {
518 struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
519 int ret;
520
521 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX);
522 if (ret < 0) {
523 ivpu_err(vdev, "Failed to allocate a MSI IRQ: %d\n", ret);
524 return ret;
525 }
526
527 ivpu_irq_handlers_init(vdev);
528
529 vdev->irq = pci_irq_vector(pdev, 0);
530
531 ret = devm_request_threaded_irq(vdev->drm.dev, vdev->irq, ivpu_hw_irq_handler,
532 ivpu_irq_thread_handler, IRQF_NO_AUTOEN, DRIVER_NAME, vdev);
533 if (ret)
534 ivpu_err(vdev, "Failed to request an IRQ %d\n", ret);
535
536 return ret;
537 }
538
ivpu_pci_init(struct ivpu_device * vdev)539 static int ivpu_pci_init(struct ivpu_device *vdev)
540 {
541 struct pci_dev *pdev = to_pci_dev(vdev->drm.dev);
542 struct resource *bar0 = &pdev->resource[0];
543 struct resource *bar4 = &pdev->resource[4];
544 int ret;
545
546 ivpu_dbg(vdev, MISC, "Mapping BAR0 (RegV) %pR\n", bar0);
547 vdev->regv = devm_ioremap_resource(vdev->drm.dev, bar0);
548 if (IS_ERR(vdev->regv)) {
549 ivpu_err(vdev, "Failed to map bar 0: %pe\n", vdev->regv);
550 return PTR_ERR(vdev->regv);
551 }
552
553 ivpu_dbg(vdev, MISC, "Mapping BAR4 (RegB) %pR\n", bar4);
554 vdev->regb = devm_ioremap_resource(vdev->drm.dev, bar4);
555 if (IS_ERR(vdev->regb)) {
556 ivpu_err(vdev, "Failed to map bar 4: %pe\n", vdev->regb);
557 return PTR_ERR(vdev->regb);
558 }
559
560 ret = dma_set_mask_and_coherent(vdev->drm.dev, DMA_BIT_MASK(vdev->hw->dma_bits));
561 if (ret) {
562 ivpu_err(vdev, "Failed to set DMA mask: %d\n", ret);
563 return ret;
564 }
565 dma_set_max_seg_size(vdev->drm.dev, UINT_MAX);
566
567 /* Clear any pending errors */
568 pcie_capability_clear_word(pdev, PCI_EXP_DEVSTA, 0x3f);
569
570 /* NPU does not require 10m D3hot delay */
571 pdev->d3hot_delay = 0;
572
573 ret = pcim_enable_device(pdev);
574 if (ret) {
575 ivpu_err(vdev, "Failed to enable PCI device: %d\n", ret);
576 return ret;
577 }
578
579 pci_set_master(pdev);
580
581 return 0;
582 }
583
ivpu_dev_init(struct ivpu_device * vdev)584 static int ivpu_dev_init(struct ivpu_device *vdev)
585 {
586 int ret;
587
588 vdev->hw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->hw), GFP_KERNEL);
589 if (!vdev->hw)
590 return -ENOMEM;
591
592 vdev->mmu = drmm_kzalloc(&vdev->drm, sizeof(*vdev->mmu), GFP_KERNEL);
593 if (!vdev->mmu)
594 return -ENOMEM;
595
596 vdev->fw = drmm_kzalloc(&vdev->drm, sizeof(*vdev->fw), GFP_KERNEL);
597 if (!vdev->fw)
598 return -ENOMEM;
599
600 vdev->ipc = drmm_kzalloc(&vdev->drm, sizeof(*vdev->ipc), GFP_KERNEL);
601 if (!vdev->ipc)
602 return -ENOMEM;
603
604 vdev->pm = drmm_kzalloc(&vdev->drm, sizeof(*vdev->pm), GFP_KERNEL);
605 if (!vdev->pm)
606 return -ENOMEM;
607
608 if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_40XX)
609 vdev->hw->dma_bits = 48;
610 else
611 vdev->hw->dma_bits = 38;
612
613 vdev->platform = IVPU_PLATFORM_INVALID;
614 vdev->context_xa_limit.min = IVPU_USER_CONTEXT_MIN_SSID;
615 vdev->context_xa_limit.max = IVPU_USER_CONTEXT_MAX_SSID;
616 atomic64_set(&vdev->unique_id_counter, 0);
617 xa_init_flags(&vdev->context_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);
618 xa_init_flags(&vdev->submitted_jobs_xa, XA_FLAGS_ALLOC1);
619 xa_init_flags(&vdev->db_xa, XA_FLAGS_ALLOC1);
620 lockdep_set_class(&vdev->submitted_jobs_xa.xa_lock, &submitted_jobs_xa_lock_class_key);
621 INIT_LIST_HEAD(&vdev->bo_list);
622
623 vdev->db_limit.min = IVPU_MIN_DB;
624 vdev->db_limit.max = IVPU_MAX_DB;
625
626 ret = drmm_mutex_init(&vdev->drm, &vdev->context_list_lock);
627 if (ret)
628 goto err_xa_destroy;
629
630 ret = drmm_mutex_init(&vdev->drm, &vdev->bo_list_lock);
631 if (ret)
632 goto err_xa_destroy;
633
634 ret = ivpu_pci_init(vdev);
635 if (ret)
636 goto err_xa_destroy;
637
638 ret = ivpu_irq_init(vdev);
639 if (ret)
640 goto err_xa_destroy;
641
642 /* Init basic HW info based on buttress registers which are accessible before power up */
643 ret = ivpu_hw_init(vdev);
644 if (ret)
645 goto err_xa_destroy;
646
647 /* Power up early so the rest of init code can access VPU registers */
648 ret = ivpu_hw_power_up(vdev);
649 if (ret)
650 goto err_shutdown;
651
652 ivpu_mmu_global_context_init(vdev);
653
654 ret = ivpu_mmu_init(vdev);
655 if (ret)
656 goto err_mmu_gctx_fini;
657
658 ret = ivpu_mmu_reserved_context_init(vdev);
659 if (ret)
660 goto err_mmu_gctx_fini;
661
662 ret = ivpu_fw_init(vdev);
663 if (ret)
664 goto err_mmu_rctx_fini;
665
666 ret = ivpu_ipc_init(vdev);
667 if (ret)
668 goto err_fw_fini;
669
670 ivpu_pm_init(vdev);
671
672 ret = ivpu_boot(vdev);
673 if (ret)
674 goto err_ipc_fini;
675
676 ivpu_job_done_consumer_init(vdev);
677 ivpu_pm_enable(vdev);
678
679 return 0;
680
681 err_ipc_fini:
682 ivpu_ipc_fini(vdev);
683 err_fw_fini:
684 ivpu_fw_fini(vdev);
685 err_mmu_rctx_fini:
686 ivpu_mmu_reserved_context_fini(vdev);
687 err_mmu_gctx_fini:
688 ivpu_mmu_global_context_fini(vdev);
689 err_shutdown:
690 ivpu_shutdown(vdev);
691 err_xa_destroy:
692 xa_destroy(&vdev->db_xa);
693 xa_destroy(&vdev->submitted_jobs_xa);
694 xa_destroy(&vdev->context_xa);
695 return ret;
696 }
697
ivpu_bo_unbind_all_user_contexts(struct ivpu_device * vdev)698 static void ivpu_bo_unbind_all_user_contexts(struct ivpu_device *vdev)
699 {
700 struct ivpu_file_priv *file_priv;
701 unsigned long ctx_id;
702
703 mutex_lock(&vdev->context_list_lock);
704
705 xa_for_each(&vdev->context_xa, ctx_id, file_priv)
706 file_priv_unbind(vdev, file_priv);
707
708 mutex_unlock(&vdev->context_list_lock);
709 }
710
ivpu_dev_fini(struct ivpu_device * vdev)711 static void ivpu_dev_fini(struct ivpu_device *vdev)
712 {
713 ivpu_jobs_abort_all(vdev);
714 ivpu_pm_cancel_recovery(vdev);
715 ivpu_pm_disable(vdev);
716 ivpu_prepare_for_reset(vdev);
717 ivpu_shutdown(vdev);
718
719 ivpu_ms_cleanup_all(vdev);
720 ivpu_job_done_consumer_fini(vdev);
721 ivpu_bo_unbind_all_user_contexts(vdev);
722
723 ivpu_ipc_fini(vdev);
724 ivpu_fw_fini(vdev);
725 ivpu_mmu_reserved_context_fini(vdev);
726 ivpu_mmu_global_context_fini(vdev);
727
728 drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->db_xa));
729 xa_destroy(&vdev->db_xa);
730 drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->submitted_jobs_xa));
731 xa_destroy(&vdev->submitted_jobs_xa);
732 drm_WARN_ON(&vdev->drm, !xa_empty(&vdev->context_xa));
733 xa_destroy(&vdev->context_xa);
734 }
735
736 static struct pci_device_id ivpu_pci_ids[] = {
737 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_MTL) },
738 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_ARL) },
739 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_LNL) },
740 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_PTL_P) },
741 { }
742 };
743 MODULE_DEVICE_TABLE(pci, ivpu_pci_ids);
744
ivpu_probe(struct pci_dev * pdev,const struct pci_device_id * id)745 static int ivpu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
746 {
747 struct ivpu_device *vdev;
748 int ret;
749
750 vdev = devm_drm_dev_alloc(&pdev->dev, &driver, struct ivpu_device, drm);
751 if (IS_ERR(vdev))
752 return PTR_ERR(vdev);
753
754 pci_set_drvdata(pdev, vdev);
755
756 ret = ivpu_dev_init(vdev);
757 if (ret)
758 return ret;
759
760 ivpu_debugfs_init(vdev);
761 ivpu_sysfs_init(vdev);
762
763 ret = drm_dev_register(&vdev->drm, 0);
764 if (ret) {
765 dev_err(&pdev->dev, "Failed to register DRM device: %d\n", ret);
766 ivpu_dev_fini(vdev);
767 }
768
769 return ret;
770 }
771
ivpu_remove(struct pci_dev * pdev)772 static void ivpu_remove(struct pci_dev *pdev)
773 {
774 struct ivpu_device *vdev = pci_get_drvdata(pdev);
775
776 drm_dev_unplug(&vdev->drm);
777 ivpu_dev_fini(vdev);
778 }
779
780 static const struct dev_pm_ops ivpu_drv_pci_pm = {
781 SET_SYSTEM_SLEEP_PM_OPS(ivpu_pm_suspend_cb, ivpu_pm_resume_cb)
782 SET_RUNTIME_PM_OPS(ivpu_pm_runtime_suspend_cb, ivpu_pm_runtime_resume_cb, NULL)
783 };
784
785 static const struct pci_error_handlers ivpu_drv_pci_err = {
786 .reset_prepare = ivpu_pm_reset_prepare_cb,
787 .reset_done = ivpu_pm_reset_done_cb,
788 };
789
790 static struct pci_driver ivpu_pci_driver = {
791 .name = KBUILD_MODNAME,
792 .id_table = ivpu_pci_ids,
793 .probe = ivpu_probe,
794 .remove = ivpu_remove,
795 .driver = {
796 .pm = &ivpu_drv_pci_pm,
797 },
798 .err_handler = &ivpu_drv_pci_err,
799 };
800
801 module_pci_driver(ivpu_pci_driver);
802
803 MODULE_AUTHOR("Intel Corporation");
804 MODULE_DESCRIPTION(DRIVER_DESC);
805 MODULE_LICENSE("GPL and additional rights");
806 MODULE_VERSION(DRIVER_VERSION_STR);
807