1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_accel.h>
8 #include <drm/drm_drv.h>
9 #include <drm/drm_gem.h>
10 #include <drm/drm_gem_shmem_helper.h>
11 #include <drm/drm_ioctl.h>
12 #include <drm/drm_managed.h>
13 #include <drm/gpu_scheduler.h>
14 #include <linux/iommu.h>
15 #include <linux/pci.h>
16
17 #include "amdxdna_cbuf.h"
18 #include "amdxdna_ctx.h"
19 #include "amdxdna_debugfs.h"
20 #include "amdxdna_gem.h"
21 #include "amdxdna_pci_drv.h"
22 #include "amdxdna_pm.h"
23
24 MODULE_FIRMWARE("amdnpu/1502_00/npu.sbin");
25 MODULE_FIRMWARE("amdnpu/17f0_10/npu.sbin");
26 MODULE_FIRMWARE("amdnpu/17f0_11/npu.sbin");
27 MODULE_FIRMWARE("amdnpu/17f0_20/npu.sbin");
28 MODULE_FIRMWARE("amdnpu/1502_00/npu_7.sbin");
29 MODULE_FIRMWARE("amdnpu/17f0_10/npu_7.sbin");
30 MODULE_FIRMWARE("amdnpu/17f0_11/npu_7.sbin");
31
32 /*
33 * 0.0: Initial version
34 * 0.1: Support getting all hardware contexts by DRM_IOCTL_AMDXDNA_GET_ARRAY
35 * 0.2: Support getting last error hardware error
36 * 0.3: Support firmware debug buffer
37 * 0.4: Support getting resource information
38 * 0.5: Support getting telemetry data
39 * 0.6: Support preemption
40 * 0.7: Support getting power and utilization data
41 * 0.8: Support BO usage query
42 * 0.9: Add new device type AMDXDNA_DEV_TYPE_PF
43 * 0.10: Support AIE4 UMQ
44 */
45 #define AMDXDNA_DRIVER_MAJOR 0
46 #define AMDXDNA_DRIVER_MINOR 10
47
48 /*
49 * Bind the driver base on (vendor_id, device_id) pair and later use the
50 * (device_id, rev_id) pair as a key to select the devices. The devices with
51 * same device_id have very similar interface to host driver.
52 */
53 static const struct pci_device_id pci_ids[] = {
54 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1502) },
55 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f0) },
56 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f2) },
57 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f3) },
58 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1B0B) },
59 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1B0C) },
60 {0}
61 };
62
63 MODULE_DEVICE_TABLE(pci, pci_ids);
64
65 static const struct amdxdna_device_id amdxdna_ids[] = {
66 { 0x1502, 0x0, &dev_npu1_info },
67 { 0x17f0, 0x10, &dev_npu4_info },
68 { 0x17f0, 0x11, &dev_npu5_info },
69 { 0x17f0, 0x20, &dev_npu6_info },
70 { 0x17f2, 0x10, &dev_npu3_pf_info },
71 { 0x17f3, 0x10, &dev_npu3_vf_info },
72 { 0x1B0B, 0x10, &dev_npu3_pf_info },
73 { 0x1B0C, 0x10, &dev_npu3_vf_info },
74 {0}
75 };
76
amdxdna_sva_init(struct amdxdna_client * client)77 static int amdxdna_sva_init(struct amdxdna_client *client)
78 {
79 struct amdxdna_dev *xdna = client->xdna;
80
81 client->sva = iommu_sva_bind_device(xdna->ddev.dev, client->mm);
82 if (IS_ERR(client->sva)) {
83 XDNA_ERR(xdna, "SVA bind device failed, ret %ld", PTR_ERR(client->sva));
84 return PTR_ERR(client->sva);
85 }
86
87 client->pasid = iommu_sva_get_pasid(client->sva);
88 if (client->pasid == IOMMU_PASID_INVALID) {
89 iommu_sva_unbind_device(client->sva);
90 client->sva = NULL;
91 XDNA_ERR(xdna, "SVA get pasid failed");
92 return -ENODEV;
93 }
94
95 return 0;
96 }
97
amdxdna_sva_fini(struct amdxdna_client * client)98 static void amdxdna_sva_fini(struct amdxdna_client *client)
99 {
100 if (IS_ERR_OR_NULL(client->sva))
101 return;
102
103 iommu_sva_unbind_device(client->sva);
104 client->sva = NULL;
105 client->pasid = IOMMU_PASID_INVALID;
106 }
107
amdxdna_drm_open(struct drm_device * ddev,struct drm_file * filp)108 static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
109 {
110 struct amdxdna_dev *xdna = to_xdna_dev(ddev);
111 struct amdxdna_client *client;
112 int ret;
113
114 client = kzalloc_obj(*client);
115 if (!client)
116 return -ENOMEM;
117
118 ret = init_srcu_struct(&client->hwctx_srcu);
119 if (ret)
120 goto free_client;
121
122 client->pid = pid_nr(rcu_access_pointer(filp->pid));
123 client->xdna = xdna;
124 client->pasid = IOMMU_PASID_INVALID;
125 client->mm = current->mm;
126
127 if (!amdxdna_iova_on(xdna)) {
128 /* No need to fail open since user may use pa + carveout later. */
129 if (amdxdna_sva_init(client)) {
130 XDNA_WARN(xdna, "PASID not available for pid %d", client->pid);
131 if (!amdxdna_use_carveout(xdna)) {
132 XDNA_ERR(xdna, "PASID unavailable and carveout not configured");
133 ret = -EINVAL;
134 goto cleanup_srcu;
135 }
136 }
137 }
138 mmgrab(client->mm);
139 xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC);
140 xa_init_flags(&client->dev_heap_xa, XA_FLAGS_ALLOC);
141 drm_mm_init(&client->dev_heap_mm, xdna->dev_info->dev_mem_base,
142 xdna->dev_info->dev_heap_max_size);
143 mutex_init(&client->mm_lock);
144
145 mutex_lock(&xdna->client_lock);
146 mutex_lock(&xdna->dev_lock);
147 list_add_tail(&client->node, &xdna->client_list);
148 mutex_unlock(&xdna->dev_lock);
149 mutex_unlock(&xdna->client_lock);
150
151 filp->driver_priv = client;
152 client->filp = filp;
153
154 XDNA_DBG(xdna, "pid %d opened", client->pid);
155 return 0;
156
157 cleanup_srcu:
158 cleanup_srcu_struct(&client->hwctx_srcu);
159 free_client:
160 kfree(client);
161 return ret;
162 }
163
amdxdna_client_cleanup(struct amdxdna_client * client)164 static void amdxdna_client_cleanup(struct amdxdna_client *client)
165 {
166 struct amdxdna_gem_obj *heap;
167 unsigned long heap_id;
168
169 list_del(&client->node);
170 amdxdna_hwctx_remove_all(client);
171 xa_destroy(&client->hwctx_xa);
172 cleanup_srcu_struct(&client->hwctx_srcu);
173
174 xa_for_each(&client->dev_heap_xa, heap_id, heap)
175 drm_gem_object_put(to_gobj(heap));
176 xa_destroy(&client->dev_heap_xa);
177 drm_mm_takedown(&client->dev_heap_mm);
178
179 mutex_destroy(&client->mm_lock);
180 mmdrop(client->mm);
181 amdxdna_sva_fini(client);
182 kfree(client);
183 }
184
amdxdna_drm_close(struct drm_device * ddev,struct drm_file * filp)185 static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp)
186 {
187 struct amdxdna_client *client = filp->driver_priv;
188 struct amdxdna_dev *xdna = to_xdna_dev(ddev);
189
190 XDNA_DBG(xdna, "closing pid %d", client->pid);
191
192 mutex_lock(&xdna->client_lock);
193 mutex_lock(&xdna->dev_lock);
194 amdxdna_client_cleanup(client);
195 mutex_unlock(&xdna->dev_lock);
196 mutex_unlock(&xdna->client_lock);
197 }
198
amdxdna_drm_get_info_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)199 static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
200 {
201 struct amdxdna_client *client = filp->driver_priv;
202 struct amdxdna_dev *xdna = to_xdna_dev(dev);
203 struct amdxdna_drm_get_info *args = data;
204 int ret;
205
206 if (!xdna->dev_info->ops->get_aie_info)
207 return -EOPNOTSUPP;
208
209 XDNA_DBG(xdna, "Request parameter %u", args->param);
210 mutex_lock(&xdna->dev_lock);
211 ret = xdna->dev_info->ops->get_aie_info(client, args);
212 mutex_unlock(&xdna->dev_lock);
213 return ret;
214 }
215
amdxdna_drm_get_array_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)216 static int amdxdna_drm_get_array_ioctl(struct drm_device *dev, void *data,
217 struct drm_file *filp)
218 {
219 struct amdxdna_client *client = filp->driver_priv;
220 struct amdxdna_dev *xdna = to_xdna_dev(dev);
221 struct amdxdna_drm_get_array *args = data;
222
223 if (!xdna->dev_info->ops->get_array)
224 return -EOPNOTSUPP;
225
226 if (args->pad || !args->num_element || !args->element_size)
227 return -EINVAL;
228
229 guard(mutex)(&xdna->dev_lock);
230 return xdna->dev_info->ops->get_array(client, args);
231 }
232
amdxdna_drm_set_state_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)233 static int amdxdna_drm_set_state_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
234 {
235 struct amdxdna_client *client = filp->driver_priv;
236 struct amdxdna_dev *xdna = to_xdna_dev(dev);
237 struct amdxdna_drm_set_state *args = data;
238 int ret;
239
240 if (!xdna->dev_info->ops->set_aie_state)
241 return -EOPNOTSUPP;
242
243 XDNA_DBG(xdna, "Request parameter %u", args->param);
244 mutex_lock(&xdna->dev_lock);
245 ret = xdna->dev_info->ops->set_aie_state(client, args);
246 mutex_unlock(&xdna->dev_lock);
247
248 return ret;
249 }
250
amdxdna_drm_gem_mmap(struct file * filp,struct vm_area_struct * vma)251 static int amdxdna_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
252 {
253 struct drm_file *drm_filp = filp->private_data;
254 struct amdxdna_client *client = drm_filp->driver_priv;
255 struct amdxdna_dev *xdna = client->xdna;
256
257 if (likely(vma->vm_pgoff >= DRM_FILE_PAGE_OFFSET_START))
258 return drm_gem_mmap(filp, vma);
259
260 if (!xdna->dev_info->ops->mmap)
261 return -EOPNOTSUPP;
262
263 return xdna->dev_info->ops->mmap(client, vma);
264 }
265
266 static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = {
267 /* Context */
268 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0),
269 DRM_IOCTL_DEF_DRV(AMDXDNA_DESTROY_HWCTX, amdxdna_drm_destroy_hwctx_ioctl, 0),
270 DRM_IOCTL_DEF_DRV(AMDXDNA_CONFIG_HWCTX, amdxdna_drm_config_hwctx_ioctl, 0),
271 /* BO */
272 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_BO, amdxdna_drm_create_bo_ioctl, 0),
273 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_BO_INFO, amdxdna_drm_get_bo_info_ioctl, 0),
274 DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0),
275 /* Execution */
276 DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0),
277 DRM_IOCTL_DEF_DRV(AMDXDNA_WAIT_CMD, amdxdna_drm_wait_cmd_ioctl, 0),
278 /* AIE hardware */
279 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0),
280 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_ARRAY, amdxdna_drm_get_array_ioctl, 0),
281 DRM_IOCTL_DEF_DRV(AMDXDNA_SET_STATE, amdxdna_drm_set_state_ioctl, DRM_ROOT_ONLY),
282 };
283
amdxdna_show_fdinfo(struct drm_printer * p,struct drm_file * filp)284 static void amdxdna_show_fdinfo(struct drm_printer *p, struct drm_file *filp)
285 {
286 struct amdxdna_client *client = filp->driver_priv;
287 size_t heap_usage, external_usage, internal_usage;
288 char *drv_name = filp->minor->dev->driver->name;
289
290 mutex_lock(&client->mm_lock);
291
292 heap_usage = client->heap_usage;
293 internal_usage = client->total_int_bo_usage;
294 external_usage = client->total_bo_usage - internal_usage;
295
296 mutex_unlock(&client->mm_lock);
297
298 /*
299 * Note for driver specific BO memory usage stat.
300 * Total memory in use = amdxdna-internal-alloc + amdxdna-external-alloc, which
301 * includes both imported and created BOs. To avoid double counts, it includes
302 * HEAP BO, but not DEV BO. DEV BO is counted by amdxdna-heap-alloc.
303 */
304 drm_fdinfo_print_size(p, drv_name, "heap", "alloc", heap_usage);
305 drm_fdinfo_print_size(p, drv_name, "internal", "alloc", internal_usage);
306 drm_fdinfo_print_size(p, drv_name, "external", "alloc", external_usage);
307 /*
308 * Note for DRM standard BO memory stat.
309 * drm-total-memory counts both DEV BO and HEAP BO. The DEV BO size is double counted.
310 * drm-shared-memory counts BO shared with other processes/devices.
311 */
312 drm_show_memory_stats(p, filp);
313 }
314
315 static const struct file_operations amdxdna_fops = {
316 .owner = THIS_MODULE,
317 .open = accel_open,
318 .release = drm_release,
319 .unlocked_ioctl = drm_ioctl,
320 .compat_ioctl = drm_compat_ioctl,
321 .poll = drm_poll,
322 .read = drm_read,
323 .llseek = noop_llseek,
324 .mmap = amdxdna_drm_gem_mmap,
325 .show_fdinfo = drm_show_fdinfo,
326 .fop_flags = FOP_UNSIGNED_OFFSET,
327 };
328
329 const struct drm_driver amdxdna_drm_drv = {
330 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL |
331 DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE,
332 .fops = &amdxdna_fops,
333 .name = "amdxdna_accel_driver",
334 .desc = "AMD XDNA DRM implementation",
335 .major = AMDXDNA_DRIVER_MAJOR,
336 .minor = AMDXDNA_DRIVER_MINOR,
337 .open = amdxdna_drm_open,
338 .postclose = amdxdna_drm_close,
339 .ioctls = amdxdna_drm_ioctls,
340 .num_ioctls = ARRAY_SIZE(amdxdna_drm_ioctls),
341 .show_fdinfo = amdxdna_show_fdinfo,
342 .gem_create_object = amdxdna_gem_create_shmem_object_cb,
343 .gem_prime_import = amdxdna_gem_prime_import,
344 };
345
346 static const struct amdxdna_dev_info *
amdxdna_get_dev_info(struct pci_dev * pdev)347 amdxdna_get_dev_info(struct pci_dev *pdev)
348 {
349 int i;
350
351 for (i = 0; i < ARRAY_SIZE(amdxdna_ids); i++) {
352 if (pdev->device == amdxdna_ids[i].device &&
353 pdev->revision == amdxdna_ids[i].revision)
354 return amdxdna_ids[i].dev_info;
355 }
356 return NULL;
357 }
358
amdxdna_xdna_drm_release(struct drm_device * drm,void * res)359 static void amdxdna_xdna_drm_release(struct drm_device *drm, void *res)
360 {
361 struct amdxdna_dev *xdna = res;
362
363 amdxdna_carveout_fini(xdna);
364 }
365
amdxdna_probe(struct pci_dev * pdev,const struct pci_device_id * id)366 static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id)
367 {
368 struct device *dev = &pdev->dev;
369 struct amdxdna_dev *xdna;
370 struct drm_device *ddev;
371 int ret;
372
373 xdna = devm_drm_dev_alloc(dev, &amdxdna_drm_drv, typeof(*xdna), ddev);
374 if (IS_ERR(xdna))
375 return PTR_ERR(xdna);
376 ddev = &xdna->ddev;
377
378 xdna->dev_info = amdxdna_get_dev_info(pdev);
379 if (!xdna->dev_info)
380 return -ENODEV;
381
382 ret = drmm_mutex_init(ddev, &xdna->client_lock);
383 if (ret)
384 return ret;
385
386 ret = drmm_mutex_init(ddev, &xdna->dev_lock);
387 if (ret)
388 return ret;
389
390 init_rwsem(&xdna->notifier_lock);
391 INIT_LIST_HEAD(&xdna->client_list);
392 pci_set_drvdata(pdev, xdna);
393
394 ret = drmm_add_action(ddev, amdxdna_xdna_drm_release, xdna);
395 if (ret)
396 return ret;
397
398 if (IS_ENABLED(CONFIG_LOCKDEP)) {
399 fs_reclaim_acquire(GFP_KERNEL);
400 might_lock(&xdna->notifier_lock);
401 fs_reclaim_release(GFP_KERNEL);
402 }
403
404 ret = amdxdna_iommu_init(xdna);
405 if (ret)
406 return ret;
407
408 xdna->notifier_wq = drmm_alloc_ordered_workqueue(ddev, "notifier_wq", WQ_MEM_RECLAIM);
409 if (IS_ERR(xdna->notifier_wq)) {
410 ret = PTR_ERR(xdna->notifier_wq);
411 goto iommu_fini;
412 }
413
414 mutex_lock(&xdna->dev_lock);
415 ret = xdna->dev_info->ops->init(xdna);
416 mutex_unlock(&xdna->dev_lock);
417 if (ret) {
418 XDNA_ERR(xdna, "Hardware init failed, ret %d", ret);
419 goto iommu_fini;
420 }
421
422 ret = amdxdna_sysfs_init(xdna);
423 if (ret) {
424 XDNA_ERR(xdna, "Create amdxdna attrs failed: %d", ret);
425 goto failed_dev_fini;
426 }
427
428 ret = drm_dev_register(ddev, 0);
429 if (ret) {
430 XDNA_ERR(xdna, "DRM register failed, ret %d", ret);
431 goto failed_sysfs_fini;
432 }
433
434 amdxdna_debugfs_init(xdna);
435 return 0;
436
437 failed_sysfs_fini:
438 amdxdna_sysfs_fini(xdna);
439 failed_dev_fini:
440 mutex_lock(&xdna->dev_lock);
441 xdna->dev_info->ops->fini(xdna);
442 mutex_unlock(&xdna->dev_lock);
443 iommu_fini:
444 amdxdna_iommu_fini(xdna);
445 return ret;
446 }
447
amdxdna_remove(struct pci_dev * pdev)448 static void amdxdna_remove(struct pci_dev *pdev)
449 {
450 struct amdxdna_dev *xdna = pci_get_drvdata(pdev);
451 struct amdxdna_client *client;
452
453 drm_dev_unplug(&xdna->ddev);
454 amdxdna_sysfs_fini(xdna);
455
456 mutex_lock(&xdna->client_lock);
457 mutex_lock(&xdna->dev_lock);
458 list_for_each_entry(client, &xdna->client_list, node) {
459 amdxdna_hwctx_remove_all(client);
460 amdxdna_sva_fini(client);
461 }
462
463 xdna->dev_info->ops->fini(xdna);
464 mutex_unlock(&xdna->dev_lock);
465 mutex_unlock(&xdna->client_lock);
466
467 amdxdna_iommu_fini(xdna);
468 }
469
470 static const struct dev_pm_ops amdxdna_pm_ops = {
471 SYSTEM_SLEEP_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume)
472 RUNTIME_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume, NULL)
473 };
474
amdxdna_sriov_configure(struct pci_dev * pdev,int num_vfs)475 static int amdxdna_sriov_configure(struct pci_dev *pdev, int num_vfs)
476 {
477 struct amdxdna_dev *xdna = pci_get_drvdata(pdev);
478
479 guard(mutex)(&xdna->dev_lock);
480 if (xdna->dev_info->ops->sriov_configure)
481 return xdna->dev_info->ops->sriov_configure(xdna, num_vfs);
482
483 return -ENOENT;
484 }
485
486 static struct pci_driver amdxdna_pci_driver = {
487 .name = KBUILD_MODNAME,
488 .id_table = pci_ids,
489 .probe = amdxdna_probe,
490 .remove = amdxdna_remove,
491 .driver.pm = &amdxdna_pm_ops,
492 .sriov_configure = amdxdna_sriov_configure,
493 };
494
495 module_pci_driver(amdxdna_pci_driver);
496
497 MODULE_LICENSE("GPL");
498 MODULE_IMPORT_NS("AMD_PMF");
499 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>");
500 MODULE_DESCRIPTION("amdxdna driver");
501