1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_accel.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_ioctl.h> 12 #include <drm/drm_managed.h> 13 #include <drm/gpu_scheduler.h> 14 #include <linux/iommu.h> 15 #include <linux/pci.h> 16 17 #include "amdxdna_cbuf.h" 18 #include "amdxdna_ctx.h" 19 #include "amdxdna_debugfs.h" 20 #include "amdxdna_gem.h" 21 #include "amdxdna_pci_drv.h" 22 #include "amdxdna_pm.h" 23 24 MODULE_FIRMWARE("amdnpu/1502_00/npu.sbin"); 25 MODULE_FIRMWARE("amdnpu/17f0_10/npu.sbin"); 26 MODULE_FIRMWARE("amdnpu/17f0_11/npu.sbin"); 27 MODULE_FIRMWARE("amdnpu/17f0_20/npu.sbin"); 28 MODULE_FIRMWARE("amdnpu/1502_00/npu_7.sbin"); 29 MODULE_FIRMWARE("amdnpu/17f0_10/npu_7.sbin"); 30 MODULE_FIRMWARE("amdnpu/17f0_11/npu_7.sbin"); 31 32 /* 33 * 0.0: Initial version 34 * 0.1: Support getting all hardware contexts by DRM_IOCTL_AMDXDNA_GET_ARRAY 35 * 0.2: Support getting last error hardware error 36 * 0.3: Support firmware debug buffer 37 * 0.4: Support getting resource information 38 * 0.5: Support getting telemetry data 39 * 0.6: Support preemption 40 * 0.7: Support getting power and utilization data 41 * 0.8: Support BO usage query 42 * 0.9: Add new device type AMDXDNA_DEV_TYPE_PF 43 * 0.10: Support AIE4 UMQ 44 */ 45 #define AMDXDNA_DRIVER_MAJOR 0 46 #define AMDXDNA_DRIVER_MINOR 10 47 48 /* 49 * Bind the driver base on (vendor_id, device_id) pair and later use the 50 * (device_id, rev_id) pair as a key to select the devices. The devices with 51 * same device_id have very similar interface to host driver. 52 */ 53 static const struct pci_device_id pci_ids[] = { 54 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1502) }, 55 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f0) }, 56 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f2) }, 57 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f3) }, 58 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1B0B) }, 59 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1B0C) }, 60 {0} 61 }; 62 63 MODULE_DEVICE_TABLE(pci, pci_ids); 64 65 static const struct amdxdna_device_id amdxdna_ids[] = { 66 { 0x1502, 0x0, &dev_npu1_info }, 67 { 0x17f0, 0x10, &dev_npu4_info }, 68 { 0x17f0, 0x11, &dev_npu5_info }, 69 { 0x17f0, 0x20, &dev_npu6_info }, 70 { 0x17f2, 0x10, &dev_npu3_pf_info }, 71 { 0x17f3, 0x10, &dev_npu3_vf_info }, 72 { 0x1B0B, 0x10, &dev_npu3_pf_info }, 73 { 0x1B0C, 0x10, &dev_npu3_vf_info }, 74 {0} 75 }; 76 77 static int amdxdna_sva_init(struct amdxdna_client *client) 78 { 79 struct amdxdna_dev *xdna = client->xdna; 80 81 client->sva = iommu_sva_bind_device(xdna->ddev.dev, client->mm); 82 if (IS_ERR(client->sva)) { 83 XDNA_ERR(xdna, "SVA bind device failed, ret %ld", PTR_ERR(client->sva)); 84 return PTR_ERR(client->sva); 85 } 86 87 client->pasid = iommu_sva_get_pasid(client->sva); 88 if (client->pasid == IOMMU_PASID_INVALID) { 89 iommu_sva_unbind_device(client->sva); 90 client->sva = NULL; 91 XDNA_ERR(xdna, "SVA get pasid failed"); 92 return -ENODEV; 93 } 94 95 return 0; 96 } 97 98 static void amdxdna_sva_fini(struct amdxdna_client *client) 99 { 100 if (IS_ERR_OR_NULL(client->sva)) 101 return; 102 103 iommu_sva_unbind_device(client->sva); 104 client->sva = NULL; 105 client->pasid = IOMMU_PASID_INVALID; 106 } 107 108 static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) 109 { 110 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 111 struct amdxdna_client *client; 112 113 client = kzalloc_obj(*client); 114 if (!client) 115 return -ENOMEM; 116 117 client->pid = pid_nr(rcu_access_pointer(filp->pid)); 118 client->xdna = xdna; 119 client->pasid = IOMMU_PASID_INVALID; 120 client->mm = current->mm; 121 122 if (!amdxdna_iova_on(xdna)) { 123 /* No need to fail open since user may use pa + carveout later. */ 124 if (amdxdna_sva_init(client)) { 125 XDNA_WARN(xdna, "PASID not available for pid %d", client->pid); 126 if (!amdxdna_use_carveout(xdna)) { 127 XDNA_ERR(xdna, "PASID unavailable and carveout not configured"); 128 kfree(client); 129 return -EINVAL; 130 } 131 } 132 } 133 mmgrab(client->mm); 134 init_srcu_struct(&client->hwctx_srcu); 135 xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); 136 xa_init_flags(&client->dev_heap_xa, XA_FLAGS_ALLOC); 137 drm_mm_init(&client->dev_heap_mm, xdna->dev_info->dev_mem_base, 138 xdna->dev_info->dev_heap_max_size); 139 mutex_init(&client->mm_lock); 140 141 mutex_lock(&xdna->dev_lock); 142 list_add_tail(&client->node, &xdna->client_list); 143 mutex_unlock(&xdna->dev_lock); 144 145 filp->driver_priv = client; 146 client->filp = filp; 147 148 XDNA_DBG(xdna, "pid %d opened", client->pid); 149 return 0; 150 } 151 152 static void amdxdna_client_cleanup(struct amdxdna_client *client) 153 { 154 struct amdxdna_gem_obj *heap; 155 unsigned long heap_id; 156 157 list_del(&client->node); 158 amdxdna_hwctx_remove_all(client); 159 xa_destroy(&client->hwctx_xa); 160 cleanup_srcu_struct(&client->hwctx_srcu); 161 162 xa_for_each(&client->dev_heap_xa, heap_id, heap) 163 drm_gem_object_put(to_gobj(heap)); 164 xa_destroy(&client->dev_heap_xa); 165 drm_mm_takedown(&client->dev_heap_mm); 166 167 mutex_destroy(&client->mm_lock); 168 mmdrop(client->mm); 169 amdxdna_sva_fini(client); 170 kfree(client); 171 } 172 173 static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) 174 { 175 struct amdxdna_client *client = filp->driver_priv; 176 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 177 int idx; 178 179 XDNA_DBG(xdna, "closing pid %d", client->pid); 180 181 if (!drm_dev_enter(&xdna->ddev, &idx)) 182 return; 183 184 mutex_lock(&xdna->dev_lock); 185 amdxdna_client_cleanup(client); 186 mutex_unlock(&xdna->dev_lock); 187 188 drm_dev_exit(idx); 189 } 190 191 static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 192 { 193 struct amdxdna_client *client = filp->driver_priv; 194 struct amdxdna_dev *xdna = to_xdna_dev(dev); 195 struct amdxdna_drm_get_info *args = data; 196 int ret; 197 198 if (!xdna->dev_info->ops->get_aie_info) 199 return -EOPNOTSUPP; 200 201 XDNA_DBG(xdna, "Request parameter %u", args->param); 202 mutex_lock(&xdna->dev_lock); 203 ret = xdna->dev_info->ops->get_aie_info(client, args); 204 mutex_unlock(&xdna->dev_lock); 205 return ret; 206 } 207 208 static int amdxdna_drm_get_array_ioctl(struct drm_device *dev, void *data, 209 struct drm_file *filp) 210 { 211 struct amdxdna_client *client = filp->driver_priv; 212 struct amdxdna_dev *xdna = to_xdna_dev(dev); 213 struct amdxdna_drm_get_array *args = data; 214 215 if (!xdna->dev_info->ops->get_array) 216 return -EOPNOTSUPP; 217 218 if (args->pad || !args->num_element || !args->element_size) 219 return -EINVAL; 220 221 guard(mutex)(&xdna->dev_lock); 222 return xdna->dev_info->ops->get_array(client, args); 223 } 224 225 static int amdxdna_drm_set_state_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 226 { 227 struct amdxdna_client *client = filp->driver_priv; 228 struct amdxdna_dev *xdna = to_xdna_dev(dev); 229 struct amdxdna_drm_set_state *args = data; 230 int ret; 231 232 if (!xdna->dev_info->ops->set_aie_state) 233 return -EOPNOTSUPP; 234 235 XDNA_DBG(xdna, "Request parameter %u", args->param); 236 mutex_lock(&xdna->dev_lock); 237 ret = xdna->dev_info->ops->set_aie_state(client, args); 238 mutex_unlock(&xdna->dev_lock); 239 240 return ret; 241 } 242 243 static int amdxdna_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma) 244 { 245 struct drm_file *drm_filp = filp->private_data; 246 struct amdxdna_client *client = drm_filp->driver_priv; 247 struct amdxdna_dev *xdna = client->xdna; 248 249 if (likely(vma->vm_pgoff >= DRM_FILE_PAGE_OFFSET_START)) 250 return drm_gem_mmap(filp, vma); 251 252 if (!xdna->dev_info->ops->mmap) 253 return -EOPNOTSUPP; 254 255 return xdna->dev_info->ops->mmap(client, vma); 256 } 257 258 static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = { 259 /* Context */ 260 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0), 261 DRM_IOCTL_DEF_DRV(AMDXDNA_DESTROY_HWCTX, amdxdna_drm_destroy_hwctx_ioctl, 0), 262 DRM_IOCTL_DEF_DRV(AMDXDNA_CONFIG_HWCTX, amdxdna_drm_config_hwctx_ioctl, 0), 263 /* BO */ 264 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_BO, amdxdna_drm_create_bo_ioctl, 0), 265 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_BO_INFO, amdxdna_drm_get_bo_info_ioctl, 0), 266 DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0), 267 /* Execution */ 268 DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0), 269 DRM_IOCTL_DEF_DRV(AMDXDNA_WAIT_CMD, amdxdna_drm_wait_cmd_ioctl, 0), 270 /* AIE hardware */ 271 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0), 272 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_ARRAY, amdxdna_drm_get_array_ioctl, 0), 273 DRM_IOCTL_DEF_DRV(AMDXDNA_SET_STATE, amdxdna_drm_set_state_ioctl, DRM_ROOT_ONLY), 274 }; 275 276 static void amdxdna_show_fdinfo(struct drm_printer *p, struct drm_file *filp) 277 { 278 struct amdxdna_client *client = filp->driver_priv; 279 size_t heap_usage, external_usage, internal_usage; 280 char *drv_name = filp->minor->dev->driver->name; 281 282 mutex_lock(&client->mm_lock); 283 284 heap_usage = client->heap_usage; 285 internal_usage = client->total_int_bo_usage; 286 external_usage = client->total_bo_usage - internal_usage; 287 288 mutex_unlock(&client->mm_lock); 289 290 /* 291 * Note for driver specific BO memory usage stat. 292 * Total memory in use = amdxdna-internal-alloc + amdxdna-external-alloc, which 293 * includes both imported and created BOs. To avoid double counts, it includes 294 * HEAP BO, but not DEV BO. DEV BO is counted by amdxdna-heap-alloc. 295 */ 296 drm_fdinfo_print_size(p, drv_name, "heap", "alloc", heap_usage); 297 drm_fdinfo_print_size(p, drv_name, "internal", "alloc", internal_usage); 298 drm_fdinfo_print_size(p, drv_name, "external", "alloc", external_usage); 299 /* 300 * Note for DRM standard BO memory stat. 301 * drm-total-memory counts both DEV BO and HEAP BO. The DEV BO size is double counted. 302 * drm-shared-memory counts BO shared with other processes/devices. 303 */ 304 drm_show_memory_stats(p, filp); 305 } 306 307 static const struct file_operations amdxdna_fops = { 308 .owner = THIS_MODULE, 309 .open = accel_open, 310 .release = drm_release, 311 .unlocked_ioctl = drm_ioctl, 312 .compat_ioctl = drm_compat_ioctl, 313 .poll = drm_poll, 314 .read = drm_read, 315 .llseek = noop_llseek, 316 .mmap = amdxdna_drm_gem_mmap, 317 .show_fdinfo = drm_show_fdinfo, 318 .fop_flags = FOP_UNSIGNED_OFFSET, 319 }; 320 321 const struct drm_driver amdxdna_drm_drv = { 322 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL | 323 DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, 324 .fops = &amdxdna_fops, 325 .name = "amdxdna_accel_driver", 326 .desc = "AMD XDNA DRM implementation", 327 .major = AMDXDNA_DRIVER_MAJOR, 328 .minor = AMDXDNA_DRIVER_MINOR, 329 .open = amdxdna_drm_open, 330 .postclose = amdxdna_drm_close, 331 .ioctls = amdxdna_drm_ioctls, 332 .num_ioctls = ARRAY_SIZE(amdxdna_drm_ioctls), 333 .show_fdinfo = amdxdna_show_fdinfo, 334 .gem_create_object = amdxdna_gem_create_shmem_object_cb, 335 .gem_prime_import = amdxdna_gem_prime_import, 336 }; 337 338 static const struct amdxdna_dev_info * 339 amdxdna_get_dev_info(struct pci_dev *pdev) 340 { 341 int i; 342 343 for (i = 0; i < ARRAY_SIZE(amdxdna_ids); i++) { 344 if (pdev->device == amdxdna_ids[i].device && 345 pdev->revision == amdxdna_ids[i].revision) 346 return amdxdna_ids[i].dev_info; 347 } 348 return NULL; 349 } 350 351 static void amdxdna_xdna_drm_release(struct drm_device *drm, void *res) 352 { 353 struct amdxdna_dev *xdna = res; 354 355 amdxdna_carveout_fini(xdna); 356 } 357 358 static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) 359 { 360 struct device *dev = &pdev->dev; 361 struct amdxdna_dev *xdna; 362 struct drm_device *ddev; 363 int ret; 364 365 xdna = devm_drm_dev_alloc(dev, &amdxdna_drm_drv, typeof(*xdna), ddev); 366 if (IS_ERR(xdna)) 367 return PTR_ERR(xdna); 368 ddev = &xdna->ddev; 369 370 xdna->dev_info = amdxdna_get_dev_info(pdev); 371 if (!xdna->dev_info) 372 return -ENODEV; 373 374 drmm_mutex_init(ddev, &xdna->dev_lock); 375 init_rwsem(&xdna->notifier_lock); 376 INIT_LIST_HEAD(&xdna->client_list); 377 pci_set_drvdata(pdev, xdna); 378 379 ret = drmm_add_action(ddev, amdxdna_xdna_drm_release, xdna); 380 if (ret) 381 return ret; 382 383 if (IS_ENABLED(CONFIG_LOCKDEP)) { 384 fs_reclaim_acquire(GFP_KERNEL); 385 might_lock(&xdna->notifier_lock); 386 fs_reclaim_release(GFP_KERNEL); 387 } 388 389 ret = amdxdna_iommu_init(xdna); 390 if (ret) 391 return ret; 392 393 xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", WQ_MEM_RECLAIM); 394 if (!xdna->notifier_wq) { 395 ret = -ENOMEM; 396 goto iommu_fini; 397 } 398 399 mutex_lock(&xdna->dev_lock); 400 ret = xdna->dev_info->ops->init(xdna); 401 mutex_unlock(&xdna->dev_lock); 402 if (ret) { 403 XDNA_ERR(xdna, "Hardware init failed, ret %d", ret); 404 goto destroy_notifier_wq; 405 } 406 407 ret = amdxdna_sysfs_init(xdna); 408 if (ret) { 409 XDNA_ERR(xdna, "Create amdxdna attrs failed: %d", ret); 410 goto failed_dev_fini; 411 } 412 413 ret = drm_dev_register(ddev, 0); 414 if (ret) { 415 XDNA_ERR(xdna, "DRM register failed, ret %d", ret); 416 goto failed_sysfs_fini; 417 } 418 419 amdxdna_debugfs_init(xdna); 420 return 0; 421 422 failed_sysfs_fini: 423 amdxdna_sysfs_fini(xdna); 424 failed_dev_fini: 425 mutex_lock(&xdna->dev_lock); 426 xdna->dev_info->ops->fini(xdna); 427 mutex_unlock(&xdna->dev_lock); 428 destroy_notifier_wq: 429 destroy_workqueue(xdna->notifier_wq); 430 iommu_fini: 431 amdxdna_iommu_fini(xdna); 432 return ret; 433 } 434 435 static void amdxdna_remove(struct pci_dev *pdev) 436 { 437 struct amdxdna_dev *xdna = pci_get_drvdata(pdev); 438 struct amdxdna_client *client; 439 440 destroy_workqueue(xdna->notifier_wq); 441 442 drm_dev_unplug(&xdna->ddev); 443 amdxdna_sysfs_fini(xdna); 444 445 mutex_lock(&xdna->dev_lock); 446 client = list_first_entry_or_null(&xdna->client_list, 447 struct amdxdna_client, node); 448 while (client) { 449 amdxdna_client_cleanup(client); 450 451 client = list_first_entry_or_null(&xdna->client_list, 452 struct amdxdna_client, node); 453 } 454 455 xdna->dev_info->ops->fini(xdna); 456 mutex_unlock(&xdna->dev_lock); 457 458 amdxdna_iommu_fini(xdna); 459 } 460 461 static const struct dev_pm_ops amdxdna_pm_ops = { 462 SYSTEM_SLEEP_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume) 463 RUNTIME_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume, NULL) 464 }; 465 466 static int amdxdna_sriov_configure(struct pci_dev *pdev, int num_vfs) 467 { 468 struct amdxdna_dev *xdna = pci_get_drvdata(pdev); 469 470 guard(mutex)(&xdna->dev_lock); 471 if (xdna->dev_info->ops->sriov_configure) 472 return xdna->dev_info->ops->sriov_configure(xdna, num_vfs); 473 474 return -ENOENT; 475 } 476 477 static struct pci_driver amdxdna_pci_driver = { 478 .name = KBUILD_MODNAME, 479 .id_table = pci_ids, 480 .probe = amdxdna_probe, 481 .remove = amdxdna_remove, 482 .driver.pm = &amdxdna_pm_ops, 483 .sriov_configure = amdxdna_sriov_configure, 484 }; 485 486 module_pci_driver(amdxdna_pci_driver); 487 488 MODULE_LICENSE("GPL"); 489 MODULE_IMPORT_NS("AMD_PMF"); 490 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); 491 MODULE_DESCRIPTION("amdxdna driver"); 492