1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_accel.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_ioctl.h> 12 #include <drm/drm_managed.h> 13 #include <drm/gpu_scheduler.h> 14 #include <linux/iommu.h> 15 #include <linux/pci.h> 16 #include <linux/pm_runtime.h> 17 18 #include "amdxdna_ctx.h" 19 #include "amdxdna_gem.h" 20 #include "amdxdna_pci_drv.h" 21 22 #define AMDXDNA_AUTOSUSPEND_DELAY 5000 /* milliseconds */ 23 24 MODULE_FIRMWARE("amdnpu/1502_00/npu.sbin"); 25 MODULE_FIRMWARE("amdnpu/17f0_10/npu.sbin"); 26 MODULE_FIRMWARE("amdnpu/17f0_11/npu.sbin"); 27 MODULE_FIRMWARE("amdnpu/17f0_20/npu.sbin"); 28 29 /* 30 * 0.0: Initial version 31 * 0.1: Support getting all hardware contexts by DRM_IOCTL_AMDXDNA_GET_ARRAY 32 */ 33 #define AMDXDNA_DRIVER_MAJOR 0 34 #define AMDXDNA_DRIVER_MINOR 1 35 36 /* 37 * Bind the driver base on (vendor_id, device_id) pair and later use the 38 * (device_id, rev_id) pair as a key to select the devices. The devices with 39 * same device_id have very similar interface to host driver. 40 */ 41 static const struct pci_device_id pci_ids[] = { 42 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1502) }, 43 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f0) }, 44 {0} 45 }; 46 47 MODULE_DEVICE_TABLE(pci, pci_ids); 48 49 static const struct amdxdna_device_id amdxdna_ids[] = { 50 { 0x1502, 0x0, &dev_npu1_info }, 51 { 0x17f0, 0x0, &dev_npu2_info }, 52 { 0x17f0, 0x10, &dev_npu4_info }, 53 { 0x17f0, 0x11, &dev_npu5_info }, 54 { 0x17f0, 0x20, &dev_npu6_info }, 55 {0} 56 }; 57 58 static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) 59 { 60 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 61 struct amdxdna_client *client; 62 int ret; 63 64 ret = pm_runtime_resume_and_get(ddev->dev); 65 if (ret) { 66 XDNA_ERR(xdna, "Failed to get rpm, ret %d", ret); 67 return ret; 68 } 69 70 client = kzalloc(sizeof(*client), GFP_KERNEL); 71 if (!client) { 72 ret = -ENOMEM; 73 goto put_rpm; 74 } 75 76 client->pid = pid_nr(rcu_access_pointer(filp->pid)); 77 client->xdna = xdna; 78 79 client->sva = iommu_sva_bind_device(xdna->ddev.dev, current->mm); 80 if (IS_ERR(client->sva)) { 81 ret = PTR_ERR(client->sva); 82 XDNA_ERR(xdna, "SVA bind device failed, ret %d", ret); 83 goto failed; 84 } 85 client->pasid = iommu_sva_get_pasid(client->sva); 86 if (client->pasid == IOMMU_PASID_INVALID) { 87 XDNA_ERR(xdna, "SVA get pasid failed"); 88 ret = -ENODEV; 89 goto unbind_sva; 90 } 91 init_srcu_struct(&client->hwctx_srcu); 92 xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); 93 mutex_init(&client->mm_lock); 94 95 mutex_lock(&xdna->dev_lock); 96 list_add_tail(&client->node, &xdna->client_list); 97 mutex_unlock(&xdna->dev_lock); 98 99 filp->driver_priv = client; 100 client->filp = filp; 101 102 XDNA_DBG(xdna, "pid %d opened", client->pid); 103 return 0; 104 105 unbind_sva: 106 iommu_sva_unbind_device(client->sva); 107 failed: 108 kfree(client); 109 put_rpm: 110 pm_runtime_mark_last_busy(ddev->dev); 111 pm_runtime_put_autosuspend(ddev->dev); 112 113 return ret; 114 } 115 116 static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) 117 { 118 struct amdxdna_client *client = filp->driver_priv; 119 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 120 121 XDNA_DBG(xdna, "closing pid %d", client->pid); 122 123 xa_destroy(&client->hwctx_xa); 124 cleanup_srcu_struct(&client->hwctx_srcu); 125 mutex_destroy(&client->mm_lock); 126 if (client->dev_heap) 127 drm_gem_object_put(to_gobj(client->dev_heap)); 128 129 iommu_sva_unbind_device(client->sva); 130 131 XDNA_DBG(xdna, "pid %d closed", client->pid); 132 kfree(client); 133 pm_runtime_mark_last_busy(ddev->dev); 134 pm_runtime_put_autosuspend(ddev->dev); 135 } 136 137 static int amdxdna_flush(struct file *f, fl_owner_t id) 138 { 139 struct drm_file *filp = f->private_data; 140 struct amdxdna_client *client = filp->driver_priv; 141 struct amdxdna_dev *xdna = client->xdna; 142 int idx; 143 144 XDNA_DBG(xdna, "PID %d flushing...", client->pid); 145 if (!drm_dev_enter(&xdna->ddev, &idx)) 146 return 0; 147 148 mutex_lock(&xdna->dev_lock); 149 list_del_init(&client->node); 150 amdxdna_hwctx_remove_all(client); 151 mutex_unlock(&xdna->dev_lock); 152 153 drm_dev_exit(idx); 154 return 0; 155 } 156 157 static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 158 { 159 struct amdxdna_client *client = filp->driver_priv; 160 struct amdxdna_dev *xdna = to_xdna_dev(dev); 161 struct amdxdna_drm_get_info *args = data; 162 int ret; 163 164 if (!xdna->dev_info->ops->get_aie_info) 165 return -EOPNOTSUPP; 166 167 XDNA_DBG(xdna, "Request parameter %u", args->param); 168 mutex_lock(&xdna->dev_lock); 169 ret = xdna->dev_info->ops->get_aie_info(client, args); 170 mutex_unlock(&xdna->dev_lock); 171 return ret; 172 } 173 174 static int amdxdna_drm_get_array_ioctl(struct drm_device *dev, void *data, 175 struct drm_file *filp) 176 { 177 struct amdxdna_client *client = filp->driver_priv; 178 struct amdxdna_dev *xdna = to_xdna_dev(dev); 179 struct amdxdna_drm_get_array *args = data; 180 181 if (!xdna->dev_info->ops->get_array) 182 return -EOPNOTSUPP; 183 184 if (args->pad || !args->num_element || !args->element_size) 185 return -EINVAL; 186 187 guard(mutex)(&xdna->dev_lock); 188 return xdna->dev_info->ops->get_array(client, args); 189 } 190 191 static int amdxdna_drm_set_state_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 192 { 193 struct amdxdna_client *client = filp->driver_priv; 194 struct amdxdna_dev *xdna = to_xdna_dev(dev); 195 struct amdxdna_drm_set_state *args = data; 196 int ret; 197 198 if (!xdna->dev_info->ops->set_aie_state) 199 return -EOPNOTSUPP; 200 201 XDNA_DBG(xdna, "Request parameter %u", args->param); 202 mutex_lock(&xdna->dev_lock); 203 ret = xdna->dev_info->ops->set_aie_state(client, args); 204 mutex_unlock(&xdna->dev_lock); 205 206 return ret; 207 } 208 209 static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = { 210 /* Context */ 211 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0), 212 DRM_IOCTL_DEF_DRV(AMDXDNA_DESTROY_HWCTX, amdxdna_drm_destroy_hwctx_ioctl, 0), 213 DRM_IOCTL_DEF_DRV(AMDXDNA_CONFIG_HWCTX, amdxdna_drm_config_hwctx_ioctl, 0), 214 /* BO */ 215 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_BO, amdxdna_drm_create_bo_ioctl, 0), 216 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_BO_INFO, amdxdna_drm_get_bo_info_ioctl, 0), 217 DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0), 218 /* Execution */ 219 DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0), 220 /* AIE hardware */ 221 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0), 222 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_ARRAY, amdxdna_drm_get_array_ioctl, 0), 223 DRM_IOCTL_DEF_DRV(AMDXDNA_SET_STATE, amdxdna_drm_set_state_ioctl, DRM_ROOT_ONLY), 224 }; 225 226 static const struct file_operations amdxdna_fops = { 227 .owner = THIS_MODULE, 228 .open = accel_open, 229 .release = drm_release, 230 .flush = amdxdna_flush, 231 .unlocked_ioctl = drm_ioctl, 232 .compat_ioctl = drm_compat_ioctl, 233 .poll = drm_poll, 234 .read = drm_read, 235 .llseek = noop_llseek, 236 .mmap = drm_gem_mmap, 237 .fop_flags = FOP_UNSIGNED_OFFSET, 238 }; 239 240 const struct drm_driver amdxdna_drm_drv = { 241 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL | 242 DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, 243 .fops = &amdxdna_fops, 244 .name = "amdxdna_accel_driver", 245 .desc = "AMD XDNA DRM implementation", 246 .major = AMDXDNA_DRIVER_MAJOR, 247 .minor = AMDXDNA_DRIVER_MINOR, 248 .open = amdxdna_drm_open, 249 .postclose = amdxdna_drm_close, 250 .ioctls = amdxdna_drm_ioctls, 251 .num_ioctls = ARRAY_SIZE(amdxdna_drm_ioctls), 252 253 .gem_create_object = amdxdna_gem_create_object_cb, 254 .gem_prime_import = amdxdna_gem_prime_import, 255 }; 256 257 static const struct amdxdna_dev_info * 258 amdxdna_get_dev_info(struct pci_dev *pdev) 259 { 260 int i; 261 262 for (i = 0; i < ARRAY_SIZE(amdxdna_ids); i++) { 263 if (pdev->device == amdxdna_ids[i].device && 264 pdev->revision == amdxdna_ids[i].revision) 265 return amdxdna_ids[i].dev_info; 266 } 267 return NULL; 268 } 269 270 static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) 271 { 272 struct device *dev = &pdev->dev; 273 struct amdxdna_dev *xdna; 274 int ret; 275 276 xdna = devm_drm_dev_alloc(dev, &amdxdna_drm_drv, typeof(*xdna), ddev); 277 if (IS_ERR(xdna)) 278 return PTR_ERR(xdna); 279 280 xdna->dev_info = amdxdna_get_dev_info(pdev); 281 if (!xdna->dev_info) 282 return -ENODEV; 283 284 drmm_mutex_init(&xdna->ddev, &xdna->dev_lock); 285 init_rwsem(&xdna->notifier_lock); 286 INIT_LIST_HEAD(&xdna->client_list); 287 pci_set_drvdata(pdev, xdna); 288 289 if (IS_ENABLED(CONFIG_LOCKDEP)) { 290 fs_reclaim_acquire(GFP_KERNEL); 291 might_lock(&xdna->notifier_lock); 292 fs_reclaim_release(GFP_KERNEL); 293 } 294 295 xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", 0); 296 if (!xdna->notifier_wq) 297 return -ENOMEM; 298 299 mutex_lock(&xdna->dev_lock); 300 ret = xdna->dev_info->ops->init(xdna); 301 mutex_unlock(&xdna->dev_lock); 302 if (ret) { 303 XDNA_ERR(xdna, "Hardware init failed, ret %d", ret); 304 goto destroy_notifier_wq; 305 } 306 307 ret = amdxdna_sysfs_init(xdna); 308 if (ret) { 309 XDNA_ERR(xdna, "Create amdxdna attrs failed: %d", ret); 310 goto failed_dev_fini; 311 } 312 313 pm_runtime_set_autosuspend_delay(dev, AMDXDNA_AUTOSUSPEND_DELAY); 314 pm_runtime_use_autosuspend(dev); 315 pm_runtime_allow(dev); 316 317 ret = drm_dev_register(&xdna->ddev, 0); 318 if (ret) { 319 XDNA_ERR(xdna, "DRM register failed, ret %d", ret); 320 pm_runtime_forbid(dev); 321 goto failed_sysfs_fini; 322 } 323 324 pm_runtime_mark_last_busy(dev); 325 pm_runtime_put_autosuspend(dev); 326 return 0; 327 328 failed_sysfs_fini: 329 amdxdna_sysfs_fini(xdna); 330 failed_dev_fini: 331 mutex_lock(&xdna->dev_lock); 332 xdna->dev_info->ops->fini(xdna); 333 mutex_unlock(&xdna->dev_lock); 334 destroy_notifier_wq: 335 destroy_workqueue(xdna->notifier_wq); 336 return ret; 337 } 338 339 static void amdxdna_remove(struct pci_dev *pdev) 340 { 341 struct amdxdna_dev *xdna = pci_get_drvdata(pdev); 342 struct device *dev = &pdev->dev; 343 struct amdxdna_client *client; 344 345 destroy_workqueue(xdna->notifier_wq); 346 347 pm_runtime_get_noresume(dev); 348 pm_runtime_forbid(dev); 349 350 drm_dev_unplug(&xdna->ddev); 351 amdxdna_sysfs_fini(xdna); 352 353 mutex_lock(&xdna->dev_lock); 354 client = list_first_entry_or_null(&xdna->client_list, 355 struct amdxdna_client, node); 356 while (client) { 357 list_del_init(&client->node); 358 amdxdna_hwctx_remove_all(client); 359 360 client = list_first_entry_or_null(&xdna->client_list, 361 struct amdxdna_client, node); 362 } 363 364 xdna->dev_info->ops->fini(xdna); 365 mutex_unlock(&xdna->dev_lock); 366 } 367 368 static int amdxdna_pmops_suspend(struct device *dev) 369 { 370 struct amdxdna_dev *xdna = pci_get_drvdata(to_pci_dev(dev)); 371 372 if (!xdna->dev_info->ops->suspend) 373 return -EOPNOTSUPP; 374 375 return xdna->dev_info->ops->suspend(xdna); 376 } 377 378 static int amdxdna_pmops_resume(struct device *dev) 379 { 380 struct amdxdna_dev *xdna = pci_get_drvdata(to_pci_dev(dev)); 381 382 if (!xdna->dev_info->ops->resume) 383 return -EOPNOTSUPP; 384 385 return xdna->dev_info->ops->resume(xdna); 386 } 387 388 static const struct dev_pm_ops amdxdna_pm_ops = { 389 SYSTEM_SLEEP_PM_OPS(amdxdna_pmops_suspend, amdxdna_pmops_resume) 390 RUNTIME_PM_OPS(amdxdna_pmops_suspend, amdxdna_pmops_resume, NULL) 391 }; 392 393 static struct pci_driver amdxdna_pci_driver = { 394 .name = KBUILD_MODNAME, 395 .id_table = pci_ids, 396 .probe = amdxdna_probe, 397 .remove = amdxdna_remove, 398 .driver.pm = &amdxdna_pm_ops, 399 }; 400 401 module_pci_driver(amdxdna_pci_driver); 402 403 MODULE_LICENSE("GPL"); 404 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); 405 MODULE_DESCRIPTION("amdxdna driver"); 406