1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_accel.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_ioctl.h> 12 #include <drm/drm_managed.h> 13 #include <drm/gpu_scheduler.h> 14 #include <linux/iommu.h> 15 #include <linux/pci.h> 16 #include <linux/pm_runtime.h> 17 18 #include "amdxdna_ctx.h" 19 #include "amdxdna_gem.h" 20 #include "amdxdna_pci_drv.h" 21 22 #define AMDXDNA_AUTOSUSPEND_DELAY 5000 /* milliseconds */ 23 24 /* 25 * Bind the driver base on (vendor_id, device_id) pair and later use the 26 * (device_id, rev_id) pair as a key to select the devices. The devices with 27 * same device_id have very similar interface to host driver. 28 */ 29 static const struct pci_device_id pci_ids[] = { 30 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1502) }, 31 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f0) }, 32 {0} 33 }; 34 35 MODULE_DEVICE_TABLE(pci, pci_ids); 36 37 static const struct amdxdna_device_id amdxdna_ids[] = { 38 { 0x1502, 0x0, &dev_npu1_info }, 39 { 0x17f0, 0x0, &dev_npu2_info }, 40 { 0x17f0, 0x10, &dev_npu4_info }, 41 { 0x17f0, 0x11, &dev_npu5_info }, 42 { 0x17f0, 0x20, &dev_npu6_info }, 43 {0} 44 }; 45 46 static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) 47 { 48 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 49 struct amdxdna_client *client; 50 int ret; 51 52 ret = pm_runtime_resume_and_get(ddev->dev); 53 if (ret) { 54 XDNA_ERR(xdna, "Failed to get rpm, ret %d", ret); 55 return ret; 56 } 57 58 client = kzalloc(sizeof(*client), GFP_KERNEL); 59 if (!client) { 60 ret = -ENOMEM; 61 goto put_rpm; 62 } 63 64 client->pid = pid_nr(rcu_access_pointer(filp->pid)); 65 client->xdna = xdna; 66 67 client->sva = iommu_sva_bind_device(xdna->ddev.dev, current->mm); 68 if (IS_ERR(client->sva)) { 69 ret = PTR_ERR(client->sva); 70 XDNA_ERR(xdna, "SVA bind device failed, ret %d", ret); 71 goto failed; 72 } 73 client->pasid = iommu_sva_get_pasid(client->sva); 74 if (client->pasid == IOMMU_PASID_INVALID) { 75 XDNA_ERR(xdna, "SVA get pasid failed"); 76 ret = -ENODEV; 77 goto unbind_sva; 78 } 79 mutex_init(&client->hwctx_lock); 80 init_srcu_struct(&client->hwctx_srcu); 81 xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); 82 mutex_init(&client->mm_lock); 83 84 mutex_lock(&xdna->dev_lock); 85 list_add_tail(&client->node, &xdna->client_list); 86 mutex_unlock(&xdna->dev_lock); 87 88 filp->driver_priv = client; 89 client->filp = filp; 90 91 XDNA_DBG(xdna, "pid %d opened", client->pid); 92 return 0; 93 94 unbind_sva: 95 iommu_sva_unbind_device(client->sva); 96 failed: 97 kfree(client); 98 put_rpm: 99 pm_runtime_mark_last_busy(ddev->dev); 100 pm_runtime_put_autosuspend(ddev->dev); 101 102 return ret; 103 } 104 105 static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) 106 { 107 struct amdxdna_client *client = filp->driver_priv; 108 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 109 110 XDNA_DBG(xdna, "closing pid %d", client->pid); 111 112 xa_destroy(&client->hwctx_xa); 113 cleanup_srcu_struct(&client->hwctx_srcu); 114 mutex_destroy(&client->hwctx_lock); 115 mutex_destroy(&client->mm_lock); 116 if (client->dev_heap) 117 drm_gem_object_put(to_gobj(client->dev_heap)); 118 119 iommu_sva_unbind_device(client->sva); 120 121 XDNA_DBG(xdna, "pid %d closed", client->pid); 122 kfree(client); 123 pm_runtime_mark_last_busy(ddev->dev); 124 pm_runtime_put_autosuspend(ddev->dev); 125 } 126 127 static int amdxdna_flush(struct file *f, fl_owner_t id) 128 { 129 struct drm_file *filp = f->private_data; 130 struct amdxdna_client *client = filp->driver_priv; 131 struct amdxdna_dev *xdna = client->xdna; 132 int idx; 133 134 XDNA_DBG(xdna, "PID %d flushing...", client->pid); 135 if (!drm_dev_enter(&xdna->ddev, &idx)) 136 return 0; 137 138 mutex_lock(&xdna->dev_lock); 139 list_del_init(&client->node); 140 mutex_unlock(&xdna->dev_lock); 141 amdxdna_hwctx_remove_all(client); 142 143 drm_dev_exit(idx); 144 return 0; 145 } 146 147 static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 148 { 149 struct amdxdna_client *client = filp->driver_priv; 150 struct amdxdna_dev *xdna = to_xdna_dev(dev); 151 struct amdxdna_drm_get_info *args = data; 152 int ret; 153 154 if (!xdna->dev_info->ops->get_aie_info) 155 return -EOPNOTSUPP; 156 157 XDNA_DBG(xdna, "Request parameter %u", args->param); 158 mutex_lock(&xdna->dev_lock); 159 ret = xdna->dev_info->ops->get_aie_info(client, args); 160 mutex_unlock(&xdna->dev_lock); 161 return ret; 162 } 163 164 static int amdxdna_drm_set_state_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 165 { 166 struct amdxdna_client *client = filp->driver_priv; 167 struct amdxdna_dev *xdna = to_xdna_dev(dev); 168 struct amdxdna_drm_set_state *args = data; 169 int ret; 170 171 if (!xdna->dev_info->ops->set_aie_state) 172 return -EOPNOTSUPP; 173 174 XDNA_DBG(xdna, "Request parameter %u", args->param); 175 mutex_lock(&xdna->dev_lock); 176 ret = xdna->dev_info->ops->set_aie_state(client, args); 177 mutex_unlock(&xdna->dev_lock); 178 179 return ret; 180 } 181 182 static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = { 183 /* Context */ 184 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0), 185 DRM_IOCTL_DEF_DRV(AMDXDNA_DESTROY_HWCTX, amdxdna_drm_destroy_hwctx_ioctl, 0), 186 DRM_IOCTL_DEF_DRV(AMDXDNA_CONFIG_HWCTX, amdxdna_drm_config_hwctx_ioctl, 0), 187 /* BO */ 188 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_BO, amdxdna_drm_create_bo_ioctl, 0), 189 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_BO_INFO, amdxdna_drm_get_bo_info_ioctl, 0), 190 DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0), 191 /* Execution */ 192 DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0), 193 /* AIE hardware */ 194 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0), 195 DRM_IOCTL_DEF_DRV(AMDXDNA_SET_STATE, amdxdna_drm_set_state_ioctl, DRM_ROOT_ONLY), 196 }; 197 198 static const struct file_operations amdxdna_fops = { 199 .owner = THIS_MODULE, 200 .open = accel_open, 201 .release = drm_release, 202 .flush = amdxdna_flush, 203 .unlocked_ioctl = drm_ioctl, 204 .compat_ioctl = drm_compat_ioctl, 205 .poll = drm_poll, 206 .read = drm_read, 207 .llseek = noop_llseek, 208 .mmap = drm_gem_mmap, 209 .fop_flags = FOP_UNSIGNED_OFFSET, 210 }; 211 212 const struct drm_driver amdxdna_drm_drv = { 213 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL | 214 DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, 215 .fops = &amdxdna_fops, 216 .name = "amdxdna_accel_driver", 217 .desc = "AMD XDNA DRM implementation", 218 .open = amdxdna_drm_open, 219 .postclose = amdxdna_drm_close, 220 .ioctls = amdxdna_drm_ioctls, 221 .num_ioctls = ARRAY_SIZE(amdxdna_drm_ioctls), 222 223 .gem_create_object = amdxdna_gem_create_object_cb, 224 }; 225 226 static const struct amdxdna_dev_info * 227 amdxdna_get_dev_info(struct pci_dev *pdev) 228 { 229 int i; 230 231 for (i = 0; i < ARRAY_SIZE(amdxdna_ids); i++) { 232 if (pdev->device == amdxdna_ids[i].device && 233 pdev->revision == amdxdna_ids[i].revision) 234 return amdxdna_ids[i].dev_info; 235 } 236 return NULL; 237 } 238 239 static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) 240 { 241 struct device *dev = &pdev->dev; 242 struct amdxdna_dev *xdna; 243 int ret; 244 245 xdna = devm_drm_dev_alloc(dev, &amdxdna_drm_drv, typeof(*xdna), ddev); 246 if (IS_ERR(xdna)) 247 return PTR_ERR(xdna); 248 249 xdna->dev_info = amdxdna_get_dev_info(pdev); 250 if (!xdna->dev_info) 251 return -ENODEV; 252 253 drmm_mutex_init(&xdna->ddev, &xdna->dev_lock); 254 init_rwsem(&xdna->notifier_lock); 255 INIT_LIST_HEAD(&xdna->client_list); 256 pci_set_drvdata(pdev, xdna); 257 258 if (IS_ENABLED(CONFIG_LOCKDEP)) { 259 fs_reclaim_acquire(GFP_KERNEL); 260 might_lock(&xdna->notifier_lock); 261 fs_reclaim_release(GFP_KERNEL); 262 } 263 264 mutex_lock(&xdna->dev_lock); 265 ret = xdna->dev_info->ops->init(xdna); 266 mutex_unlock(&xdna->dev_lock); 267 if (ret) { 268 XDNA_ERR(xdna, "Hardware init failed, ret %d", ret); 269 return ret; 270 } 271 272 ret = amdxdna_sysfs_init(xdna); 273 if (ret) { 274 XDNA_ERR(xdna, "Create amdxdna attrs failed: %d", ret); 275 goto failed_dev_fini; 276 } 277 278 pm_runtime_set_autosuspend_delay(dev, AMDXDNA_AUTOSUSPEND_DELAY); 279 pm_runtime_use_autosuspend(dev); 280 pm_runtime_allow(dev); 281 282 ret = drm_dev_register(&xdna->ddev, 0); 283 if (ret) { 284 XDNA_ERR(xdna, "DRM register failed, ret %d", ret); 285 pm_runtime_forbid(dev); 286 goto failed_sysfs_fini; 287 } 288 289 pm_runtime_mark_last_busy(dev); 290 pm_runtime_put_autosuspend(dev); 291 return 0; 292 293 failed_sysfs_fini: 294 amdxdna_sysfs_fini(xdna); 295 failed_dev_fini: 296 mutex_lock(&xdna->dev_lock); 297 xdna->dev_info->ops->fini(xdna); 298 mutex_unlock(&xdna->dev_lock); 299 return ret; 300 } 301 302 static void amdxdna_remove(struct pci_dev *pdev) 303 { 304 struct amdxdna_dev *xdna = pci_get_drvdata(pdev); 305 struct device *dev = &pdev->dev; 306 struct amdxdna_client *client; 307 308 pm_runtime_get_noresume(dev); 309 pm_runtime_forbid(dev); 310 311 drm_dev_unplug(&xdna->ddev); 312 amdxdna_sysfs_fini(xdna); 313 314 mutex_lock(&xdna->dev_lock); 315 client = list_first_entry_or_null(&xdna->client_list, 316 struct amdxdna_client, node); 317 while (client) { 318 list_del_init(&client->node); 319 mutex_unlock(&xdna->dev_lock); 320 321 amdxdna_hwctx_remove_all(client); 322 323 mutex_lock(&xdna->dev_lock); 324 client = list_first_entry_or_null(&xdna->client_list, 325 struct amdxdna_client, node); 326 } 327 328 xdna->dev_info->ops->fini(xdna); 329 mutex_unlock(&xdna->dev_lock); 330 } 331 332 static int amdxdna_dev_suspend_nolock(struct amdxdna_dev *xdna) 333 { 334 if (xdna->dev_info->ops->suspend) 335 xdna->dev_info->ops->suspend(xdna); 336 337 return 0; 338 } 339 340 static int amdxdna_dev_resume_nolock(struct amdxdna_dev *xdna) 341 { 342 if (xdna->dev_info->ops->resume) 343 return xdna->dev_info->ops->resume(xdna); 344 345 return 0; 346 } 347 348 static int amdxdna_pmops_suspend(struct device *dev) 349 { 350 struct amdxdna_dev *xdna = pci_get_drvdata(to_pci_dev(dev)); 351 struct amdxdna_client *client; 352 353 mutex_lock(&xdna->dev_lock); 354 list_for_each_entry(client, &xdna->client_list, node) 355 amdxdna_hwctx_suspend(client); 356 357 amdxdna_dev_suspend_nolock(xdna); 358 mutex_unlock(&xdna->dev_lock); 359 360 return 0; 361 } 362 363 static int amdxdna_pmops_resume(struct device *dev) 364 { 365 struct amdxdna_dev *xdna = pci_get_drvdata(to_pci_dev(dev)); 366 struct amdxdna_client *client; 367 int ret; 368 369 XDNA_INFO(xdna, "firmware resuming..."); 370 mutex_lock(&xdna->dev_lock); 371 ret = amdxdna_dev_resume_nolock(xdna); 372 if (ret) { 373 XDNA_ERR(xdna, "resume NPU firmware failed"); 374 mutex_unlock(&xdna->dev_lock); 375 return ret; 376 } 377 378 XDNA_INFO(xdna, "hardware context resuming..."); 379 list_for_each_entry(client, &xdna->client_list, node) 380 amdxdna_hwctx_resume(client); 381 mutex_unlock(&xdna->dev_lock); 382 383 return 0; 384 } 385 386 static int amdxdna_rpmops_suspend(struct device *dev) 387 { 388 struct amdxdna_dev *xdna = pci_get_drvdata(to_pci_dev(dev)); 389 int ret; 390 391 mutex_lock(&xdna->dev_lock); 392 ret = amdxdna_dev_suspend_nolock(xdna); 393 mutex_unlock(&xdna->dev_lock); 394 395 XDNA_DBG(xdna, "Runtime suspend done ret: %d", ret); 396 return ret; 397 } 398 399 static int amdxdna_rpmops_resume(struct device *dev) 400 { 401 struct amdxdna_dev *xdna = pci_get_drvdata(to_pci_dev(dev)); 402 int ret; 403 404 mutex_lock(&xdna->dev_lock); 405 ret = amdxdna_dev_resume_nolock(xdna); 406 mutex_unlock(&xdna->dev_lock); 407 408 XDNA_DBG(xdna, "Runtime resume done ret: %d", ret); 409 return ret; 410 } 411 412 static const struct dev_pm_ops amdxdna_pm_ops = { 413 SYSTEM_SLEEP_PM_OPS(amdxdna_pmops_suspend, amdxdna_pmops_resume) 414 RUNTIME_PM_OPS(amdxdna_rpmops_suspend, amdxdna_rpmops_resume, NULL) 415 }; 416 417 static struct pci_driver amdxdna_pci_driver = { 418 .name = KBUILD_MODNAME, 419 .id_table = pci_ids, 420 .probe = amdxdna_probe, 421 .remove = amdxdna_remove, 422 .driver.pm = &amdxdna_pm_ops, 423 }; 424 425 module_pci_driver(amdxdna_pci_driver); 426 427 MODULE_LICENSE("GPL"); 428 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); 429 MODULE_DESCRIPTION("amdxdna driver"); 430