1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_accel.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_ioctl.h> 12 #include <drm/drm_managed.h> 13 #include <drm/gpu_scheduler.h> 14 #include <linux/iommu.h> 15 #include <linux/pci.h> 16 17 #include "amdxdna_ctx.h" 18 #include "amdxdna_gem.h" 19 #include "amdxdna_pci_drv.h" 20 #include "amdxdna_pm.h" 21 22 MODULE_FIRMWARE("amdnpu/1502_00/npu.sbin"); 23 MODULE_FIRMWARE("amdnpu/17f0_10/npu.sbin"); 24 MODULE_FIRMWARE("amdnpu/17f0_11/npu.sbin"); 25 MODULE_FIRMWARE("amdnpu/17f0_20/npu.sbin"); 26 MODULE_FIRMWARE("amdnpu/1502_00/npu_7.sbin"); 27 MODULE_FIRMWARE("amdnpu/17f0_10/npu_7.sbin"); 28 MODULE_FIRMWARE("amdnpu/17f0_11/npu_7.sbin"); 29 30 /* 31 * 0.0: Initial version 32 * 0.1: Support getting all hardware contexts by DRM_IOCTL_AMDXDNA_GET_ARRAY 33 * 0.2: Support getting last error hardware error 34 * 0.3: Support firmware debug buffer 35 * 0.4: Support getting resource information 36 * 0.5: Support getting telemetry data 37 * 0.6: Support preemption 38 */ 39 #define AMDXDNA_DRIVER_MAJOR 0 40 #define AMDXDNA_DRIVER_MINOR 6 41 42 /* 43 * Bind the driver base on (vendor_id, device_id) pair and later use the 44 * (device_id, rev_id) pair as a key to select the devices. The devices with 45 * same device_id have very similar interface to host driver. 46 */ 47 static const struct pci_device_id pci_ids[] = { 48 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1502) }, 49 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f0) }, 50 {0} 51 }; 52 53 MODULE_DEVICE_TABLE(pci, pci_ids); 54 55 static const struct amdxdna_device_id amdxdna_ids[] = { 56 { 0x1502, 0x0, &dev_npu1_info }, 57 { 0x17f0, 0x10, &dev_npu4_info }, 58 { 0x17f0, 0x11, &dev_npu5_info }, 59 { 0x17f0, 0x20, &dev_npu6_info }, 60 {0} 61 }; 62 63 static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) 64 { 65 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 66 struct amdxdna_client *client; 67 int ret; 68 69 client = kzalloc_obj(*client); 70 if (!client) 71 return -ENOMEM; 72 73 client->pid = pid_nr(rcu_access_pointer(filp->pid)); 74 client->xdna = xdna; 75 76 client->sva = iommu_sva_bind_device(xdna->ddev.dev, current->mm); 77 if (IS_ERR(client->sva)) { 78 ret = PTR_ERR(client->sva); 79 XDNA_ERR(xdna, "SVA bind device failed, ret %d", ret); 80 goto failed; 81 } 82 client->pasid = iommu_sva_get_pasid(client->sva); 83 if (client->pasid == IOMMU_PASID_INVALID) { 84 XDNA_ERR(xdna, "SVA get pasid failed"); 85 ret = -ENODEV; 86 goto unbind_sva; 87 } 88 client->mm = current->mm; 89 mmgrab(client->mm); 90 init_srcu_struct(&client->hwctx_srcu); 91 xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); 92 mutex_init(&client->mm_lock); 93 94 mutex_lock(&xdna->dev_lock); 95 list_add_tail(&client->node, &xdna->client_list); 96 mutex_unlock(&xdna->dev_lock); 97 98 filp->driver_priv = client; 99 client->filp = filp; 100 101 XDNA_DBG(xdna, "pid %d opened", client->pid); 102 return 0; 103 104 unbind_sva: 105 iommu_sva_unbind_device(client->sva); 106 failed: 107 kfree(client); 108 109 return ret; 110 } 111 112 static void amdxdna_client_cleanup(struct amdxdna_client *client) 113 { 114 list_del(&client->node); 115 amdxdna_hwctx_remove_all(client); 116 xa_destroy(&client->hwctx_xa); 117 cleanup_srcu_struct(&client->hwctx_srcu); 118 mutex_destroy(&client->mm_lock); 119 120 if (client->dev_heap) 121 drm_gem_object_put(to_gobj(client->dev_heap)); 122 123 iommu_sva_unbind_device(client->sva); 124 mmdrop(client->mm); 125 126 kfree(client); 127 } 128 129 static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) 130 { 131 struct amdxdna_client *client = filp->driver_priv; 132 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 133 int idx; 134 135 XDNA_DBG(xdna, "closing pid %d", client->pid); 136 137 if (!drm_dev_enter(&xdna->ddev, &idx)) 138 return; 139 140 mutex_lock(&xdna->dev_lock); 141 amdxdna_client_cleanup(client); 142 mutex_unlock(&xdna->dev_lock); 143 144 drm_dev_exit(idx); 145 } 146 147 static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 148 { 149 struct amdxdna_client *client = filp->driver_priv; 150 struct amdxdna_dev *xdna = to_xdna_dev(dev); 151 struct amdxdna_drm_get_info *args = data; 152 int ret; 153 154 if (!xdna->dev_info->ops->get_aie_info) 155 return -EOPNOTSUPP; 156 157 XDNA_DBG(xdna, "Request parameter %u", args->param); 158 mutex_lock(&xdna->dev_lock); 159 ret = xdna->dev_info->ops->get_aie_info(client, args); 160 mutex_unlock(&xdna->dev_lock); 161 return ret; 162 } 163 164 static int amdxdna_drm_get_array_ioctl(struct drm_device *dev, void *data, 165 struct drm_file *filp) 166 { 167 struct amdxdna_client *client = filp->driver_priv; 168 struct amdxdna_dev *xdna = to_xdna_dev(dev); 169 struct amdxdna_drm_get_array *args = data; 170 171 if (!xdna->dev_info->ops->get_array) 172 return -EOPNOTSUPP; 173 174 if (args->pad || !args->num_element || !args->element_size) 175 return -EINVAL; 176 177 guard(mutex)(&xdna->dev_lock); 178 return xdna->dev_info->ops->get_array(client, args); 179 } 180 181 static int amdxdna_drm_set_state_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 182 { 183 struct amdxdna_client *client = filp->driver_priv; 184 struct amdxdna_dev *xdna = to_xdna_dev(dev); 185 struct amdxdna_drm_set_state *args = data; 186 int ret; 187 188 if (!xdna->dev_info->ops->set_aie_state) 189 return -EOPNOTSUPP; 190 191 XDNA_DBG(xdna, "Request parameter %u", args->param); 192 mutex_lock(&xdna->dev_lock); 193 ret = xdna->dev_info->ops->set_aie_state(client, args); 194 mutex_unlock(&xdna->dev_lock); 195 196 return ret; 197 } 198 199 static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = { 200 /* Context */ 201 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0), 202 DRM_IOCTL_DEF_DRV(AMDXDNA_DESTROY_HWCTX, amdxdna_drm_destroy_hwctx_ioctl, 0), 203 DRM_IOCTL_DEF_DRV(AMDXDNA_CONFIG_HWCTX, amdxdna_drm_config_hwctx_ioctl, 0), 204 /* BO */ 205 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_BO, amdxdna_drm_create_bo_ioctl, 0), 206 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_BO_INFO, amdxdna_drm_get_bo_info_ioctl, 0), 207 DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0), 208 /* Execution */ 209 DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0), 210 /* AIE hardware */ 211 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0), 212 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_ARRAY, amdxdna_drm_get_array_ioctl, 0), 213 DRM_IOCTL_DEF_DRV(AMDXDNA_SET_STATE, amdxdna_drm_set_state_ioctl, DRM_ROOT_ONLY), 214 }; 215 216 static const struct file_operations amdxdna_fops = { 217 .owner = THIS_MODULE, 218 .open = accel_open, 219 .release = drm_release, 220 .unlocked_ioctl = drm_ioctl, 221 .compat_ioctl = drm_compat_ioctl, 222 .poll = drm_poll, 223 .read = drm_read, 224 .llseek = noop_llseek, 225 .mmap = drm_gem_mmap, 226 .fop_flags = FOP_UNSIGNED_OFFSET, 227 }; 228 229 const struct drm_driver amdxdna_drm_drv = { 230 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL | 231 DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, 232 .fops = &amdxdna_fops, 233 .name = "amdxdna_accel_driver", 234 .desc = "AMD XDNA DRM implementation", 235 .major = AMDXDNA_DRIVER_MAJOR, 236 .minor = AMDXDNA_DRIVER_MINOR, 237 .open = amdxdna_drm_open, 238 .postclose = amdxdna_drm_close, 239 .ioctls = amdxdna_drm_ioctls, 240 .num_ioctls = ARRAY_SIZE(amdxdna_drm_ioctls), 241 242 .gem_create_object = amdxdna_gem_create_object_cb, 243 .gem_prime_import = amdxdna_gem_prime_import, 244 }; 245 246 static const struct amdxdna_dev_info * 247 amdxdna_get_dev_info(struct pci_dev *pdev) 248 { 249 int i; 250 251 for (i = 0; i < ARRAY_SIZE(amdxdna_ids); i++) { 252 if (pdev->device == amdxdna_ids[i].device && 253 pdev->revision == amdxdna_ids[i].revision) 254 return amdxdna_ids[i].dev_info; 255 } 256 return NULL; 257 } 258 259 static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) 260 { 261 struct device *dev = &pdev->dev; 262 struct amdxdna_dev *xdna; 263 int ret; 264 265 xdna = devm_drm_dev_alloc(dev, &amdxdna_drm_drv, typeof(*xdna), ddev); 266 if (IS_ERR(xdna)) 267 return PTR_ERR(xdna); 268 269 xdna->dev_info = amdxdna_get_dev_info(pdev); 270 if (!xdna->dev_info) 271 return -ENODEV; 272 273 drmm_mutex_init(&xdna->ddev, &xdna->dev_lock); 274 init_rwsem(&xdna->notifier_lock); 275 INIT_LIST_HEAD(&xdna->client_list); 276 pci_set_drvdata(pdev, xdna); 277 278 if (IS_ENABLED(CONFIG_LOCKDEP)) { 279 fs_reclaim_acquire(GFP_KERNEL); 280 might_lock(&xdna->notifier_lock); 281 fs_reclaim_release(GFP_KERNEL); 282 } 283 284 xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", WQ_MEM_RECLAIM); 285 if (!xdna->notifier_wq) 286 return -ENOMEM; 287 288 mutex_lock(&xdna->dev_lock); 289 ret = xdna->dev_info->ops->init(xdna); 290 mutex_unlock(&xdna->dev_lock); 291 if (ret) { 292 XDNA_ERR(xdna, "Hardware init failed, ret %d", ret); 293 goto destroy_notifier_wq; 294 } 295 296 ret = amdxdna_sysfs_init(xdna); 297 if (ret) { 298 XDNA_ERR(xdna, "Create amdxdna attrs failed: %d", ret); 299 goto failed_dev_fini; 300 } 301 302 ret = drm_dev_register(&xdna->ddev, 0); 303 if (ret) { 304 XDNA_ERR(xdna, "DRM register failed, ret %d", ret); 305 goto failed_sysfs_fini; 306 } 307 308 return 0; 309 310 failed_sysfs_fini: 311 amdxdna_sysfs_fini(xdna); 312 failed_dev_fini: 313 mutex_lock(&xdna->dev_lock); 314 xdna->dev_info->ops->fini(xdna); 315 mutex_unlock(&xdna->dev_lock); 316 destroy_notifier_wq: 317 destroy_workqueue(xdna->notifier_wq); 318 return ret; 319 } 320 321 static void amdxdna_remove(struct pci_dev *pdev) 322 { 323 struct amdxdna_dev *xdna = pci_get_drvdata(pdev); 324 struct amdxdna_client *client; 325 326 destroy_workqueue(xdna->notifier_wq); 327 328 drm_dev_unplug(&xdna->ddev); 329 amdxdna_sysfs_fini(xdna); 330 331 mutex_lock(&xdna->dev_lock); 332 client = list_first_entry_or_null(&xdna->client_list, 333 struct amdxdna_client, node); 334 while (client) { 335 amdxdna_client_cleanup(client); 336 337 client = list_first_entry_or_null(&xdna->client_list, 338 struct amdxdna_client, node); 339 } 340 341 xdna->dev_info->ops->fini(xdna); 342 mutex_unlock(&xdna->dev_lock); 343 } 344 345 static const struct dev_pm_ops amdxdna_pm_ops = { 346 SYSTEM_SLEEP_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume) 347 RUNTIME_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume, NULL) 348 }; 349 350 static struct pci_driver amdxdna_pci_driver = { 351 .name = KBUILD_MODNAME, 352 .id_table = pci_ids, 353 .probe = amdxdna_probe, 354 .remove = amdxdna_remove, 355 .driver.pm = &amdxdna_pm_ops, 356 }; 357 358 module_pci_driver(amdxdna_pci_driver); 359 360 MODULE_LICENSE("GPL"); 361 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); 362 MODULE_DESCRIPTION("amdxdna driver"); 363