1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/amdxdna_accel.h> 7 #include <drm/drm_accel.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_gem.h> 10 #include <drm/drm_gem_shmem_helper.h> 11 #include <drm/drm_ioctl.h> 12 #include <drm/drm_managed.h> 13 #include <drm/gpu_scheduler.h> 14 #include <linux/iommu.h> 15 #include <linux/pci.h> 16 17 #include "amdxdna_ctx.h" 18 #include "amdxdna_gem.h" 19 #include "amdxdna_pci_drv.h" 20 #include "amdxdna_pm.h" 21 22 MODULE_FIRMWARE("amdnpu/1502_00/npu.sbin"); 23 MODULE_FIRMWARE("amdnpu/17f0_10/npu.sbin"); 24 MODULE_FIRMWARE("amdnpu/17f0_11/npu.sbin"); 25 MODULE_FIRMWARE("amdnpu/17f0_20/npu.sbin"); 26 27 /* 28 * 0.0: Initial version 29 * 0.1: Support getting all hardware contexts by DRM_IOCTL_AMDXDNA_GET_ARRAY 30 * 0.2: Support getting last error hardware error 31 * 0.3: Support firmware debug buffer 32 */ 33 #define AMDXDNA_DRIVER_MAJOR 0 34 #define AMDXDNA_DRIVER_MINOR 3 35 36 /* 37 * Bind the driver base on (vendor_id, device_id) pair and later use the 38 * (device_id, rev_id) pair as a key to select the devices. The devices with 39 * same device_id have very similar interface to host driver. 40 */ 41 static const struct pci_device_id pci_ids[] = { 42 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1502) }, 43 { PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f0) }, 44 {0} 45 }; 46 47 MODULE_DEVICE_TABLE(pci, pci_ids); 48 49 static const struct amdxdna_device_id amdxdna_ids[] = { 50 { 0x1502, 0x0, &dev_npu1_info }, 51 { 0x17f0, 0x0, &dev_npu2_info }, 52 { 0x17f0, 0x10, &dev_npu4_info }, 53 { 0x17f0, 0x11, &dev_npu5_info }, 54 { 0x17f0, 0x20, &dev_npu6_info }, 55 {0} 56 }; 57 58 static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp) 59 { 60 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 61 struct amdxdna_client *client; 62 int ret; 63 64 client = kzalloc(sizeof(*client), GFP_KERNEL); 65 if (!client) 66 return -ENOMEM; 67 68 client->pid = pid_nr(rcu_access_pointer(filp->pid)); 69 client->xdna = xdna; 70 71 client->sva = iommu_sva_bind_device(xdna->ddev.dev, current->mm); 72 if (IS_ERR(client->sva)) { 73 ret = PTR_ERR(client->sva); 74 XDNA_ERR(xdna, "SVA bind device failed, ret %d", ret); 75 goto failed; 76 } 77 client->pasid = iommu_sva_get_pasid(client->sva); 78 if (client->pasid == IOMMU_PASID_INVALID) { 79 XDNA_ERR(xdna, "SVA get pasid failed"); 80 ret = -ENODEV; 81 goto unbind_sva; 82 } 83 init_srcu_struct(&client->hwctx_srcu); 84 xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC); 85 mutex_init(&client->mm_lock); 86 87 mutex_lock(&xdna->dev_lock); 88 list_add_tail(&client->node, &xdna->client_list); 89 mutex_unlock(&xdna->dev_lock); 90 91 filp->driver_priv = client; 92 client->filp = filp; 93 94 XDNA_DBG(xdna, "pid %d opened", client->pid); 95 return 0; 96 97 unbind_sva: 98 iommu_sva_unbind_device(client->sva); 99 failed: 100 kfree(client); 101 102 return ret; 103 } 104 105 static void amdxdna_drm_close(struct drm_device *ddev, struct drm_file *filp) 106 { 107 struct amdxdna_client *client = filp->driver_priv; 108 struct amdxdna_dev *xdna = to_xdna_dev(ddev); 109 110 XDNA_DBG(xdna, "closing pid %d", client->pid); 111 112 xa_destroy(&client->hwctx_xa); 113 cleanup_srcu_struct(&client->hwctx_srcu); 114 mutex_destroy(&client->mm_lock); 115 if (client->dev_heap) 116 drm_gem_object_put(to_gobj(client->dev_heap)); 117 118 iommu_sva_unbind_device(client->sva); 119 120 XDNA_DBG(xdna, "pid %d closed", client->pid); 121 kfree(client); 122 } 123 124 static int amdxdna_flush(struct file *f, fl_owner_t id) 125 { 126 struct drm_file *filp = f->private_data; 127 struct amdxdna_client *client = filp->driver_priv; 128 struct amdxdna_dev *xdna = client->xdna; 129 int idx; 130 131 XDNA_DBG(xdna, "PID %d flushing...", client->pid); 132 if (!drm_dev_enter(&xdna->ddev, &idx)) 133 return 0; 134 135 mutex_lock(&xdna->dev_lock); 136 list_del_init(&client->node); 137 amdxdna_hwctx_remove_all(client); 138 mutex_unlock(&xdna->dev_lock); 139 140 drm_dev_exit(idx); 141 return 0; 142 } 143 144 static int amdxdna_drm_get_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 145 { 146 struct amdxdna_client *client = filp->driver_priv; 147 struct amdxdna_dev *xdna = to_xdna_dev(dev); 148 struct amdxdna_drm_get_info *args = data; 149 int ret; 150 151 if (!xdna->dev_info->ops->get_aie_info) 152 return -EOPNOTSUPP; 153 154 XDNA_DBG(xdna, "Request parameter %u", args->param); 155 mutex_lock(&xdna->dev_lock); 156 ret = xdna->dev_info->ops->get_aie_info(client, args); 157 mutex_unlock(&xdna->dev_lock); 158 return ret; 159 } 160 161 static int amdxdna_drm_get_array_ioctl(struct drm_device *dev, void *data, 162 struct drm_file *filp) 163 { 164 struct amdxdna_client *client = filp->driver_priv; 165 struct amdxdna_dev *xdna = to_xdna_dev(dev); 166 struct amdxdna_drm_get_array *args = data; 167 168 if (!xdna->dev_info->ops->get_array) 169 return -EOPNOTSUPP; 170 171 if (args->pad || !args->num_element || !args->element_size) 172 return -EINVAL; 173 174 guard(mutex)(&xdna->dev_lock); 175 return xdna->dev_info->ops->get_array(client, args); 176 } 177 178 static int amdxdna_drm_set_state_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 179 { 180 struct amdxdna_client *client = filp->driver_priv; 181 struct amdxdna_dev *xdna = to_xdna_dev(dev); 182 struct amdxdna_drm_set_state *args = data; 183 int ret; 184 185 if (!xdna->dev_info->ops->set_aie_state) 186 return -EOPNOTSUPP; 187 188 XDNA_DBG(xdna, "Request parameter %u", args->param); 189 mutex_lock(&xdna->dev_lock); 190 ret = xdna->dev_info->ops->set_aie_state(client, args); 191 mutex_unlock(&xdna->dev_lock); 192 193 return ret; 194 } 195 196 static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = { 197 /* Context */ 198 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0), 199 DRM_IOCTL_DEF_DRV(AMDXDNA_DESTROY_HWCTX, amdxdna_drm_destroy_hwctx_ioctl, 0), 200 DRM_IOCTL_DEF_DRV(AMDXDNA_CONFIG_HWCTX, amdxdna_drm_config_hwctx_ioctl, 0), 201 /* BO */ 202 DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_BO, amdxdna_drm_create_bo_ioctl, 0), 203 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_BO_INFO, amdxdna_drm_get_bo_info_ioctl, 0), 204 DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0), 205 /* Execution */ 206 DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0), 207 /* AIE hardware */ 208 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0), 209 DRM_IOCTL_DEF_DRV(AMDXDNA_GET_ARRAY, amdxdna_drm_get_array_ioctl, 0), 210 DRM_IOCTL_DEF_DRV(AMDXDNA_SET_STATE, amdxdna_drm_set_state_ioctl, DRM_ROOT_ONLY), 211 }; 212 213 static const struct file_operations amdxdna_fops = { 214 .owner = THIS_MODULE, 215 .open = accel_open, 216 .release = drm_release, 217 .flush = amdxdna_flush, 218 .unlocked_ioctl = drm_ioctl, 219 .compat_ioctl = drm_compat_ioctl, 220 .poll = drm_poll, 221 .read = drm_read, 222 .llseek = noop_llseek, 223 .mmap = drm_gem_mmap, 224 .fop_flags = FOP_UNSIGNED_OFFSET, 225 }; 226 227 const struct drm_driver amdxdna_drm_drv = { 228 .driver_features = DRIVER_GEM | DRIVER_COMPUTE_ACCEL | 229 DRIVER_SYNCOBJ | DRIVER_SYNCOBJ_TIMELINE, 230 .fops = &amdxdna_fops, 231 .name = "amdxdna_accel_driver", 232 .desc = "AMD XDNA DRM implementation", 233 .major = AMDXDNA_DRIVER_MAJOR, 234 .minor = AMDXDNA_DRIVER_MINOR, 235 .open = amdxdna_drm_open, 236 .postclose = amdxdna_drm_close, 237 .ioctls = amdxdna_drm_ioctls, 238 .num_ioctls = ARRAY_SIZE(amdxdna_drm_ioctls), 239 240 .gem_create_object = amdxdna_gem_create_object_cb, 241 .gem_prime_import = amdxdna_gem_prime_import, 242 }; 243 244 static const struct amdxdna_dev_info * 245 amdxdna_get_dev_info(struct pci_dev *pdev) 246 { 247 int i; 248 249 for (i = 0; i < ARRAY_SIZE(amdxdna_ids); i++) { 250 if (pdev->device == amdxdna_ids[i].device && 251 pdev->revision == amdxdna_ids[i].revision) 252 return amdxdna_ids[i].dev_info; 253 } 254 return NULL; 255 } 256 257 static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id) 258 { 259 struct device *dev = &pdev->dev; 260 struct amdxdna_dev *xdna; 261 int ret; 262 263 xdna = devm_drm_dev_alloc(dev, &amdxdna_drm_drv, typeof(*xdna), ddev); 264 if (IS_ERR(xdna)) 265 return PTR_ERR(xdna); 266 267 xdna->dev_info = amdxdna_get_dev_info(pdev); 268 if (!xdna->dev_info) 269 return -ENODEV; 270 271 drmm_mutex_init(&xdna->ddev, &xdna->dev_lock); 272 init_rwsem(&xdna->notifier_lock); 273 INIT_LIST_HEAD(&xdna->client_list); 274 pci_set_drvdata(pdev, xdna); 275 276 if (IS_ENABLED(CONFIG_LOCKDEP)) { 277 fs_reclaim_acquire(GFP_KERNEL); 278 might_lock(&xdna->notifier_lock); 279 fs_reclaim_release(GFP_KERNEL); 280 } 281 282 xdna->notifier_wq = alloc_ordered_workqueue("notifier_wq", 0); 283 if (!xdna->notifier_wq) 284 return -ENOMEM; 285 286 mutex_lock(&xdna->dev_lock); 287 ret = xdna->dev_info->ops->init(xdna); 288 mutex_unlock(&xdna->dev_lock); 289 if (ret) { 290 XDNA_ERR(xdna, "Hardware init failed, ret %d", ret); 291 goto destroy_notifier_wq; 292 } 293 294 ret = amdxdna_sysfs_init(xdna); 295 if (ret) { 296 XDNA_ERR(xdna, "Create amdxdna attrs failed: %d", ret); 297 goto failed_dev_fini; 298 } 299 300 ret = drm_dev_register(&xdna->ddev, 0); 301 if (ret) { 302 XDNA_ERR(xdna, "DRM register failed, ret %d", ret); 303 goto failed_sysfs_fini; 304 } 305 306 return 0; 307 308 failed_sysfs_fini: 309 amdxdna_sysfs_fini(xdna); 310 failed_dev_fini: 311 mutex_lock(&xdna->dev_lock); 312 xdna->dev_info->ops->fini(xdna); 313 mutex_unlock(&xdna->dev_lock); 314 destroy_notifier_wq: 315 destroy_workqueue(xdna->notifier_wq); 316 return ret; 317 } 318 319 static void amdxdna_remove(struct pci_dev *pdev) 320 { 321 struct amdxdna_dev *xdna = pci_get_drvdata(pdev); 322 struct amdxdna_client *client; 323 324 destroy_workqueue(xdna->notifier_wq); 325 326 drm_dev_unplug(&xdna->ddev); 327 amdxdna_sysfs_fini(xdna); 328 329 mutex_lock(&xdna->dev_lock); 330 client = list_first_entry_or_null(&xdna->client_list, 331 struct amdxdna_client, node); 332 while (client) { 333 list_del_init(&client->node); 334 amdxdna_hwctx_remove_all(client); 335 336 client = list_first_entry_or_null(&xdna->client_list, 337 struct amdxdna_client, node); 338 } 339 340 xdna->dev_info->ops->fini(xdna); 341 mutex_unlock(&xdna->dev_lock); 342 } 343 344 static const struct dev_pm_ops amdxdna_pm_ops = { 345 SYSTEM_SLEEP_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume) 346 RUNTIME_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume, NULL) 347 }; 348 349 static struct pci_driver amdxdna_pci_driver = { 350 .name = KBUILD_MODNAME, 351 .id_table = pci_ids, 352 .probe = amdxdna_probe, 353 .remove = amdxdna_remove, 354 .driver.pm = &amdxdna_pm_ops, 355 }; 356 357 module_pci_driver(amdxdna_pci_driver); 358 359 MODULE_LICENSE("GPL"); 360 MODULE_AUTHOR("XRT Team <runtimeca39d@amd.com>"); 361 MODULE_DESCRIPTION("amdxdna driver"); 362