1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/highmem.h> 8 #include <linux/module.h> 9 #include <linux/set_memory.h> 10 #include <linux/xarray.h> 11 12 #include <drm/drm_cache.h> 13 #include <drm/drm_debugfs.h> 14 #include <drm/drm_file.h> 15 #include <drm/drm_utils.h> 16 17 #include "ivpu_drv.h" 18 #include "ivpu_gem.h" 19 #include "ivpu_hw.h" 20 #include "ivpu_mmu.h" 21 #include "ivpu_mmu_context.h" 22 23 static const struct drm_gem_object_funcs ivpu_gem_funcs; 24 25 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) 26 { 27 ivpu_dbg(vdev, BO, 28 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n", 29 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx ? bo->ctx->id : 0, 30 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, 31 (bool)bo->base.base.import_attach); 32 } 33 34 /* 35 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU. 36 * 37 * This function pins physical memory pages, then maps the physical pages 38 * to IOMMU address space and finally updates the VPU MMU page tables 39 * to allow the VPU to translate VPU address to IOMMU address. 40 */ 41 int __must_check ivpu_bo_pin(struct ivpu_bo *bo) 42 { 43 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 44 int ret = 0; 45 46 mutex_lock(&bo->lock); 47 48 ivpu_dbg_bo(vdev, bo, "pin"); 49 drm_WARN_ON(&vdev->drm, !bo->ctx); 50 51 if (!bo->mmu_mapped) { 52 struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base); 53 54 if (IS_ERR(sgt)) { 55 ret = PTR_ERR(sgt); 56 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); 57 goto unlock; 58 } 59 60 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt, 61 ivpu_bo_is_snooped(bo)); 62 if (ret) { 63 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret); 64 goto unlock; 65 } 66 bo->mmu_mapped = true; 67 } 68 69 unlock: 70 mutex_unlock(&bo->lock); 71 72 return ret; 73 } 74 75 static int 76 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, 77 const struct ivpu_addr_range *range) 78 { 79 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 80 int idx, ret; 81 82 if (!drm_dev_enter(&vdev->drm, &idx)) 83 return -ENODEV; 84 85 mutex_lock(&bo->lock); 86 87 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node); 88 if (!ret) { 89 bo->ctx = ctx; 90 bo->vpu_addr = bo->mm_node.start; 91 } else { 92 ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret); 93 } 94 95 ivpu_dbg_bo(vdev, bo, "alloc"); 96 97 mutex_unlock(&bo->lock); 98 99 drm_dev_exit(idx); 100 101 return ret; 102 } 103 104 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) 105 { 106 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 107 108 lockdep_assert(lockdep_is_held(&bo->lock) || !kref_read(&bo->base.base.refcount)); 109 110 if (bo->mmu_mapped) { 111 drm_WARN_ON(&vdev->drm, !bo->ctx); 112 drm_WARN_ON(&vdev->drm, !bo->vpu_addr); 113 drm_WARN_ON(&vdev->drm, !bo->base.sgt); 114 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt); 115 bo->mmu_mapped = false; 116 } 117 118 if (bo->ctx) { 119 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node); 120 bo->ctx = NULL; 121 } 122 123 if (bo->base.base.import_attach) 124 return; 125 126 dma_resv_lock(bo->base.base.resv, NULL); 127 if (bo->base.sgt) { 128 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); 129 sg_free_table(bo->base.sgt); 130 kfree(bo->base.sgt); 131 bo->base.sgt = NULL; 132 } 133 dma_resv_unlock(bo->base.base.resv); 134 } 135 136 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 137 { 138 struct ivpu_bo *bo; 139 140 if (drm_WARN_ON(&vdev->drm, !ctx)) 141 return; 142 143 mutex_lock(&vdev->bo_list_lock); 144 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) { 145 mutex_lock(&bo->lock); 146 if (bo->ctx == ctx) { 147 ivpu_dbg_bo(vdev, bo, "unbind"); 148 ivpu_bo_unbind_locked(bo); 149 } 150 mutex_unlock(&bo->lock); 151 } 152 mutex_unlock(&vdev->bo_list_lock); 153 } 154 155 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size) 156 { 157 struct ivpu_bo *bo; 158 159 if (size == 0 || !PAGE_ALIGNED(size)) 160 return ERR_PTR(-EINVAL); 161 162 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 163 if (!bo) 164 return ERR_PTR(-ENOMEM); 165 166 bo->base.base.funcs = &ivpu_gem_funcs; 167 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */ 168 169 INIT_LIST_HEAD(&bo->bo_list_node); 170 mutex_init(&bo->lock); 171 172 return &bo->base.base; 173 } 174 175 static struct ivpu_bo * 176 ivpu_bo_create(struct ivpu_device *vdev, u64 size, u32 flags) 177 { 178 struct drm_gem_shmem_object *shmem; 179 struct ivpu_bo *bo; 180 181 switch (flags & DRM_IVPU_BO_CACHE_MASK) { 182 case DRM_IVPU_BO_CACHED: 183 case DRM_IVPU_BO_WC: 184 break; 185 default: 186 return ERR_PTR(-EINVAL); 187 } 188 189 shmem = drm_gem_shmem_create(&vdev->drm, size); 190 if (IS_ERR(shmem)) 191 return ERR_CAST(shmem); 192 193 bo = to_ivpu_bo(&shmem->base); 194 bo->base.map_wc = flags & DRM_IVPU_BO_WC; 195 bo->flags = flags; 196 197 mutex_lock(&vdev->bo_list_lock); 198 list_add_tail(&bo->bo_list_node, &vdev->bo_list); 199 mutex_unlock(&vdev->bo_list_lock); 200 201 return bo; 202 } 203 204 static int ivpu_bo_open(struct drm_gem_object *obj, struct drm_file *file) 205 { 206 struct ivpu_file_priv *file_priv = file->driver_priv; 207 struct ivpu_device *vdev = file_priv->vdev; 208 struct ivpu_bo *bo = to_ivpu_bo(obj); 209 struct ivpu_addr_range *range; 210 211 if (bo->ctx) { 212 ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n", 213 file_priv->ctx.id, bo->ctx->id); 214 return -EALREADY; 215 } 216 217 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) 218 range = &vdev->hw->ranges.shave; 219 else if (bo->flags & DRM_IVPU_BO_DMA_MEM) 220 range = &vdev->hw->ranges.dma; 221 else 222 range = &vdev->hw->ranges.user; 223 224 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range); 225 } 226 227 static void ivpu_bo_free(struct drm_gem_object *obj) 228 { 229 struct ivpu_device *vdev = to_ivpu_device(obj->dev); 230 struct ivpu_bo *bo = to_ivpu_bo(obj); 231 232 ivpu_dbg_bo(vdev, bo, "free"); 233 234 mutex_lock(&vdev->bo_list_lock); 235 list_del(&bo->bo_list_node); 236 mutex_unlock(&vdev->bo_list_lock); 237 238 drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); 239 240 ivpu_bo_unbind_locked(bo); 241 mutex_destroy(&bo->lock); 242 243 drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1); 244 drm_gem_shmem_free(&bo->base); 245 } 246 247 static const struct drm_gem_object_funcs ivpu_gem_funcs = { 248 .free = ivpu_bo_free, 249 .open = ivpu_bo_open, 250 .print_info = drm_gem_shmem_object_print_info, 251 .pin = drm_gem_shmem_object_pin, 252 .unpin = drm_gem_shmem_object_unpin, 253 .get_sg_table = drm_gem_shmem_object_get_sg_table, 254 .vmap = drm_gem_shmem_object_vmap, 255 .vunmap = drm_gem_shmem_object_vunmap, 256 .mmap = drm_gem_shmem_object_mmap, 257 .vm_ops = &drm_gem_shmem_vm_ops, 258 }; 259 260 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 261 { 262 struct ivpu_file_priv *file_priv = file->driver_priv; 263 struct ivpu_device *vdev = file_priv->vdev; 264 struct drm_ivpu_bo_create *args = data; 265 u64 size = PAGE_ALIGN(args->size); 266 struct ivpu_bo *bo; 267 int ret; 268 269 if (args->flags & ~DRM_IVPU_BO_FLAGS) 270 return -EINVAL; 271 272 if (size == 0) 273 return -EINVAL; 274 275 bo = ivpu_bo_create(vdev, size, args->flags); 276 if (IS_ERR(bo)) { 277 ivpu_err(vdev, "Failed to create BO: %pe (ctx %u size %llu flags 0x%x)", 278 bo, file_priv->ctx.id, args->size, args->flags); 279 return PTR_ERR(bo); 280 } 281 282 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 283 if (!ret) 284 args->vpu_addr = bo->vpu_addr; 285 286 drm_gem_object_put(&bo->base.base); 287 288 return ret; 289 } 290 291 struct ivpu_bo * 292 ivpu_bo_alloc_internal(struct ivpu_device *vdev, u64 vpu_addr, u64 size, u32 flags) 293 { 294 const struct ivpu_addr_range *range; 295 struct ivpu_addr_range fixed_range; 296 struct iosys_map map; 297 struct ivpu_bo *bo; 298 int ret; 299 300 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(vpu_addr)); 301 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); 302 303 if (vpu_addr) { 304 fixed_range.start = vpu_addr; 305 fixed_range.end = vpu_addr + size; 306 range = &fixed_range; 307 } else { 308 range = &vdev->hw->ranges.global; 309 } 310 311 bo = ivpu_bo_create(vdev, size, flags); 312 if (IS_ERR(bo)) { 313 ivpu_err(vdev, "Failed to create BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)", 314 bo, vpu_addr, size, flags); 315 return NULL; 316 } 317 318 ret = ivpu_bo_alloc_vpu_addr(bo, &vdev->gctx, range); 319 if (ret) 320 goto err_put; 321 322 ret = ivpu_bo_pin(bo); 323 if (ret) 324 goto err_put; 325 326 dma_resv_lock(bo->base.base.resv, NULL); 327 ret = drm_gem_shmem_vmap(&bo->base, &map); 328 dma_resv_unlock(bo->base.base.resv); 329 if (ret) 330 goto err_put; 331 332 return bo; 333 334 err_put: 335 drm_gem_object_put(&bo->base.base); 336 return NULL; 337 } 338 339 void ivpu_bo_free_internal(struct ivpu_bo *bo) 340 { 341 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr); 342 343 dma_resv_lock(bo->base.base.resv, NULL); 344 drm_gem_shmem_vunmap(&bo->base, &map); 345 dma_resv_unlock(bo->base.base.resv); 346 347 drm_gem_object_put(&bo->base.base); 348 } 349 350 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 351 { 352 struct drm_ivpu_bo_info *args = data; 353 struct drm_gem_object *obj; 354 struct ivpu_bo *bo; 355 int ret = 0; 356 357 obj = drm_gem_object_lookup(file, args->handle); 358 if (!obj) 359 return -ENOENT; 360 361 bo = to_ivpu_bo(obj); 362 363 mutex_lock(&bo->lock); 364 args->flags = bo->flags; 365 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node); 366 args->vpu_addr = bo->vpu_addr; 367 args->size = obj->size; 368 mutex_unlock(&bo->lock); 369 370 drm_gem_object_put(obj); 371 return ret; 372 } 373 374 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 375 { 376 struct drm_ivpu_bo_wait *args = data; 377 struct drm_gem_object *obj; 378 unsigned long timeout; 379 long ret; 380 381 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 382 383 obj = drm_gem_object_lookup(file, args->handle); 384 if (!obj) 385 return -EINVAL; 386 387 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout); 388 if (ret == 0) { 389 ret = -ETIMEDOUT; 390 } else if (ret > 0) { 391 ret = 0; 392 args->job_status = to_ivpu_bo(obj)->job_status; 393 } 394 395 drm_gem_object_put(obj); 396 397 return ret; 398 } 399 400 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) 401 { 402 mutex_lock(&bo->lock); 403 404 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u", 405 bo, bo->ctx->id, bo->vpu_addr, bo->base.base.size, 406 bo->flags, kref_read(&bo->base.base.refcount)); 407 408 if (bo->base.pages) 409 drm_printf(p, " has_pages"); 410 411 if (bo->mmu_mapped) 412 drm_printf(p, " mmu_mapped"); 413 414 if (bo->base.base.import_attach) 415 drm_printf(p, " imported"); 416 417 drm_printf(p, "\n"); 418 419 mutex_unlock(&bo->lock); 420 } 421 422 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) 423 { 424 struct ivpu_device *vdev = to_ivpu_device(dev); 425 struct ivpu_bo *bo; 426 427 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n", 428 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs"); 429 430 mutex_lock(&vdev->bo_list_lock); 431 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) 432 ivpu_bo_print_info(bo, p); 433 mutex_unlock(&vdev->bo_list_lock); 434 } 435 436 void ivpu_bo_list_print(struct drm_device *dev) 437 { 438 struct drm_printer p = drm_info_printer(dev->dev); 439 440 ivpu_bo_list(dev, &p); 441 } 442