1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/highmem.h> 8 #include <linux/module.h> 9 #include <linux/set_memory.h> 10 #include <linux/xarray.h> 11 12 #include <drm/drm_cache.h> 13 #include <drm/drm_debugfs.h> 14 #include <drm/drm_file.h> 15 #include <drm/drm_utils.h> 16 17 #include "ivpu_drv.h" 18 #include "ivpu_gem.h" 19 #include "ivpu_hw.h" 20 #include "ivpu_mmu.h" 21 #include "ivpu_mmu_context.h" 22 23 MODULE_IMPORT_NS("DMA_BUF"); 24 25 static const struct drm_gem_object_funcs ivpu_gem_funcs; 26 27 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) 28 { 29 ivpu_dbg(vdev, BO, 30 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n", 31 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx_id, 32 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, 33 (bool)drm_gem_is_imported(&bo->base.base)); 34 } 35 36 static inline int ivpu_bo_lock(struct ivpu_bo *bo) 37 { 38 return dma_resv_lock(bo->base.base.resv, NULL); 39 } 40 41 static inline void ivpu_bo_unlock(struct ivpu_bo *bo) 42 { 43 dma_resv_unlock(bo->base.base.resv); 44 } 45 46 /* 47 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU. 48 * 49 * This function pins physical memory pages, then maps the physical pages 50 * to IOMMU address space and finally updates the VPU MMU page tables 51 * to allow the VPU to translate VPU address to IOMMU address. 52 */ 53 int __must_check ivpu_bo_pin(struct ivpu_bo *bo) 54 { 55 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 56 struct sg_table *sgt; 57 int ret = 0; 58 59 ivpu_dbg_bo(vdev, bo, "pin"); 60 61 sgt = drm_gem_shmem_get_pages_sgt(&bo->base); 62 if (IS_ERR(sgt)) { 63 ret = PTR_ERR(sgt); 64 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); 65 return ret; 66 } 67 68 ivpu_bo_lock(bo); 69 70 if (!bo->mmu_mapped) { 71 drm_WARN_ON(&vdev->drm, !bo->ctx); 72 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt, 73 ivpu_bo_is_snooped(bo)); 74 if (ret) { 75 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret); 76 goto unlock; 77 } 78 bo->mmu_mapped = true; 79 } 80 81 unlock: 82 ivpu_bo_unlock(bo); 83 84 return ret; 85 } 86 87 static int 88 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, 89 const struct ivpu_addr_range *range) 90 { 91 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 92 int idx, ret; 93 94 if (!drm_dev_enter(&vdev->drm, &idx)) 95 return -ENODEV; 96 97 ivpu_bo_lock(bo); 98 99 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node); 100 if (!ret) { 101 bo->ctx = ctx; 102 bo->vpu_addr = bo->mm_node.start; 103 } else { 104 ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret); 105 } 106 107 ivpu_bo_unlock(bo); 108 109 drm_dev_exit(idx); 110 111 return ret; 112 } 113 114 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) 115 { 116 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 117 118 lockdep_assert(dma_resv_held(bo->base.base.resv) || !kref_read(&bo->base.base.refcount)); 119 120 if (bo->mmu_mapped) { 121 drm_WARN_ON(&vdev->drm, !bo->ctx); 122 drm_WARN_ON(&vdev->drm, !bo->vpu_addr); 123 drm_WARN_ON(&vdev->drm, !bo->base.sgt); 124 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt); 125 bo->mmu_mapped = false; 126 } 127 128 if (bo->ctx) { 129 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node); 130 bo->ctx = NULL; 131 } 132 133 if (drm_gem_is_imported(&bo->base.base)) 134 return; 135 136 if (bo->base.sgt) { 137 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); 138 sg_free_table(bo->base.sgt); 139 kfree(bo->base.sgt); 140 bo->base.sgt = NULL; 141 } 142 } 143 144 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 145 { 146 struct ivpu_bo *bo; 147 148 if (drm_WARN_ON(&vdev->drm, !ctx)) 149 return; 150 151 mutex_lock(&vdev->bo_list_lock); 152 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) { 153 ivpu_bo_lock(bo); 154 if (bo->ctx == ctx) { 155 ivpu_dbg_bo(vdev, bo, "unbind"); 156 ivpu_bo_unbind_locked(bo); 157 } 158 ivpu_bo_unlock(bo); 159 } 160 mutex_unlock(&vdev->bo_list_lock); 161 } 162 163 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size) 164 { 165 struct ivpu_bo *bo; 166 167 if (size == 0 || !PAGE_ALIGNED(size)) 168 return ERR_PTR(-EINVAL); 169 170 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 171 if (!bo) 172 return ERR_PTR(-ENOMEM); 173 174 bo->base.base.funcs = &ivpu_gem_funcs; 175 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */ 176 177 INIT_LIST_HEAD(&bo->bo_list_node); 178 179 return &bo->base.base; 180 } 181 182 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, 183 struct dma_buf *dma_buf) 184 { 185 struct device *attach_dev = dev->dev; 186 struct dma_buf_attachment *attach; 187 struct sg_table *sgt; 188 struct drm_gem_object *obj; 189 int ret; 190 191 attach = dma_buf_attach(dma_buf, attach_dev); 192 if (IS_ERR(attach)) 193 return ERR_CAST(attach); 194 195 get_dma_buf(dma_buf); 196 197 sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL); 198 if (IS_ERR(sgt)) { 199 ret = PTR_ERR(sgt); 200 goto fail_detach; 201 } 202 203 obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); 204 if (IS_ERR(obj)) { 205 ret = PTR_ERR(obj); 206 goto fail_unmap; 207 } 208 209 obj->import_attach = attach; 210 obj->resv = dma_buf->resv; 211 212 return obj; 213 214 fail_unmap: 215 dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL); 216 fail_detach: 217 dma_buf_detach(dma_buf, attach); 218 dma_buf_put(dma_buf); 219 220 return ERR_PTR(ret); 221 } 222 223 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags, u32 ctx_id) 224 { 225 struct drm_gem_shmem_object *shmem; 226 struct ivpu_bo *bo; 227 228 switch (flags & DRM_IVPU_BO_CACHE_MASK) { 229 case DRM_IVPU_BO_CACHED: 230 case DRM_IVPU_BO_WC: 231 break; 232 default: 233 return ERR_PTR(-EINVAL); 234 } 235 236 shmem = drm_gem_shmem_create(&vdev->drm, size); 237 if (IS_ERR(shmem)) 238 return ERR_CAST(shmem); 239 240 bo = to_ivpu_bo(&shmem->base); 241 bo->ctx_id = ctx_id; 242 bo->base.map_wc = flags & DRM_IVPU_BO_WC; 243 bo->flags = flags; 244 245 mutex_lock(&vdev->bo_list_lock); 246 list_add_tail(&bo->bo_list_node, &vdev->bo_list); 247 mutex_unlock(&vdev->bo_list_lock); 248 249 ivpu_dbg_bo(vdev, bo, "alloc"); 250 251 return bo; 252 } 253 254 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file) 255 { 256 struct ivpu_file_priv *file_priv = file->driver_priv; 257 struct ivpu_device *vdev = file_priv->vdev; 258 struct ivpu_bo *bo = to_ivpu_bo(obj); 259 struct ivpu_addr_range *range; 260 261 if (bo->ctx) { 262 ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n", 263 file_priv->ctx.id, bo->ctx->id); 264 return -EALREADY; 265 } 266 267 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) 268 range = &vdev->hw->ranges.shave; 269 else if (bo->flags & DRM_IVPU_BO_DMA_MEM) 270 range = &vdev->hw->ranges.dma; 271 else 272 range = &vdev->hw->ranges.user; 273 274 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range); 275 } 276 277 static void ivpu_gem_bo_free(struct drm_gem_object *obj) 278 { 279 struct ivpu_device *vdev = to_ivpu_device(obj->dev); 280 struct ivpu_bo *bo = to_ivpu_bo(obj); 281 282 ivpu_dbg_bo(vdev, bo, "free"); 283 284 mutex_lock(&vdev->bo_list_lock); 285 list_del(&bo->bo_list_node); 286 mutex_unlock(&vdev->bo_list_lock); 287 288 drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) && 289 !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); 290 drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0); 291 drm_WARN_ON(&vdev->drm, bo->base.vaddr); 292 293 ivpu_bo_unbind_locked(bo); 294 drm_WARN_ON(&vdev->drm, bo->mmu_mapped); 295 drm_WARN_ON(&vdev->drm, bo->ctx); 296 297 drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1); 298 drm_gem_shmem_free(&bo->base); 299 } 300 301 static const struct drm_gem_object_funcs ivpu_gem_funcs = { 302 .free = ivpu_gem_bo_free, 303 .open = ivpu_gem_bo_open, 304 .print_info = drm_gem_shmem_object_print_info, 305 .pin = drm_gem_shmem_object_pin, 306 .unpin = drm_gem_shmem_object_unpin, 307 .get_sg_table = drm_gem_shmem_object_get_sg_table, 308 .vmap = drm_gem_shmem_object_vmap, 309 .vunmap = drm_gem_shmem_object_vunmap, 310 .mmap = drm_gem_shmem_object_mmap, 311 .vm_ops = &drm_gem_shmem_vm_ops, 312 }; 313 314 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 315 { 316 struct ivpu_file_priv *file_priv = file->driver_priv; 317 struct ivpu_device *vdev = file_priv->vdev; 318 struct drm_ivpu_bo_create *args = data; 319 u64 size = PAGE_ALIGN(args->size); 320 struct ivpu_bo *bo; 321 int ret; 322 323 if (args->flags & ~DRM_IVPU_BO_FLAGS) 324 return -EINVAL; 325 326 if (size == 0) 327 return -EINVAL; 328 329 bo = ivpu_bo_alloc(vdev, size, args->flags, file_priv->ctx.id); 330 if (IS_ERR(bo)) { 331 ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)", 332 bo, file_priv->ctx.id, args->size, args->flags); 333 return PTR_ERR(bo); 334 } 335 336 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 337 if (ret) 338 ivpu_err(vdev, "Failed to create handle for BO: %pe (ctx %u size %llu flags 0x%x)", 339 bo, file_priv->ctx.id, args->size, args->flags); 340 else 341 args->vpu_addr = bo->vpu_addr; 342 343 drm_gem_object_put(&bo->base.base); 344 345 return ret; 346 } 347 348 struct ivpu_bo * 349 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 350 struct ivpu_addr_range *range, u64 size, u32 flags) 351 { 352 struct iosys_map map; 353 struct ivpu_bo *bo; 354 int ret; 355 356 if (drm_WARN_ON(&vdev->drm, !range)) 357 return NULL; 358 359 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start)); 360 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end)); 361 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); 362 363 bo = ivpu_bo_alloc(vdev, size, flags, IVPU_GLOBAL_CONTEXT_MMU_SSID); 364 if (IS_ERR(bo)) { 365 ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)", 366 bo, range->start, size, flags); 367 return NULL; 368 } 369 370 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range); 371 if (ret) 372 goto err_put; 373 374 ret = ivpu_bo_pin(bo); 375 if (ret) 376 goto err_put; 377 378 if (flags & DRM_IVPU_BO_MAPPABLE) { 379 ivpu_bo_lock(bo); 380 ret = drm_gem_shmem_vmap_locked(&bo->base, &map); 381 ivpu_bo_unlock(bo); 382 383 if (ret) 384 goto err_put; 385 } 386 387 return bo; 388 389 err_put: 390 drm_gem_object_put(&bo->base.base); 391 return NULL; 392 } 393 394 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags) 395 { 396 return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags); 397 } 398 399 void ivpu_bo_free(struct ivpu_bo *bo) 400 { 401 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr); 402 403 if (bo->flags & DRM_IVPU_BO_MAPPABLE) { 404 ivpu_bo_lock(bo); 405 drm_gem_shmem_vunmap_locked(&bo->base, &map); 406 ivpu_bo_unlock(bo); 407 } 408 409 drm_gem_object_put(&bo->base.base); 410 } 411 412 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 413 { 414 struct drm_ivpu_bo_info *args = data; 415 struct drm_gem_object *obj; 416 struct ivpu_bo *bo; 417 int ret = 0; 418 419 obj = drm_gem_object_lookup(file, args->handle); 420 if (!obj) 421 return -ENOENT; 422 423 bo = to_ivpu_bo(obj); 424 425 ivpu_bo_lock(bo); 426 args->flags = bo->flags; 427 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node); 428 args->vpu_addr = bo->vpu_addr; 429 args->size = obj->size; 430 ivpu_bo_unlock(bo); 431 432 drm_gem_object_put(obj); 433 return ret; 434 } 435 436 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 437 { 438 struct drm_ivpu_bo_wait *args = data; 439 struct drm_gem_object *obj; 440 unsigned long timeout; 441 long ret; 442 443 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 444 445 /* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */ 446 timeout += 1; 447 448 obj = drm_gem_object_lookup(file, args->handle); 449 if (!obj) 450 return -EINVAL; 451 452 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout); 453 if (ret == 0) { 454 ret = -ETIMEDOUT; 455 } else if (ret > 0) { 456 ret = 0; 457 args->job_status = to_ivpu_bo(obj)->job_status; 458 } 459 460 drm_gem_object_put(obj); 461 462 return ret; 463 } 464 465 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) 466 { 467 ivpu_bo_lock(bo); 468 469 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u", 470 bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size, 471 bo->flags, kref_read(&bo->base.base.refcount)); 472 473 if (bo->base.pages) 474 drm_printf(p, " has_pages"); 475 476 if (bo->mmu_mapped) 477 drm_printf(p, " mmu_mapped"); 478 479 if (drm_gem_is_imported(&bo->base.base)) 480 drm_printf(p, " imported"); 481 482 drm_printf(p, "\n"); 483 484 ivpu_bo_unlock(bo); 485 } 486 487 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) 488 { 489 struct ivpu_device *vdev = to_ivpu_device(dev); 490 struct ivpu_bo *bo; 491 492 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n", 493 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs"); 494 495 mutex_lock(&vdev->bo_list_lock); 496 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) 497 ivpu_bo_print_info(bo, p); 498 mutex_unlock(&vdev->bo_list_lock); 499 } 500 501 void ivpu_bo_list_print(struct drm_device *dev) 502 { 503 struct drm_printer p = drm_info_printer(dev->dev); 504 505 ivpu_bo_list(dev, &p); 506 } 507