1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/highmem.h> 8 #include <linux/module.h> 9 #include <linux/set_memory.h> 10 #include <linux/xarray.h> 11 12 #include <drm/drm_cache.h> 13 #include <drm/drm_debugfs.h> 14 #include <drm/drm_file.h> 15 #include <drm/drm_utils.h> 16 17 #include "ivpu_drv.h" 18 #include "ivpu_fw.h" 19 #include "ivpu_gem.h" 20 #include "ivpu_hw.h" 21 #include "ivpu_mmu.h" 22 #include "ivpu_mmu_context.h" 23 24 MODULE_IMPORT_NS("DMA_BUF"); 25 26 static const struct drm_gem_object_funcs ivpu_gem_funcs; 27 28 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) 29 { 30 ivpu_dbg(vdev, BO, 31 "%6s: bo %8p size %9zu ctx %d vpu_addr %9llx pages %d sgt %d mmu_mapped %d wc %d imported %d\n", 32 action, bo, ivpu_bo_size(bo), bo->ctx_id, bo->vpu_addr, 33 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, 34 (bool)drm_gem_is_imported(&bo->base.base)); 35 } 36 37 static inline int ivpu_bo_lock(struct ivpu_bo *bo) 38 { 39 return dma_resv_lock(bo->base.base.resv, NULL); 40 } 41 42 static inline void ivpu_bo_unlock(struct ivpu_bo *bo) 43 { 44 dma_resv_unlock(bo->base.base.resv); 45 } 46 47 static struct sg_table *ivpu_bo_map_attachment(struct ivpu_device *vdev, struct ivpu_bo *bo) 48 { 49 struct sg_table *sgt; 50 51 drm_WARN_ON(&vdev->drm, !bo->base.base.import_attach); 52 53 ivpu_bo_lock(bo); 54 55 sgt = bo->base.sgt; 56 if (!sgt) { 57 sgt = dma_buf_map_attachment(bo->base.base.import_attach, DMA_BIDIRECTIONAL); 58 if (IS_ERR(sgt)) 59 ivpu_err(vdev, "Failed to map BO in IOMMU: %ld\n", PTR_ERR(sgt)); 60 else 61 bo->base.sgt = sgt; 62 } 63 64 ivpu_bo_unlock(bo); 65 66 return sgt; 67 } 68 69 /* 70 * ivpu_bo_bind() - pin the backing physical pages and map them to VPU. 71 * 72 * This function pins physical memory pages, then maps the physical pages 73 * to IOMMU address space and finally updates the VPU MMU page tables 74 * to allow the VPU to translate VPU address to IOMMU address. 75 */ 76 int __must_check ivpu_bo_bind(struct ivpu_bo *bo) 77 { 78 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 79 struct sg_table *sgt; 80 int ret = 0; 81 82 ivpu_dbg_bo(vdev, bo, "bind"); 83 84 if (bo->base.base.import_attach) 85 sgt = ivpu_bo_map_attachment(vdev, bo); 86 else 87 sgt = drm_gem_shmem_get_pages_sgt(&bo->base); 88 if (IS_ERR(sgt)) { 89 ret = PTR_ERR(sgt); 90 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); 91 return ret; 92 } 93 94 ivpu_bo_lock(bo); 95 96 if (!bo->mmu_mapped) { 97 drm_WARN_ON(&vdev->drm, !bo->ctx); 98 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt, 99 ivpu_bo_is_snooped(bo), ivpu_bo_is_read_only(bo)); 100 if (ret) { 101 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret); 102 goto unlock; 103 } 104 bo->mmu_mapped = true; 105 } 106 107 unlock: 108 ivpu_bo_unlock(bo); 109 110 return ret; 111 } 112 113 static int 114 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, 115 const struct ivpu_addr_range *range) 116 { 117 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 118 int idx, ret; 119 120 if (!drm_dev_enter(&vdev->drm, &idx)) 121 return -ENODEV; 122 123 ivpu_bo_lock(bo); 124 125 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node); 126 if (!ret) { 127 bo->ctx = ctx; 128 bo->ctx_id = ctx->id; 129 bo->vpu_addr = bo->mm_node.start; 130 ivpu_dbg_bo(vdev, bo, "vaddr"); 131 } 132 133 ivpu_bo_unlock(bo); 134 135 drm_dev_exit(idx); 136 137 return ret; 138 } 139 140 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) 141 { 142 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 143 144 dma_resv_assert_held(bo->base.base.resv); 145 146 if (bo->mmu_mapped) { 147 drm_WARN_ON(&vdev->drm, !bo->ctx); 148 drm_WARN_ON(&vdev->drm, !bo->vpu_addr); 149 drm_WARN_ON(&vdev->drm, !bo->base.sgt); 150 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt); 151 bo->mmu_mapped = false; 152 } 153 154 if (bo->ctx) { 155 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node); 156 bo->ctx = NULL; 157 } 158 159 if (bo->base.sgt) { 160 if (bo->base.base.import_attach) { 161 dma_buf_unmap_attachment(bo->base.base.import_attach, 162 bo->base.sgt, DMA_BIDIRECTIONAL); 163 } else { 164 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); 165 sg_free_table(bo->base.sgt); 166 kfree(bo->base.sgt); 167 } 168 bo->base.sgt = NULL; 169 } 170 } 171 172 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 173 { 174 struct ivpu_bo *bo; 175 176 if (drm_WARN_ON(&vdev->drm, !ctx)) 177 return; 178 179 mutex_lock(&vdev->bo_list_lock); 180 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) { 181 ivpu_bo_lock(bo); 182 if (bo->ctx == ctx) { 183 ivpu_dbg_bo(vdev, bo, "unbind"); 184 ivpu_bo_unbind_locked(bo); 185 } 186 ivpu_bo_unlock(bo); 187 } 188 mutex_unlock(&vdev->bo_list_lock); 189 } 190 191 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size) 192 { 193 struct ivpu_bo *bo; 194 195 if (size == 0 || !PAGE_ALIGNED(size)) 196 return ERR_PTR(-EINVAL); 197 198 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 199 if (!bo) 200 return ERR_PTR(-ENOMEM); 201 202 bo->base.base.funcs = &ivpu_gem_funcs; 203 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */ 204 205 INIT_LIST_HEAD(&bo->bo_list_node); 206 207 return &bo->base.base; 208 } 209 210 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, 211 struct dma_buf *dma_buf) 212 { 213 struct ivpu_device *vdev = to_ivpu_device(dev); 214 struct device *attach_dev = dev->dev; 215 struct dma_buf_attachment *attach; 216 struct drm_gem_object *obj; 217 struct ivpu_bo *bo; 218 int ret; 219 220 attach = dma_buf_attach(dma_buf, attach_dev); 221 if (IS_ERR(attach)) 222 return ERR_CAST(attach); 223 224 get_dma_buf(dma_buf); 225 226 obj = drm_gem_shmem_prime_import_sg_table(dev, attach, NULL); 227 if (IS_ERR(obj)) { 228 ret = PTR_ERR(obj); 229 goto fail_detach; 230 } 231 232 obj->import_attach = attach; 233 obj->resv = dma_buf->resv; 234 235 bo = to_ivpu_bo(obj); 236 237 mutex_lock(&vdev->bo_list_lock); 238 list_add_tail(&bo->bo_list_node, &vdev->bo_list); 239 mutex_unlock(&vdev->bo_list_lock); 240 241 ivpu_dbg(vdev, BO, "import: bo %8p size %9zu\n", bo, ivpu_bo_size(bo)); 242 243 return obj; 244 245 fail_detach: 246 dma_buf_detach(dma_buf, attach); 247 dma_buf_put(dma_buf); 248 249 return ERR_PTR(ret); 250 } 251 252 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags) 253 { 254 struct drm_gem_shmem_object *shmem; 255 struct ivpu_bo *bo; 256 257 switch (flags & DRM_IVPU_BO_CACHE_MASK) { 258 case DRM_IVPU_BO_CACHED: 259 case DRM_IVPU_BO_WC: 260 break; 261 default: 262 return ERR_PTR(-EINVAL); 263 } 264 265 shmem = drm_gem_shmem_create(&vdev->drm, size); 266 if (IS_ERR(shmem)) 267 return ERR_CAST(shmem); 268 269 bo = to_ivpu_bo(&shmem->base); 270 bo->base.map_wc = flags & DRM_IVPU_BO_WC; 271 bo->flags = flags; 272 273 mutex_lock(&vdev->bo_list_lock); 274 list_add_tail(&bo->bo_list_node, &vdev->bo_list); 275 mutex_unlock(&vdev->bo_list_lock); 276 277 ivpu_dbg(vdev, BO, " alloc: bo %8p size %9llu\n", bo, size); 278 279 return bo; 280 } 281 282 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file) 283 { 284 struct ivpu_file_priv *file_priv = file->driver_priv; 285 struct ivpu_device *vdev = file_priv->vdev; 286 struct ivpu_bo *bo = to_ivpu_bo(obj); 287 struct ivpu_addr_range *range; 288 289 if (bo->ctx) { 290 ivpu_dbg(vdev, IOCTL, "Can't add BO %pe to ctx %u: already in ctx %u\n", 291 bo, file_priv->ctx.id, bo->ctx->id); 292 return -EALREADY; 293 } 294 295 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) 296 range = &vdev->hw->ranges.shave; 297 else if (bo->flags & DRM_IVPU_BO_DMA_MEM) 298 range = &vdev->hw->ranges.dma; 299 else 300 range = &vdev->hw->ranges.user; 301 302 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range); 303 } 304 305 static void ivpu_gem_bo_free(struct drm_gem_object *obj) 306 { 307 struct ivpu_device *vdev = to_ivpu_device(obj->dev); 308 struct ivpu_bo *bo = to_ivpu_bo(obj); 309 310 ivpu_dbg_bo(vdev, bo, "free"); 311 312 drm_WARN_ON(&vdev->drm, list_empty(&bo->bo_list_node)); 313 314 mutex_lock(&vdev->bo_list_lock); 315 list_del(&bo->bo_list_node); 316 317 drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) && 318 !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); 319 drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0); 320 drm_WARN_ON(&vdev->drm, bo->base.vaddr); 321 322 ivpu_bo_lock(bo); 323 ivpu_bo_unbind_locked(bo); 324 ivpu_bo_unlock(bo); 325 326 mutex_unlock(&vdev->bo_list_lock); 327 328 drm_WARN_ON(&vdev->drm, bo->mmu_mapped); 329 drm_WARN_ON(&vdev->drm, bo->ctx); 330 331 drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1); 332 drm_WARN_ON(obj->dev, bo->base.base.vma_node.vm_files.rb_node); 333 drm_gem_shmem_free(&bo->base); 334 } 335 336 static const struct drm_gem_object_funcs ivpu_gem_funcs = { 337 .free = ivpu_gem_bo_free, 338 .open = ivpu_gem_bo_open, 339 .print_info = drm_gem_shmem_object_print_info, 340 .pin = drm_gem_shmem_object_pin, 341 .unpin = drm_gem_shmem_object_unpin, 342 .get_sg_table = drm_gem_shmem_object_get_sg_table, 343 .vmap = drm_gem_shmem_object_vmap, 344 .vunmap = drm_gem_shmem_object_vunmap, 345 .mmap = drm_gem_shmem_object_mmap, 346 .vm_ops = &drm_gem_shmem_vm_ops, 347 }; 348 349 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 350 { 351 struct ivpu_file_priv *file_priv = file->driver_priv; 352 struct ivpu_device *vdev = file_priv->vdev; 353 struct drm_ivpu_bo_create *args = data; 354 u64 size = PAGE_ALIGN(args->size); 355 struct ivpu_bo *bo; 356 int ret; 357 358 if (args->flags & ~DRM_IVPU_BO_FLAGS) { 359 ivpu_dbg(vdev, IOCTL, "Invalid BO flags 0x%x\n", args->flags); 360 return -EINVAL; 361 } 362 363 if (size == 0) { 364 ivpu_dbg(vdev, IOCTL, "Invalid BO size %llu\n", args->size); 365 return -EINVAL; 366 } 367 368 bo = ivpu_bo_alloc(vdev, size, args->flags); 369 if (IS_ERR(bo)) { 370 ivpu_dbg(vdev, IOCTL, "Failed to allocate BO: %pe ctx %u size %llu flags 0x%x\n", 371 bo, file_priv->ctx.id, args->size, args->flags); 372 return PTR_ERR(bo); 373 } 374 375 drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 0); 376 377 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 378 if (ret) { 379 ivpu_dbg(vdev, IOCTL, "Failed to create handle for BO: %pe ctx %u size %llu flags 0x%x\n", 380 bo, file_priv->ctx.id, args->size, args->flags); 381 } else { 382 args->vpu_addr = bo->vpu_addr; 383 drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 1); 384 } 385 386 drm_gem_object_put(&bo->base.base); 387 388 return ret; 389 } 390 391 struct ivpu_bo * 392 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 393 struct ivpu_addr_range *range, u64 size, u32 flags) 394 { 395 struct iosys_map map; 396 struct ivpu_bo *bo; 397 int ret; 398 399 if (drm_WARN_ON(&vdev->drm, !range)) 400 return NULL; 401 402 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start)); 403 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end)); 404 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); 405 406 bo = ivpu_bo_alloc(vdev, size, flags); 407 if (IS_ERR(bo)) { 408 ivpu_err(vdev, "Failed to allocate BO: %pe vpu_addr 0x%llx size %llu flags 0x%x\n", 409 bo, range->start, size, flags); 410 return NULL; 411 } 412 413 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range); 414 if (ret) { 415 ivpu_err(vdev, "Failed to allocate NPU address for BO: %pe ctx %u size %llu: %d\n", 416 bo, ctx->id, size, ret); 417 goto err_put; 418 } 419 420 ret = ivpu_bo_bind(bo); 421 if (ret) 422 goto err_put; 423 424 if (flags & DRM_IVPU_BO_MAPPABLE) { 425 ivpu_bo_lock(bo); 426 ret = drm_gem_shmem_vmap_locked(&bo->base, &map); 427 ivpu_bo_unlock(bo); 428 429 if (ret) 430 goto err_put; 431 } 432 433 return bo; 434 435 err_put: 436 drm_gem_object_put(&bo->base.base); 437 return NULL; 438 } 439 440 struct ivpu_bo *ivpu_bo_create_runtime(struct ivpu_device *vdev, u64 addr, u64 size, u32 flags) 441 { 442 struct ivpu_addr_range range; 443 444 if (!ivpu_is_within_range(addr, size, &vdev->hw->ranges.runtime)) { 445 ivpu_err(vdev, "Invalid runtime BO address 0x%llx size %llu\n", addr, size); 446 return NULL; 447 } 448 449 if (ivpu_hw_range_init(vdev, &range, addr, size)) 450 return NULL; 451 452 return ivpu_bo_create(vdev, &vdev->gctx, &range, size, flags); 453 } 454 455 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags) 456 { 457 return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags); 458 } 459 460 void ivpu_bo_free(struct ivpu_bo *bo) 461 { 462 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr); 463 464 if (bo->flags & DRM_IVPU_BO_MAPPABLE) { 465 ivpu_bo_lock(bo); 466 drm_gem_shmem_vunmap_locked(&bo->base, &map); 467 ivpu_bo_unlock(bo); 468 } 469 470 drm_gem_object_put(&bo->base.base); 471 } 472 473 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 474 { 475 struct drm_ivpu_bo_info *args = data; 476 struct drm_gem_object *obj; 477 struct ivpu_bo *bo; 478 int ret = 0; 479 480 obj = drm_gem_object_lookup(file, args->handle); 481 if (!obj) 482 return -ENOENT; 483 484 bo = to_ivpu_bo(obj); 485 486 ivpu_bo_lock(bo); 487 args->flags = bo->flags; 488 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node); 489 args->vpu_addr = bo->vpu_addr; 490 args->size = obj->size; 491 ivpu_bo_unlock(bo); 492 493 drm_gem_object_put(obj); 494 return ret; 495 } 496 497 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 498 { 499 struct drm_ivpu_bo_wait *args = data; 500 struct drm_gem_object *obj; 501 unsigned long timeout; 502 long ret; 503 504 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 505 506 /* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */ 507 timeout += 1; 508 509 obj = drm_gem_object_lookup(file, args->handle); 510 if (!obj) 511 return -EINVAL; 512 513 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout); 514 if (ret == 0) { 515 ret = -ETIMEDOUT; 516 } else if (ret > 0) { 517 ret = 0; 518 args->job_status = to_ivpu_bo(obj)->job_status; 519 } 520 521 drm_gem_object_put(obj); 522 523 return ret; 524 } 525 526 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) 527 { 528 ivpu_bo_lock(bo); 529 530 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u", 531 bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size, 532 bo->flags, kref_read(&bo->base.base.refcount)); 533 534 if (bo->base.pages) 535 drm_printf(p, " has_pages"); 536 537 if (bo->mmu_mapped) 538 drm_printf(p, " mmu_mapped"); 539 540 if (drm_gem_is_imported(&bo->base.base)) 541 drm_printf(p, " imported"); 542 543 drm_printf(p, "\n"); 544 545 ivpu_bo_unlock(bo); 546 } 547 548 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) 549 { 550 struct ivpu_device *vdev = to_ivpu_device(dev); 551 struct ivpu_bo *bo; 552 553 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n", 554 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs"); 555 556 mutex_lock(&vdev->bo_list_lock); 557 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) 558 ivpu_bo_print_info(bo, p); 559 mutex_unlock(&vdev->bo_list_lock); 560 } 561 562 void ivpu_bo_list_print(struct drm_device *dev) 563 { 564 struct drm_printer p = drm_info_printer(dev->dev); 565 566 ivpu_bo_list(dev, &p); 567 } 568