1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/highmem.h> 8 #include <linux/module.h> 9 #include <linux/set_memory.h> 10 #include <linux/xarray.h> 11 12 #include <drm/drm_cache.h> 13 #include <drm/drm_debugfs.h> 14 #include <drm/drm_file.h> 15 #include <drm/drm_utils.h> 16 17 #include "ivpu_drv.h" 18 #include "ivpu_fw.h" 19 #include "ivpu_gem.h" 20 #include "ivpu_hw.h" 21 #include "ivpu_mmu.h" 22 #include "ivpu_mmu_context.h" 23 24 MODULE_IMPORT_NS("DMA_BUF"); 25 26 static const struct drm_gem_object_funcs ivpu_gem_funcs; 27 28 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) 29 { 30 ivpu_dbg(vdev, BO, 31 "%6s: bo %8p size %9zu ctx %d vpu_addr %9llx pages %d sgt %d mmu_mapped %d wc %d imported %d\n", 32 action, bo, ivpu_bo_size(bo), bo->ctx_id, bo->vpu_addr, 33 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, 34 (bool)drm_gem_is_imported(&bo->base.base)); 35 } 36 37 static inline int ivpu_bo_lock(struct ivpu_bo *bo) 38 { 39 return dma_resv_lock(bo->base.base.resv, NULL); 40 } 41 42 static inline void ivpu_bo_unlock(struct ivpu_bo *bo) 43 { 44 dma_resv_unlock(bo->base.base.resv); 45 } 46 47 static struct sg_table *ivpu_bo_map_attachment(struct ivpu_device *vdev, struct ivpu_bo *bo) 48 { 49 struct sg_table *sgt; 50 51 drm_WARN_ON(&vdev->drm, !bo->base.base.import_attach); 52 53 ivpu_bo_lock(bo); 54 55 sgt = bo->base.sgt; 56 if (!sgt) { 57 sgt = dma_buf_map_attachment(bo->base.base.import_attach, DMA_BIDIRECTIONAL); 58 if (IS_ERR(sgt)) 59 ivpu_err(vdev, "Failed to map BO in IOMMU: %ld\n", PTR_ERR(sgt)); 60 else 61 bo->base.sgt = sgt; 62 } 63 64 ivpu_bo_unlock(bo); 65 66 return sgt; 67 } 68 69 /* 70 * ivpu_bo_bind() - pin the backing physical pages and map them to VPU. 71 * 72 * This function pins physical memory pages, then maps the physical pages 73 * to IOMMU address space and finally updates the VPU MMU page tables 74 * to allow the VPU to translate VPU address to IOMMU address. 75 */ 76 int __must_check ivpu_bo_bind(struct ivpu_bo *bo) 77 { 78 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 79 struct sg_table *sgt; 80 int ret = 0; 81 82 ivpu_dbg_bo(vdev, bo, "bind"); 83 84 if (bo->base.base.import_attach) 85 sgt = ivpu_bo_map_attachment(vdev, bo); 86 else 87 sgt = drm_gem_shmem_get_pages_sgt(&bo->base); 88 if (IS_ERR(sgt)) { 89 ret = PTR_ERR(sgt); 90 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); 91 return ret; 92 } 93 94 ivpu_bo_lock(bo); 95 96 if (!bo->mmu_mapped) { 97 drm_WARN_ON(&vdev->drm, !bo->ctx); 98 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt, 99 ivpu_bo_is_snooped(bo), ivpu_bo_is_read_only(bo)); 100 if (ret) { 101 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret); 102 goto unlock; 103 } 104 bo->mmu_mapped = true; 105 } 106 107 unlock: 108 ivpu_bo_unlock(bo); 109 110 return ret; 111 } 112 113 static int 114 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, 115 const struct ivpu_addr_range *range) 116 { 117 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 118 int idx, ret; 119 120 if (!drm_dev_enter(&vdev->drm, &idx)) 121 return -ENODEV; 122 123 ivpu_bo_lock(bo); 124 125 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node); 126 if (!ret) { 127 bo->ctx = ctx; 128 bo->ctx_id = ctx->id; 129 bo->vpu_addr = bo->mm_node.start; 130 ivpu_dbg_bo(vdev, bo, "vaddr"); 131 } 132 133 ivpu_bo_unlock(bo); 134 135 drm_dev_exit(idx); 136 137 return ret; 138 } 139 140 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) 141 { 142 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 143 144 dma_resv_assert_held(bo->base.base.resv); 145 146 if (bo->mmu_mapped) { 147 drm_WARN_ON(&vdev->drm, !bo->ctx); 148 drm_WARN_ON(&vdev->drm, !bo->vpu_addr); 149 drm_WARN_ON(&vdev->drm, !bo->base.sgt); 150 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt); 151 bo->mmu_mapped = false; 152 } 153 154 if (bo->ctx) { 155 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node); 156 bo->ctx = NULL; 157 } 158 159 if (bo->base.sgt) { 160 if (bo->base.base.import_attach) { 161 dma_buf_unmap_attachment(bo->base.base.import_attach, 162 bo->base.sgt, DMA_BIDIRECTIONAL); 163 } else { 164 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); 165 sg_free_table(bo->base.sgt); 166 kfree(bo->base.sgt); 167 } 168 bo->base.sgt = NULL; 169 } 170 } 171 172 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 173 { 174 struct ivpu_bo *bo; 175 176 if (drm_WARN_ON(&vdev->drm, !ctx)) 177 return; 178 179 mutex_lock(&vdev->bo_list_lock); 180 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) { 181 ivpu_bo_lock(bo); 182 if (bo->ctx == ctx) { 183 ivpu_dbg_bo(vdev, bo, "unbind"); 184 ivpu_bo_unbind_locked(bo); 185 } 186 ivpu_bo_unlock(bo); 187 } 188 mutex_unlock(&vdev->bo_list_lock); 189 } 190 191 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size) 192 { 193 struct ivpu_bo *bo; 194 195 if (size == 0 || !PAGE_ALIGNED(size)) 196 return ERR_PTR(-EINVAL); 197 198 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 199 if (!bo) 200 return ERR_PTR(-ENOMEM); 201 202 bo->base.base.funcs = &ivpu_gem_funcs; 203 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */ 204 205 INIT_LIST_HEAD(&bo->bo_list_node); 206 207 return &bo->base.base; 208 } 209 210 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, 211 struct dma_buf *dma_buf) 212 { 213 struct ivpu_device *vdev = to_ivpu_device(dev); 214 struct device *attach_dev = dev->dev; 215 struct dma_buf_attachment *attach; 216 struct drm_gem_object *obj; 217 struct ivpu_bo *bo; 218 int ret; 219 220 attach = dma_buf_attach(dma_buf, attach_dev); 221 if (IS_ERR(attach)) 222 return ERR_CAST(attach); 223 224 get_dma_buf(dma_buf); 225 226 obj = drm_gem_shmem_prime_import_sg_table(dev, attach, NULL); 227 if (IS_ERR(obj)) { 228 ret = PTR_ERR(obj); 229 goto fail_detach; 230 } 231 232 obj->import_attach = attach; 233 obj->resv = dma_buf->resv; 234 235 bo = to_ivpu_bo(obj); 236 237 mutex_lock(&vdev->bo_list_lock); 238 list_add_tail(&bo->bo_list_node, &vdev->bo_list); 239 mutex_unlock(&vdev->bo_list_lock); 240 241 ivpu_dbg(vdev, BO, "import: bo %8p size %9zu\n", bo, ivpu_bo_size(bo)); 242 243 return obj; 244 245 fail_detach: 246 dma_buf_detach(dma_buf, attach); 247 dma_buf_put(dma_buf); 248 249 return ERR_PTR(ret); 250 } 251 252 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags) 253 { 254 struct drm_gem_shmem_object *shmem; 255 struct ivpu_bo *bo; 256 257 switch (flags & DRM_IVPU_BO_CACHE_MASK) { 258 case DRM_IVPU_BO_CACHED: 259 case DRM_IVPU_BO_WC: 260 break; 261 default: 262 return ERR_PTR(-EINVAL); 263 } 264 265 shmem = drm_gem_shmem_create(&vdev->drm, size); 266 if (IS_ERR(shmem)) 267 return ERR_CAST(shmem); 268 269 bo = to_ivpu_bo(&shmem->base); 270 bo->base.map_wc = flags & DRM_IVPU_BO_WC; 271 bo->flags = flags; 272 273 mutex_lock(&vdev->bo_list_lock); 274 list_add_tail(&bo->bo_list_node, &vdev->bo_list); 275 mutex_unlock(&vdev->bo_list_lock); 276 277 ivpu_dbg(vdev, BO, " alloc: bo %8p size %9llu\n", bo, size); 278 279 return bo; 280 } 281 282 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file) 283 { 284 struct ivpu_file_priv *file_priv = file->driver_priv; 285 struct ivpu_device *vdev = file_priv->vdev; 286 struct ivpu_bo *bo = to_ivpu_bo(obj); 287 struct ivpu_addr_range *range; 288 289 if (bo->ctx) { 290 ivpu_dbg(vdev, IOCTL, "Can't add BO %pe to ctx %u: already in ctx %u\n", 291 bo, file_priv->ctx.id, bo->ctx->id); 292 return -EALREADY; 293 } 294 295 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) 296 range = &vdev->hw->ranges.shave; 297 else if (bo->flags & DRM_IVPU_BO_DMA_MEM) 298 range = &vdev->hw->ranges.dma; 299 else 300 range = &vdev->hw->ranges.user; 301 302 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range); 303 } 304 305 static void ivpu_gem_bo_free(struct drm_gem_object *obj) 306 { 307 struct ivpu_device *vdev = to_ivpu_device(obj->dev); 308 struct ivpu_bo *bo = to_ivpu_bo(obj); 309 310 ivpu_dbg_bo(vdev, bo, "free"); 311 312 drm_WARN_ON(&vdev->drm, list_empty(&bo->bo_list_node)); 313 314 mutex_lock(&vdev->bo_list_lock); 315 list_del(&bo->bo_list_node); 316 317 drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) && 318 !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); 319 drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0); 320 drm_WARN_ON(&vdev->drm, bo->base.vaddr); 321 322 ivpu_bo_lock(bo); 323 ivpu_bo_unbind_locked(bo); 324 ivpu_bo_unlock(bo); 325 326 mutex_unlock(&vdev->bo_list_lock); 327 328 drm_WARN_ON(&vdev->drm, bo->mmu_mapped); 329 drm_WARN_ON(&vdev->drm, bo->ctx); 330 331 drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1); 332 drm_WARN_ON(obj->dev, bo->base.base.vma_node.vm_files.rb_node); 333 drm_gem_shmem_free(&bo->base); 334 } 335 336 static enum drm_gem_object_status ivpu_gem_status(struct drm_gem_object *obj) 337 { 338 struct ivpu_bo *bo = to_ivpu_bo(obj); 339 enum drm_gem_object_status status = 0; 340 341 if (ivpu_bo_is_resident(bo)) 342 status |= DRM_GEM_OBJECT_RESIDENT; 343 344 return status; 345 } 346 347 static const struct drm_gem_object_funcs ivpu_gem_funcs = { 348 .free = ivpu_gem_bo_free, 349 .open = ivpu_gem_bo_open, 350 .print_info = drm_gem_shmem_object_print_info, 351 .pin = drm_gem_shmem_object_pin, 352 .unpin = drm_gem_shmem_object_unpin, 353 .get_sg_table = drm_gem_shmem_object_get_sg_table, 354 .vmap = drm_gem_shmem_object_vmap, 355 .vunmap = drm_gem_shmem_object_vunmap, 356 .mmap = drm_gem_shmem_object_mmap, 357 .status = ivpu_gem_status, 358 .vm_ops = &drm_gem_shmem_vm_ops, 359 }; 360 361 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 362 { 363 struct ivpu_file_priv *file_priv = file->driver_priv; 364 struct ivpu_device *vdev = file_priv->vdev; 365 struct drm_ivpu_bo_create *args = data; 366 u64 size = PAGE_ALIGN(args->size); 367 struct ivpu_bo *bo; 368 int ret; 369 370 if (args->flags & ~DRM_IVPU_BO_FLAGS) { 371 ivpu_dbg(vdev, IOCTL, "Invalid BO flags 0x%x\n", args->flags); 372 return -EINVAL; 373 } 374 375 if (size == 0) { 376 ivpu_dbg(vdev, IOCTL, "Invalid BO size %llu\n", args->size); 377 return -EINVAL; 378 } 379 380 bo = ivpu_bo_alloc(vdev, size, args->flags); 381 if (IS_ERR(bo)) { 382 ivpu_dbg(vdev, IOCTL, "Failed to allocate BO: %pe ctx %u size %llu flags 0x%x\n", 383 bo, file_priv->ctx.id, args->size, args->flags); 384 return PTR_ERR(bo); 385 } 386 387 drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 0); 388 389 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 390 if (ret) { 391 ivpu_dbg(vdev, IOCTL, "Failed to create handle for BO: %pe ctx %u size %llu flags 0x%x\n", 392 bo, file_priv->ctx.id, args->size, args->flags); 393 } else { 394 args->vpu_addr = bo->vpu_addr; 395 drm_WARN_ON(&vdev->drm, bo->base.base.handle_count != 1); 396 } 397 398 drm_gem_object_put(&bo->base.base); 399 400 return ret; 401 } 402 403 struct ivpu_bo * 404 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 405 struct ivpu_addr_range *range, u64 size, u32 flags) 406 { 407 struct iosys_map map; 408 struct ivpu_bo *bo; 409 int ret; 410 411 if (drm_WARN_ON(&vdev->drm, !range)) 412 return NULL; 413 414 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start)); 415 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end)); 416 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); 417 418 bo = ivpu_bo_alloc(vdev, size, flags); 419 if (IS_ERR(bo)) { 420 ivpu_err(vdev, "Failed to allocate BO: %pe vpu_addr 0x%llx size %llu flags 0x%x\n", 421 bo, range->start, size, flags); 422 return NULL; 423 } 424 425 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range); 426 if (ret) { 427 ivpu_err(vdev, "Failed to allocate NPU address for BO: %pe ctx %u size %llu: %d\n", 428 bo, ctx->id, size, ret); 429 goto err_put; 430 } 431 432 ret = ivpu_bo_bind(bo); 433 if (ret) 434 goto err_put; 435 436 if (flags & DRM_IVPU_BO_MAPPABLE) { 437 ivpu_bo_lock(bo); 438 ret = drm_gem_shmem_vmap_locked(&bo->base, &map); 439 ivpu_bo_unlock(bo); 440 441 if (ret) 442 goto err_put; 443 } 444 445 return bo; 446 447 err_put: 448 drm_gem_object_put(&bo->base.base); 449 return NULL; 450 } 451 452 struct ivpu_bo *ivpu_bo_create_runtime(struct ivpu_device *vdev, u64 addr, u64 size, u32 flags) 453 { 454 struct ivpu_addr_range range; 455 456 if (!ivpu_is_within_range(addr, size, &vdev->hw->ranges.runtime)) { 457 ivpu_err(vdev, "Invalid runtime BO address 0x%llx size %llu\n", addr, size); 458 return NULL; 459 } 460 461 if (ivpu_hw_range_init(vdev, &range, addr, size)) 462 return NULL; 463 464 return ivpu_bo_create(vdev, &vdev->gctx, &range, size, flags); 465 } 466 467 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags) 468 { 469 return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags); 470 } 471 472 void ivpu_bo_free(struct ivpu_bo *bo) 473 { 474 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr); 475 476 if (bo->flags & DRM_IVPU_BO_MAPPABLE) { 477 ivpu_bo_lock(bo); 478 drm_gem_shmem_vunmap_locked(&bo->base, &map); 479 ivpu_bo_unlock(bo); 480 } 481 482 drm_gem_object_put(&bo->base.base); 483 } 484 485 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 486 { 487 struct drm_ivpu_bo_info *args = data; 488 struct drm_gem_object *obj; 489 struct ivpu_bo *bo; 490 int ret = 0; 491 492 obj = drm_gem_object_lookup(file, args->handle); 493 if (!obj) 494 return -ENOENT; 495 496 bo = to_ivpu_bo(obj); 497 498 ivpu_bo_lock(bo); 499 args->flags = bo->flags; 500 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node); 501 args->vpu_addr = bo->vpu_addr; 502 args->size = obj->size; 503 ivpu_bo_unlock(bo); 504 505 drm_gem_object_put(obj); 506 return ret; 507 } 508 509 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 510 { 511 struct drm_ivpu_bo_wait *args = data; 512 struct drm_gem_object *obj; 513 unsigned long timeout; 514 long ret; 515 516 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 517 518 /* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */ 519 timeout += 1; 520 521 obj = drm_gem_object_lookup(file, args->handle); 522 if (!obj) 523 return -EINVAL; 524 525 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout); 526 if (ret == 0) { 527 ret = -ETIMEDOUT; 528 } else if (ret > 0) { 529 ret = 0; 530 args->job_status = to_ivpu_bo(obj)->job_status; 531 } 532 533 drm_gem_object_put(obj); 534 535 return ret; 536 } 537 538 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) 539 { 540 ivpu_bo_lock(bo); 541 542 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u", 543 bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size, 544 bo->flags, kref_read(&bo->base.base.refcount)); 545 546 if (bo->base.pages) 547 drm_printf(p, " has_pages"); 548 549 if (bo->mmu_mapped) 550 drm_printf(p, " mmu_mapped"); 551 552 if (drm_gem_is_imported(&bo->base.base)) 553 drm_printf(p, " imported"); 554 555 drm_printf(p, "\n"); 556 557 ivpu_bo_unlock(bo); 558 } 559 560 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) 561 { 562 struct ivpu_device *vdev = to_ivpu_device(dev); 563 struct ivpu_bo *bo; 564 565 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n", 566 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs"); 567 568 mutex_lock(&vdev->bo_list_lock); 569 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) 570 ivpu_bo_print_info(bo, p); 571 mutex_unlock(&vdev->bo_list_lock); 572 } 573 574 void ivpu_bo_list_print(struct drm_device *dev) 575 { 576 struct drm_printer p = drm_info_printer(dev->dev); 577 578 ivpu_bo_list(dev, &p); 579 } 580