1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/highmem.h> 8 #include <linux/module.h> 9 #include <linux/set_memory.h> 10 #include <linux/xarray.h> 11 12 #include <drm/drm_cache.h> 13 #include <drm/drm_debugfs.h> 14 #include <drm/drm_file.h> 15 #include <drm/drm_utils.h> 16 17 #include "ivpu_drv.h" 18 #include "ivpu_fw.h" 19 #include "ivpu_gem.h" 20 #include "ivpu_hw.h" 21 #include "ivpu_mmu.h" 22 #include "ivpu_mmu_context.h" 23 24 MODULE_IMPORT_NS("DMA_BUF"); 25 26 static const struct drm_gem_object_funcs ivpu_gem_funcs; 27 28 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) 29 { 30 ivpu_dbg(vdev, BO, 31 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n", 32 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx_id, 33 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, 34 (bool)drm_gem_is_imported(&bo->base.base)); 35 } 36 37 static inline int ivpu_bo_lock(struct ivpu_bo *bo) 38 { 39 return dma_resv_lock(bo->base.base.resv, NULL); 40 } 41 42 static inline void ivpu_bo_unlock(struct ivpu_bo *bo) 43 { 44 dma_resv_unlock(bo->base.base.resv); 45 } 46 47 /* 48 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU. 49 * 50 * This function pins physical memory pages, then maps the physical pages 51 * to IOMMU address space and finally updates the VPU MMU page tables 52 * to allow the VPU to translate VPU address to IOMMU address. 53 */ 54 int __must_check ivpu_bo_pin(struct ivpu_bo *bo) 55 { 56 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 57 struct sg_table *sgt; 58 int ret = 0; 59 60 ivpu_dbg_bo(vdev, bo, "pin"); 61 62 sgt = drm_gem_shmem_get_pages_sgt(&bo->base); 63 if (IS_ERR(sgt)) { 64 ret = PTR_ERR(sgt); 65 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); 66 return ret; 67 } 68 69 ivpu_bo_lock(bo); 70 71 if (!bo->mmu_mapped) { 72 drm_WARN_ON(&vdev->drm, !bo->ctx); 73 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt, 74 ivpu_bo_is_snooped(bo)); 75 if (ret) { 76 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret); 77 goto unlock; 78 } 79 bo->mmu_mapped = true; 80 } 81 82 unlock: 83 ivpu_bo_unlock(bo); 84 85 return ret; 86 } 87 88 static int 89 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, 90 const struct ivpu_addr_range *range) 91 { 92 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 93 int idx, ret; 94 95 if (!drm_dev_enter(&vdev->drm, &idx)) 96 return -ENODEV; 97 98 ivpu_bo_lock(bo); 99 100 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node); 101 if (!ret) { 102 bo->ctx = ctx; 103 bo->vpu_addr = bo->mm_node.start; 104 } else { 105 ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret); 106 } 107 108 ivpu_bo_unlock(bo); 109 110 drm_dev_exit(idx); 111 112 return ret; 113 } 114 115 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) 116 { 117 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 118 119 lockdep_assert(dma_resv_held(bo->base.base.resv) || !kref_read(&bo->base.base.refcount)); 120 121 if (bo->mmu_mapped) { 122 drm_WARN_ON(&vdev->drm, !bo->ctx); 123 drm_WARN_ON(&vdev->drm, !bo->vpu_addr); 124 drm_WARN_ON(&vdev->drm, !bo->base.sgt); 125 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt); 126 bo->mmu_mapped = false; 127 } 128 129 if (bo->ctx) { 130 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node); 131 bo->ctx = NULL; 132 } 133 134 if (drm_gem_is_imported(&bo->base.base)) 135 return; 136 137 if (bo->base.sgt) { 138 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); 139 sg_free_table(bo->base.sgt); 140 kfree(bo->base.sgt); 141 bo->base.sgt = NULL; 142 } 143 } 144 145 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 146 { 147 struct ivpu_bo *bo; 148 149 if (drm_WARN_ON(&vdev->drm, !ctx)) 150 return; 151 152 mutex_lock(&vdev->bo_list_lock); 153 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) { 154 ivpu_bo_lock(bo); 155 if (bo->ctx == ctx) { 156 ivpu_dbg_bo(vdev, bo, "unbind"); 157 ivpu_bo_unbind_locked(bo); 158 } 159 ivpu_bo_unlock(bo); 160 } 161 mutex_unlock(&vdev->bo_list_lock); 162 } 163 164 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size) 165 { 166 struct ivpu_bo *bo; 167 168 if (size == 0 || !PAGE_ALIGNED(size)) 169 return ERR_PTR(-EINVAL); 170 171 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 172 if (!bo) 173 return ERR_PTR(-ENOMEM); 174 175 bo->base.base.funcs = &ivpu_gem_funcs; 176 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */ 177 178 INIT_LIST_HEAD(&bo->bo_list_node); 179 180 return &bo->base.base; 181 } 182 183 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev, 184 struct dma_buf *dma_buf) 185 { 186 struct device *attach_dev = dev->dev; 187 struct dma_buf_attachment *attach; 188 struct sg_table *sgt; 189 struct drm_gem_object *obj; 190 int ret; 191 192 attach = dma_buf_attach(dma_buf, attach_dev); 193 if (IS_ERR(attach)) 194 return ERR_CAST(attach); 195 196 get_dma_buf(dma_buf); 197 198 sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL); 199 if (IS_ERR(sgt)) { 200 ret = PTR_ERR(sgt); 201 goto fail_detach; 202 } 203 204 obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt); 205 if (IS_ERR(obj)) { 206 ret = PTR_ERR(obj); 207 goto fail_unmap; 208 } 209 210 obj->import_attach = attach; 211 obj->resv = dma_buf->resv; 212 213 return obj; 214 215 fail_unmap: 216 dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL); 217 fail_detach: 218 dma_buf_detach(dma_buf, attach); 219 dma_buf_put(dma_buf); 220 221 return ERR_PTR(ret); 222 } 223 224 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags, u32 ctx_id) 225 { 226 struct drm_gem_shmem_object *shmem; 227 struct ivpu_bo *bo; 228 229 switch (flags & DRM_IVPU_BO_CACHE_MASK) { 230 case DRM_IVPU_BO_CACHED: 231 case DRM_IVPU_BO_WC: 232 break; 233 default: 234 return ERR_PTR(-EINVAL); 235 } 236 237 shmem = drm_gem_shmem_create(&vdev->drm, size); 238 if (IS_ERR(shmem)) 239 return ERR_CAST(shmem); 240 241 bo = to_ivpu_bo(&shmem->base); 242 bo->ctx_id = ctx_id; 243 bo->base.map_wc = flags & DRM_IVPU_BO_WC; 244 bo->flags = flags; 245 246 mutex_lock(&vdev->bo_list_lock); 247 list_add_tail(&bo->bo_list_node, &vdev->bo_list); 248 mutex_unlock(&vdev->bo_list_lock); 249 250 ivpu_dbg_bo(vdev, bo, "alloc"); 251 252 return bo; 253 } 254 255 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file) 256 { 257 struct ivpu_file_priv *file_priv = file->driver_priv; 258 struct ivpu_device *vdev = file_priv->vdev; 259 struct ivpu_bo *bo = to_ivpu_bo(obj); 260 struct ivpu_addr_range *range; 261 262 if (bo->ctx) { 263 ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n", 264 file_priv->ctx.id, bo->ctx->id); 265 return -EALREADY; 266 } 267 268 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) 269 range = &vdev->hw->ranges.shave; 270 else if (bo->flags & DRM_IVPU_BO_DMA_MEM) 271 range = &vdev->hw->ranges.dma; 272 else 273 range = &vdev->hw->ranges.user; 274 275 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range); 276 } 277 278 static void ivpu_gem_bo_free(struct drm_gem_object *obj) 279 { 280 struct ivpu_device *vdev = to_ivpu_device(obj->dev); 281 struct ivpu_bo *bo = to_ivpu_bo(obj); 282 283 ivpu_dbg_bo(vdev, bo, "free"); 284 285 mutex_lock(&vdev->bo_list_lock); 286 list_del(&bo->bo_list_node); 287 mutex_unlock(&vdev->bo_list_lock); 288 289 drm_WARN_ON(&vdev->drm, !drm_gem_is_imported(&bo->base.base) && 290 !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); 291 drm_WARN_ON(&vdev->drm, ivpu_bo_size(bo) == 0); 292 drm_WARN_ON(&vdev->drm, bo->base.vaddr); 293 294 ivpu_bo_unbind_locked(bo); 295 drm_WARN_ON(&vdev->drm, bo->mmu_mapped); 296 drm_WARN_ON(&vdev->drm, bo->ctx); 297 298 drm_WARN_ON(obj->dev, refcount_read(&bo->base.pages_use_count) > 1); 299 drm_gem_shmem_free(&bo->base); 300 } 301 302 static const struct drm_gem_object_funcs ivpu_gem_funcs = { 303 .free = ivpu_gem_bo_free, 304 .open = ivpu_gem_bo_open, 305 .print_info = drm_gem_shmem_object_print_info, 306 .pin = drm_gem_shmem_object_pin, 307 .unpin = drm_gem_shmem_object_unpin, 308 .get_sg_table = drm_gem_shmem_object_get_sg_table, 309 .vmap = drm_gem_shmem_object_vmap, 310 .vunmap = drm_gem_shmem_object_vunmap, 311 .mmap = drm_gem_shmem_object_mmap, 312 .vm_ops = &drm_gem_shmem_vm_ops, 313 }; 314 315 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 316 { 317 struct ivpu_file_priv *file_priv = file->driver_priv; 318 struct ivpu_device *vdev = file_priv->vdev; 319 struct drm_ivpu_bo_create *args = data; 320 u64 size = PAGE_ALIGN(args->size); 321 struct ivpu_bo *bo; 322 int ret; 323 324 if (args->flags & ~DRM_IVPU_BO_FLAGS) 325 return -EINVAL; 326 327 if (size == 0) 328 return -EINVAL; 329 330 bo = ivpu_bo_alloc(vdev, size, args->flags, file_priv->ctx.id); 331 if (IS_ERR(bo)) { 332 ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)", 333 bo, file_priv->ctx.id, args->size, args->flags); 334 return PTR_ERR(bo); 335 } 336 337 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 338 if (ret) 339 ivpu_err(vdev, "Failed to create handle for BO: %pe (ctx %u size %llu flags 0x%x)", 340 bo, file_priv->ctx.id, args->size, args->flags); 341 else 342 args->vpu_addr = bo->vpu_addr; 343 344 drm_gem_object_put(&bo->base.base); 345 346 return ret; 347 } 348 349 struct ivpu_bo * 350 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 351 struct ivpu_addr_range *range, u64 size, u32 flags) 352 { 353 struct iosys_map map; 354 struct ivpu_bo *bo; 355 int ret; 356 357 if (drm_WARN_ON(&vdev->drm, !range)) 358 return NULL; 359 360 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start)); 361 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end)); 362 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); 363 364 bo = ivpu_bo_alloc(vdev, size, flags, IVPU_GLOBAL_CONTEXT_MMU_SSID); 365 if (IS_ERR(bo)) { 366 ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)", 367 bo, range->start, size, flags); 368 return NULL; 369 } 370 371 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range); 372 if (ret) 373 goto err_put; 374 375 ret = ivpu_bo_pin(bo); 376 if (ret) 377 goto err_put; 378 379 if (flags & DRM_IVPU_BO_MAPPABLE) { 380 ivpu_bo_lock(bo); 381 ret = drm_gem_shmem_vmap_locked(&bo->base, &map); 382 ivpu_bo_unlock(bo); 383 384 if (ret) 385 goto err_put; 386 } 387 388 return bo; 389 390 err_put: 391 drm_gem_object_put(&bo->base.base); 392 return NULL; 393 } 394 395 struct ivpu_bo *ivpu_bo_create_runtime(struct ivpu_device *vdev, u64 addr, u64 size, u32 flags) 396 { 397 struct ivpu_addr_range range; 398 399 if (!ivpu_is_within_range(addr, size, &vdev->hw->ranges.runtime)) { 400 ivpu_err(vdev, "Invalid runtime BO address 0x%llx size %llu\n", addr, size); 401 return NULL; 402 } 403 404 if (ivpu_hw_range_init(vdev, &range, addr, size)) 405 return NULL; 406 407 return ivpu_bo_create(vdev, &vdev->gctx, &range, size, flags); 408 } 409 410 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags) 411 { 412 return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags); 413 } 414 415 void ivpu_bo_free(struct ivpu_bo *bo) 416 { 417 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr); 418 419 if (bo->flags & DRM_IVPU_BO_MAPPABLE) { 420 ivpu_bo_lock(bo); 421 drm_gem_shmem_vunmap_locked(&bo->base, &map); 422 ivpu_bo_unlock(bo); 423 } 424 425 drm_gem_object_put(&bo->base.base); 426 } 427 428 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 429 { 430 struct drm_ivpu_bo_info *args = data; 431 struct drm_gem_object *obj; 432 struct ivpu_bo *bo; 433 int ret = 0; 434 435 obj = drm_gem_object_lookup(file, args->handle); 436 if (!obj) 437 return -ENOENT; 438 439 bo = to_ivpu_bo(obj); 440 441 ivpu_bo_lock(bo); 442 args->flags = bo->flags; 443 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node); 444 args->vpu_addr = bo->vpu_addr; 445 args->size = obj->size; 446 ivpu_bo_unlock(bo); 447 448 drm_gem_object_put(obj); 449 return ret; 450 } 451 452 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 453 { 454 struct drm_ivpu_bo_wait *args = data; 455 struct drm_gem_object *obj; 456 unsigned long timeout; 457 long ret; 458 459 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 460 461 /* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */ 462 timeout += 1; 463 464 obj = drm_gem_object_lookup(file, args->handle); 465 if (!obj) 466 return -EINVAL; 467 468 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout); 469 if (ret == 0) { 470 ret = -ETIMEDOUT; 471 } else if (ret > 0) { 472 ret = 0; 473 args->job_status = to_ivpu_bo(obj)->job_status; 474 } 475 476 drm_gem_object_put(obj); 477 478 return ret; 479 } 480 481 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) 482 { 483 ivpu_bo_lock(bo); 484 485 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u", 486 bo, bo->ctx_id, bo->vpu_addr, bo->base.base.size, 487 bo->flags, kref_read(&bo->base.base.refcount)); 488 489 if (bo->base.pages) 490 drm_printf(p, " has_pages"); 491 492 if (bo->mmu_mapped) 493 drm_printf(p, " mmu_mapped"); 494 495 if (drm_gem_is_imported(&bo->base.base)) 496 drm_printf(p, " imported"); 497 498 drm_printf(p, "\n"); 499 500 ivpu_bo_unlock(bo); 501 } 502 503 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) 504 { 505 struct ivpu_device *vdev = to_ivpu_device(dev); 506 struct ivpu_bo *bo; 507 508 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n", 509 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs"); 510 511 mutex_lock(&vdev->bo_list_lock); 512 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) 513 ivpu_bo_print_info(bo, p); 514 mutex_unlock(&vdev->bo_list_lock); 515 } 516 517 void ivpu_bo_list_print(struct drm_device *dev) 518 { 519 struct drm_printer p = drm_info_printer(dev->dev); 520 521 ivpu_bo_list(dev, &p); 522 } 523