1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020-2023 Intel Corporation 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/highmem.h> 8 #include <linux/module.h> 9 #include <linux/set_memory.h> 10 #include <linux/xarray.h> 11 12 #include <drm/drm_cache.h> 13 #include <drm/drm_debugfs.h> 14 #include <drm/drm_file.h> 15 #include <drm/drm_utils.h> 16 17 #include "ivpu_drv.h" 18 #include "ivpu_gem.h" 19 #include "ivpu_hw.h" 20 #include "ivpu_mmu.h" 21 #include "ivpu_mmu_context.h" 22 23 static const struct drm_gem_object_funcs ivpu_gem_funcs; 24 25 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action) 26 { 27 ivpu_dbg(vdev, BO, 28 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n", 29 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx ? bo->ctx->id : 0, 30 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc, 31 (bool)bo->base.base.import_attach); 32 } 33 34 /* 35 * ivpu_bo_pin() - pin the backing physical pages and map them to VPU. 36 * 37 * This function pins physical memory pages, then maps the physical pages 38 * to IOMMU address space and finally updates the VPU MMU page tables 39 * to allow the VPU to translate VPU address to IOMMU address. 40 */ 41 int __must_check ivpu_bo_pin(struct ivpu_bo *bo) 42 { 43 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 44 int ret = 0; 45 46 mutex_lock(&bo->lock); 47 48 ivpu_dbg_bo(vdev, bo, "pin"); 49 drm_WARN_ON(&vdev->drm, !bo->ctx); 50 51 if (!bo->mmu_mapped) { 52 struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base); 53 54 if (IS_ERR(sgt)) { 55 ret = PTR_ERR(sgt); 56 ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret); 57 goto unlock; 58 } 59 60 ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt, 61 ivpu_bo_is_snooped(bo)); 62 if (ret) { 63 ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret); 64 goto unlock; 65 } 66 bo->mmu_mapped = true; 67 } 68 69 unlock: 70 mutex_unlock(&bo->lock); 71 72 return ret; 73 } 74 75 static int 76 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx, 77 const struct ivpu_addr_range *range) 78 { 79 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 80 int idx, ret; 81 82 if (!drm_dev_enter(&vdev->drm, &idx)) 83 return -ENODEV; 84 85 mutex_lock(&bo->lock); 86 87 ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node); 88 if (!ret) { 89 bo->ctx = ctx; 90 bo->vpu_addr = bo->mm_node.start; 91 } else { 92 ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret); 93 } 94 95 ivpu_dbg_bo(vdev, bo, "alloc"); 96 97 mutex_unlock(&bo->lock); 98 99 drm_dev_exit(idx); 100 101 return ret; 102 } 103 104 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo) 105 { 106 struct ivpu_device *vdev = ivpu_bo_to_vdev(bo); 107 108 lockdep_assert(lockdep_is_held(&bo->lock) || !kref_read(&bo->base.base.refcount)); 109 110 if (bo->mmu_mapped) { 111 drm_WARN_ON(&vdev->drm, !bo->ctx); 112 drm_WARN_ON(&vdev->drm, !bo->vpu_addr); 113 drm_WARN_ON(&vdev->drm, !bo->base.sgt); 114 ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt); 115 bo->mmu_mapped = false; 116 } 117 118 if (bo->ctx) { 119 ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node); 120 bo->ctx = NULL; 121 } 122 123 if (bo->base.base.import_attach) 124 return; 125 126 dma_resv_lock(bo->base.base.resv, NULL); 127 if (bo->base.sgt) { 128 dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); 129 sg_free_table(bo->base.sgt); 130 kfree(bo->base.sgt); 131 bo->base.sgt = NULL; 132 } 133 dma_resv_unlock(bo->base.base.resv); 134 } 135 136 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx) 137 { 138 struct ivpu_bo *bo; 139 140 if (drm_WARN_ON(&vdev->drm, !ctx)) 141 return; 142 143 mutex_lock(&vdev->bo_list_lock); 144 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) { 145 mutex_lock(&bo->lock); 146 if (bo->ctx == ctx) { 147 ivpu_dbg_bo(vdev, bo, "unbind"); 148 ivpu_bo_unbind_locked(bo); 149 } 150 mutex_unlock(&bo->lock); 151 } 152 mutex_unlock(&vdev->bo_list_lock); 153 } 154 155 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size) 156 { 157 struct ivpu_bo *bo; 158 159 if (size == 0 || !PAGE_ALIGNED(size)) 160 return ERR_PTR(-EINVAL); 161 162 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 163 if (!bo) 164 return ERR_PTR(-ENOMEM); 165 166 bo->base.base.funcs = &ivpu_gem_funcs; 167 bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */ 168 169 INIT_LIST_HEAD(&bo->bo_list_node); 170 mutex_init(&bo->lock); 171 172 return &bo->base.base; 173 } 174 175 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags) 176 { 177 struct drm_gem_shmem_object *shmem; 178 struct ivpu_bo *bo; 179 180 switch (flags & DRM_IVPU_BO_CACHE_MASK) { 181 case DRM_IVPU_BO_CACHED: 182 case DRM_IVPU_BO_WC: 183 break; 184 default: 185 return ERR_PTR(-EINVAL); 186 } 187 188 shmem = drm_gem_shmem_create(&vdev->drm, size); 189 if (IS_ERR(shmem)) 190 return ERR_CAST(shmem); 191 192 bo = to_ivpu_bo(&shmem->base); 193 bo->base.map_wc = flags & DRM_IVPU_BO_WC; 194 bo->flags = flags; 195 196 mutex_lock(&vdev->bo_list_lock); 197 list_add_tail(&bo->bo_list_node, &vdev->bo_list); 198 mutex_unlock(&vdev->bo_list_lock); 199 200 return bo; 201 } 202 203 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file) 204 { 205 struct ivpu_file_priv *file_priv = file->driver_priv; 206 struct ivpu_device *vdev = file_priv->vdev; 207 struct ivpu_bo *bo = to_ivpu_bo(obj); 208 struct ivpu_addr_range *range; 209 210 if (bo->ctx) { 211 ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n", 212 file_priv->ctx.id, bo->ctx->id); 213 return -EALREADY; 214 } 215 216 if (bo->flags & DRM_IVPU_BO_SHAVE_MEM) 217 range = &vdev->hw->ranges.shave; 218 else if (bo->flags & DRM_IVPU_BO_DMA_MEM) 219 range = &vdev->hw->ranges.dma; 220 else 221 range = &vdev->hw->ranges.user; 222 223 return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range); 224 } 225 226 static void ivpu_gem_bo_free(struct drm_gem_object *obj) 227 { 228 struct ivpu_device *vdev = to_ivpu_device(obj->dev); 229 struct ivpu_bo *bo = to_ivpu_bo(obj); 230 231 ivpu_dbg_bo(vdev, bo, "free"); 232 233 mutex_lock(&vdev->bo_list_lock); 234 list_del(&bo->bo_list_node); 235 mutex_unlock(&vdev->bo_list_lock); 236 237 drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ)); 238 239 ivpu_bo_unbind_locked(bo); 240 mutex_destroy(&bo->lock); 241 242 drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1); 243 drm_gem_shmem_free(&bo->base); 244 } 245 246 static const struct drm_gem_object_funcs ivpu_gem_funcs = { 247 .free = ivpu_gem_bo_free, 248 .open = ivpu_gem_bo_open, 249 .print_info = drm_gem_shmem_object_print_info, 250 .pin = drm_gem_shmem_object_pin, 251 .unpin = drm_gem_shmem_object_unpin, 252 .get_sg_table = drm_gem_shmem_object_get_sg_table, 253 .vmap = drm_gem_shmem_object_vmap, 254 .vunmap = drm_gem_shmem_object_vunmap, 255 .mmap = drm_gem_shmem_object_mmap, 256 .vm_ops = &drm_gem_shmem_vm_ops, 257 }; 258 259 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 260 { 261 struct ivpu_file_priv *file_priv = file->driver_priv; 262 struct ivpu_device *vdev = file_priv->vdev; 263 struct drm_ivpu_bo_create *args = data; 264 u64 size = PAGE_ALIGN(args->size); 265 struct ivpu_bo *bo; 266 int ret; 267 268 if (args->flags & ~DRM_IVPU_BO_FLAGS) 269 return -EINVAL; 270 271 if (size == 0) 272 return -EINVAL; 273 274 bo = ivpu_bo_alloc(vdev, size, args->flags); 275 if (IS_ERR(bo)) { 276 ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)", 277 bo, file_priv->ctx.id, args->size, args->flags); 278 return PTR_ERR(bo); 279 } 280 281 ret = drm_gem_handle_create(file, &bo->base.base, &args->handle); 282 if (!ret) 283 args->vpu_addr = bo->vpu_addr; 284 285 drm_gem_object_put(&bo->base.base); 286 287 return ret; 288 } 289 290 struct ivpu_bo * 291 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx, 292 struct ivpu_addr_range *range, u64 size, u32 flags) 293 { 294 struct iosys_map map; 295 struct ivpu_bo *bo; 296 int ret; 297 298 if (drm_WARN_ON(&vdev->drm, !range)) 299 return NULL; 300 301 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start)); 302 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end)); 303 drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size)); 304 305 bo = ivpu_bo_alloc(vdev, size, flags); 306 if (IS_ERR(bo)) { 307 ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)", 308 bo, range->start, size, flags); 309 return NULL; 310 } 311 312 ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range); 313 if (ret) 314 goto err_put; 315 316 ret = ivpu_bo_pin(bo); 317 if (ret) 318 goto err_put; 319 320 if (flags & DRM_IVPU_BO_MAPPABLE) { 321 dma_resv_lock(bo->base.base.resv, NULL); 322 ret = drm_gem_shmem_vmap(&bo->base, &map); 323 dma_resv_unlock(bo->base.base.resv); 324 325 if (ret) 326 goto err_put; 327 } 328 329 return bo; 330 331 err_put: 332 drm_gem_object_put(&bo->base.base); 333 return NULL; 334 } 335 336 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags) 337 { 338 return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags); 339 } 340 341 void ivpu_bo_free(struct ivpu_bo *bo) 342 { 343 struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr); 344 345 if (bo->flags & DRM_IVPU_BO_MAPPABLE) { 346 dma_resv_lock(bo->base.base.resv, NULL); 347 drm_gem_shmem_vunmap(&bo->base, &map); 348 dma_resv_unlock(bo->base.base.resv); 349 } 350 351 drm_gem_object_put(&bo->base.base); 352 } 353 354 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 355 { 356 struct drm_ivpu_bo_info *args = data; 357 struct drm_gem_object *obj; 358 struct ivpu_bo *bo; 359 int ret = 0; 360 361 obj = drm_gem_object_lookup(file, args->handle); 362 if (!obj) 363 return -ENOENT; 364 365 bo = to_ivpu_bo(obj); 366 367 mutex_lock(&bo->lock); 368 args->flags = bo->flags; 369 args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node); 370 args->vpu_addr = bo->vpu_addr; 371 args->size = obj->size; 372 mutex_unlock(&bo->lock); 373 374 drm_gem_object_put(obj); 375 return ret; 376 } 377 378 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 379 { 380 struct drm_ivpu_bo_wait *args = data; 381 struct drm_gem_object *obj; 382 unsigned long timeout; 383 long ret; 384 385 timeout = drm_timeout_abs_to_jiffies(args->timeout_ns); 386 387 obj = drm_gem_object_lookup(file, args->handle); 388 if (!obj) 389 return -EINVAL; 390 391 ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout); 392 if (ret == 0) { 393 ret = -ETIMEDOUT; 394 } else if (ret > 0) { 395 ret = 0; 396 args->job_status = to_ivpu_bo(obj)->job_status; 397 } 398 399 drm_gem_object_put(obj); 400 401 return ret; 402 } 403 404 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p) 405 { 406 mutex_lock(&bo->lock); 407 408 drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u", 409 bo, bo->ctx->id, bo->vpu_addr, bo->base.base.size, 410 bo->flags, kref_read(&bo->base.base.refcount)); 411 412 if (bo->base.pages) 413 drm_printf(p, " has_pages"); 414 415 if (bo->mmu_mapped) 416 drm_printf(p, " mmu_mapped"); 417 418 if (bo->base.base.import_attach) 419 drm_printf(p, " imported"); 420 421 drm_printf(p, "\n"); 422 423 mutex_unlock(&bo->lock); 424 } 425 426 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p) 427 { 428 struct ivpu_device *vdev = to_ivpu_device(dev); 429 struct ivpu_bo *bo; 430 431 drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n", 432 "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs"); 433 434 mutex_lock(&vdev->bo_list_lock); 435 list_for_each_entry(bo, &vdev->bo_list, bo_list_node) 436 ivpu_bo_print_info(bo, p); 437 mutex_unlock(&vdev->bo_list_lock); 438 } 439 440 void ivpu_bo_list_print(struct drm_device *dev) 441 { 442 struct drm_printer p = drm_info_printer(dev->dev); 443 444 ivpu_bo_list(dev, &p); 445 } 446