1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */ 3 4 #include <linux/mm.h> 5 #include <linux/iosys-map.h> 6 #include <linux/sync_file.h> 7 #include <linux/pagemap.h> 8 #include <linux/shmem_fs.h> 9 #include <linux/dma-mapping.h> 10 11 #include <drm/drm_file.h> 12 #include <drm/drm_syncobj.h> 13 #include <drm/drm_utils.h> 14 15 #include <drm/lima_drm.h> 16 17 #include "lima_drv.h" 18 #include "lima_gem.h" 19 #include "lima_vm.h" 20 21 int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) 22 { 23 struct page **pages; 24 struct address_space *mapping = bo->base.base.filp->f_mapping; 25 struct device *dev = bo->base.base.dev->dev; 26 size_t old_size = bo->heap_size; 27 size_t new_size = bo->heap_size ? bo->heap_size * 2 : 28 (lima_heap_init_nr_pages << PAGE_SHIFT); 29 struct sg_table sgt; 30 int i, ret; 31 32 if (bo->heap_size >= bo->base.base.size) 33 return -ENOSPC; 34 35 new_size = min(new_size, bo->base.base.size); 36 37 mutex_lock(&bo->base.pages_lock); 38 39 if (bo->base.pages) { 40 pages = bo->base.pages; 41 } else { 42 pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, 43 sizeof(*pages), GFP_KERNEL | __GFP_ZERO); 44 if (!pages) { 45 mutex_unlock(&bo->base.pages_lock); 46 return -ENOMEM; 47 } 48 49 bo->base.pages = pages; 50 bo->base.pages_use_count = 1; 51 52 mapping_set_unevictable(mapping); 53 } 54 55 for (i = old_size >> PAGE_SHIFT; i < new_size >> PAGE_SHIFT; i++) { 56 struct page *page = shmem_read_mapping_page(mapping, i); 57 58 if (IS_ERR(page)) { 59 mutex_unlock(&bo->base.pages_lock); 60 return PTR_ERR(page); 61 } 62 pages[i] = page; 63 } 64 65 mutex_unlock(&bo->base.pages_lock); 66 67 ret = sg_alloc_table_from_pages(&sgt, pages, i, 0, 68 new_size, GFP_KERNEL); 69 if (ret) 70 return ret; 71 72 if (bo->base.sgt) { 73 dma_unmap_sgtable(dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0); 74 sg_free_table(bo->base.sgt); 75 } else { 76 bo->base.sgt = kmalloc(sizeof(*bo->base.sgt), GFP_KERNEL); 77 if (!bo->base.sgt) { 78 sg_free_table(&sgt); 79 return -ENOMEM; 80 } 81 } 82 83 ret = dma_map_sgtable(dev, &sgt, DMA_BIDIRECTIONAL, 0); 84 if (ret) { 85 sg_free_table(&sgt); 86 kfree(bo->base.sgt); 87 bo->base.sgt = NULL; 88 return ret; 89 } 90 91 *bo->base.sgt = sgt; 92 93 if (vm) { 94 ret = lima_vm_map_bo(vm, bo, old_size >> PAGE_SHIFT); 95 if (ret) 96 return ret; 97 } 98 99 bo->heap_size = new_size; 100 return 0; 101 } 102 103 int lima_gem_create_handle(struct drm_device *dev, struct drm_file *file, 104 u32 size, u32 flags, u32 *handle) 105 { 106 int err; 107 gfp_t mask; 108 struct drm_gem_shmem_object *shmem; 109 struct drm_gem_object *obj; 110 struct lima_bo *bo; 111 bool is_heap = flags & LIMA_BO_FLAG_HEAP; 112 113 shmem = drm_gem_shmem_create(dev, size); 114 if (IS_ERR(shmem)) 115 return PTR_ERR(shmem); 116 117 obj = &shmem->base; 118 119 /* Mali Utgard GPU can only support 32bit address space */ 120 mask = mapping_gfp_mask(obj->filp->f_mapping); 121 mask &= ~__GFP_HIGHMEM; 122 mask |= __GFP_DMA32; 123 mapping_set_gfp_mask(obj->filp->f_mapping, mask); 124 125 if (is_heap) { 126 bo = to_lima_bo(obj); 127 err = lima_heap_alloc(bo, NULL); 128 if (err) 129 goto out; 130 } else { 131 struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(shmem); 132 133 if (IS_ERR(sgt)) { 134 err = PTR_ERR(sgt); 135 goto out; 136 } 137 } 138 139 err = drm_gem_handle_create(file, obj, handle); 140 141 out: 142 /* drop reference from allocate - handle holds it now */ 143 drm_gem_object_put(obj); 144 145 return err; 146 } 147 148 static void lima_gem_free_object(struct drm_gem_object *obj) 149 { 150 struct lima_bo *bo = to_lima_bo(obj); 151 152 if (!list_empty(&bo->va)) 153 dev_err(obj->dev->dev, "lima gem free bo still has va\n"); 154 155 drm_gem_shmem_free(&bo->base); 156 } 157 158 static int lima_gem_object_open(struct drm_gem_object *obj, struct drm_file *file) 159 { 160 struct lima_bo *bo = to_lima_bo(obj); 161 struct lima_drm_priv *priv = to_lima_drm_priv(file); 162 struct lima_vm *vm = priv->vm; 163 164 return lima_vm_bo_add(vm, bo, true); 165 } 166 167 static void lima_gem_object_close(struct drm_gem_object *obj, struct drm_file *file) 168 { 169 struct lima_bo *bo = to_lima_bo(obj); 170 struct lima_drm_priv *priv = to_lima_drm_priv(file); 171 struct lima_vm *vm = priv->vm; 172 173 lima_vm_bo_del(vm, bo); 174 } 175 176 static int lima_gem_pin(struct drm_gem_object *obj) 177 { 178 struct lima_bo *bo = to_lima_bo(obj); 179 180 if (bo->heap_size) 181 return -EINVAL; 182 183 return drm_gem_shmem_pin(&bo->base); 184 } 185 186 static int lima_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map) 187 { 188 struct lima_bo *bo = to_lima_bo(obj); 189 190 if (bo->heap_size) 191 return -EINVAL; 192 193 return drm_gem_shmem_vmap(&bo->base, map); 194 } 195 196 static int lima_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 197 { 198 struct lima_bo *bo = to_lima_bo(obj); 199 200 if (bo->heap_size) 201 return -EINVAL; 202 203 return drm_gem_shmem_mmap(&bo->base, vma); 204 } 205 206 static const struct drm_gem_object_funcs lima_gem_funcs = { 207 .free = lima_gem_free_object, 208 .open = lima_gem_object_open, 209 .close = lima_gem_object_close, 210 .print_info = drm_gem_shmem_object_print_info, 211 .pin = lima_gem_pin, 212 .unpin = drm_gem_shmem_object_unpin, 213 .get_sg_table = drm_gem_shmem_object_get_sg_table, 214 .vmap = lima_gem_vmap, 215 .vunmap = drm_gem_shmem_object_vunmap, 216 .mmap = lima_gem_mmap, 217 .vm_ops = &drm_gem_shmem_vm_ops, 218 }; 219 220 struct drm_gem_object *lima_gem_create_object(struct drm_device *dev, size_t size) 221 { 222 struct lima_bo *bo; 223 224 bo = kzalloc(sizeof(*bo), GFP_KERNEL); 225 if (!bo) 226 return ERR_PTR(-ENOMEM); 227 228 mutex_init(&bo->lock); 229 INIT_LIST_HEAD(&bo->va); 230 bo->base.map_wc = true; 231 bo->base.base.funcs = &lima_gem_funcs; 232 233 return &bo->base.base; 234 } 235 236 int lima_gem_get_info(struct drm_file *file, u32 handle, u32 *va, u64 *offset) 237 { 238 struct drm_gem_object *obj; 239 struct lima_bo *bo; 240 struct lima_drm_priv *priv = to_lima_drm_priv(file); 241 struct lima_vm *vm = priv->vm; 242 243 obj = drm_gem_object_lookup(file, handle); 244 if (!obj) 245 return -ENOENT; 246 247 bo = to_lima_bo(obj); 248 249 *va = lima_vm_get_va(vm, bo); 250 251 *offset = drm_vma_node_offset_addr(&obj->vma_node); 252 253 drm_gem_object_put(obj); 254 return 0; 255 } 256 257 static int lima_gem_sync_bo(struct lima_sched_task *task, struct lima_bo *bo, 258 bool write, bool explicit) 259 { 260 int err; 261 262 err = dma_resv_reserve_fences(lima_bo_resv(bo), 1); 263 if (err) 264 return err; 265 266 /* explicit sync use user passed dep fence */ 267 if (explicit) 268 return 0; 269 270 return drm_sched_job_add_implicit_dependencies(&task->base, 271 &bo->base.base, 272 write); 273 } 274 275 static int lima_gem_add_deps(struct drm_file *file, struct lima_submit *submit) 276 { 277 int i, err; 278 279 for (i = 0; i < ARRAY_SIZE(submit->in_sync); i++) { 280 struct dma_fence *fence = NULL; 281 282 if (!submit->in_sync[i]) 283 continue; 284 285 err = drm_syncobj_find_fence(file, submit->in_sync[i], 286 0, 0, &fence); 287 if (err) 288 return err; 289 290 err = drm_sched_job_add_dependency(&submit->task->base, fence); 291 if (err) { 292 dma_fence_put(fence); 293 return err; 294 } 295 } 296 297 return 0; 298 } 299 300 int lima_gem_submit(struct drm_file *file, struct lima_submit *submit) 301 { 302 int i, err = 0; 303 struct ww_acquire_ctx ctx; 304 struct lima_drm_priv *priv = to_lima_drm_priv(file); 305 struct lima_vm *vm = priv->vm; 306 struct drm_syncobj *out_sync = NULL; 307 struct dma_fence *fence; 308 struct lima_bo **bos = submit->lbos; 309 310 if (submit->out_sync) { 311 out_sync = drm_syncobj_find(file, submit->out_sync); 312 if (!out_sync) 313 return -ENOENT; 314 } 315 316 for (i = 0; i < submit->nr_bos; i++) { 317 struct drm_gem_object *obj; 318 struct lima_bo *bo; 319 320 obj = drm_gem_object_lookup(file, submit->bos[i].handle); 321 if (!obj) { 322 err = -ENOENT; 323 goto err_out0; 324 } 325 326 bo = to_lima_bo(obj); 327 328 /* increase refcnt of gpu va map to prevent unmapped when executing, 329 * will be decreased when task done 330 */ 331 err = lima_vm_bo_add(vm, bo, false); 332 if (err) { 333 drm_gem_object_put(obj); 334 goto err_out0; 335 } 336 337 bos[i] = bo; 338 } 339 340 err = drm_gem_lock_reservations((struct drm_gem_object **)bos, 341 submit->nr_bos, &ctx); 342 if (err) 343 goto err_out0; 344 345 err = lima_sched_task_init( 346 submit->task, submit->ctx->context + submit->pipe, 347 bos, submit->nr_bos, vm); 348 if (err) 349 goto err_out1; 350 351 err = lima_gem_add_deps(file, submit); 352 if (err) 353 goto err_out2; 354 355 for (i = 0; i < submit->nr_bos; i++) { 356 err = lima_gem_sync_bo( 357 submit->task, bos[i], 358 submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE, 359 submit->flags & LIMA_SUBMIT_FLAG_EXPLICIT_FENCE); 360 if (err) 361 goto err_out2; 362 } 363 364 fence = lima_sched_context_queue_task(submit->task); 365 366 for (i = 0; i < submit->nr_bos; i++) { 367 dma_resv_add_fence(lima_bo_resv(bos[i]), fence, 368 submit->bos[i].flags & LIMA_SUBMIT_BO_WRITE ? 369 DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ); 370 } 371 372 drm_gem_unlock_reservations((struct drm_gem_object **)bos, 373 submit->nr_bos, &ctx); 374 375 for (i = 0; i < submit->nr_bos; i++) 376 drm_gem_object_put(&bos[i]->base.base); 377 378 if (out_sync) { 379 drm_syncobj_replace_fence(out_sync, fence); 380 drm_syncobj_put(out_sync); 381 } 382 383 dma_fence_put(fence); 384 385 return 0; 386 387 err_out2: 388 lima_sched_task_fini(submit->task); 389 err_out1: 390 drm_gem_unlock_reservations((struct drm_gem_object **)bos, 391 submit->nr_bos, &ctx); 392 err_out0: 393 for (i = 0; i < submit->nr_bos; i++) { 394 if (!bos[i]) 395 break; 396 lima_vm_bo_del(vm, bos[i]); 397 drm_gem_object_put(&bos[i]->base.base); 398 } 399 if (out_sync) 400 drm_syncobj_put(out_sync); 401 return err; 402 } 403 404 int lima_gem_wait(struct drm_file *file, u32 handle, u32 op, s64 timeout_ns) 405 { 406 bool write = op & LIMA_GEM_WAIT_WRITE; 407 long ret, timeout; 408 409 if (!op) 410 return 0; 411 412 timeout = drm_timeout_abs_to_jiffies(timeout_ns); 413 414 ret = drm_gem_dma_resv_wait(file, handle, write, timeout); 415 if (ret == -ETIME) 416 ret = timeout ? -ETIMEDOUT : -EBUSY; 417 418 return ret; 419 } 420