1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015 Etnaviv Project 4 */ 5 6 #include <drm/drm_file.h> 7 #include <drm/drm_print.h> 8 #include <linux/dma-fence-array.h> 9 #include <linux/file.h> 10 #include <linux/dma-resv.h> 11 #include <linux/sync_file.h> 12 #include <linux/uaccess.h> 13 #include <linux/vmalloc.h> 14 15 #include "etnaviv_cmdbuf.h" 16 #include "etnaviv_drv.h" 17 #include "etnaviv_gpu.h" 18 #include "etnaviv_gem.h" 19 #include "etnaviv_perfmon.h" 20 #include "etnaviv_sched.h" 21 22 /* 23 * Cmdstream submission: 24 */ 25 26 #define BO_INVALID_FLAGS ~(ETNA_SUBMIT_BO_READ | ETNA_SUBMIT_BO_WRITE) 27 /* make sure these don't conflict w/ ETNAVIV_SUBMIT_BO_x */ 28 #define BO_LOCKED 0x4000 29 #define BO_PINNED 0x2000 30 31 static struct etnaviv_gem_submit *submit_create(struct drm_device *dev, 32 struct etnaviv_gpu *gpu, size_t nr_bos, size_t nr_pmrs) 33 { 34 struct etnaviv_gem_submit *submit; 35 36 submit = kzalloc_flex(*submit, bos, nr_bos); 37 if (!submit) 38 return NULL; 39 40 submit->pmrs = kzalloc_objs(struct etnaviv_perfmon_request, nr_pmrs); 41 if (!submit->pmrs) { 42 kfree(submit); 43 return NULL; 44 } 45 submit->nr_pmrs = nr_pmrs; 46 47 submit->gpu = gpu; 48 kref_init(&submit->refcount); 49 50 return submit; 51 } 52 53 static int submit_lookup_objects(struct etnaviv_gem_submit *submit, 54 struct drm_file *file, struct drm_etnaviv_gem_submit_bo *submit_bos, 55 unsigned nr_bos) 56 { 57 struct drm_etnaviv_gem_submit_bo *bo; 58 unsigned i; 59 int ret = 0; 60 61 spin_lock(&file->table_lock); 62 63 for (i = 0, bo = submit_bos; i < nr_bos; i++, bo++) { 64 struct drm_gem_object *obj; 65 66 if (bo->flags & BO_INVALID_FLAGS) { 67 DRM_ERROR("invalid flags: %x\n", bo->flags); 68 ret = -EINVAL; 69 goto out_unlock; 70 } 71 72 submit->bos[i].flags = bo->flags; 73 if (submit->flags & ETNA_SUBMIT_SOFTPIN) { 74 if (bo->presumed < ETNAVIV_SOFTPIN_START_ADDRESS) { 75 DRM_ERROR("invalid softpin address\n"); 76 ret = -EINVAL; 77 goto out_unlock; 78 } 79 submit->bos[i].va = bo->presumed; 80 } 81 82 /* normally use drm_gem_object_lookup(), but for bulk lookup 83 * all under single table_lock just hit object_idr directly: 84 */ 85 obj = idr_find(&file->object_idr, bo->handle); 86 if (!obj) { 87 DRM_ERROR("invalid handle %u at index %u\n", 88 bo->handle, i); 89 ret = -EINVAL; 90 goto out_unlock; 91 } 92 93 /* 94 * Take a refcount on the object. The file table lock 95 * prevents the object_idr's refcount on this being dropped. 96 */ 97 drm_gem_object_get(obj); 98 99 submit->bos[i].obj = to_etnaviv_bo(obj); 100 } 101 102 out_unlock: 103 submit->nr_bos = i; 104 spin_unlock(&file->table_lock); 105 106 return ret; 107 } 108 109 static void submit_unlock_object(struct etnaviv_gem_submit *submit, int i) 110 { 111 if (submit->bos[i].flags & BO_LOCKED) { 112 struct drm_gem_object *obj = &submit->bos[i].obj->base; 113 114 dma_resv_unlock(obj->resv); 115 submit->bos[i].flags &= ~BO_LOCKED; 116 } 117 } 118 119 static int submit_lock_objects(struct etnaviv_gem_submit *submit, 120 struct ww_acquire_ctx *ticket) 121 { 122 int contended, slow_locked = -1, i, ret = 0; 123 124 retry: 125 for (i = 0; i < submit->nr_bos; i++) { 126 struct drm_gem_object *obj = &submit->bos[i].obj->base; 127 128 if (slow_locked == i) 129 slow_locked = -1; 130 131 contended = i; 132 133 if (!(submit->bos[i].flags & BO_LOCKED)) { 134 ret = dma_resv_lock_interruptible(obj->resv, ticket); 135 if (ret == -EALREADY) 136 DRM_ERROR("BO at index %u already on submit list\n", 137 i); 138 if (ret) 139 goto fail; 140 submit->bos[i].flags |= BO_LOCKED; 141 } 142 } 143 144 ww_acquire_done(ticket); 145 146 return 0; 147 148 fail: 149 for (; i >= 0; i--) 150 submit_unlock_object(submit, i); 151 152 if (slow_locked > 0) 153 submit_unlock_object(submit, slow_locked); 154 155 if (ret == -EDEADLK) { 156 struct drm_gem_object *obj; 157 158 obj = &submit->bos[contended].obj->base; 159 160 /* we lost out in a seqno race, lock and retry.. */ 161 ret = dma_resv_lock_slow_interruptible(obj->resv, ticket); 162 if (!ret) { 163 submit->bos[contended].flags |= BO_LOCKED; 164 slow_locked = contended; 165 goto retry; 166 } 167 } 168 169 return ret; 170 } 171 172 static int submit_fence_sync(struct etnaviv_gem_submit *submit) 173 { 174 int i, ret = 0; 175 176 for (i = 0; i < submit->nr_bos; i++) { 177 struct etnaviv_gem_submit_bo *bo = &submit->bos[i]; 178 struct dma_resv *robj = bo->obj->base.resv; 179 180 ret = dma_resv_reserve_fences(robj, 1); 181 if (ret) 182 return ret; 183 184 if (submit->flags & ETNA_SUBMIT_NO_IMPLICIT) 185 continue; 186 187 ret = drm_sched_job_add_implicit_dependencies(&submit->sched_job, 188 &bo->obj->base, 189 bo->flags & ETNA_SUBMIT_BO_WRITE); 190 if (ret) 191 return ret; 192 } 193 194 return ret; 195 } 196 197 static void submit_attach_object_fences(struct etnaviv_gem_submit *submit) 198 { 199 int i; 200 201 for (i = 0; i < submit->nr_bos; i++) { 202 struct drm_gem_object *obj = &submit->bos[i].obj->base; 203 bool write = submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE; 204 205 dma_resv_add_fence(obj->resv, submit->out_fence, write ? 206 DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ); 207 submit_unlock_object(submit, i); 208 } 209 } 210 211 static int submit_pin_objects(struct etnaviv_gem_submit *submit) 212 { 213 int i, ret = 0; 214 215 for (i = 0; i < submit->nr_bos; i++) { 216 struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; 217 struct etnaviv_vram_mapping *mapping; 218 219 mapping = etnaviv_gem_mapping_get(&etnaviv_obj->base, 220 submit->mmu_context, 221 submit->bos[i].va); 222 if (IS_ERR(mapping)) { 223 ret = PTR_ERR(mapping); 224 break; 225 } 226 227 if ((submit->flags & ETNA_SUBMIT_SOFTPIN) && 228 submit->bos[i].va != mapping->iova) { 229 etnaviv_gem_mapping_unreference(mapping); 230 return -EINVAL; 231 } 232 233 atomic_inc(&etnaviv_obj->gpu_active); 234 235 submit->bos[i].flags |= BO_PINNED; 236 submit->bos[i].mapping = mapping; 237 } 238 239 return ret; 240 } 241 242 static int submit_bo(struct etnaviv_gem_submit *submit, u32 idx, 243 struct etnaviv_gem_submit_bo **bo) 244 { 245 if (idx >= submit->nr_bos) { 246 DRM_ERROR("invalid buffer index: %u (out of %u)\n", 247 idx, submit->nr_bos); 248 return -EINVAL; 249 } 250 251 *bo = &submit->bos[idx]; 252 253 return 0; 254 } 255 256 /* process the reloc's and patch up the cmdstream as needed: */ 257 static int submit_reloc(struct etnaviv_gem_submit *submit, void *stream, 258 u32 size, const struct drm_etnaviv_gem_submit_reloc *relocs, 259 u32 nr_relocs) 260 { 261 u32 i, last_offset = 0; 262 u32 *ptr = stream; 263 int ret; 264 265 /* Submits using softpin don't blend with relocs */ 266 if ((submit->flags & ETNA_SUBMIT_SOFTPIN) && nr_relocs != 0) 267 return -EINVAL; 268 269 for (i = 0; i < nr_relocs; i++) { 270 const struct drm_etnaviv_gem_submit_reloc *r = relocs + i; 271 struct etnaviv_gem_submit_bo *bo; 272 u32 off; 273 274 if (unlikely(r->flags)) { 275 DRM_ERROR("invalid reloc flags\n"); 276 return -EINVAL; 277 } 278 279 if (r->submit_offset % 4) { 280 DRM_ERROR("non-aligned reloc offset: %u\n", 281 r->submit_offset); 282 return -EINVAL; 283 } 284 285 /* offset in dwords: */ 286 off = r->submit_offset / 4; 287 288 if ((off >= size ) || 289 (off < last_offset)) { 290 DRM_ERROR("invalid offset %u at reloc %u\n", off, i); 291 return -EINVAL; 292 } 293 294 ret = submit_bo(submit, r->reloc_idx, &bo); 295 if (ret) 296 return ret; 297 298 if (r->reloc_offset > bo->obj->base.size - sizeof(*ptr)) { 299 DRM_ERROR("relocation %u outside object\n", i); 300 return -EINVAL; 301 } 302 303 ptr[off] = bo->mapping->iova + r->reloc_offset; 304 305 last_offset = off; 306 } 307 308 return 0; 309 } 310 311 static int submit_perfmon_validate(struct etnaviv_gem_submit *submit, 312 u32 exec_state, const struct drm_etnaviv_gem_submit_pmr *pmrs) 313 { 314 u32 i; 315 316 for (i = 0; i < submit->nr_pmrs; i++) { 317 const struct drm_etnaviv_gem_submit_pmr *r = pmrs + i; 318 struct etnaviv_gem_submit_bo *bo; 319 int ret; 320 321 ret = submit_bo(submit, r->read_idx, &bo); 322 if (ret) 323 return ret; 324 325 /* at offset 0 a sequence number gets stored used for userspace sync */ 326 if (r->read_offset == 0) { 327 DRM_ERROR("perfmon request: offset is 0"); 328 return -EINVAL; 329 } 330 331 if (r->read_offset >= bo->obj->base.size - sizeof(u32)) { 332 DRM_ERROR("perfmon request: offset %u outside object", i); 333 return -EINVAL; 334 } 335 336 if (r->flags & ~(ETNA_PM_PROCESS_PRE | ETNA_PM_PROCESS_POST)) { 337 DRM_ERROR("perfmon request: flags are not valid"); 338 return -EINVAL; 339 } 340 341 if (etnaviv_pm_req_validate(r, exec_state)) { 342 DRM_ERROR("perfmon request: domain or signal not valid"); 343 return -EINVAL; 344 } 345 346 submit->pmrs[i].flags = r->flags; 347 submit->pmrs[i].domain = r->domain; 348 submit->pmrs[i].signal = r->signal; 349 submit->pmrs[i].sequence = r->sequence; 350 submit->pmrs[i].offset = r->read_offset; 351 submit->pmrs[i].bo_vma = etnaviv_gem_vmap(&bo->obj->base); 352 } 353 354 return 0; 355 } 356 357 static void submit_cleanup(struct kref *kref) 358 { 359 struct etnaviv_gem_submit *submit = 360 container_of(kref, struct etnaviv_gem_submit, refcount); 361 unsigned i; 362 363 if (submit->cmdbuf.suballoc) 364 etnaviv_cmdbuf_free(&submit->cmdbuf); 365 366 if (submit->mmu_context) 367 etnaviv_iommu_context_put(submit->mmu_context); 368 369 if (submit->prev_mmu_context) 370 etnaviv_iommu_context_put(submit->prev_mmu_context); 371 372 for (i = 0; i < submit->nr_bos; i++) { 373 struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; 374 375 /* unpin all objects */ 376 if (submit->bos[i].flags & BO_PINNED) { 377 etnaviv_gem_mapping_unreference(submit->bos[i].mapping); 378 atomic_dec(&etnaviv_obj->gpu_active); 379 submit->bos[i].mapping = NULL; 380 submit->bos[i].flags &= ~BO_PINNED; 381 } 382 383 /* if the GPU submit failed, objects might still be locked */ 384 submit_unlock_object(submit, i); 385 drm_gem_object_put(&etnaviv_obj->base); 386 } 387 388 wake_up_all(&submit->gpu->fence_event); 389 390 if (submit->out_fence) { 391 /* 392 * Remove from user fence array before dropping the reference, 393 * so fence can not be found in lookup anymore. 394 */ 395 xa_erase(&submit->gpu->user_fences, submit->out_fence_id); 396 dma_fence_put(submit->out_fence); 397 } 398 399 put_pid(submit->pid); 400 401 kfree(submit->pmrs); 402 kfree(submit); 403 } 404 405 void etnaviv_submit_put(struct etnaviv_gem_submit *submit) 406 { 407 kref_put(&submit->refcount, submit_cleanup); 408 } 409 410 int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, 411 struct drm_file *file) 412 { 413 struct etnaviv_file_private *ctx = file->driver_priv; 414 struct etnaviv_drm_private *priv = dev->dev_private; 415 struct drm_etnaviv_gem_submit *args = data; 416 struct drm_etnaviv_gem_submit_reloc *relocs; 417 struct drm_etnaviv_gem_submit_pmr *pmrs; 418 struct drm_etnaviv_gem_submit_bo *bos; 419 struct etnaviv_gem_submit *submit; 420 struct etnaviv_gpu *gpu; 421 struct sync_file *sync_file = NULL; 422 struct ww_acquire_ctx ticket; 423 int out_fence_fd = -1; 424 struct pid *pid = get_pid(task_pid(current)); 425 void *stream; 426 int ret; 427 428 if (args->pipe >= ETNA_MAX_PIPES) 429 return -EINVAL; 430 431 gpu = priv->gpu[args->pipe]; 432 if (!gpu) 433 return -ENXIO; 434 435 if (args->stream_size % 4) { 436 DRM_ERROR("non-aligned cmdstream buffer size: %u\n", 437 args->stream_size); 438 return -EINVAL; 439 } 440 441 if (args->exec_state != ETNA_PIPE_3D && 442 args->exec_state != ETNA_PIPE_2D && 443 args->exec_state != ETNA_PIPE_VG) { 444 DRM_ERROR("invalid exec_state: 0x%x\n", args->exec_state); 445 return -EINVAL; 446 } 447 448 if (args->flags & ~ETNA_SUBMIT_FLAGS) { 449 DRM_ERROR("invalid flags: 0x%x\n", args->flags); 450 return -EINVAL; 451 } 452 453 if ((args->flags & ETNA_SUBMIT_SOFTPIN) && 454 priv->mmu_global->version != ETNAVIV_IOMMU_V2) { 455 DRM_ERROR("softpin requested on incompatible MMU\n"); 456 return -EINVAL; 457 } 458 459 if (args->stream_size > SZ_128K || args->nr_relocs > SZ_128K || 460 args->nr_bos > SZ_128K || args->nr_pmrs > 128) { 461 DRM_ERROR("submit arguments out of size limits\n"); 462 return -EINVAL; 463 } 464 465 /* 466 * Copy the command submission and bo array to kernel space in 467 * one go, and do this outside of any locks. 468 */ 469 bos = kvmalloc_objs(*bos, args->nr_bos); 470 relocs = kvmalloc_objs(*relocs, args->nr_relocs); 471 pmrs = kvmalloc_objs(*pmrs, args->nr_pmrs); 472 stream = kvmalloc_array(1, args->stream_size, GFP_KERNEL); 473 if (!bos || !relocs || !pmrs || !stream) { 474 ret = -ENOMEM; 475 goto err_submit_cmds; 476 } 477 478 ret = copy_from_user(bos, u64_to_user_ptr(args->bos), 479 args->nr_bos * sizeof(*bos)); 480 if (ret) { 481 ret = -EFAULT; 482 goto err_submit_cmds; 483 } 484 485 ret = copy_from_user(relocs, u64_to_user_ptr(args->relocs), 486 args->nr_relocs * sizeof(*relocs)); 487 if (ret) { 488 ret = -EFAULT; 489 goto err_submit_cmds; 490 } 491 492 ret = copy_from_user(pmrs, u64_to_user_ptr(args->pmrs), 493 args->nr_pmrs * sizeof(*pmrs)); 494 if (ret) { 495 ret = -EFAULT; 496 goto err_submit_cmds; 497 } 498 499 ret = copy_from_user(stream, u64_to_user_ptr(args->stream), 500 args->stream_size); 501 if (ret) { 502 ret = -EFAULT; 503 goto err_submit_cmds; 504 } 505 506 if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { 507 out_fence_fd = get_unused_fd_flags(O_CLOEXEC); 508 if (out_fence_fd < 0) { 509 ret = out_fence_fd; 510 goto err_submit_cmds; 511 } 512 } 513 514 ww_acquire_init(&ticket, &reservation_ww_class); 515 516 submit = submit_create(dev, gpu, args->nr_bos, args->nr_pmrs); 517 if (!submit) { 518 ret = -ENOMEM; 519 goto err_submit_ww_acquire; 520 } 521 522 submit->pid = pid; 523 524 ret = etnaviv_cmdbuf_init(priv->cmdbuf_suballoc, &submit->cmdbuf, 525 ALIGN(args->stream_size, 8) + 8); 526 if (ret) 527 goto err_submit_put; 528 529 submit->ctx = file->driver_priv; 530 submit->mmu_context = etnaviv_iommu_context_get(submit->ctx->mmu); 531 submit->exec_state = args->exec_state; 532 submit->flags = args->flags; 533 534 ret = drm_sched_job_init(&submit->sched_job, 535 &ctx->sched_entity[args->pipe], 536 1, submit->ctx, file->client_id); 537 if (ret) 538 goto err_submit_put; 539 540 ret = submit_lookup_objects(submit, file, bos, args->nr_bos); 541 if (ret) 542 goto err_submit_job; 543 544 if ((priv->mmu_global->version != ETNAVIV_IOMMU_V2) && 545 !etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4, 546 relocs, args->nr_relocs)) { 547 ret = -EINVAL; 548 goto err_submit_job; 549 } 550 551 if (args->flags & ETNA_SUBMIT_FENCE_FD_IN) { 552 struct dma_fence *in_fence = sync_file_get_fence(args->fence_fd); 553 if (!in_fence) { 554 ret = -EINVAL; 555 goto err_submit_job; 556 } 557 558 ret = drm_sched_job_add_dependency(&submit->sched_job, 559 in_fence); 560 if (ret) 561 goto err_submit_job; 562 } 563 564 ret = submit_pin_objects(submit); 565 if (ret) 566 goto err_submit_job; 567 568 ret = submit_reloc(submit, stream, args->stream_size / 4, 569 relocs, args->nr_relocs); 570 if (ret) 571 goto err_submit_job; 572 573 ret = submit_perfmon_validate(submit, args->exec_state, pmrs); 574 if (ret) 575 goto err_submit_job; 576 577 memcpy(submit->cmdbuf.vaddr, stream, args->stream_size); 578 579 ret = submit_lock_objects(submit, &ticket); 580 if (ret) 581 goto err_submit_job; 582 583 ret = submit_fence_sync(submit); 584 if (ret) 585 goto err_submit_job; 586 587 ret = etnaviv_sched_push_job(submit); 588 if (ret) 589 goto err_submit_job; 590 591 submit_attach_object_fences(submit); 592 593 if (args->flags & ETNA_SUBMIT_FENCE_FD_OUT) { 594 /* 595 * This can be improved: ideally we want to allocate the sync 596 * file before kicking off the GPU job and just attach the 597 * fence to the sync file here, eliminating the ENOMEM 598 * possibility at this stage. 599 */ 600 sync_file = sync_file_create(submit->out_fence); 601 if (!sync_file) { 602 ret = -ENOMEM; 603 /* 604 * When this late error is hit, the submit has already 605 * been handed over to the scheduler. At this point 606 * the sched_job must not be cleaned up. 607 */ 608 goto err_submit_put; 609 } 610 fd_install(out_fence_fd, sync_file->file); 611 } 612 613 args->fence_fd = out_fence_fd; 614 args->fence = submit->out_fence_id; 615 616 err_submit_job: 617 if (ret) 618 drm_sched_job_cleanup(&submit->sched_job); 619 err_submit_put: 620 etnaviv_submit_put(submit); 621 622 err_submit_ww_acquire: 623 ww_acquire_fini(&ticket); 624 625 err_submit_cmds: 626 if (ret && (out_fence_fd >= 0)) 627 put_unused_fd(out_fence_fd); 628 kvfree(stream); 629 kvfree(bos); 630 kvfree(relocs); 631 kvfree(pmrs); 632 633 return ret; 634 } 635