1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2020 NVIDIA Corporation */ 3 4 #include <linux/dma-fence-unwrap.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/file.h> 7 #include <linux/host1x.h> 8 #include <linux/iommu.h> 9 #include <linux/kref.h> 10 #include <linux/list.h> 11 #include <linux/nospec.h> 12 #include <linux/pm_runtime.h> 13 #include <linux/scatterlist.h> 14 #include <linux/slab.h> 15 16 #include <drm/drm_drv.h> 17 #include <drm/drm_file.h> 18 #include <drm/drm_syncobj.h> 19 20 #include "drm.h" 21 #include "gem.h" 22 #include "submit.h" 23 #include "uapi.h" 24 25 #define SUBMIT_ERR(context, fmt, ...) \ 26 dev_err_ratelimited(context->client->base.dev, \ 27 "%s: job submission failed: " fmt "\n", \ 28 current->comm, ##__VA_ARGS__) 29 30 struct gather_bo { 31 struct host1x_bo base; 32 33 struct kref ref; 34 35 struct device *dev; 36 u32 *gather_data; 37 dma_addr_t gather_data_dma; 38 size_t gather_data_words; 39 }; 40 41 static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo) 42 { 43 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 44 45 kref_get(&bo->ref); 46 47 return host_bo; 48 } 49 50 static void gather_bo_release(struct kref *ref) 51 { 52 struct gather_bo *bo = container_of(ref, struct gather_bo, ref); 53 54 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma, 55 0); 56 kfree(bo); 57 } 58 59 static void gather_bo_put(struct host1x_bo *host_bo) 60 { 61 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 62 63 kref_put(&bo->ref, gather_bo_release); 64 } 65 66 static struct host1x_bo_mapping * 67 gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) 68 { 69 struct gather_bo *gather = container_of(bo, struct gather_bo, base); 70 struct host1x_bo_mapping *map; 71 int err; 72 73 map = kzalloc_obj(*map); 74 if (!map) 75 return ERR_PTR(-ENOMEM); 76 77 kref_init(&map->ref); 78 map->bo = bo; 79 map->direction = direction; 80 map->dev = dev; 81 82 map->sgt = kzalloc_obj(*map->sgt); 83 if (!map->sgt) { 84 err = -ENOMEM; 85 goto free; 86 } 87 88 err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, 89 gather->gather_data_words * 4); 90 if (err) 91 goto free_sgt; 92 93 err = dma_map_sgtable(dev, map->sgt, direction, 0); 94 if (err) 95 goto free_sgt; 96 97 map->phys = sg_dma_address(map->sgt->sgl); 98 map->size = gather->gather_data_words * 4; 99 map->chunks = err; 100 101 return map; 102 103 free_sgt: 104 sg_free_table(map->sgt); 105 kfree(map->sgt); 106 free: 107 kfree(map); 108 return ERR_PTR(err); 109 } 110 111 static void gather_bo_unpin(struct host1x_bo_mapping *map) 112 { 113 if (!map) 114 return; 115 116 dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); 117 sg_free_table(map->sgt); 118 kfree(map->sgt); 119 120 kfree(map); 121 } 122 123 static void *gather_bo_mmap(struct host1x_bo *host_bo) 124 { 125 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 126 127 return bo->gather_data; 128 } 129 130 static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr) 131 { 132 } 133 134 static const struct host1x_bo_ops gather_bo_ops = { 135 .get = gather_bo_get, 136 .put = gather_bo_put, 137 .pin = gather_bo_pin, 138 .unpin = gather_bo_unpin, 139 .mmap = gather_bo_mmap, 140 .munmap = gather_bo_munmap, 141 }; 142 143 static struct tegra_drm_mapping * 144 tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id) 145 { 146 struct tegra_drm_mapping *mapping; 147 148 xa_lock(&context->mappings); 149 150 mapping = xa_load(&context->mappings, id); 151 if (mapping) 152 kref_get(&mapping->ref); 153 154 xa_unlock(&context->mappings); 155 156 return mapping; 157 } 158 159 static void *alloc_copy_user_array(void __user *from, size_t count, size_t size) 160 { 161 size_t copy_len; 162 void *data; 163 164 if (check_mul_overflow(count, size, ©_len)) 165 return ERR_PTR(-EINVAL); 166 167 if (copy_len > 0x4000) 168 return ERR_PTR(-E2BIG); 169 170 data = vmemdup_user(from, copy_len); 171 if (IS_ERR(data)) 172 return ERR_CAST(data); 173 174 return data; 175 } 176 177 static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev, 178 struct tegra_drm_context *context, 179 struct drm_tegra_channel_submit *args) 180 { 181 struct gather_bo *bo; 182 size_t copy_len; 183 184 if (args->gather_data_words == 0) { 185 SUBMIT_ERR(context, "gather_data_words cannot be zero"); 186 return -EINVAL; 187 } 188 189 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) { 190 SUBMIT_ERR(context, "gather_data_words is too large"); 191 return -EINVAL; 192 } 193 194 bo = kzalloc_obj(*bo); 195 if (!bo) { 196 SUBMIT_ERR(context, "failed to allocate memory for bo info"); 197 return -ENOMEM; 198 } 199 200 host1x_bo_init(&bo->base, &gather_bo_ops); 201 kref_init(&bo->ref); 202 bo->dev = dev; 203 204 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma, 205 GFP_KERNEL | __GFP_NOWARN, 0); 206 if (!bo->gather_data) { 207 SUBMIT_ERR(context, "failed to allocate memory for gather data"); 208 kfree(bo); 209 return -ENOMEM; 210 } 211 212 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) { 213 SUBMIT_ERR(context, "failed to copy gather data from userspace"); 214 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0); 215 kfree(bo); 216 return -EFAULT; 217 } 218 219 bo->gather_data_words = args->gather_data_words; 220 221 *pbo = bo; 222 223 return 0; 224 } 225 226 static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo, 227 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping) 228 { 229 /* TODO check that target_offset is within bounds */ 230 dma_addr_t iova = buf->reloc.target_offset; 231 u32 written_ptr; 232 233 if (mapping->bo_map) 234 iova += mapping->iova; 235 else 236 iova += mapping->ctx_map->mapping->phys; 237 238 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 239 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) 240 iova |= BIT_ULL(39); 241 #endif 242 243 written_ptr = iova >> buf->reloc.shift; 244 245 if (buf->reloc.gather_offset_words >= bo->gather_data_words) { 246 SUBMIT_ERR(context, 247 "relocation has too large gather offset (%u vs gather length %zu)", 248 buf->reloc.gather_offset_words, bo->gather_data_words); 249 return -EINVAL; 250 } 251 252 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words, 253 bo->gather_data_words); 254 255 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr; 256 257 return 0; 258 } 259 260 static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo, 261 struct drm_tegra_channel_submit *args, 262 struct tegra_drm_submit_data *job_data) 263 { 264 struct tegra_drm_used_mapping *mappings; 265 struct drm_tegra_submit_buf *bufs; 266 int err; 267 u32 i; 268 269 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs, 270 sizeof(*bufs)); 271 if (IS_ERR(bufs)) { 272 SUBMIT_ERR(context, "failed to copy bufs array from userspace"); 273 return PTR_ERR(bufs); 274 } 275 276 mappings = kzalloc_objs(*mappings, args->num_bufs); 277 if (!mappings) { 278 SUBMIT_ERR(context, "failed to allocate memory for mapping info"); 279 err = -ENOMEM; 280 goto done; 281 } 282 283 for (i = 0; i < args->num_bufs; i++) { 284 struct drm_tegra_submit_buf *buf = &bufs[i]; 285 struct tegra_drm_mapping *mapping; 286 287 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) { 288 SUBMIT_ERR(context, "invalid flag specified for buffer"); 289 err = -EINVAL; 290 goto drop_refs; 291 } 292 293 mapping = tegra_drm_mapping_get(context, buf->mapping); 294 if (!mapping) { 295 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping); 296 err = -EINVAL; 297 goto drop_refs; 298 } 299 300 err = submit_write_reloc(context, bo, buf, mapping); 301 if (err) { 302 tegra_drm_mapping_put(mapping); 303 goto drop_refs; 304 } 305 306 mappings[i].mapping = mapping; 307 mappings[i].flags = buf->flags; 308 } 309 310 job_data->used_mappings = mappings; 311 job_data->num_used_mappings = i; 312 313 err = 0; 314 315 goto done; 316 317 drop_refs: 318 while (i--) 319 tegra_drm_mapping_put(mappings[i].mapping); 320 321 kfree(mappings); 322 job_data->used_mappings = NULL; 323 324 done: 325 kvfree(bufs); 326 327 return err; 328 } 329 330 static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job, 331 struct xarray *syncpoints, struct drm_tegra_channel_submit *args) 332 { 333 struct host1x_syncpt *sp; 334 335 if (args->syncpt.flags) { 336 SUBMIT_ERR(context, "invalid flag specified for syncpt"); 337 return -EINVAL; 338 } 339 340 /* Syncpt ref will be dropped on job release */ 341 sp = xa_load(syncpoints, args->syncpt.id); 342 if (!sp) { 343 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated"); 344 return -EINVAL; 345 } 346 347 job->syncpt = host1x_syncpt_get(sp); 348 job->syncpt_incrs = args->syncpt.increments; 349 350 return 0; 351 } 352 353 static int submit_handle_in_fence(struct tegra_drm_context *context, struct host1x *host1x, 354 struct dma_fence *fence, u32 *num_waits) 355 { 356 struct dma_fence_unwrap iter; 357 struct host1x *fence_host1x; 358 struct dma_fence *f; 359 int err = 0; 360 361 *num_waits = 0; 362 363 dma_fence_unwrap_for_each(f, &iter, fence) { 364 long wait_err; 365 366 if (err) 367 continue; 368 369 if (dma_fence_is_signaled(f)) 370 continue; 371 372 err = host1x_fence_extract(f, &fence_host1x, NULL, NULL); 373 if (!err && fence_host1x == host1x) { 374 (*num_waits)++; 375 continue; 376 } 377 378 wait_err = dma_fence_wait_timeout(f, true, msecs_to_jiffies(10000)); 379 if (wait_err == 0) { 380 SUBMIT_ERR(context, "wait for syncobj_in timed out"); 381 err = -ETIMEDOUT; 382 continue; 383 } else if (wait_err < 0) { 384 /* In practice, -ERESTARTSYS */ 385 err = wait_err; 386 continue; 387 } else { 388 err = 0; 389 } 390 } 391 392 return err; 393 } 394 395 static void submit_job_add_prefence(struct host1x_job *job, struct dma_fence *fence, u32 class) 396 { 397 struct dma_fence_unwrap iter; 398 struct dma_fence *f; 399 400 dma_fence_unwrap_for_each(f, &iter, fence) { 401 u32 id, threshold; 402 403 if (dma_fence_is_signaled(f)) 404 continue; 405 406 if (host1x_fence_extract(f, NULL, &id, &threshold)) { 407 /* 408 * The fence may have signalled since the check above, and 409 * the dma_fence framework then detached its ops. 410 */ 411 continue; 412 } 413 414 host1x_job_add_wait(job, id, threshold, false, class); 415 } 416 } 417 418 static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context, 419 struct drm_tegra_submit_cmd_gather_uptr *cmd, 420 struct gather_bo *bo, u32 *offset, 421 struct tegra_drm_submit_data *job_data, 422 u32 *class) 423 { 424 u32 next_offset; 425 426 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) { 427 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command"); 428 return -EINVAL; 429 } 430 431 /* Check for maximum gather size */ 432 if (cmd->words > 16383) { 433 SUBMIT_ERR(context, "too many words in GATHER_UPTR command"); 434 return -EINVAL; 435 } 436 437 if (check_add_overflow(*offset, cmd->words, &next_offset)) { 438 SUBMIT_ERR(context, "too many total words in job"); 439 return -EINVAL; 440 } 441 442 if (next_offset > bo->gather_data_words) { 443 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data"); 444 return -EINVAL; 445 } 446 447 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset, 448 cmd->words, job_data, class)) { 449 SUBMIT_ERR(context, "job was rejected by firewall"); 450 return -EINVAL; 451 } 452 453 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4); 454 455 *offset = next_offset; 456 457 return 0; 458 } 459 460 static struct host1x_job * 461 submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo, 462 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data, 463 struct xarray *syncpoints, struct dma_fence *in_fence, u32 num_prefence_waits) 464 { 465 struct drm_tegra_submit_cmd *cmds; 466 u32 i, gather_offset = 0, class; 467 struct host1x_job *job; 468 int err; 469 470 /* Set initial class for firewall. */ 471 class = context->client->base.class; 472 473 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds, 474 sizeof(*cmds)); 475 if (IS_ERR(cmds)) { 476 SUBMIT_ERR(context, "failed to copy cmds array from userspace"); 477 return ERR_CAST(cmds); 478 } 479 480 job = host1x_job_alloc(context->channel, args->num_cmds + num_prefence_waits, 0, true); 481 if (!job) { 482 SUBMIT_ERR(context, "failed to allocate memory for job"); 483 job = ERR_PTR(-ENOMEM); 484 goto done; 485 } 486 487 err = submit_get_syncpt(context, job, syncpoints, args); 488 if (err < 0) 489 goto free_job; 490 491 job->client = &context->client->base; 492 job->class = context->client->base.class; 493 job->serialize = true; 494 495 if (in_fence) 496 submit_job_add_prefence(job, in_fence, class); 497 498 for (i = 0; i < args->num_cmds; i++) { 499 struct drm_tegra_submit_cmd *cmd = &cmds[i]; 500 501 if (cmd->flags) { 502 SUBMIT_ERR(context, "unknown flags given for cmd"); 503 err = -EINVAL; 504 goto free_job; 505 } 506 507 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) { 508 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo, 509 &gather_offset, job_data, &class); 510 if (err) 511 goto free_job; 512 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) { 513 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 514 SUBMIT_ERR(context, "non-zero reserved value"); 515 err = -EINVAL; 516 goto free_job; 517 } 518 519 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 520 false, class); 521 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) { 522 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 523 SUBMIT_ERR(context, "non-zero reserved value"); 524 err = -EINVAL; 525 goto free_job; 526 } 527 528 if (cmd->wait_syncpt.id != args->syncpt.id) { 529 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job"); 530 err = -EINVAL; 531 goto free_job; 532 } 533 534 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 535 true, class); 536 } else { 537 SUBMIT_ERR(context, "unknown cmd type"); 538 err = -EINVAL; 539 goto free_job; 540 } 541 } 542 543 if (gather_offset == 0) { 544 SUBMIT_ERR(context, "job must have at least one gather"); 545 err = -EINVAL; 546 goto free_job; 547 } 548 549 goto done; 550 551 free_job: 552 host1x_job_put(job); 553 job = ERR_PTR(err); 554 555 done: 556 kvfree(cmds); 557 558 return job; 559 } 560 561 static void release_job(struct host1x_job *job) 562 { 563 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base); 564 struct tegra_drm_submit_data *job_data = job->user_data; 565 u32 i; 566 567 for (i = 0; i < job_data->num_used_mappings; i++) 568 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 569 570 if (job->memory_context) { 571 host1x_memory_context_inactive(job->memory_context); 572 host1x_memory_context_put(job->memory_context); 573 } 574 575 kfree(job_data->used_mappings); 576 kfree(job_data); 577 578 pm_runtime_mark_last_busy(client->base.dev); 579 pm_runtime_put_autosuspend(client->base.dev); 580 } 581 582 int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, 583 struct drm_file *file) 584 { 585 struct host1x *host1x = dev_get_drvdata(drm->dev->parent); 586 struct host1x_memory_context *active_memctx = NULL; 587 struct tegra_drm_file *fpriv = file->driver_priv; 588 struct drm_tegra_channel_submit *args = data; 589 struct tegra_drm_submit_data *job_data; 590 struct drm_syncobj *syncobj = NULL; 591 struct dma_fence *in_fence = NULL; 592 struct tegra_drm_context *context; 593 u32 num_prefence_waits = 0; 594 struct host1x_job *job; 595 struct gather_bo *bo; 596 u32 i; 597 int err; 598 599 mutex_lock(&fpriv->lock); 600 601 context = xa_load(&fpriv->contexts, args->context); 602 if (!context) { 603 mutex_unlock(&fpriv->lock); 604 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__, 605 current->comm, args->context); 606 return -EINVAL; 607 } 608 609 if (context->memory_context) { 610 err = host1x_memory_context_active(context->memory_context); 611 if (err) { 612 mutex_unlock(&fpriv->lock); 613 SUBMIT_ERR(context, "failed to activate memory context"); 614 return err; 615 } 616 617 active_memctx = context->memory_context; 618 } 619 620 if (args->syncobj_in) { 621 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &in_fence); 622 if (err) { 623 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in); 624 goto unlock; 625 } 626 627 err = submit_handle_in_fence(context, host1x, in_fence, &num_prefence_waits); 628 if (err) 629 goto put_in_fence; 630 } 631 632 if (args->syncobj_out) { 633 syncobj = drm_syncobj_find(file, args->syncobj_out); 634 if (!syncobj) { 635 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out); 636 err = -ENOENT; 637 goto put_in_fence; 638 } 639 } 640 641 /* Allocate gather BO and copy gather words in. */ 642 err = submit_copy_gather_data(&bo, drm->dev, context, args); 643 if (err) 644 goto put_in_fence; 645 646 job_data = kzalloc_obj(*job_data); 647 if (!job_data) { 648 SUBMIT_ERR(context, "failed to allocate memory for job data"); 649 err = -ENOMEM; 650 goto put_bo; 651 } 652 653 /* Get data buffer mappings and do relocation patching. */ 654 err = submit_process_bufs(context, bo, args, job_data); 655 if (err) 656 goto free_job_data; 657 658 /* Allocate host1x_job and add gathers and waits to it. */ 659 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints, in_fence, 660 num_prefence_waits); 661 if (IS_ERR(job)) { 662 err = PTR_ERR(job); 663 goto free_job_data; 664 } 665 666 /* 667 * The prefence has been recorded as plain syncpoint waits, so the fence 668 * itself is not needed anymore. 669 */ 670 dma_fence_put(in_fence); 671 in_fence = NULL; 672 673 /* Map gather data for Host1x. */ 674 err = host1x_job_pin(job, context->client->base.dev); 675 if (err) { 676 SUBMIT_ERR(context, "failed to pin job: %d", err); 677 goto put_job; 678 } 679 680 if (context->client->ops->get_streamid_offset) { 681 err = context->client->ops->get_streamid_offset( 682 context->client, &job->engine_streamid_offset); 683 if (err) { 684 SUBMIT_ERR(context, "failed to get streamid offset: %d", err); 685 goto unpin_job; 686 } 687 } 688 689 if (context->memory_context && context->client->ops->can_use_memory_ctx) { 690 bool supported; 691 692 err = context->client->ops->can_use_memory_ctx(context->client, &supported); 693 if (err) { 694 SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d", err); 695 goto unpin_job; 696 } 697 698 if (supported) { 699 job->memory_context = active_memctx; 700 active_memctx = NULL; 701 host1x_memory_context_get(job->memory_context); 702 } 703 } else if (context->client->ops->get_streamid_offset) { 704 /* 705 * Job submission will need to temporarily change stream ID, 706 * so need to tell it what to change it back to. 707 */ 708 if (!tegra_dev_iommu_get_stream_id(context->client->base.dev, 709 &job->engine_fallback_streamid)) 710 job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS; 711 } 712 713 /* Boot engine. */ 714 err = pm_runtime_resume_and_get(context->client->base.dev); 715 if (err < 0) { 716 SUBMIT_ERR(context, "could not power up engine: %d", err); 717 goto put_memory_context; 718 } 719 720 job->user_data = job_data; 721 job->release = release_job; 722 job->timeout = 10000; 723 724 /* 725 * job_data is now part of job reference counting, so don't release 726 * it from here. 727 */ 728 job_data = NULL; 729 730 /* Submit job to hardware. */ 731 err = host1x_job_submit(job); 732 if (err) { 733 SUBMIT_ERR(context, "host1x job submission failed: %d", err); 734 goto unpin_job; 735 } 736 737 /* Return postfences to userspace and add fences to DMA reservations. */ 738 args->syncpt.value = job->syncpt_end; 739 740 if (syncobj) { 741 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end, true); 742 if (IS_ERR(fence)) { 743 err = PTR_ERR(fence); 744 SUBMIT_ERR(context, "failed to create postfence: %d", err); 745 goto put_job; 746 } 747 748 drm_syncobj_replace_fence(syncobj, fence); 749 dma_fence_put(fence); 750 } 751 752 goto put_job; 753 754 put_memory_context: 755 if (job->memory_context) { 756 host1x_memory_context_inactive(job->memory_context); 757 host1x_memory_context_put(job->memory_context); 758 } 759 unpin_job: 760 host1x_job_unpin(job); 761 put_job: 762 host1x_job_put(job); 763 free_job_data: 764 if (job_data && job_data->used_mappings) { 765 for (i = 0; i < job_data->num_used_mappings; i++) 766 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 767 768 kfree(job_data->used_mappings); 769 } 770 771 kfree(job_data); 772 put_bo: 773 gather_bo_put(&bo->base); 774 put_in_fence: 775 dma_fence_put(in_fence); 776 unlock: 777 if (syncobj) 778 drm_syncobj_put(syncobj); 779 if (active_memctx) 780 host1x_memory_context_inactive(active_memctx); 781 782 mutex_unlock(&fpriv->lock); 783 return err; 784 } 785