1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2020 NVIDIA Corporation */ 3 4 #include <linux/dma-fence-array.h> 5 #include <linux/dma-mapping.h> 6 #include <linux/file.h> 7 #include <linux/host1x.h> 8 #include <linux/iommu.h> 9 #include <linux/kref.h> 10 #include <linux/list.h> 11 #include <linux/nospec.h> 12 #include <linux/pm_runtime.h> 13 #include <linux/scatterlist.h> 14 #include <linux/slab.h> 15 #include <linux/sync_file.h> 16 17 #include <drm/drm_drv.h> 18 #include <drm/drm_file.h> 19 #include <drm/drm_syncobj.h> 20 21 #include "drm.h" 22 #include "gem.h" 23 #include "submit.h" 24 #include "uapi.h" 25 26 #define SUBMIT_ERR(context, fmt, ...) \ 27 dev_err_ratelimited(context->client->base.dev, \ 28 "%s: job submission failed: " fmt "\n", \ 29 current->comm, ##__VA_ARGS__) 30 31 struct gather_bo { 32 struct host1x_bo base; 33 34 struct kref ref; 35 36 struct device *dev; 37 u32 *gather_data; 38 dma_addr_t gather_data_dma; 39 size_t gather_data_words; 40 }; 41 42 static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo) 43 { 44 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 45 46 kref_get(&bo->ref); 47 48 return host_bo; 49 } 50 51 static void gather_bo_release(struct kref *ref) 52 { 53 struct gather_bo *bo = container_of(ref, struct gather_bo, ref); 54 55 dma_free_attrs(bo->dev, bo->gather_data_words * 4, bo->gather_data, bo->gather_data_dma, 56 0); 57 kfree(bo); 58 } 59 60 static void gather_bo_put(struct host1x_bo *host_bo) 61 { 62 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 63 64 kref_put(&bo->ref, gather_bo_release); 65 } 66 67 static struct host1x_bo_mapping * 68 gather_bo_pin(struct device *dev, struct host1x_bo *bo, enum dma_data_direction direction) 69 { 70 struct gather_bo *gather = container_of(bo, struct gather_bo, base); 71 struct host1x_bo_mapping *map; 72 int err; 73 74 map = kzalloc_obj(*map); 75 if (!map) 76 return ERR_PTR(-ENOMEM); 77 78 kref_init(&map->ref); 79 map->bo = bo; 80 map->direction = direction; 81 map->dev = dev; 82 83 map->sgt = kzalloc_obj(*map->sgt); 84 if (!map->sgt) { 85 err = -ENOMEM; 86 goto free; 87 } 88 89 err = dma_get_sgtable(gather->dev, map->sgt, gather->gather_data, gather->gather_data_dma, 90 gather->gather_data_words * 4); 91 if (err) 92 goto free_sgt; 93 94 err = dma_map_sgtable(dev, map->sgt, direction, 0); 95 if (err) 96 goto free_sgt; 97 98 map->phys = sg_dma_address(map->sgt->sgl); 99 map->size = gather->gather_data_words * 4; 100 map->chunks = err; 101 102 return map; 103 104 free_sgt: 105 sg_free_table(map->sgt); 106 kfree(map->sgt); 107 free: 108 kfree(map); 109 return ERR_PTR(err); 110 } 111 112 static void gather_bo_unpin(struct host1x_bo_mapping *map) 113 { 114 if (!map) 115 return; 116 117 dma_unmap_sgtable(map->dev, map->sgt, map->direction, 0); 118 sg_free_table(map->sgt); 119 kfree(map->sgt); 120 121 kfree(map); 122 } 123 124 static void *gather_bo_mmap(struct host1x_bo *host_bo) 125 { 126 struct gather_bo *bo = container_of(host_bo, struct gather_bo, base); 127 128 return bo->gather_data; 129 } 130 131 static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr) 132 { 133 } 134 135 static const struct host1x_bo_ops gather_bo_ops = { 136 .get = gather_bo_get, 137 .put = gather_bo_put, 138 .pin = gather_bo_pin, 139 .unpin = gather_bo_unpin, 140 .mmap = gather_bo_mmap, 141 .munmap = gather_bo_munmap, 142 }; 143 144 static struct tegra_drm_mapping * 145 tegra_drm_mapping_get(struct tegra_drm_context *context, u32 id) 146 { 147 struct tegra_drm_mapping *mapping; 148 149 xa_lock(&context->mappings); 150 151 mapping = xa_load(&context->mappings, id); 152 if (mapping) 153 kref_get(&mapping->ref); 154 155 xa_unlock(&context->mappings); 156 157 return mapping; 158 } 159 160 static void *alloc_copy_user_array(void __user *from, size_t count, size_t size) 161 { 162 size_t copy_len; 163 void *data; 164 165 if (check_mul_overflow(count, size, ©_len)) 166 return ERR_PTR(-EINVAL); 167 168 if (copy_len > 0x4000) 169 return ERR_PTR(-E2BIG); 170 171 data = vmemdup_user(from, copy_len); 172 if (IS_ERR(data)) 173 return ERR_CAST(data); 174 175 return data; 176 } 177 178 static int submit_copy_gather_data(struct gather_bo **pbo, struct device *dev, 179 struct tegra_drm_context *context, 180 struct drm_tegra_channel_submit *args) 181 { 182 struct gather_bo *bo; 183 size_t copy_len; 184 185 if (args->gather_data_words == 0) { 186 SUBMIT_ERR(context, "gather_data_words cannot be zero"); 187 return -EINVAL; 188 } 189 190 if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len)) { 191 SUBMIT_ERR(context, "gather_data_words is too large"); 192 return -EINVAL; 193 } 194 195 bo = kzalloc_obj(*bo); 196 if (!bo) { 197 SUBMIT_ERR(context, "failed to allocate memory for bo info"); 198 return -ENOMEM; 199 } 200 201 host1x_bo_init(&bo->base, &gather_bo_ops); 202 kref_init(&bo->ref); 203 bo->dev = dev; 204 205 bo->gather_data = dma_alloc_attrs(dev, copy_len, &bo->gather_data_dma, 206 GFP_KERNEL | __GFP_NOWARN, 0); 207 if (!bo->gather_data) { 208 SUBMIT_ERR(context, "failed to allocate memory for gather data"); 209 kfree(bo); 210 return -ENOMEM; 211 } 212 213 if (copy_from_user(bo->gather_data, u64_to_user_ptr(args->gather_data_ptr), copy_len)) { 214 SUBMIT_ERR(context, "failed to copy gather data from userspace"); 215 dma_free_attrs(dev, copy_len, bo->gather_data, bo->gather_data_dma, 0); 216 kfree(bo); 217 return -EFAULT; 218 } 219 220 bo->gather_data_words = args->gather_data_words; 221 222 *pbo = bo; 223 224 return 0; 225 } 226 227 static int submit_write_reloc(struct tegra_drm_context *context, struct gather_bo *bo, 228 struct drm_tegra_submit_buf *buf, struct tegra_drm_mapping *mapping) 229 { 230 /* TODO check that target_offset is within bounds */ 231 dma_addr_t iova = mapping->iova + buf->reloc.target_offset; 232 u32 written_ptr; 233 234 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 235 if (buf->flags & DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) 236 iova |= BIT_ULL(39); 237 #endif 238 239 written_ptr = iova >> buf->reloc.shift; 240 241 if (buf->reloc.gather_offset_words >= bo->gather_data_words) { 242 SUBMIT_ERR(context, 243 "relocation has too large gather offset (%u vs gather length %zu)", 244 buf->reloc.gather_offset_words, bo->gather_data_words); 245 return -EINVAL; 246 } 247 248 buf->reloc.gather_offset_words = array_index_nospec(buf->reloc.gather_offset_words, 249 bo->gather_data_words); 250 251 bo->gather_data[buf->reloc.gather_offset_words] = written_ptr; 252 253 return 0; 254 } 255 256 static int submit_process_bufs(struct tegra_drm_context *context, struct gather_bo *bo, 257 struct drm_tegra_channel_submit *args, 258 struct tegra_drm_submit_data *job_data) 259 { 260 struct tegra_drm_used_mapping *mappings; 261 struct drm_tegra_submit_buf *bufs; 262 int err; 263 u32 i; 264 265 bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr), args->num_bufs, 266 sizeof(*bufs)); 267 if (IS_ERR(bufs)) { 268 SUBMIT_ERR(context, "failed to copy bufs array from userspace"); 269 return PTR_ERR(bufs); 270 } 271 272 mappings = kzalloc_objs(*mappings, args->num_bufs); 273 if (!mappings) { 274 SUBMIT_ERR(context, "failed to allocate memory for mapping info"); 275 err = -ENOMEM; 276 goto done; 277 } 278 279 for (i = 0; i < args->num_bufs; i++) { 280 struct drm_tegra_submit_buf *buf = &bufs[i]; 281 struct tegra_drm_mapping *mapping; 282 283 if (buf->flags & ~DRM_TEGRA_SUBMIT_RELOC_SECTOR_LAYOUT) { 284 SUBMIT_ERR(context, "invalid flag specified for buffer"); 285 err = -EINVAL; 286 goto drop_refs; 287 } 288 289 mapping = tegra_drm_mapping_get(context, buf->mapping); 290 if (!mapping) { 291 SUBMIT_ERR(context, "invalid mapping ID '%u' for buffer", buf->mapping); 292 err = -EINVAL; 293 goto drop_refs; 294 } 295 296 err = submit_write_reloc(context, bo, buf, mapping); 297 if (err) { 298 tegra_drm_mapping_put(mapping); 299 goto drop_refs; 300 } 301 302 mappings[i].mapping = mapping; 303 mappings[i].flags = buf->flags; 304 } 305 306 job_data->used_mappings = mappings; 307 job_data->num_used_mappings = i; 308 309 err = 0; 310 311 goto done; 312 313 drop_refs: 314 while (i--) 315 tegra_drm_mapping_put(mappings[i].mapping); 316 317 kfree(mappings); 318 job_data->used_mappings = NULL; 319 320 done: 321 kvfree(bufs); 322 323 return err; 324 } 325 326 static int submit_get_syncpt(struct tegra_drm_context *context, struct host1x_job *job, 327 struct xarray *syncpoints, struct drm_tegra_channel_submit *args) 328 { 329 struct host1x_syncpt *sp; 330 331 if (args->syncpt.flags) { 332 SUBMIT_ERR(context, "invalid flag specified for syncpt"); 333 return -EINVAL; 334 } 335 336 /* Syncpt ref will be dropped on job release */ 337 sp = xa_load(syncpoints, args->syncpt.id); 338 if (!sp) { 339 SUBMIT_ERR(context, "syncpoint specified in syncpt was not allocated"); 340 return -EINVAL; 341 } 342 343 job->syncpt = host1x_syncpt_get(sp); 344 job->syncpt_incrs = args->syncpt.increments; 345 346 return 0; 347 } 348 349 static int submit_job_add_gather(struct host1x_job *job, struct tegra_drm_context *context, 350 struct drm_tegra_submit_cmd_gather_uptr *cmd, 351 struct gather_bo *bo, u32 *offset, 352 struct tegra_drm_submit_data *job_data, 353 u32 *class) 354 { 355 u32 next_offset; 356 357 if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2]) { 358 SUBMIT_ERR(context, "non-zero reserved field in GATHER_UPTR command"); 359 return -EINVAL; 360 } 361 362 /* Check for maximum gather size */ 363 if (cmd->words > 16383) { 364 SUBMIT_ERR(context, "too many words in GATHER_UPTR command"); 365 return -EINVAL; 366 } 367 368 if (check_add_overflow(*offset, cmd->words, &next_offset)) { 369 SUBMIT_ERR(context, "too many total words in job"); 370 return -EINVAL; 371 } 372 373 if (next_offset > bo->gather_data_words) { 374 SUBMIT_ERR(context, "GATHER_UPTR command overflows gather data"); 375 return -EINVAL; 376 } 377 378 if (tegra_drm_fw_validate(context->client, bo->gather_data, *offset, 379 cmd->words, job_data, class)) { 380 SUBMIT_ERR(context, "job was rejected by firewall"); 381 return -EINVAL; 382 } 383 384 host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4); 385 386 *offset = next_offset; 387 388 return 0; 389 } 390 391 static struct host1x_job * 392 submit_create_job(struct tegra_drm_context *context, struct gather_bo *bo, 393 struct drm_tegra_channel_submit *args, struct tegra_drm_submit_data *job_data, 394 struct xarray *syncpoints) 395 { 396 struct drm_tegra_submit_cmd *cmds; 397 u32 i, gather_offset = 0, class; 398 struct host1x_job *job; 399 int err; 400 401 /* Set initial class for firewall. */ 402 class = context->client->base.class; 403 404 cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr), args->num_cmds, 405 sizeof(*cmds)); 406 if (IS_ERR(cmds)) { 407 SUBMIT_ERR(context, "failed to copy cmds array from userspace"); 408 return ERR_CAST(cmds); 409 } 410 411 job = host1x_job_alloc(context->channel, args->num_cmds, 0, true); 412 if (!job) { 413 SUBMIT_ERR(context, "failed to allocate memory for job"); 414 job = ERR_PTR(-ENOMEM); 415 goto done; 416 } 417 418 err = submit_get_syncpt(context, job, syncpoints, args); 419 if (err < 0) 420 goto free_job; 421 422 job->client = &context->client->base; 423 job->class = context->client->base.class; 424 job->serialize = true; 425 426 for (i = 0; i < args->num_cmds; i++) { 427 struct drm_tegra_submit_cmd *cmd = &cmds[i]; 428 429 if (cmd->flags) { 430 SUBMIT_ERR(context, "unknown flags given for cmd"); 431 err = -EINVAL; 432 goto free_job; 433 } 434 435 if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) { 436 err = submit_job_add_gather(job, context, &cmd->gather_uptr, bo, 437 &gather_offset, job_data, &class); 438 if (err) 439 goto free_job; 440 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) { 441 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 442 SUBMIT_ERR(context, "non-zero reserved value"); 443 err = -EINVAL; 444 goto free_job; 445 } 446 447 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 448 false, class); 449 } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT_RELATIVE) { 450 if (cmd->wait_syncpt.reserved[0] || cmd->wait_syncpt.reserved[1]) { 451 SUBMIT_ERR(context, "non-zero reserved value"); 452 err = -EINVAL; 453 goto free_job; 454 } 455 456 if (cmd->wait_syncpt.id != args->syncpt.id) { 457 SUBMIT_ERR(context, "syncpoint ID in CMD_WAIT_SYNCPT_RELATIVE is not used by the job"); 458 err = -EINVAL; 459 goto free_job; 460 } 461 462 host1x_job_add_wait(job, cmd->wait_syncpt.id, cmd->wait_syncpt.value, 463 true, class); 464 } else { 465 SUBMIT_ERR(context, "unknown cmd type"); 466 err = -EINVAL; 467 goto free_job; 468 } 469 } 470 471 if (gather_offset == 0) { 472 SUBMIT_ERR(context, "job must have at least one gather"); 473 err = -EINVAL; 474 goto free_job; 475 } 476 477 goto done; 478 479 free_job: 480 host1x_job_put(job); 481 job = ERR_PTR(err); 482 483 done: 484 kvfree(cmds); 485 486 return job; 487 } 488 489 static void release_job(struct host1x_job *job) 490 { 491 struct tegra_drm_client *client = container_of(job->client, struct tegra_drm_client, base); 492 struct tegra_drm_submit_data *job_data = job->user_data; 493 u32 i; 494 495 if (job->memory_context) 496 host1x_memory_context_put(job->memory_context); 497 498 for (i = 0; i < job_data->num_used_mappings; i++) 499 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 500 501 kfree(job_data->used_mappings); 502 kfree(job_data); 503 504 pm_runtime_mark_last_busy(client->base.dev); 505 pm_runtime_put_autosuspend(client->base.dev); 506 } 507 508 int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data, 509 struct drm_file *file) 510 { 511 struct tegra_drm_file *fpriv = file->driver_priv; 512 struct drm_tegra_channel_submit *args = data; 513 struct tegra_drm_submit_data *job_data; 514 struct drm_syncobj *syncobj = NULL; 515 struct tegra_drm_context *context; 516 struct host1x_job *job; 517 struct gather_bo *bo; 518 u32 i; 519 int err; 520 521 mutex_lock(&fpriv->lock); 522 523 context = xa_load(&fpriv->contexts, args->context); 524 if (!context) { 525 mutex_unlock(&fpriv->lock); 526 pr_err_ratelimited("%s: %s: invalid channel context '%#x'", __func__, 527 current->comm, args->context); 528 return -EINVAL; 529 } 530 531 if (args->syncobj_in) { 532 struct dma_fence *fence; 533 534 err = drm_syncobj_find_fence(file, args->syncobj_in, 0, 0, &fence); 535 if (err) { 536 SUBMIT_ERR(context, "invalid syncobj_in '%#x'", args->syncobj_in); 537 goto unlock; 538 } 539 540 err = dma_fence_wait_timeout(fence, true, msecs_to_jiffies(10000)); 541 dma_fence_put(fence); 542 if (err) { 543 SUBMIT_ERR(context, "wait for syncobj_in timed out"); 544 goto unlock; 545 } 546 } 547 548 if (args->syncobj_out) { 549 syncobj = drm_syncobj_find(file, args->syncobj_out); 550 if (!syncobj) { 551 SUBMIT_ERR(context, "invalid syncobj_out '%#x'", args->syncobj_out); 552 err = -ENOENT; 553 goto unlock; 554 } 555 } 556 557 /* Allocate gather BO and copy gather words in. */ 558 err = submit_copy_gather_data(&bo, drm->dev, context, args); 559 if (err) 560 goto unlock; 561 562 job_data = kzalloc_obj(*job_data); 563 if (!job_data) { 564 SUBMIT_ERR(context, "failed to allocate memory for job data"); 565 err = -ENOMEM; 566 goto put_bo; 567 } 568 569 /* Get data buffer mappings and do relocation patching. */ 570 err = submit_process_bufs(context, bo, args, job_data); 571 if (err) 572 goto free_job_data; 573 574 /* Allocate host1x_job and add gathers and waits to it. */ 575 job = submit_create_job(context, bo, args, job_data, &fpriv->syncpoints); 576 if (IS_ERR(job)) { 577 err = PTR_ERR(job); 578 goto free_job_data; 579 } 580 581 /* Map gather data for Host1x. */ 582 err = host1x_job_pin(job, context->client->base.dev); 583 if (err) { 584 SUBMIT_ERR(context, "failed to pin job: %d", err); 585 goto put_job; 586 } 587 588 if (context->client->ops->get_streamid_offset) { 589 err = context->client->ops->get_streamid_offset( 590 context->client, &job->engine_streamid_offset); 591 if (err) { 592 SUBMIT_ERR(context, "failed to get streamid offset: %d", err); 593 goto unpin_job; 594 } 595 } 596 597 if (context->memory_context && context->client->ops->can_use_memory_ctx) { 598 bool supported; 599 600 err = context->client->ops->can_use_memory_ctx(context->client, &supported); 601 if (err) { 602 SUBMIT_ERR(context, "failed to detect if engine can use memory context: %d", err); 603 goto unpin_job; 604 } 605 606 if (supported) { 607 job->memory_context = context->memory_context; 608 host1x_memory_context_get(job->memory_context); 609 } 610 } else if (context->client->ops->get_streamid_offset) { 611 /* 612 * Job submission will need to temporarily change stream ID, 613 * so need to tell it what to change it back to. 614 */ 615 if (!tegra_dev_iommu_get_stream_id(context->client->base.dev, 616 &job->engine_fallback_streamid)) 617 job->engine_fallback_streamid = TEGRA_STREAM_ID_BYPASS; 618 } 619 620 /* Boot engine. */ 621 err = pm_runtime_resume_and_get(context->client->base.dev); 622 if (err < 0) { 623 SUBMIT_ERR(context, "could not power up engine: %d", err); 624 goto put_memory_context; 625 } 626 627 job->user_data = job_data; 628 job->release = release_job; 629 job->timeout = 10000; 630 631 /* 632 * job_data is now part of job reference counting, so don't release 633 * it from here. 634 */ 635 job_data = NULL; 636 637 /* Submit job to hardware. */ 638 err = host1x_job_submit(job); 639 if (err) { 640 SUBMIT_ERR(context, "host1x job submission failed: %d", err); 641 goto unpin_job; 642 } 643 644 /* Return postfences to userspace and add fences to DMA reservations. */ 645 args->syncpt.value = job->syncpt_end; 646 647 if (syncobj) { 648 struct dma_fence *fence = host1x_fence_create(job->syncpt, job->syncpt_end, true); 649 if (IS_ERR(fence)) { 650 err = PTR_ERR(fence); 651 SUBMIT_ERR(context, "failed to create postfence: %d", err); 652 } 653 654 drm_syncobj_replace_fence(syncobj, fence); 655 } 656 657 goto put_job; 658 659 put_memory_context: 660 if (job->memory_context) 661 host1x_memory_context_put(job->memory_context); 662 unpin_job: 663 host1x_job_unpin(job); 664 put_job: 665 host1x_job_put(job); 666 free_job_data: 667 if (job_data && job_data->used_mappings) { 668 for (i = 0; i < job_data->num_used_mappings; i++) 669 tegra_drm_mapping_put(job_data->used_mappings[i].mapping); 670 671 kfree(job_data->used_mappings); 672 } 673 674 kfree(job_data); 675 put_bo: 676 gather_bo_put(&bo->base); 677 unlock: 678 if (syncobj) 679 drm_syncobj_put(syncobj); 680 681 mutex_unlock(&fpriv->lock); 682 return err; 683 } 684