1 /* 2 * Copyright 2008 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22 * DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: 25 * Jerome Glisse <glisse@freedesktop.org> 26 */ 27 28 #include <linux/file.h> 29 #include <linux/pagemap.h> 30 #include <linux/sync_file.h> 31 #include <linux/dma-buf.h> 32 #include <linux/pm_runtime.h> 33 34 #include <drm/amdgpu_drm.h> 35 #include <drm/drm_syncobj.h> 36 #include <drm/ttm/ttm_tt.h> 37 38 #include "amdgpu_cs.h" 39 #include "amdgpu.h" 40 #include "amdgpu_trace.h" 41 #include "amdgpu_gmc.h" 42 #include "amdgpu_gem.h" 43 #include "amdgpu_ras.h" 44 #include "amdgpu_hmm.h" 45 46 /* 47 * Maximum IB length (dwords) for rings whose emit_ib packet format 48 * documents a 20-bit size field. 49 */ 50 #define AMDGPU_GFX_SDMA_IB_PACKET_SIZE_MAX_DW 0xFFFFF 51 #define AMDGPU_MM_IB_PACKET_SIZE_MAX_DW 0x7FFFF0 52 53 static u32 amdgpu_cs_ib_packet_size_max_dw(enum amdgpu_ring_type type) 54 { 55 switch (type) { 56 case AMDGPU_RING_TYPE_GFX: 57 case AMDGPU_RING_TYPE_COMPUTE: 58 case AMDGPU_RING_TYPE_SDMA: 59 case AMDGPU_RING_TYPE_VPE: 60 return AMDGPU_GFX_SDMA_IB_PACKET_SIZE_MAX_DW; 61 default: 62 return AMDGPU_MM_IB_PACKET_SIZE_MAX_DW; 63 } 64 } 65 66 static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, 67 struct amdgpu_device *adev, 68 struct drm_file *filp, 69 union drm_amdgpu_cs *cs) 70 { 71 struct amdgpu_fpriv *fpriv = filp->driver_priv; 72 73 if (cs->in.num_chunks == 0) 74 return -EINVAL; 75 76 memset(p, 0, sizeof(*p)); 77 p->adev = adev; 78 p->filp = filp; 79 80 p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id); 81 if (!p->ctx) 82 return -EINVAL; 83 84 amdgpu_sync_create(&p->sync); 85 drm_exec_init(&p->exec, DRM_EXEC_INTERRUPTIBLE_WAIT | 86 DRM_EXEC_IGNORE_DUPLICATES, 0); 87 return 0; 88 } 89 90 static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p, 91 struct drm_amdgpu_cs_chunk_ib *chunk_ib) 92 { 93 struct drm_sched_entity *entity; 94 unsigned int i; 95 int r; 96 97 r = amdgpu_ctx_get_entity(p->ctx, chunk_ib->ip_type, 98 chunk_ib->ip_instance, 99 chunk_ib->ring, &entity); 100 if (r) 101 return r; 102 103 /* Check if we can add this IB to some existing job */ 104 for (i = 0; i < p->gang_size; ++i) 105 if (p->entities[i] == entity) 106 return i; 107 108 /* If not increase the gang size if possible */ 109 if (i == AMDGPU_CS_GANG_SIZE) 110 return -EINVAL; 111 112 p->entities[i] = entity; 113 p->gang_size = i + 1; 114 return i; 115 } 116 117 static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p, 118 struct drm_amdgpu_cs_chunk_ib *chunk_ib, 119 unsigned int *num_ibs) 120 { 121 int r; 122 123 r = amdgpu_cs_job_idx(p, chunk_ib); 124 if (r < 0) 125 return r; 126 127 if (num_ibs[r] >= amdgpu_ring_max_ibs(chunk_ib->ip_type)) 128 return -EINVAL; 129 130 ++(num_ibs[r]); 131 p->gang_leader_idx = r; 132 return 0; 133 } 134 135 static int amdgpu_cs_p1_user_fence(struct amdgpu_cs_parser *p, 136 struct drm_amdgpu_cs_chunk_fence *data, 137 uint32_t *offset) 138 { 139 struct drm_gem_object *gobj; 140 unsigned long size; 141 142 gobj = drm_gem_object_lookup(p->filp, data->handle); 143 if (gobj == NULL) 144 return -EINVAL; 145 146 p->uf_bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj)); 147 drm_gem_object_put(gobj); 148 149 size = amdgpu_bo_size(p->uf_bo); 150 if (size != PAGE_SIZE || data->offset > (size - 8)) 151 return -EINVAL; 152 153 if (amdgpu_ttm_tt_get_usermm(p->uf_bo->tbo.ttm)) 154 return -EINVAL; 155 156 *offset = data->offset; 157 return 0; 158 } 159 160 static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p, 161 struct drm_amdgpu_bo_list_in *data) 162 { 163 struct drm_amdgpu_bo_list_entry *info; 164 struct amdgpu_bo_list *list; 165 166 info = amdgpu_bo_create_list_entry_array(data); 167 if (IS_ERR(info)) 168 return PTR_ERR(info); 169 170 list = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number); 171 kvfree(info); 172 if (IS_ERR(list)) 173 return PTR_ERR(list); 174 175 p->bo_list = list; 176 return 0; 177 } 178 179 /* Copy the data from userspace and go over it the first time */ 180 static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p, 181 union drm_amdgpu_cs *cs) 182 { 183 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 184 unsigned int num_ibs[AMDGPU_CS_GANG_SIZE] = { }; 185 struct amdgpu_vm *vm = &fpriv->vm; 186 uint64_t *chunk_array; 187 uint32_t uf_offset = 0; 188 size_t size; 189 int ret; 190 int i; 191 192 chunk_array = memdup_array_user(u64_to_user_ptr(cs->in.chunks), 193 cs->in.num_chunks, 194 sizeof(uint64_t)); 195 if (IS_ERR(chunk_array)) 196 return PTR_ERR(chunk_array); 197 198 p->nchunks = cs->in.num_chunks; 199 p->chunks = kvmalloc_objs(struct amdgpu_cs_chunk, p->nchunks); 200 if (!p->chunks) { 201 ret = -ENOMEM; 202 goto free_chunk; 203 } 204 205 for (i = 0; i < p->nchunks; i++) { 206 struct drm_amdgpu_cs_chunk __user *chunk_ptr = NULL; 207 struct drm_amdgpu_cs_chunk user_chunk; 208 209 chunk_ptr = u64_to_user_ptr(chunk_array[i]); 210 if (copy_from_user(&user_chunk, chunk_ptr, 211 sizeof(struct drm_amdgpu_cs_chunk))) { 212 ret = -EFAULT; 213 i--; 214 goto free_partial_kdata; 215 } 216 p->chunks[i].chunk_id = user_chunk.chunk_id; 217 p->chunks[i].length_dw = user_chunk.length_dw; 218 219 size = p->chunks[i].length_dw; 220 221 if (size > INT_MAX / sizeof(uint32_t)) { 222 ret = -EINVAL; 223 i--; 224 goto free_partial_kdata; 225 } 226 227 p->chunks[i].kdata = vmemdup_array_user(u64_to_user_ptr(user_chunk.chunk_data), 228 size, 229 sizeof(uint32_t)); 230 if (IS_ERR(p->chunks[i].kdata)) { 231 ret = PTR_ERR(p->chunks[i].kdata); 232 i--; 233 goto free_partial_kdata; 234 } 235 size *= sizeof(uint32_t); 236 237 /* Assume the worst on the following checks */ 238 ret = -EINVAL; 239 switch (p->chunks[i].chunk_id) { 240 case AMDGPU_CHUNK_ID_IB: 241 if (size < sizeof(struct drm_amdgpu_cs_chunk_ib)) 242 goto free_partial_kdata; 243 244 ret = amdgpu_cs_p1_ib(p, p->chunks[i].kdata, num_ibs); 245 if (ret) 246 goto free_partial_kdata; 247 break; 248 249 case AMDGPU_CHUNK_ID_FENCE: 250 if (size < sizeof(struct drm_amdgpu_cs_chunk_fence)) 251 goto free_partial_kdata; 252 253 /* Only a single user fence is allowed to simplify handling. */ 254 if (p->uf_bo) 255 goto free_partial_kdata; 256 257 ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata, 258 &uf_offset); 259 if (ret) 260 goto free_partial_kdata; 261 break; 262 263 case AMDGPU_CHUNK_ID_BO_HANDLES: 264 if (size < sizeof(struct drm_amdgpu_bo_list_in)) 265 goto free_partial_kdata; 266 267 /* Only a single BO list is allowed to simplify handling. */ 268 if (p->bo_list) 269 goto free_partial_kdata; 270 271 ret = amdgpu_cs_p1_bo_handles(p, p->chunks[i].kdata); 272 if (ret) 273 goto free_partial_kdata; 274 break; 275 276 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: 277 if (size < sizeof(struct drm_amdgpu_cs_chunk_cp_gfx_shadow)) 278 goto free_partial_kdata; 279 break; 280 281 case AMDGPU_CHUNK_ID_DEPENDENCIES: 282 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 283 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 284 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: 285 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: 286 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: 287 break; 288 289 default: 290 goto free_partial_kdata; 291 } 292 } 293 294 if (!p->gang_size || (amdgpu_sriov_vf(p->adev) && p->gang_size > 1)) { 295 ret = -EINVAL; 296 goto free_all_kdata; 297 } 298 299 for (i = 0; i < p->gang_size; ++i) { 300 ret = amdgpu_job_alloc(p->adev, vm, p->entities[i], vm, 301 num_ibs[i], p->filp->client_id, 302 GFP_KERNEL, &p->jobs[i]); 303 if (ret) 304 goto free_all_kdata; 305 switch (p->adev->enforce_isolation[fpriv->xcp_id]) { 306 case AMDGPU_ENFORCE_ISOLATION_DISABLE: 307 default: 308 p->jobs[i]->enforce_isolation = false; 309 p->jobs[i]->run_cleaner_shader = false; 310 break; 311 case AMDGPU_ENFORCE_ISOLATION_ENABLE: 312 p->jobs[i]->enforce_isolation = true; 313 p->jobs[i]->run_cleaner_shader = true; 314 break; 315 case AMDGPU_ENFORCE_ISOLATION_ENABLE_LEGACY: 316 p->jobs[i]->enforce_isolation = true; 317 p->jobs[i]->run_cleaner_shader = false; 318 break; 319 case AMDGPU_ENFORCE_ISOLATION_NO_CLEANER_SHADER: 320 p->jobs[i]->enforce_isolation = true; 321 p->jobs[i]->run_cleaner_shader = false; 322 break; 323 } 324 } 325 p->gang_leader = p->jobs[p->gang_leader_idx]; 326 327 if (p->ctx->generation != p->gang_leader->generation) { 328 ret = -ECANCELED; 329 goto free_all_kdata; 330 } 331 332 if (p->uf_bo) 333 p->gang_leader->uf_addr = uf_offset; 334 kvfree(chunk_array); 335 336 /* Use this opportunity to fill in task info for the vm */ 337 amdgpu_vm_set_task_info(vm); 338 339 return 0; 340 341 free_all_kdata: 342 i = p->nchunks - 1; 343 free_partial_kdata: 344 for (; i >= 0; i--) 345 kvfree(p->chunks[i].kdata); 346 kvfree(p->chunks); 347 p->chunks = NULL; 348 p->nchunks = 0; 349 free_chunk: 350 kvfree(chunk_array); 351 352 return ret; 353 } 354 355 static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p, 356 struct amdgpu_cs_chunk *chunk, 357 unsigned int *ce_preempt, 358 unsigned int *de_preempt) 359 { 360 struct drm_amdgpu_cs_chunk_ib *chunk_ib = chunk->kdata; 361 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 362 struct amdgpu_vm *vm = &fpriv->vm; 363 struct amdgpu_ring *ring; 364 struct amdgpu_job *job; 365 struct amdgpu_ib *ib; 366 int r; 367 368 r = amdgpu_cs_job_idx(p, chunk_ib); 369 if (r < 0) 370 return r; 371 372 job = p->jobs[r]; 373 ring = amdgpu_job_ring(job); 374 375 /* submissions to kernel queues are disabled */ 376 if (ring->no_user_submission) 377 return -EINVAL; 378 379 /* MM engine doesn't support user fences */ 380 if (p->uf_bo && ring->funcs->no_user_fence) 381 return -EINVAL; 382 383 if (!p->adev->debug_enable_ce_cs && 384 chunk_ib->flags & AMDGPU_IB_FLAG_CE) { 385 dev_err_ratelimited(p->adev->dev, "CE CS is blocked, use debug=0x400 to override\n"); 386 return -EINVAL; 387 } 388 389 if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX && 390 chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) { 391 if (chunk_ib->flags & AMDGPU_IB_FLAG_CE) 392 (*ce_preempt)++; 393 else 394 (*de_preempt)++; 395 396 /* Each GFX command submit allows only 1 IB max 397 * preemptible for CE & DE */ 398 if (*ce_preempt > 1 || *de_preempt > 1) 399 return -EINVAL; 400 } 401 402 if (chunk_ib->ib_bytes / 4 > 403 amdgpu_cs_ib_packet_size_max_dw(ring->funcs->type)) 404 return -EINVAL; 405 406 ib = &job->ibs[job->num_ibs++]; 407 408 if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE) 409 job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT; 410 411 r = amdgpu_ib_get(p->adev, vm, ring->funcs->parse_cs ? 412 chunk_ib->ib_bytes : 0, 413 AMDGPU_IB_POOL_DELAYED, ib); 414 if (r) { 415 drm_err(adev_to_drm(p->adev), "Failed to get ib !\n"); 416 return r; 417 } 418 419 ib->gpu_addr = chunk_ib->va_start; 420 ib->length_dw = chunk_ib->ib_bytes / 4; 421 ib->flags = chunk_ib->flags; 422 return 0; 423 } 424 425 static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p, 426 struct amdgpu_cs_chunk *chunk) 427 { 428 struct drm_amdgpu_cs_chunk_dep *deps = chunk->kdata; 429 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 430 unsigned int num_deps; 431 int i, r; 432 433 num_deps = chunk->length_dw * 4 / 434 sizeof(struct drm_amdgpu_cs_chunk_dep); 435 436 for (i = 0; i < num_deps; ++i) { 437 struct amdgpu_ctx *ctx; 438 struct drm_sched_entity *entity; 439 struct dma_fence *fence; 440 441 ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id); 442 if (ctx == NULL) 443 return -EINVAL; 444 445 r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type, 446 deps[i].ip_instance, 447 deps[i].ring, &entity); 448 if (r) { 449 amdgpu_ctx_put(ctx); 450 return r; 451 } 452 453 fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle); 454 amdgpu_ctx_put(ctx); 455 456 if (IS_ERR(fence)) 457 return PTR_ERR(fence); 458 else if (!fence) 459 continue; 460 461 if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) { 462 struct drm_sched_fence *s_fence; 463 struct dma_fence *old = fence; 464 465 s_fence = to_drm_sched_fence(fence); 466 fence = dma_fence_get(&s_fence->scheduled); 467 dma_fence_put(old); 468 } 469 470 r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL); 471 dma_fence_put(fence); 472 if (r) 473 return r; 474 } 475 return 0; 476 } 477 478 static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p, 479 uint32_t handle, u64 point, 480 u64 flags) 481 { 482 struct dma_fence *fence; 483 int r; 484 485 r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence); 486 if (r) { 487 drm_err(adev_to_drm(p->adev), "syncobj %u failed to find fence @ %llu (%d)!\n", 488 handle, point, r); 489 return r; 490 } 491 492 r = amdgpu_sync_fence(&p->sync, fence, GFP_KERNEL); 493 dma_fence_put(fence); 494 return r; 495 } 496 497 static int amdgpu_cs_p2_syncobj_in(struct amdgpu_cs_parser *p, 498 struct amdgpu_cs_chunk *chunk) 499 { 500 struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata; 501 unsigned int num_deps; 502 int i, r; 503 504 num_deps = chunk->length_dw * 4 / 505 sizeof(struct drm_amdgpu_cs_chunk_sem); 506 for (i = 0; i < num_deps; ++i) { 507 r = amdgpu_syncobj_lookup_and_add(p, deps[i].handle, 0, 0); 508 if (r) 509 return r; 510 } 511 512 return 0; 513 } 514 515 static int amdgpu_cs_p2_syncobj_timeline_wait(struct amdgpu_cs_parser *p, 516 struct amdgpu_cs_chunk *chunk) 517 { 518 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata; 519 unsigned int num_deps; 520 int i, r; 521 522 num_deps = chunk->length_dw * 4 / 523 sizeof(struct drm_amdgpu_cs_chunk_syncobj); 524 for (i = 0; i < num_deps; ++i) { 525 r = amdgpu_syncobj_lookup_and_add(p, syncobj_deps[i].handle, 526 syncobj_deps[i].point, 527 syncobj_deps[i].flags); 528 if (r) 529 return r; 530 } 531 532 return 0; 533 } 534 535 static int amdgpu_cs_p2_syncobj_out(struct amdgpu_cs_parser *p, 536 struct amdgpu_cs_chunk *chunk) 537 { 538 struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata; 539 unsigned int num_deps; 540 int i; 541 542 num_deps = chunk->length_dw * 4 / 543 sizeof(struct drm_amdgpu_cs_chunk_sem); 544 545 if (p->post_deps) 546 return -EINVAL; 547 548 p->post_deps = kmalloc_objs(*p->post_deps, num_deps); 549 p->num_post_deps = 0; 550 551 if (!p->post_deps) 552 return -ENOMEM; 553 554 555 for (i = 0; i < num_deps; ++i) { 556 p->post_deps[i].syncobj = 557 drm_syncobj_find(p->filp, deps[i].handle); 558 if (!p->post_deps[i].syncobj) 559 return -EINVAL; 560 p->post_deps[i].chain = NULL; 561 p->post_deps[i].point = 0; 562 p->num_post_deps++; 563 } 564 565 return 0; 566 } 567 568 static int amdgpu_cs_p2_syncobj_timeline_signal(struct amdgpu_cs_parser *p, 569 struct amdgpu_cs_chunk *chunk) 570 { 571 struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata; 572 unsigned int num_deps; 573 int i; 574 575 num_deps = chunk->length_dw * 4 / 576 sizeof(struct drm_amdgpu_cs_chunk_syncobj); 577 578 if (p->post_deps) 579 return -EINVAL; 580 581 p->post_deps = kmalloc_objs(*p->post_deps, num_deps); 582 p->num_post_deps = 0; 583 584 if (!p->post_deps) 585 return -ENOMEM; 586 587 for (i = 0; i < num_deps; ++i) { 588 struct amdgpu_cs_post_dep *dep = &p->post_deps[i]; 589 590 dep->chain = NULL; 591 if (syncobj_deps[i].point) { 592 dep->chain = dma_fence_chain_alloc(); 593 if (!dep->chain) 594 return -ENOMEM; 595 } 596 597 dep->syncobj = drm_syncobj_find(p->filp, 598 syncobj_deps[i].handle); 599 if (!dep->syncobj) { 600 dma_fence_chain_free(dep->chain); 601 return -EINVAL; 602 } 603 dep->point = syncobj_deps[i].point; 604 p->num_post_deps++; 605 } 606 607 return 0; 608 } 609 610 static int amdgpu_cs_p2_shadow(struct amdgpu_cs_parser *p, 611 struct amdgpu_cs_chunk *chunk) 612 { 613 struct drm_amdgpu_cs_chunk_cp_gfx_shadow *shadow = chunk->kdata; 614 int i; 615 616 if (shadow->flags & ~AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW) 617 return -EINVAL; 618 619 for (i = 0; i < p->gang_size; ++i) { 620 p->jobs[i]->shadow_va = shadow->shadow_va; 621 p->jobs[i]->csa_va = shadow->csa_va; 622 p->jobs[i]->gds_va = shadow->gds_va; 623 p->jobs[i]->init_shadow = 624 shadow->flags & AMDGPU_CS_CHUNK_CP_GFX_SHADOW_FLAGS_INIT_SHADOW; 625 } 626 627 return 0; 628 } 629 630 static int amdgpu_cs_pass2(struct amdgpu_cs_parser *p) 631 { 632 unsigned int ce_preempt = 0, de_preempt = 0; 633 int i, r; 634 635 for (i = 0; i < p->nchunks; ++i) { 636 struct amdgpu_cs_chunk *chunk; 637 638 chunk = &p->chunks[i]; 639 640 switch (chunk->chunk_id) { 641 case AMDGPU_CHUNK_ID_IB: 642 r = amdgpu_cs_p2_ib(p, chunk, &ce_preempt, &de_preempt); 643 if (r) 644 return r; 645 break; 646 case AMDGPU_CHUNK_ID_DEPENDENCIES: 647 case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES: 648 r = amdgpu_cs_p2_dependencies(p, chunk); 649 if (r) 650 return r; 651 break; 652 case AMDGPU_CHUNK_ID_SYNCOBJ_IN: 653 r = amdgpu_cs_p2_syncobj_in(p, chunk); 654 if (r) 655 return r; 656 break; 657 case AMDGPU_CHUNK_ID_SYNCOBJ_OUT: 658 r = amdgpu_cs_p2_syncobj_out(p, chunk); 659 if (r) 660 return r; 661 break; 662 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT: 663 r = amdgpu_cs_p2_syncobj_timeline_wait(p, chunk); 664 if (r) 665 return r; 666 break; 667 case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL: 668 r = amdgpu_cs_p2_syncobj_timeline_signal(p, chunk); 669 if (r) 670 return r; 671 break; 672 case AMDGPU_CHUNK_ID_CP_GFX_SHADOW: 673 r = amdgpu_cs_p2_shadow(p, chunk); 674 if (r) 675 return r; 676 break; 677 } 678 } 679 680 return 0; 681 } 682 683 /* Convert microseconds to bytes. */ 684 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us) 685 { 686 if (us <= 0 || !adev->mm_stats.log2_max_MBps) 687 return 0; 688 689 /* Since accum_us is incremented by a million per second, just 690 * multiply it by the number of MB/s to get the number of bytes. 691 */ 692 return us << adev->mm_stats.log2_max_MBps; 693 } 694 695 static s64 bytes_to_us(struct amdgpu_device *adev, u64 bytes) 696 { 697 if (!adev->mm_stats.log2_max_MBps) 698 return 0; 699 700 return bytes >> adev->mm_stats.log2_max_MBps; 701 } 702 703 /* Returns how many bytes TTM can move right now. If no bytes can be moved, 704 * it returns 0. If it returns non-zero, it's OK to move at least one buffer, 705 * which means it can go over the threshold once. If that happens, the driver 706 * will be in debt and no other buffer migrations can be done until that debt 707 * is repaid. 708 * 709 * This approach allows moving a buffer of any size (it's important to allow 710 * that). 711 * 712 * The currency is simply time in microseconds and it increases as the clock 713 * ticks. The accumulated microseconds (us) are converted to bytes and 714 * returned. 715 */ 716 static void amdgpu_cs_get_threshold_for_moves(struct amdgpu_device *adev, 717 u64 *max_bytes, 718 u64 *max_vis_bytes) 719 { 720 s64 time_us, increment_us; 721 u64 free_vram, total_vram, used_vram; 722 /* Allow a maximum of 200 accumulated ms. This is basically per-IB 723 * throttling. 724 * 725 * It means that in order to get full max MBps, at least 5 IBs per 726 * second must be submitted and not more than 200ms apart from each 727 * other. 728 */ 729 const s64 us_upper_bound = 200000; 730 731 if ((!adev->mm_stats.log2_max_MBps) || !ttm_resource_manager_used(&adev->mman.vram_mgr.manager)) { 732 *max_bytes = 0; 733 *max_vis_bytes = 0; 734 return; 735 } 736 737 total_vram = adev->gmc.real_vram_size - atomic64_read(&adev->vram_pin_size); 738 used_vram = ttm_resource_manager_usage(&adev->mman.vram_mgr.manager); 739 free_vram = used_vram >= total_vram ? 0 : total_vram - used_vram; 740 741 spin_lock(&adev->mm_stats.lock); 742 743 /* Increase the amount of accumulated us. */ 744 time_us = ktime_to_us(ktime_get()); 745 increment_us = time_us - adev->mm_stats.last_update_us; 746 adev->mm_stats.last_update_us = time_us; 747 adev->mm_stats.accum_us = min(adev->mm_stats.accum_us + increment_us, 748 us_upper_bound); 749 750 /* This prevents the short period of low performance when the VRAM 751 * usage is low and the driver is in debt or doesn't have enough 752 * accumulated us to fill VRAM quickly. 753 * 754 * The situation can occur in these cases: 755 * - a lot of VRAM is freed by userspace 756 * - the presence of a big buffer causes a lot of evictions 757 * (solution: split buffers into smaller ones) 758 * 759 * If 128 MB or 1/8th of VRAM is free, start filling it now by setting 760 * accum_us to a positive number. 761 */ 762 if (free_vram >= 128 * 1024 * 1024 || free_vram >= total_vram / 8) { 763 s64 min_us; 764 765 /* Be more aggressive on dGPUs. Try to fill a portion of free 766 * VRAM now. 767 */ 768 if (!(adev->flags & AMD_IS_APU)) 769 min_us = bytes_to_us(adev, free_vram / 4); 770 else 771 min_us = 0; /* Reset accum_us on APUs. */ 772 773 adev->mm_stats.accum_us = max(min_us, adev->mm_stats.accum_us); 774 } 775 776 /* This is set to 0 if the driver is in debt to disallow (optional) 777 * buffer moves. 778 */ 779 *max_bytes = us_to_bytes(adev, adev->mm_stats.accum_us); 780 781 /* Do the same for visible VRAM if half of it is free */ 782 if (!amdgpu_gmc_vram_full_visible(&adev->gmc)) { 783 u64 total_vis_vram = adev->gmc.visible_vram_size; 784 u64 used_vis_vram = 785 amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr); 786 787 if (used_vis_vram < total_vis_vram) { 788 u64 free_vis_vram = total_vis_vram - used_vis_vram; 789 790 adev->mm_stats.accum_us_vis = min(adev->mm_stats.accum_us_vis + 791 increment_us, us_upper_bound); 792 793 if (free_vis_vram >= total_vis_vram / 2) 794 adev->mm_stats.accum_us_vis = 795 max(bytes_to_us(adev, free_vis_vram / 2), 796 adev->mm_stats.accum_us_vis); 797 } 798 799 *max_vis_bytes = us_to_bytes(adev, adev->mm_stats.accum_us_vis); 800 } else { 801 *max_vis_bytes = 0; 802 } 803 804 spin_unlock(&adev->mm_stats.lock); 805 } 806 807 /* Report how many bytes have really been moved for the last command 808 * submission. This can result in a debt that can stop buffer migrations 809 * temporarily. 810 */ 811 void amdgpu_cs_report_moved_bytes(struct amdgpu_device *adev, u64 num_bytes, 812 u64 num_vis_bytes) 813 { 814 spin_lock(&adev->mm_stats.lock); 815 adev->mm_stats.accum_us -= bytes_to_us(adev, num_bytes); 816 adev->mm_stats.accum_us_vis -= bytes_to_us(adev, num_vis_bytes); 817 spin_unlock(&adev->mm_stats.lock); 818 } 819 820 static int amdgpu_cs_bo_validate(void *param, struct amdgpu_bo *bo) 821 { 822 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 823 struct amdgpu_cs_parser *p = param; 824 struct ttm_operation_ctx ctx = { 825 .interruptible = true, 826 .no_wait_gpu = false, 827 .resv = bo->tbo.base.resv 828 }; 829 uint32_t domain; 830 int r; 831 832 if (bo->tbo.pin_count) 833 return 0; 834 835 /* Don't move this buffer if we have depleted our allowance 836 * to move it. Don't move anything if the threshold is zero. 837 */ 838 if (p->bytes_moved < p->bytes_moved_threshold && 839 (!bo->tbo.base.dma_buf || 840 list_empty(&bo->tbo.base.dma_buf->attachments))) { 841 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 842 (bo->flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED)) { 843 /* And don't move a CPU_ACCESS_REQUIRED BO to limited 844 * visible VRAM if we've depleted our allowance to do 845 * that. 846 */ 847 if (p->bytes_moved_vis < p->bytes_moved_vis_threshold) 848 domain = bo->preferred_domains; 849 else 850 domain = bo->allowed_domains; 851 } else { 852 domain = bo->preferred_domains; 853 } 854 } else { 855 domain = bo->allowed_domains; 856 } 857 858 retry: 859 amdgpu_bo_placement_from_domain(bo, domain); 860 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 861 862 p->bytes_moved += ctx.bytes_moved; 863 if (!amdgpu_gmc_vram_full_visible(&adev->gmc) && 864 amdgpu_res_cpu_visible(adev, bo->tbo.resource)) 865 p->bytes_moved_vis += ctx.bytes_moved; 866 867 if (unlikely(r == -ENOMEM) && domain != bo->allowed_domains) { 868 domain = bo->allowed_domains; 869 goto retry; 870 } 871 872 return r; 873 } 874 875 static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, 876 union drm_amdgpu_cs *cs) 877 { 878 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 879 struct ttm_operation_ctx ctx = { true, false }; 880 struct amdgpu_bo_list *list = NULL; 881 struct amdgpu_vm *vm = &fpriv->vm; 882 struct amdgpu_bo_list_entry *e; 883 struct drm_gem_object *obj; 884 unsigned int i; 885 int r; 886 887 /* p->bo_list could already be assigned if AMDGPU_CHUNK_ID_BO_HANDLES is present */ 888 if (cs->in.bo_list_handle) { 889 if (p->bo_list) 890 return -EINVAL; 891 892 list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle); 893 } else if (!p->bo_list) { 894 /* Create a empty bo_list when no handle is provided */ 895 list = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0); 896 } 897 898 if (IS_ERR(list)) 899 return PTR_ERR(list); 900 else if (list) 901 p->bo_list = list; 902 else 903 list = p->bo_list; 904 905 /* Get userptr backing pages. If pages are updated after registered 906 * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do 907 * amdgpu_ttm_backend_bind() to flush and invalidate new pages 908 */ 909 amdgpu_bo_list_for_each_userptr_entry(e, list) { 910 bool userpage_invalidated = false; 911 struct amdgpu_bo *bo = e->bo; 912 913 e->range = amdgpu_hmm_range_alloc(NULL); 914 if (unlikely(!e->range)) { 915 r = -ENOMEM; 916 goto out_free_user_pages; 917 } 918 919 r = amdgpu_ttm_tt_get_user_pages(bo, e->range); 920 if (r) 921 goto out_free_user_pages; 922 923 for (i = 0; i < bo->tbo.ttm->num_pages; i++) { 924 if (bo->tbo.ttm->pages[i] != 925 hmm_pfn_to_page(e->range->hmm_range.hmm_pfns[i])) { 926 userpage_invalidated = true; 927 break; 928 } 929 } 930 e->user_invalidated = userpage_invalidated; 931 } 932 933 drm_exec_until_all_locked(&p->exec) { 934 r = amdgpu_vm_lock_pd(&fpriv->vm, &p->exec, 1 + p->gang_size); 935 drm_exec_retry_on_contention(&p->exec); 936 if (unlikely(r)) 937 goto out_free_user_pages; 938 939 amdgpu_bo_list_for_each_entry(e, list) { 940 r = drm_exec_prepare_obj(&p->exec, &e->bo->tbo.base, 941 TTM_NUM_MOVE_FENCES + p->gang_size); 942 drm_exec_retry_on_contention(&p->exec); 943 if (unlikely(r)) 944 goto out_free_user_pages; 945 946 e->bo_va = amdgpu_vm_bo_find(vm, e->bo); 947 } 948 949 if (p->uf_bo) { 950 r = drm_exec_prepare_obj(&p->exec, &p->uf_bo->tbo.base, 951 TTM_NUM_MOVE_FENCES + p->gang_size); 952 drm_exec_retry_on_contention(&p->exec); 953 if (unlikely(r)) 954 goto out_free_user_pages; 955 } 956 } 957 958 amdgpu_bo_list_for_each_userptr_entry(e, list) { 959 struct mm_struct *usermm; 960 961 usermm = amdgpu_ttm_tt_get_usermm(e->bo->tbo.ttm); 962 if (usermm && usermm != current->mm) { 963 r = -EPERM; 964 goto out_free_user_pages; 965 } 966 967 if (amdgpu_ttm_tt_is_userptr(e->bo->tbo.ttm) && 968 e->user_invalidated) { 969 amdgpu_bo_placement_from_domain(e->bo, 970 AMDGPU_GEM_DOMAIN_CPU); 971 r = ttm_bo_validate(&e->bo->tbo, &e->bo->placement, 972 &ctx); 973 if (r) 974 goto out_free_user_pages; 975 976 amdgpu_ttm_tt_set_user_pages(e->bo->tbo.ttm, 977 e->range); 978 } 979 } 980 981 amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold, 982 &p->bytes_moved_vis_threshold); 983 p->bytes_moved = 0; 984 p->bytes_moved_vis = 0; 985 986 r = amdgpu_vm_validate(p->adev, &fpriv->vm, NULL, 987 amdgpu_cs_bo_validate, p); 988 if (r) { 989 drm_err(adev_to_drm(p->adev), "amdgpu_vm_validate() failed.\n"); 990 goto out_free_user_pages; 991 } 992 993 drm_exec_for_each_locked_object(&p->exec, obj) { 994 r = amdgpu_cs_bo_validate(p, gem_to_amdgpu_bo(obj)); 995 if (unlikely(r)) 996 goto out_free_user_pages; 997 } 998 999 if (p->uf_bo) { 1000 r = amdgpu_ttm_alloc_gart(&p->uf_bo->tbo); 1001 if (unlikely(r)) 1002 goto out_free_user_pages; 1003 1004 p->gang_leader->uf_addr += amdgpu_bo_gpu_offset(p->uf_bo); 1005 } 1006 1007 amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved, 1008 p->bytes_moved_vis); 1009 1010 for (i = 0; i < p->gang_size; ++i) 1011 amdgpu_job_set_resources(p->jobs[i], list->gds_obj, 1012 list->gws_obj, list->oa_obj); 1013 return 0; 1014 1015 out_free_user_pages: 1016 amdgpu_bo_list_for_each_userptr_entry(e, list) { 1017 amdgpu_hmm_range_free(e->range); 1018 e->range = NULL; 1019 } 1020 return r; 1021 } 1022 1023 static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *p) 1024 { 1025 int i, j; 1026 1027 if (!trace_amdgpu_cs_enabled()) 1028 return; 1029 1030 for (i = 0; i < p->gang_size; ++i) { 1031 struct amdgpu_job *job = p->jobs[i]; 1032 1033 for (j = 0; j < job->num_ibs; ++j) 1034 trace_amdgpu_cs(p, job, &job->ibs[j]); 1035 } 1036 } 1037 1038 static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p, 1039 struct amdgpu_job *job) 1040 { 1041 struct amdgpu_ring *ring = amdgpu_job_ring(job); 1042 struct amdgpu_device *adev = ring->adev; 1043 unsigned int i; 1044 int r; 1045 1046 /* Only for UVD/VCE VM emulation */ 1047 if (!ring->funcs->parse_cs && !ring->funcs->patch_cs_in_place) 1048 return 0; 1049 1050 for (i = 0; i < job->num_ibs; ++i) { 1051 struct amdgpu_ib *ib = &job->ibs[i]; 1052 struct amdgpu_bo_va_mapping *m; 1053 struct amdgpu_bo *aobj; 1054 uint64_t va_start; 1055 uint8_t *kptr; 1056 1057 va_start = ib->gpu_addr & AMDGPU_GMC_HOLE_MASK; 1058 r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m); 1059 if (r) { 1060 drm_err(adev_to_drm(p->adev), "IB va_start is invalid\n"); 1061 return r; 1062 } 1063 1064 if ((va_start + ib->length_dw * 4) > 1065 (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) { 1066 drm_err(adev_to_drm(p->adev), "IB va_start+ib_bytes is invalid\n"); 1067 return -EINVAL; 1068 } 1069 1070 /* the IB should be reserved at this point */ 1071 r = amdgpu_bo_kmap(aobj, (void **)&kptr); 1072 if (r) 1073 return r; 1074 1075 kptr += va_start - (m->start * AMDGPU_GPU_PAGE_SIZE); 1076 1077 if (ring->funcs->parse_cs) { 1078 memcpy(ib->ptr, kptr, ib->length_dw * 4); 1079 amdgpu_bo_kunmap(aobj); 1080 1081 r = amdgpu_ring_parse_cs(ring, p, job, ib); 1082 if (r) 1083 return r; 1084 1085 if (ib->sa_bo) 1086 ib->gpu_addr = amdgpu_sa_bo_gpu_addr(ib->sa_bo); 1087 } else { 1088 ib->ptr = (uint32_t *)kptr; 1089 r = amdgpu_ring_patch_cs_in_place(ring, p, job, ib); 1090 amdgpu_bo_kunmap(aobj); 1091 if (r) 1092 return r; 1093 } 1094 } 1095 1096 return 0; 1097 } 1098 1099 static int amdgpu_cs_patch_jobs(struct amdgpu_cs_parser *p) 1100 { 1101 unsigned int i; 1102 int r; 1103 1104 for (i = 0; i < p->gang_size; ++i) { 1105 r = amdgpu_cs_patch_ibs(p, p->jobs[i]); 1106 if (r) 1107 return r; 1108 } 1109 return 0; 1110 } 1111 1112 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p) 1113 { 1114 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1115 struct amdgpu_job *job = p->gang_leader; 1116 struct amdgpu_device *adev = p->adev; 1117 struct amdgpu_vm *vm = &fpriv->vm; 1118 struct amdgpu_bo_list_entry *e; 1119 struct amdgpu_bo_va *bo_va; 1120 unsigned int i; 1121 int r; 1122 1123 /* 1124 * We can't use gang submit on with reserved VMIDs when the VM changes 1125 * can't be invalidated by more than one engine at the same time. 1126 */ 1127 if (p->gang_size > 1 && !adev->vm_manager.concurrent_flush) { 1128 for (i = 0; i < p->gang_size; ++i) { 1129 struct drm_sched_entity *entity = p->entities[i]; 1130 struct drm_gpu_scheduler *sched = entity->rq->sched; 1131 struct amdgpu_ring *ring = to_amdgpu_ring(sched); 1132 1133 if (amdgpu_vmid_uses_reserved(vm, ring->vm_hub)) 1134 return -EINVAL; 1135 } 1136 } 1137 1138 if (!amdgpu_vm_ready(vm)) 1139 return -EINVAL; 1140 1141 r = amdgpu_vm_clear_freed(adev, vm, NULL); 1142 if (r) 1143 return r; 1144 1145 r = amdgpu_vm_bo_update(adev, fpriv->prt_va, false); 1146 if (r) 1147 return r; 1148 1149 r = amdgpu_sync_fence(&p->sync, fpriv->prt_va->last_pt_update, 1150 GFP_KERNEL); 1151 if (r) 1152 return r; 1153 1154 if (fpriv->csa_va) { 1155 bo_va = fpriv->csa_va; 1156 r = amdgpu_vm_bo_update(adev, bo_va, false); 1157 if (r) 1158 return r; 1159 1160 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update, 1161 GFP_KERNEL); 1162 if (r) 1163 return r; 1164 } 1165 1166 /* FIXME: In theory this loop shouldn't be needed any more when 1167 * amdgpu_vm_handle_moved handles all moved BOs that are reserved 1168 * with p->ticket. But removing it caused test regressions, so I'm 1169 * leaving it here for now. 1170 */ 1171 amdgpu_bo_list_for_each_entry(e, p->bo_list) { 1172 bo_va = e->bo_va; 1173 if (bo_va == NULL) 1174 continue; 1175 1176 r = amdgpu_vm_bo_update(adev, bo_va, false); 1177 if (r) 1178 return r; 1179 1180 r = amdgpu_sync_fence(&p->sync, bo_va->last_pt_update, 1181 GFP_KERNEL); 1182 if (r) 1183 return r; 1184 } 1185 1186 r = amdgpu_vm_handle_moved(adev, vm, drm_exec_ticket(&p->exec)); 1187 if (r) 1188 return r; 1189 1190 r = amdgpu_vm_update_pdes(adev, vm, false); 1191 if (r) 1192 return r; 1193 1194 r = amdgpu_sync_fence(&p->sync, vm->last_update, GFP_KERNEL); 1195 if (r) 1196 return r; 1197 1198 for (i = 0; i < p->gang_size; ++i) { 1199 job = p->jobs[i]; 1200 1201 if (!job->vm) 1202 continue; 1203 1204 job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo); 1205 } 1206 1207 return 0; 1208 } 1209 1210 static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) 1211 { 1212 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1213 struct drm_gpu_scheduler *sched; 1214 struct drm_gem_object *obj; 1215 struct dma_fence *fence; 1216 unsigned int i; 1217 int r; 1218 1219 r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entities[p->gang_leader_idx]); 1220 if (r) { 1221 if (r != -ERESTARTSYS) 1222 drm_err(adev_to_drm(p->adev), "amdgpu_ctx_wait_prev_fence failed.\n"); 1223 return r; 1224 } 1225 1226 drm_exec_for_each_locked_object(&p->exec, obj) { 1227 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 1228 1229 struct dma_resv *resv = bo->tbo.base.resv; 1230 enum amdgpu_sync_mode sync_mode; 1231 1232 sync_mode = amdgpu_bo_explicit_sync(bo) ? 1233 AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER; 1234 r = amdgpu_sync_resv(p->adev, &p->sync, resv, sync_mode, 1235 &fpriv->vm); 1236 if (r) 1237 return r; 1238 } 1239 1240 for (i = 0; i < p->gang_size; ++i) { 1241 r = amdgpu_sync_push_to_job(&p->sync, p->jobs[i]); 1242 if (r) 1243 return r; 1244 } 1245 1246 sched = p->gang_leader->base.entity->rq->sched; 1247 while ((fence = amdgpu_sync_get_fence(&p->sync))) { 1248 struct drm_sched_fence *s_fence = to_drm_sched_fence(fence); 1249 1250 /* 1251 * When we have an dependency it might be necessary to insert a 1252 * pipeline sync to make sure that all caches etc are flushed and the 1253 * next job actually sees the results from the previous one 1254 * before we start executing on the same scheduler ring. 1255 */ 1256 if (!s_fence || s_fence->sched != sched) { 1257 dma_fence_put(fence); 1258 continue; 1259 } 1260 1261 r = amdgpu_sync_fence(&p->gang_leader->explicit_sync, fence, 1262 GFP_KERNEL); 1263 dma_fence_put(fence); 1264 if (r) 1265 return r; 1266 } 1267 return 0; 1268 } 1269 1270 static void amdgpu_cs_post_dependencies(struct amdgpu_cs_parser *p) 1271 { 1272 int i; 1273 1274 for (i = 0; i < p->num_post_deps; ++i) { 1275 if (p->post_deps[i].chain && p->post_deps[i].point) { 1276 drm_syncobj_add_point(p->post_deps[i].syncobj, 1277 p->post_deps[i].chain, 1278 p->fence, p->post_deps[i].point); 1279 p->post_deps[i].chain = NULL; 1280 } else { 1281 drm_syncobj_replace_fence(p->post_deps[i].syncobj, 1282 p->fence); 1283 } 1284 } 1285 } 1286 1287 static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, 1288 union drm_amdgpu_cs *cs) 1289 { 1290 struct amdgpu_fpriv *fpriv = p->filp->driver_priv; 1291 struct amdgpu_job *leader = p->gang_leader; 1292 struct amdgpu_vm *vm = &fpriv->vm; 1293 struct amdgpu_bo_list_entry *e; 1294 struct drm_gem_object *gobj; 1295 unsigned int i; 1296 uint64_t seq; 1297 int r; 1298 1299 for (i = 0; i < p->gang_size; ++i) 1300 drm_sched_job_arm(&p->jobs[i]->base); 1301 1302 for (i = 0; i < p->gang_size; ++i) { 1303 struct dma_fence *fence; 1304 1305 if (p->jobs[i] == leader) 1306 continue; 1307 1308 fence = &p->jobs[i]->base.s_fence->scheduled; 1309 dma_fence_get(fence); 1310 r = drm_sched_job_add_dependency(&leader->base, fence); 1311 if (r) { 1312 dma_fence_put(fence); 1313 return r; 1314 } 1315 } 1316 1317 if (p->gang_size > 1) { 1318 for (i = 0; i < p->gang_size; ++i) 1319 amdgpu_job_set_gang_leader(p->jobs[i], leader); 1320 } 1321 1322 /* No memory allocation is allowed while holding the notifier lock. 1323 * The lock is held until amdgpu_cs_submit is finished and fence is 1324 * added to BOs. 1325 */ 1326 mutex_lock(&p->adev->notifier_lock); 1327 1328 /* If userptr are invalidated after amdgpu_cs_parser_bos(), return 1329 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl. 1330 */ 1331 r = 0; 1332 amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { 1333 r |= !amdgpu_hmm_range_valid(e->range); 1334 amdgpu_hmm_range_free(e->range); 1335 e->range = NULL; 1336 } 1337 1338 if (r || !list_empty(&vm->individual.needs_update)) { 1339 r = -EAGAIN; 1340 mutex_unlock(&p->adev->notifier_lock); 1341 return r; 1342 } 1343 1344 p->fence = dma_fence_get(&leader->base.s_fence->finished); 1345 drm_exec_for_each_locked_object(&p->exec, gobj) { 1346 1347 ttm_bo_move_to_lru_tail_unlocked(&gem_to_amdgpu_bo(gobj)->tbo); 1348 1349 /* Everybody except for the gang leader uses READ */ 1350 for (i = 0; i < p->gang_size; ++i) { 1351 if (p->jobs[i] == leader) 1352 continue; 1353 1354 dma_resv_add_fence(gobj->resv, 1355 &p->jobs[i]->base.s_fence->finished, 1356 DMA_RESV_USAGE_READ); 1357 } 1358 1359 /* The gang leader as remembered as writer */ 1360 dma_resv_add_fence(gobj->resv, p->fence, DMA_RESV_USAGE_WRITE); 1361 } 1362 1363 seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_leader_idx], 1364 p->fence); 1365 amdgpu_cs_post_dependencies(p); 1366 1367 if ((leader->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) && 1368 !p->ctx->preamble_presented) { 1369 leader->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST; 1370 p->ctx->preamble_presented = true; 1371 } 1372 1373 cs->out.handle = seq; 1374 leader->uf_sequence = seq; 1375 1376 amdgpu_vm_bo_trace_cs(&fpriv->vm, drm_exec_ticket(&p->exec)); 1377 for (i = 0; i < p->gang_size; ++i) { 1378 amdgpu_job_free_resources(p->jobs[i]); 1379 trace_amdgpu_cs_ioctl(p->jobs[i]); 1380 drm_sched_entity_push_job(&p->jobs[i]->base); 1381 p->jobs[i] = NULL; 1382 } 1383 1384 amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm); 1385 1386 mutex_unlock(&p->adev->notifier_lock); 1387 return 0; 1388 } 1389 1390 /* Cleanup the parser structure */ 1391 static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser) 1392 { 1393 struct amdgpu_device *adev = parser->adev; 1394 struct amdgpu_bo_list_entry *e; 1395 unsigned int i; 1396 1397 amdgpu_sync_free(&parser->sync); 1398 drm_exec_fini(&parser->exec); 1399 1400 for (i = 0; i < parser->num_post_deps; i++) { 1401 drm_syncobj_put(parser->post_deps[i].syncobj); 1402 kfree(parser->post_deps[i].chain); 1403 } 1404 kfree(parser->post_deps); 1405 1406 dma_fence_put(parser->fence); 1407 1408 if (parser->ctx) 1409 amdgpu_ctx_put(parser->ctx); 1410 if (parser->bo_list) { 1411 if (adev->debug_vm) { 1412 /* Invalidate all BOs to test for userspace bugs */ 1413 amdgpu_bo_list_for_each_entry(e, parser->bo_list) { 1414 struct amdgpu_bo *bo = e->bo; 1415 1416 /* ignore duplicates */ 1417 if (!bo) 1418 continue; 1419 1420 amdgpu_vm_bo_invalidate(bo, false); 1421 } 1422 } 1423 amdgpu_bo_list_put(parser->bo_list); 1424 } 1425 1426 for (i = 0; i < parser->nchunks; i++) 1427 kvfree(parser->chunks[i].kdata); 1428 kvfree(parser->chunks); 1429 for (i = 0; i < parser->gang_size; ++i) { 1430 if (parser->jobs[i]) 1431 amdgpu_job_free(parser->jobs[i]); 1432 } 1433 amdgpu_bo_unref(&parser->uf_bo); 1434 } 1435 1436 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 1437 { 1438 struct amdgpu_device *adev = drm_to_adev(dev); 1439 struct amdgpu_bo_list_entry *e; 1440 struct amdgpu_cs_parser parser; 1441 int r; 1442 1443 if (amdgpu_ras_intr_triggered()) 1444 return -EHWPOISON; 1445 1446 if (!adev->accel_working) 1447 return -EBUSY; 1448 1449 PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock); 1450 1451 r = PM_RUNTIME_ACQUIRE_ERR(&lock); 1452 if (r) 1453 return r; 1454 1455 r = amdgpu_cs_parser_init(&parser, adev, filp, data); 1456 if (r) { 1457 drm_err_ratelimited(dev, "Failed to initialize parser %d!\n", r); 1458 return r; 1459 } 1460 1461 r = amdgpu_cs_pass1(&parser, data); 1462 if (r) 1463 goto error_fini; 1464 1465 r = amdgpu_cs_pass2(&parser); 1466 if (r) 1467 goto error_fini; 1468 1469 r = amdgpu_cs_parser_bos(&parser, data); 1470 if (r) { 1471 if (r == -ENOMEM) 1472 drm_err(dev, "Not enough memory for command submission!\n"); 1473 else if (r != -ERESTARTSYS && r != -EAGAIN) 1474 drm_dbg(dev, "Failed to process the buffer list %d!\n", r); 1475 goto error_fini; 1476 } 1477 1478 r = amdgpu_cs_patch_jobs(&parser); 1479 if (r) 1480 goto error_free_user_pages; 1481 1482 r = amdgpu_cs_vm_handling(&parser); 1483 if (r) 1484 goto error_free_user_pages; 1485 1486 r = amdgpu_cs_sync_rings(&parser); 1487 if (r) 1488 goto error_free_user_pages; 1489 1490 trace_amdgpu_cs_ibs(&parser); 1491 1492 r = amdgpu_cs_submit(&parser, data); 1493 if (r) 1494 goto error_free_user_pages; 1495 1496 amdgpu_cs_parser_fini(&parser); 1497 return 0; 1498 1499 error_free_user_pages: 1500 /* amdgpu_cs_submit() only drops the ranges once it got that far. */ 1501 amdgpu_bo_list_for_each_userptr_entry(e, parser.bo_list) { 1502 amdgpu_hmm_range_free(e->range); 1503 e->range = NULL; 1504 } 1505 error_fini: 1506 amdgpu_cs_parser_fini(&parser); 1507 return r; 1508 } 1509 1510 /** 1511 * amdgpu_cs_wait_ioctl - wait for a command submission to finish 1512 * 1513 * @dev: drm device 1514 * @data: data from userspace 1515 * @filp: file private 1516 * 1517 * Wait for the command submission identified by handle to finish. 1518 */ 1519 int amdgpu_cs_wait_ioctl(struct drm_device *dev, void *data, 1520 struct drm_file *filp) 1521 { 1522 union drm_amdgpu_wait_cs *wait = data; 1523 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout); 1524 struct drm_sched_entity *entity; 1525 struct amdgpu_ctx *ctx; 1526 struct dma_fence *fence; 1527 long r; 1528 1529 ctx = amdgpu_ctx_get(filp->driver_priv, wait->in.ctx_id); 1530 if (ctx == NULL) 1531 return -EINVAL; 1532 1533 r = amdgpu_ctx_get_entity(ctx, wait->in.ip_type, wait->in.ip_instance, 1534 wait->in.ring, &entity); 1535 if (r) { 1536 amdgpu_ctx_put(ctx); 1537 return r; 1538 } 1539 1540 fence = amdgpu_ctx_get_fence(ctx, entity, wait->in.handle); 1541 if (IS_ERR(fence)) 1542 r = PTR_ERR(fence); 1543 else if (fence) { 1544 r = dma_fence_wait_timeout(fence, true, timeout); 1545 if (r > 0 && fence->error) 1546 r = fence->error; 1547 dma_fence_put(fence); 1548 } else 1549 r = 1; 1550 1551 amdgpu_ctx_put(ctx); 1552 if (r < 0) 1553 return r; 1554 1555 memset(wait, 0, sizeof(*wait)); 1556 wait->out.status = (r == 0); 1557 1558 return 0; 1559 } 1560 1561 /** 1562 * amdgpu_cs_get_fence - helper to get fence from drm_amdgpu_fence 1563 * 1564 * @adev: amdgpu device 1565 * @filp: file private 1566 * @user: drm_amdgpu_fence copied from user space 1567 */ 1568 static struct dma_fence *amdgpu_cs_get_fence(struct amdgpu_device *adev, 1569 struct drm_file *filp, 1570 struct drm_amdgpu_fence *user) 1571 { 1572 struct drm_sched_entity *entity; 1573 struct amdgpu_ctx *ctx; 1574 struct dma_fence *fence; 1575 int r; 1576 1577 ctx = amdgpu_ctx_get(filp->driver_priv, user->ctx_id); 1578 if (ctx == NULL) 1579 return ERR_PTR(-EINVAL); 1580 1581 r = amdgpu_ctx_get_entity(ctx, user->ip_type, user->ip_instance, 1582 user->ring, &entity); 1583 if (r) { 1584 amdgpu_ctx_put(ctx); 1585 return ERR_PTR(r); 1586 } 1587 1588 fence = amdgpu_ctx_get_fence(ctx, entity, user->seq_no); 1589 amdgpu_ctx_put(ctx); 1590 1591 return fence; 1592 } 1593 1594 int amdgpu_cs_fence_to_handle_ioctl(struct drm_device *dev, void *data, 1595 struct drm_file *filp) 1596 { 1597 struct amdgpu_device *adev = drm_to_adev(dev); 1598 union drm_amdgpu_fence_to_handle *info = data; 1599 struct dma_fence *fence; 1600 struct drm_syncobj *syncobj; 1601 struct sync_file *sync_file; 1602 int fd, r; 1603 1604 fence = amdgpu_cs_get_fence(adev, filp, &info->in.fence); 1605 if (IS_ERR(fence)) 1606 return PTR_ERR(fence); 1607 1608 if (!fence) 1609 fence = dma_fence_get_stub(); 1610 1611 switch (info->in.what) { 1612 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ: 1613 r = drm_syncobj_create(&syncobj, 0, fence); 1614 dma_fence_put(fence); 1615 if (r) 1616 return r; 1617 r = drm_syncobj_get_handle(filp, syncobj, &info->out.handle); 1618 drm_syncobj_put(syncobj); 1619 return r; 1620 1621 case AMDGPU_FENCE_TO_HANDLE_GET_SYNCOBJ_FD: 1622 r = drm_syncobj_create(&syncobj, 0, fence); 1623 dma_fence_put(fence); 1624 if (r) 1625 return r; 1626 r = drm_syncobj_get_fd(syncobj, (int *)&info->out.handle); 1627 drm_syncobj_put(syncobj); 1628 return r; 1629 1630 case AMDGPU_FENCE_TO_HANDLE_GET_SYNC_FILE_FD: 1631 fd = get_unused_fd_flags(O_CLOEXEC); 1632 if (fd < 0) { 1633 dma_fence_put(fence); 1634 return fd; 1635 } 1636 1637 sync_file = sync_file_create(fence); 1638 dma_fence_put(fence); 1639 if (!sync_file) { 1640 put_unused_fd(fd); 1641 return -ENOMEM; 1642 } 1643 1644 fd_install(fd, sync_file->file); 1645 info->out.handle = fd; 1646 return 0; 1647 1648 default: 1649 dma_fence_put(fence); 1650 return -EINVAL; 1651 } 1652 } 1653 1654 /** 1655 * amdgpu_cs_wait_all_fences - wait on all fences to signal 1656 * 1657 * @adev: amdgpu device 1658 * @filp: file private 1659 * @wait: wait parameters 1660 * @fences: array of drm_amdgpu_fence 1661 */ 1662 static int amdgpu_cs_wait_all_fences(struct amdgpu_device *adev, 1663 struct drm_file *filp, 1664 union drm_amdgpu_wait_fences *wait, 1665 struct drm_amdgpu_fence *fences) 1666 { 1667 uint32_t fence_count = wait->in.fence_count; 1668 unsigned int i; 1669 long r = 1; 1670 1671 for (i = 0; i < fence_count; i++) { 1672 struct dma_fence *fence; 1673 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1674 1675 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1676 if (IS_ERR(fence)) 1677 return PTR_ERR(fence); 1678 else if (!fence) 1679 continue; 1680 1681 r = dma_fence_wait_timeout(fence, true, timeout); 1682 if (r > 0 && fence->error) 1683 r = fence->error; 1684 1685 dma_fence_put(fence); 1686 if (r < 0) 1687 return r; 1688 1689 if (r == 0) 1690 break; 1691 } 1692 1693 memset(wait, 0, sizeof(*wait)); 1694 wait->out.status = (r > 0); 1695 1696 return 0; 1697 } 1698 1699 /** 1700 * amdgpu_cs_wait_any_fence - wait on any fence to signal 1701 * 1702 * @adev: amdgpu device 1703 * @filp: file private 1704 * @wait: wait parameters 1705 * @fences: array of drm_amdgpu_fence 1706 */ 1707 static int amdgpu_cs_wait_any_fence(struct amdgpu_device *adev, 1708 struct drm_file *filp, 1709 union drm_amdgpu_wait_fences *wait, 1710 struct drm_amdgpu_fence *fences) 1711 { 1712 unsigned long timeout = amdgpu_gem_timeout(wait->in.timeout_ns); 1713 uint32_t fence_count = wait->in.fence_count; 1714 uint32_t first = ~0; 1715 struct dma_fence **array; 1716 unsigned int i; 1717 long r; 1718 1719 /* Prepare the fence array */ 1720 array = kzalloc_objs(struct dma_fence *, fence_count); 1721 1722 if (array == NULL) 1723 return -ENOMEM; 1724 1725 for (i = 0; i < fence_count; i++) { 1726 struct dma_fence *fence; 1727 1728 fence = amdgpu_cs_get_fence(adev, filp, &fences[i]); 1729 if (IS_ERR(fence)) { 1730 r = PTR_ERR(fence); 1731 goto err_free_fence_array; 1732 } else if (fence) { 1733 array[i] = fence; 1734 } else { /* NULL, the fence has been already signaled */ 1735 r = 1; 1736 first = i; 1737 goto out; 1738 } 1739 } 1740 1741 r = dma_fence_wait_any_timeout(array, fence_count, true, timeout, 1742 &first); 1743 if (r < 0) 1744 goto err_free_fence_array; 1745 1746 out: 1747 memset(wait, 0, sizeof(*wait)); 1748 wait->out.status = (r > 0); 1749 wait->out.first_signaled = first; 1750 1751 if (first < fence_count && array[first]) 1752 r = array[first]->error; 1753 else 1754 r = 0; 1755 1756 err_free_fence_array: 1757 for (i = 0; i < fence_count; i++) 1758 dma_fence_put(array[i]); 1759 kfree(array); 1760 1761 return r; 1762 } 1763 1764 /** 1765 * amdgpu_cs_wait_fences_ioctl - wait for multiple command submissions to finish 1766 * 1767 * @dev: drm device 1768 * @data: data from userspace 1769 * @filp: file private 1770 */ 1771 int amdgpu_cs_wait_fences_ioctl(struct drm_device *dev, void *data, 1772 struct drm_file *filp) 1773 { 1774 struct amdgpu_device *adev = drm_to_adev(dev); 1775 union drm_amdgpu_wait_fences *wait = data; 1776 struct drm_amdgpu_fence *fences; 1777 int r; 1778 1779 /* 1780 * fence_count must be non-zero; dma_fence_wait_any_timeout() 1781 * does not accept an empty fence array. 1782 */ 1783 if (!wait->in.fence_count) 1784 return -EINVAL; 1785 1786 /* Get the fences from userspace */ 1787 fences = memdup_array_user(u64_to_user_ptr(wait->in.fences), 1788 wait->in.fence_count, 1789 sizeof(struct drm_amdgpu_fence)); 1790 if (IS_ERR(fences)) 1791 return PTR_ERR(fences); 1792 1793 if (wait->in.wait_all) 1794 r = amdgpu_cs_wait_all_fences(adev, filp, wait, fences); 1795 else 1796 r = amdgpu_cs_wait_any_fence(adev, filp, wait, fences); 1797 1798 kfree(fences); 1799 1800 return r; 1801 } 1802 1803 /** 1804 * amdgpu_cs_find_mapping - find bo_va for VM address 1805 * 1806 * @parser: command submission parser context 1807 * @addr: VM address 1808 * @bo: resulting BO of the mapping found 1809 * @map: Placeholder to return found BO mapping 1810 * 1811 * Search the buffer objects in the command submission context for a certain 1812 * virtual memory address. Returns allocation structure when found, NULL 1813 * otherwise. 1814 */ 1815 int amdgpu_cs_find_mapping(struct amdgpu_cs_parser *parser, 1816 uint64_t addr, struct amdgpu_bo **bo, 1817 struct amdgpu_bo_va_mapping **map) 1818 { 1819 struct amdgpu_fpriv *fpriv = parser->filp->driver_priv; 1820 struct ttm_operation_ctx ctx = { false, false }; 1821 struct amdgpu_vm *vm = &fpriv->vm; 1822 struct amdgpu_bo_va_mapping *mapping; 1823 int i, r; 1824 1825 mapping = amdgpu_vm_bo_lookup_mapping(vm, addr); 1826 if (!mapping || !mapping->bo_va || !mapping->bo_va->base.bo) 1827 return -EINVAL; 1828 1829 *bo = mapping->bo_va->base.bo; 1830 *map = mapping; 1831 1832 /* Double check that the BO is reserved by this CS */ 1833 if (dma_resv_locking_ctx((*bo)->tbo.base.resv) != drm_exec_ticket(&parser->exec)) 1834 return -EINVAL; 1835 1836 /* Make sure VRAM is allocated contigiously */ 1837 (*bo)->flags |= AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS; 1838 if ((*bo)->tbo.resource->mem_type == TTM_PL_VRAM && 1839 !((*bo)->tbo.resource->placement & TTM_PL_FLAG_CONTIGUOUS)) { 1840 1841 amdgpu_bo_placement_from_domain(*bo, (*bo)->allowed_domains); 1842 for (i = 0; i < (*bo)->placement.num_placement; i++) 1843 (*bo)->placements[i].flags |= TTM_PL_FLAG_CONTIGUOUS; 1844 r = ttm_bo_validate(&(*bo)->tbo, &(*bo)->placement, &ctx); 1845 if (r) 1846 return r; 1847 } 1848 1849 return amdgpu_ttm_alloc_gart(&(*bo)->tbo); 1850 } 1851