1 /* 2 * Copyright 2008 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22 * DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: 25 * Jerome Glisse <glisse@freedesktop.org> 26 */ 27 #include <linux/list_sort.h> 28 #include <drm/drmP.h> 29 #include <drm/radeon_drm.h> 30 #include "radeon_reg.h" 31 #include "radeon.h" 32 #include "radeon_trace.h" 33 34 #define RADEON_CS_MAX_PRIORITY 32u 35 #define RADEON_CS_NUM_BUCKETS (RADEON_CS_MAX_PRIORITY + 1) 36 37 /* This is based on the bucket sort with O(n) time complexity. 38 * An item with priority "i" is added to bucket[i]. The lists are then 39 * concatenated in descending order. 40 */ 41 struct radeon_cs_buckets { 42 struct list_head bucket[RADEON_CS_NUM_BUCKETS]; 43 }; 44 45 static void radeon_cs_buckets_init(struct radeon_cs_buckets *b) 46 { 47 unsigned i; 48 49 for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++) 50 INIT_LIST_HEAD(&b->bucket[i]); 51 } 52 53 static void radeon_cs_buckets_add(struct radeon_cs_buckets *b, 54 struct list_head *item, unsigned priority) 55 { 56 /* Since buffers which appear sooner in the relocation list are 57 * likely to be used more often than buffers which appear later 58 * in the list, the sort mustn't change the ordering of buffers 59 * with the same priority, i.e. it must be stable. 60 */ 61 list_add_tail(item, &b->bucket[min(priority, RADEON_CS_MAX_PRIORITY)]); 62 } 63 64 static void radeon_cs_buckets_get_list(struct radeon_cs_buckets *b, 65 struct list_head *out_list) 66 { 67 unsigned i; 68 69 /* Connect the sorted buckets in the output list. */ 70 for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++) { 71 list_splice(&b->bucket[i], out_list); 72 } 73 } 74 75 static int radeon_cs_parser_relocs(struct radeon_cs_parser *p) 76 { 77 struct drm_device *ddev = p->rdev->ddev; 78 struct radeon_cs_chunk *chunk; 79 struct radeon_cs_buckets buckets; 80 unsigned i, j; 81 bool duplicate, need_mmap_lock = false; 82 int r; 83 84 if (p->chunk_relocs_idx == -1) { 85 return 0; 86 } 87 chunk = &p->chunks[p->chunk_relocs_idx]; 88 p->dma_reloc_idx = 0; 89 /* FIXME: we assume that each relocs use 4 dwords */ 90 p->nrelocs = chunk->length_dw / 4; 91 p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL); 92 if (p->relocs_ptr == NULL) { 93 return -ENOMEM; 94 } 95 p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL); 96 if (p->relocs == NULL) { 97 return -ENOMEM; 98 } 99 100 radeon_cs_buckets_init(&buckets); 101 102 for (i = 0; i < p->nrelocs; i++) { 103 struct drm_radeon_cs_reloc *r; 104 unsigned priority; 105 106 duplicate = false; 107 r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4]; 108 for (j = 0; j < i; j++) { 109 if (r->handle == p->relocs[j].handle) { 110 p->relocs_ptr[i] = &p->relocs[j]; 111 duplicate = true; 112 break; 113 } 114 } 115 if (duplicate) { 116 p->relocs[i].handle = 0; 117 continue; 118 } 119 120 p->relocs[i].gobj = drm_gem_object_lookup(ddev, p->filp, 121 r->handle); 122 if (p->relocs[i].gobj == NULL) { 123 DRM_ERROR("gem object lookup failed 0x%x\n", 124 r->handle); 125 return -ENOENT; 126 } 127 p->relocs_ptr[i] = &p->relocs[i]; 128 p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj); 129 130 /* The userspace buffer priorities are from 0 to 15. A higher 131 * number means the buffer is more important. 132 * Also, the buffers used for write have a higher priority than 133 * the buffers used for read only, which doubles the range 134 * to 0 to 31. 32 is reserved for the kernel driver. 135 */ 136 priority = (r->flags & RADEON_RELOC_PRIO_MASK) * 2 137 + !!r->write_domain; 138 139 /* the first reloc of an UVD job is the msg and that must be in 140 VRAM, also but everything into VRAM on AGP cards and older 141 IGP chips to avoid image corruptions */ 142 if (p->ring == R600_RING_TYPE_UVD_INDEX && 143 (i == 0 || drm_pci_device_is_agp(p->rdev->ddev) || 144 p->rdev->family == CHIP_RS780 || 145 p->rdev->family == CHIP_RS880)) { 146 147 /* TODO: is this still needed for NI+ ? */ 148 p->relocs[i].prefered_domains = 149 RADEON_GEM_DOMAIN_VRAM; 150 151 p->relocs[i].allowed_domains = 152 RADEON_GEM_DOMAIN_VRAM; 153 154 /* prioritize this over any other relocation */ 155 priority = RADEON_CS_MAX_PRIORITY; 156 } else { 157 uint32_t domain = r->write_domain ? 158 r->write_domain : r->read_domains; 159 160 if (domain & RADEON_GEM_DOMAIN_CPU) { 161 DRM_ERROR("RADEON_GEM_DOMAIN_CPU is not valid " 162 "for command submission\n"); 163 return -EINVAL; 164 } 165 166 p->relocs[i].prefered_domains = domain; 167 if (domain == RADEON_GEM_DOMAIN_VRAM) 168 domain |= RADEON_GEM_DOMAIN_GTT; 169 p->relocs[i].allowed_domains = domain; 170 } 171 172 if (radeon_ttm_tt_has_userptr(p->relocs[i].robj->tbo.ttm)) { 173 uint32_t domain = p->relocs[i].prefered_domains; 174 if (!(domain & RADEON_GEM_DOMAIN_GTT)) { 175 DRM_ERROR("Only RADEON_GEM_DOMAIN_GTT is " 176 "allowed for userptr BOs\n"); 177 return -EINVAL; 178 } 179 need_mmap_lock = true; 180 domain = RADEON_GEM_DOMAIN_GTT; 181 p->relocs[i].prefered_domains = domain; 182 p->relocs[i].allowed_domains = domain; 183 } 184 185 p->relocs[i].tv.bo = &p->relocs[i].robj->tbo; 186 p->relocs[i].tv.shared = !r->write_domain; 187 p->relocs[i].handle = r->handle; 188 189 radeon_cs_buckets_add(&buckets, &p->relocs[i].tv.head, 190 priority); 191 } 192 193 radeon_cs_buckets_get_list(&buckets, &p->validated); 194 195 if (p->cs_flags & RADEON_CS_USE_VM) 196 p->vm_bos = radeon_vm_get_bos(p->rdev, p->ib.vm, 197 &p->validated); 198 if (need_mmap_lock) 199 down_read(¤t->mm->mmap_sem); 200 201 r = radeon_bo_list_validate(p->rdev, &p->ticket, &p->validated, p->ring); 202 203 if (need_mmap_lock) 204 up_read(¤t->mm->mmap_sem); 205 206 return r; 207 } 208 209 static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority) 210 { 211 p->priority = priority; 212 213 switch (ring) { 214 default: 215 DRM_ERROR("unknown ring id: %d\n", ring); 216 return -EINVAL; 217 case RADEON_CS_RING_GFX: 218 p->ring = RADEON_RING_TYPE_GFX_INDEX; 219 break; 220 case RADEON_CS_RING_COMPUTE: 221 if (p->rdev->family >= CHIP_TAHITI) { 222 if (p->priority > 0) 223 p->ring = CAYMAN_RING_TYPE_CP1_INDEX; 224 else 225 p->ring = CAYMAN_RING_TYPE_CP2_INDEX; 226 } else 227 p->ring = RADEON_RING_TYPE_GFX_INDEX; 228 break; 229 case RADEON_CS_RING_DMA: 230 if (p->rdev->family >= CHIP_CAYMAN) { 231 if (p->priority > 0) 232 p->ring = R600_RING_TYPE_DMA_INDEX; 233 else 234 p->ring = CAYMAN_RING_TYPE_DMA1_INDEX; 235 } else if (p->rdev->family >= CHIP_RV770) { 236 p->ring = R600_RING_TYPE_DMA_INDEX; 237 } else { 238 return -EINVAL; 239 } 240 break; 241 case RADEON_CS_RING_UVD: 242 p->ring = R600_RING_TYPE_UVD_INDEX; 243 break; 244 case RADEON_CS_RING_VCE: 245 /* TODO: only use the low priority ring for now */ 246 p->ring = TN_RING_TYPE_VCE1_INDEX; 247 break; 248 } 249 return 0; 250 } 251 252 static int radeon_cs_sync_rings(struct radeon_cs_parser *p) 253 { 254 struct radeon_cs_reloc *reloc; 255 int r; 256 257 list_for_each_entry(reloc, &p->validated, tv.head) { 258 struct reservation_object *resv; 259 260 resv = reloc->robj->tbo.resv; 261 r = radeon_semaphore_sync_resv(p->rdev, p->ib.semaphore, resv, 262 reloc->tv.shared); 263 if (r) 264 return r; 265 } 266 return 0; 267 } 268 269 /* XXX: note that this is called from the legacy UMS CS ioctl as well */ 270 int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data) 271 { 272 struct drm_radeon_cs *cs = data; 273 uint64_t *chunk_array_ptr; 274 unsigned size, i; 275 u32 ring = RADEON_CS_RING_GFX; 276 s32 priority = 0; 277 278 if (!cs->num_chunks) { 279 return 0; 280 } 281 /* get chunks */ 282 INIT_LIST_HEAD(&p->validated); 283 p->idx = 0; 284 p->ib.sa_bo = NULL; 285 p->ib.semaphore = NULL; 286 p->const_ib.sa_bo = NULL; 287 p->const_ib.semaphore = NULL; 288 p->chunk_ib_idx = -1; 289 p->chunk_relocs_idx = -1; 290 p->chunk_flags_idx = -1; 291 p->chunk_const_ib_idx = -1; 292 p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL); 293 if (p->chunks_array == NULL) { 294 return -ENOMEM; 295 } 296 chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks); 297 if (copy_from_user(p->chunks_array, chunk_array_ptr, 298 sizeof(uint64_t)*cs->num_chunks)) { 299 return -EFAULT; 300 } 301 p->cs_flags = 0; 302 p->nchunks = cs->num_chunks; 303 p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL); 304 if (p->chunks == NULL) { 305 return -ENOMEM; 306 } 307 for (i = 0; i < p->nchunks; i++) { 308 struct drm_radeon_cs_chunk __user **chunk_ptr = NULL; 309 struct drm_radeon_cs_chunk user_chunk; 310 uint32_t __user *cdata; 311 312 chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i]; 313 if (copy_from_user(&user_chunk, chunk_ptr, 314 sizeof(struct drm_radeon_cs_chunk))) { 315 return -EFAULT; 316 } 317 p->chunks[i].length_dw = user_chunk.length_dw; 318 p->chunks[i].chunk_id = user_chunk.chunk_id; 319 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) { 320 p->chunk_relocs_idx = i; 321 } 322 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) { 323 p->chunk_ib_idx = i; 324 /* zero length IB isn't useful */ 325 if (p->chunks[i].length_dw == 0) 326 return -EINVAL; 327 } 328 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) { 329 p->chunk_const_ib_idx = i; 330 /* zero length CONST IB isn't useful */ 331 if (p->chunks[i].length_dw == 0) 332 return -EINVAL; 333 } 334 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) { 335 p->chunk_flags_idx = i; 336 /* zero length flags aren't useful */ 337 if (p->chunks[i].length_dw == 0) 338 return -EINVAL; 339 } 340 341 size = p->chunks[i].length_dw; 342 cdata = (void __user *)(unsigned long)user_chunk.chunk_data; 343 p->chunks[i].user_ptr = cdata; 344 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) 345 continue; 346 347 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) { 348 if (!p->rdev || !(p->rdev->flags & RADEON_IS_AGP)) 349 continue; 350 } 351 352 p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t)); 353 size *= sizeof(uint32_t); 354 if (p->chunks[i].kdata == NULL) { 355 return -ENOMEM; 356 } 357 if (copy_from_user(p->chunks[i].kdata, cdata, size)) { 358 return -EFAULT; 359 } 360 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) { 361 p->cs_flags = p->chunks[i].kdata[0]; 362 if (p->chunks[i].length_dw > 1) 363 ring = p->chunks[i].kdata[1]; 364 if (p->chunks[i].length_dw > 2) 365 priority = (s32)p->chunks[i].kdata[2]; 366 } 367 } 368 369 /* these are KMS only */ 370 if (p->rdev) { 371 if ((p->cs_flags & RADEON_CS_USE_VM) && 372 !p->rdev->vm_manager.enabled) { 373 DRM_ERROR("VM not active on asic!\n"); 374 return -EINVAL; 375 } 376 377 if (radeon_cs_get_ring(p, ring, priority)) 378 return -EINVAL; 379 380 /* we only support VM on some SI+ rings */ 381 if ((p->cs_flags & RADEON_CS_USE_VM) == 0) { 382 if (p->rdev->asic->ring[p->ring]->cs_parse == NULL) { 383 DRM_ERROR("Ring %d requires VM!\n", p->ring); 384 return -EINVAL; 385 } 386 } else { 387 if (p->rdev->asic->ring[p->ring]->ib_parse == NULL) { 388 DRM_ERROR("VM not supported on ring %d!\n", 389 p->ring); 390 return -EINVAL; 391 } 392 } 393 } 394 395 return 0; 396 } 397 398 static int cmp_size_smaller_first(void *priv, struct list_head *a, 399 struct list_head *b) 400 { 401 struct radeon_cs_reloc *la = list_entry(a, struct radeon_cs_reloc, tv.head); 402 struct radeon_cs_reloc *lb = list_entry(b, struct radeon_cs_reloc, tv.head); 403 404 /* Sort A before B if A is smaller. */ 405 return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages; 406 } 407 408 /** 409 * cs_parser_fini() - clean parser states 410 * @parser: parser structure holding parsing context. 411 * @error: error number 412 * 413 * If error is set than unvalidate buffer, otherwise just free memory 414 * used by parsing context. 415 **/ 416 static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bool backoff) 417 { 418 unsigned i; 419 420 if (!error) { 421 /* Sort the buffer list from the smallest to largest buffer, 422 * which affects the order of buffers in the LRU list. 423 * This assures that the smallest buffers are added first 424 * to the LRU list, so they are likely to be later evicted 425 * first, instead of large buffers whose eviction is more 426 * expensive. 427 * 428 * This slightly lowers the number of bytes moved by TTM 429 * per frame under memory pressure. 430 */ 431 list_sort(NULL, &parser->validated, cmp_size_smaller_first); 432 433 ttm_eu_fence_buffer_objects(&parser->ticket, 434 &parser->validated, 435 &parser->ib.fence->base); 436 } else if (backoff) { 437 ttm_eu_backoff_reservation(&parser->ticket, 438 &parser->validated); 439 } 440 441 if (parser->relocs != NULL) { 442 for (i = 0; i < parser->nrelocs; i++) { 443 if (parser->relocs[i].gobj) 444 drm_gem_object_unreference_unlocked(parser->relocs[i].gobj); 445 } 446 } 447 kfree(parser->track); 448 kfree(parser->relocs); 449 kfree(parser->relocs_ptr); 450 drm_free_large(parser->vm_bos); 451 for (i = 0; i < parser->nchunks; i++) 452 drm_free_large(parser->chunks[i].kdata); 453 kfree(parser->chunks); 454 kfree(parser->chunks_array); 455 radeon_ib_free(parser->rdev, &parser->ib); 456 radeon_ib_free(parser->rdev, &parser->const_ib); 457 } 458 459 static int radeon_cs_ib_chunk(struct radeon_device *rdev, 460 struct radeon_cs_parser *parser) 461 { 462 int r; 463 464 if (parser->chunk_ib_idx == -1) 465 return 0; 466 467 if (parser->cs_flags & RADEON_CS_USE_VM) 468 return 0; 469 470 r = radeon_cs_parse(rdev, parser->ring, parser); 471 if (r || parser->parser_error) { 472 DRM_ERROR("Invalid command stream !\n"); 473 return r; 474 } 475 476 r = radeon_cs_sync_rings(parser); 477 if (r) { 478 if (r != -ERESTARTSYS) 479 DRM_ERROR("Failed to sync rings: %i\n", r); 480 return r; 481 } 482 483 if (parser->ring == R600_RING_TYPE_UVD_INDEX) 484 radeon_uvd_note_usage(rdev); 485 else if ((parser->ring == TN_RING_TYPE_VCE1_INDEX) || 486 (parser->ring == TN_RING_TYPE_VCE2_INDEX)) 487 radeon_vce_note_usage(rdev); 488 489 r = radeon_ib_schedule(rdev, &parser->ib, NULL, true); 490 if (r) { 491 DRM_ERROR("Failed to schedule IB !\n"); 492 } 493 return r; 494 } 495 496 static int radeon_bo_vm_update_pte(struct radeon_cs_parser *p, 497 struct radeon_vm *vm) 498 { 499 struct radeon_device *rdev = p->rdev; 500 struct radeon_bo_va *bo_va; 501 int i, r; 502 503 r = radeon_vm_update_page_directory(rdev, vm); 504 if (r) 505 return r; 506 507 r = radeon_vm_clear_freed(rdev, vm); 508 if (r) 509 return r; 510 511 if (vm->ib_bo_va == NULL) { 512 DRM_ERROR("Tmp BO not in VM!\n"); 513 return -EINVAL; 514 } 515 516 r = radeon_vm_bo_update(rdev, vm->ib_bo_va, 517 &rdev->ring_tmp_bo.bo->tbo.mem); 518 if (r) 519 return r; 520 521 for (i = 0; i < p->nrelocs; i++) { 522 struct radeon_bo *bo; 523 524 /* ignore duplicates */ 525 if (p->relocs_ptr[i] != &p->relocs[i]) 526 continue; 527 528 bo = p->relocs[i].robj; 529 bo_va = radeon_vm_bo_find(vm, bo); 530 if (bo_va == NULL) { 531 dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm); 532 return -EINVAL; 533 } 534 535 r = radeon_vm_bo_update(rdev, bo_va, &bo->tbo.mem); 536 if (r) 537 return r; 538 } 539 540 return radeon_vm_clear_invalids(rdev, vm); 541 } 542 543 static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, 544 struct radeon_cs_parser *parser) 545 { 546 struct radeon_fpriv *fpriv = parser->filp->driver_priv; 547 struct radeon_vm *vm = &fpriv->vm; 548 int r; 549 550 if (parser->chunk_ib_idx == -1) 551 return 0; 552 if ((parser->cs_flags & RADEON_CS_USE_VM) == 0) 553 return 0; 554 555 if (parser->const_ib.length_dw) { 556 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib); 557 if (r) { 558 return r; 559 } 560 } 561 562 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib); 563 if (r) { 564 return r; 565 } 566 567 if (parser->ring == R600_RING_TYPE_UVD_INDEX) 568 radeon_uvd_note_usage(rdev); 569 570 mutex_lock(&vm->mutex); 571 r = radeon_bo_vm_update_pte(parser, vm); 572 if (r) { 573 goto out; 574 } 575 576 r = radeon_cs_sync_rings(parser); 577 if (r) { 578 if (r != -ERESTARTSYS) 579 DRM_ERROR("Failed to sync rings: %i\n", r); 580 goto out; 581 } 582 radeon_semaphore_sync_fence(parser->ib.semaphore, vm->fence); 583 584 if ((rdev->family >= CHIP_TAHITI) && 585 (parser->chunk_const_ib_idx != -1)) { 586 r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib, true); 587 } else { 588 r = radeon_ib_schedule(rdev, &parser->ib, NULL, true); 589 } 590 591 out: 592 mutex_unlock(&vm->mutex); 593 return r; 594 } 595 596 static int radeon_cs_handle_lockup(struct radeon_device *rdev, int r) 597 { 598 if (r == -EDEADLK) { 599 r = radeon_gpu_reset(rdev); 600 if (!r) 601 r = -EAGAIN; 602 } 603 return r; 604 } 605 606 static int radeon_cs_ib_fill(struct radeon_device *rdev, struct radeon_cs_parser *parser) 607 { 608 struct radeon_cs_chunk *ib_chunk; 609 struct radeon_vm *vm = NULL; 610 int r; 611 612 if (parser->chunk_ib_idx == -1) 613 return 0; 614 615 if (parser->cs_flags & RADEON_CS_USE_VM) { 616 struct radeon_fpriv *fpriv = parser->filp->driver_priv; 617 vm = &fpriv->vm; 618 619 if ((rdev->family >= CHIP_TAHITI) && 620 (parser->chunk_const_ib_idx != -1)) { 621 ib_chunk = &parser->chunks[parser->chunk_const_ib_idx]; 622 if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { 623 DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw); 624 return -EINVAL; 625 } 626 r = radeon_ib_get(rdev, parser->ring, &parser->const_ib, 627 vm, ib_chunk->length_dw * 4); 628 if (r) { 629 DRM_ERROR("Failed to get const ib !\n"); 630 return r; 631 } 632 parser->const_ib.is_const_ib = true; 633 parser->const_ib.length_dw = ib_chunk->length_dw; 634 if (copy_from_user(parser->const_ib.ptr, 635 ib_chunk->user_ptr, 636 ib_chunk->length_dw * 4)) 637 return -EFAULT; 638 } 639 640 ib_chunk = &parser->chunks[parser->chunk_ib_idx]; 641 if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { 642 DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw); 643 return -EINVAL; 644 } 645 } 646 ib_chunk = &parser->chunks[parser->chunk_ib_idx]; 647 648 r = radeon_ib_get(rdev, parser->ring, &parser->ib, 649 vm, ib_chunk->length_dw * 4); 650 if (r) { 651 DRM_ERROR("Failed to get ib !\n"); 652 return r; 653 } 654 parser->ib.length_dw = ib_chunk->length_dw; 655 if (ib_chunk->kdata) 656 memcpy(parser->ib.ptr, ib_chunk->kdata, ib_chunk->length_dw * 4); 657 else if (copy_from_user(parser->ib.ptr, ib_chunk->user_ptr, ib_chunk->length_dw * 4)) 658 return -EFAULT; 659 return 0; 660 } 661 662 int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 663 { 664 struct radeon_device *rdev = dev->dev_private; 665 struct radeon_cs_parser parser; 666 int r; 667 668 down_read(&rdev->exclusive_lock); 669 if (!rdev->accel_working) { 670 up_read(&rdev->exclusive_lock); 671 return -EBUSY; 672 } 673 if (rdev->in_reset) { 674 up_read(&rdev->exclusive_lock); 675 r = radeon_gpu_reset(rdev); 676 if (!r) 677 r = -EAGAIN; 678 return r; 679 } 680 /* initialize parser */ 681 memset(&parser, 0, sizeof(struct radeon_cs_parser)); 682 parser.filp = filp; 683 parser.rdev = rdev; 684 parser.dev = rdev->dev; 685 parser.family = rdev->family; 686 r = radeon_cs_parser_init(&parser, data); 687 if (r) { 688 DRM_ERROR("Failed to initialize parser !\n"); 689 radeon_cs_parser_fini(&parser, r, false); 690 up_read(&rdev->exclusive_lock); 691 r = radeon_cs_handle_lockup(rdev, r); 692 return r; 693 } 694 695 r = radeon_cs_ib_fill(rdev, &parser); 696 if (!r) { 697 r = radeon_cs_parser_relocs(&parser); 698 if (r && r != -ERESTARTSYS) 699 DRM_ERROR("Failed to parse relocation %d!\n", r); 700 } 701 702 if (r) { 703 radeon_cs_parser_fini(&parser, r, false); 704 up_read(&rdev->exclusive_lock); 705 r = radeon_cs_handle_lockup(rdev, r); 706 return r; 707 } 708 709 trace_radeon_cs(&parser); 710 711 r = radeon_cs_ib_chunk(rdev, &parser); 712 if (r) { 713 goto out; 714 } 715 r = radeon_cs_ib_vm_chunk(rdev, &parser); 716 if (r) { 717 goto out; 718 } 719 out: 720 radeon_cs_parser_fini(&parser, r, true); 721 up_read(&rdev->exclusive_lock); 722 r = radeon_cs_handle_lockup(rdev, r); 723 return r; 724 } 725 726 /** 727 * radeon_cs_packet_parse() - parse cp packet and point ib index to next packet 728 * @parser: parser structure holding parsing context. 729 * @pkt: where to store packet information 730 * 731 * Assume that chunk_ib_index is properly set. Will return -EINVAL 732 * if packet is bigger than remaining ib size. or if packets is unknown. 733 **/ 734 int radeon_cs_packet_parse(struct radeon_cs_parser *p, 735 struct radeon_cs_packet *pkt, 736 unsigned idx) 737 { 738 struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx]; 739 struct radeon_device *rdev = p->rdev; 740 uint32_t header; 741 742 if (idx >= ib_chunk->length_dw) { 743 DRM_ERROR("Can not parse packet at %d after CS end %d !\n", 744 idx, ib_chunk->length_dw); 745 return -EINVAL; 746 } 747 header = radeon_get_ib_value(p, idx); 748 pkt->idx = idx; 749 pkt->type = RADEON_CP_PACKET_GET_TYPE(header); 750 pkt->count = RADEON_CP_PACKET_GET_COUNT(header); 751 pkt->one_reg_wr = 0; 752 switch (pkt->type) { 753 case RADEON_PACKET_TYPE0: 754 if (rdev->family < CHIP_R600) { 755 pkt->reg = R100_CP_PACKET0_GET_REG(header); 756 pkt->one_reg_wr = 757 RADEON_CP_PACKET0_GET_ONE_REG_WR(header); 758 } else 759 pkt->reg = R600_CP_PACKET0_GET_REG(header); 760 break; 761 case RADEON_PACKET_TYPE3: 762 pkt->opcode = RADEON_CP_PACKET3_GET_OPCODE(header); 763 break; 764 case RADEON_PACKET_TYPE2: 765 pkt->count = -1; 766 break; 767 default: 768 DRM_ERROR("Unknown packet type %d at %d !\n", pkt->type, idx); 769 return -EINVAL; 770 } 771 if ((pkt->count + 1 + pkt->idx) >= ib_chunk->length_dw) { 772 DRM_ERROR("Packet (%d:%d:%d) end after CS buffer (%d) !\n", 773 pkt->idx, pkt->type, pkt->count, ib_chunk->length_dw); 774 return -EINVAL; 775 } 776 return 0; 777 } 778 779 /** 780 * radeon_cs_packet_next_is_pkt3_nop() - test if the next packet is P3 NOP 781 * @p: structure holding the parser context. 782 * 783 * Check if the next packet is NOP relocation packet3. 784 **/ 785 bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p) 786 { 787 struct radeon_cs_packet p3reloc; 788 int r; 789 790 r = radeon_cs_packet_parse(p, &p3reloc, p->idx); 791 if (r) 792 return false; 793 if (p3reloc.type != RADEON_PACKET_TYPE3) 794 return false; 795 if (p3reloc.opcode != RADEON_PACKET3_NOP) 796 return false; 797 return true; 798 } 799 800 /** 801 * radeon_cs_dump_packet() - dump raw packet context 802 * @p: structure holding the parser context. 803 * @pkt: structure holding the packet. 804 * 805 * Used mostly for debugging and error reporting. 806 **/ 807 void radeon_cs_dump_packet(struct radeon_cs_parser *p, 808 struct radeon_cs_packet *pkt) 809 { 810 volatile uint32_t *ib; 811 unsigned i; 812 unsigned idx; 813 814 ib = p->ib.ptr; 815 idx = pkt->idx; 816 for (i = 0; i <= (pkt->count + 1); i++, idx++) 817 DRM_INFO("ib[%d]=0x%08X\n", idx, ib[idx]); 818 } 819 820 /** 821 * radeon_cs_packet_next_reloc() - parse next (should be reloc) packet 822 * @parser: parser structure holding parsing context. 823 * @data: pointer to relocation data 824 * @offset_start: starting offset 825 * @offset_mask: offset mask (to align start offset on) 826 * @reloc: reloc informations 827 * 828 * Check if next packet is relocation packet3, do bo validation and compute 829 * GPU offset using the provided start. 830 **/ 831 int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p, 832 struct radeon_cs_reloc **cs_reloc, 833 int nomm) 834 { 835 struct radeon_cs_chunk *relocs_chunk; 836 struct radeon_cs_packet p3reloc; 837 unsigned idx; 838 int r; 839 840 if (p->chunk_relocs_idx == -1) { 841 DRM_ERROR("No relocation chunk !\n"); 842 return -EINVAL; 843 } 844 *cs_reloc = NULL; 845 relocs_chunk = &p->chunks[p->chunk_relocs_idx]; 846 r = radeon_cs_packet_parse(p, &p3reloc, p->idx); 847 if (r) 848 return r; 849 p->idx += p3reloc.count + 2; 850 if (p3reloc.type != RADEON_PACKET_TYPE3 || 851 p3reloc.opcode != RADEON_PACKET3_NOP) { 852 DRM_ERROR("No packet3 for relocation for packet at %d.\n", 853 p3reloc.idx); 854 radeon_cs_dump_packet(p, &p3reloc); 855 return -EINVAL; 856 } 857 idx = radeon_get_ib_value(p, p3reloc.idx + 1); 858 if (idx >= relocs_chunk->length_dw) { 859 DRM_ERROR("Relocs at %d after relocations chunk end %d !\n", 860 idx, relocs_chunk->length_dw); 861 radeon_cs_dump_packet(p, &p3reloc); 862 return -EINVAL; 863 } 864 /* FIXME: we assume reloc size is 4 dwords */ 865 if (nomm) { 866 *cs_reloc = p->relocs; 867 (*cs_reloc)->gpu_offset = 868 (u64)relocs_chunk->kdata[idx + 3] << 32; 869 (*cs_reloc)->gpu_offset |= relocs_chunk->kdata[idx + 0]; 870 } else 871 *cs_reloc = p->relocs_ptr[(idx / 4)]; 872 return 0; 873 } 874