1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <drm/drmP.h> 34 #include "radeon_drm.h" 35 #include "radeon.h" 36 37 struct radeon_object { 38 struct ttm_buffer_object tobj; 39 struct list_head list; 40 struct radeon_device *rdev; 41 struct drm_gem_object *gobj; 42 struct ttm_bo_kmap_obj kmap; 43 unsigned pin_count; 44 uint64_t gpu_addr; 45 void *kptr; 46 bool is_iomem; 47 uint32_t tiling_flags; 48 uint32_t pitch; 49 int surface_reg; 50 }; 51 52 int radeon_ttm_init(struct radeon_device *rdev); 53 void radeon_ttm_fini(struct radeon_device *rdev); 54 55 /* 56 * To exclude mutual BO access we rely on bo_reserve exclusion, as all 57 * function are calling it. 58 */ 59 60 static int radeon_object_reserve(struct radeon_object *robj, bool interruptible) 61 { 62 return ttm_bo_reserve(&robj->tobj, interruptible, false, false, 0); 63 } 64 65 static void radeon_object_unreserve(struct radeon_object *robj) 66 { 67 ttm_bo_unreserve(&robj->tobj); 68 } 69 70 static void radeon_ttm_object_object_destroy(struct ttm_buffer_object *tobj) 71 { 72 struct radeon_object *robj; 73 74 robj = container_of(tobj, struct radeon_object, tobj); 75 list_del_init(&robj->list); 76 radeon_object_clear_surface_reg(robj); 77 kfree(robj); 78 } 79 80 static inline void radeon_object_gpu_addr(struct radeon_object *robj) 81 { 82 /* Default gpu address */ 83 robj->gpu_addr = 0xFFFFFFFFFFFFFFFFULL; 84 if (robj->tobj.mem.mm_node == NULL) { 85 return; 86 } 87 robj->gpu_addr = ((u64)robj->tobj.mem.mm_node->start) << PAGE_SHIFT; 88 switch (robj->tobj.mem.mem_type) { 89 case TTM_PL_VRAM: 90 robj->gpu_addr += (u64)robj->rdev->mc.vram_location; 91 break; 92 case TTM_PL_TT: 93 robj->gpu_addr += (u64)robj->rdev->mc.gtt_location; 94 break; 95 default: 96 DRM_ERROR("Unknown placement %d\n", robj->tobj.mem.mem_type); 97 robj->gpu_addr = 0xFFFFFFFFFFFFFFFFULL; 98 return; 99 } 100 } 101 102 static inline uint32_t radeon_object_flags_from_domain(uint32_t domain) 103 { 104 uint32_t flags = 0; 105 if (domain & RADEON_GEM_DOMAIN_VRAM) { 106 flags |= TTM_PL_FLAG_VRAM | TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED; 107 } 108 if (domain & RADEON_GEM_DOMAIN_GTT) { 109 flags |= TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 110 } 111 if (domain & RADEON_GEM_DOMAIN_CPU) { 112 flags |= TTM_PL_FLAG_SYSTEM | TTM_PL_MASK_CACHING; 113 } 114 if (!flags) { 115 flags |= TTM_PL_FLAG_SYSTEM | TTM_PL_MASK_CACHING; 116 } 117 return flags; 118 } 119 120 int radeon_object_create(struct radeon_device *rdev, 121 struct drm_gem_object *gobj, 122 unsigned long size, 123 bool kernel, 124 uint32_t domain, 125 bool interruptible, 126 struct radeon_object **robj_ptr) 127 { 128 struct radeon_object *robj; 129 enum ttm_bo_type type; 130 uint32_t flags; 131 int r; 132 133 if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) { 134 rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping; 135 } 136 if (kernel) { 137 type = ttm_bo_type_kernel; 138 } else { 139 type = ttm_bo_type_device; 140 } 141 *robj_ptr = NULL; 142 robj = kzalloc(sizeof(struct radeon_object), GFP_KERNEL); 143 if (robj == NULL) { 144 return -ENOMEM; 145 } 146 robj->rdev = rdev; 147 robj->gobj = gobj; 148 robj->surface_reg = -1; 149 INIT_LIST_HEAD(&robj->list); 150 151 flags = radeon_object_flags_from_domain(domain); 152 r = ttm_buffer_object_init(&rdev->mman.bdev, &robj->tobj, size, type, flags, 153 0, 0, false, NULL, size, 154 &radeon_ttm_object_object_destroy); 155 if (unlikely(r != 0)) { 156 /* ttm call radeon_ttm_object_object_destroy if error happen */ 157 DRM_ERROR("Failed to allocate TTM object (%ld, 0x%08X, %u)\n", 158 size, flags, 0); 159 return r; 160 } 161 *robj_ptr = robj; 162 if (gobj) { 163 list_add_tail(&robj->list, &rdev->gem.objects); 164 } 165 return 0; 166 } 167 168 int radeon_object_kmap(struct radeon_object *robj, void **ptr) 169 { 170 int r; 171 172 spin_lock(&robj->tobj.lock); 173 if (robj->kptr) { 174 if (ptr) { 175 *ptr = robj->kptr; 176 } 177 spin_unlock(&robj->tobj.lock); 178 return 0; 179 } 180 spin_unlock(&robj->tobj.lock); 181 r = ttm_bo_kmap(&robj->tobj, 0, robj->tobj.num_pages, &robj->kmap); 182 if (r) { 183 return r; 184 } 185 spin_lock(&robj->tobj.lock); 186 robj->kptr = ttm_kmap_obj_virtual(&robj->kmap, &robj->is_iomem); 187 spin_unlock(&robj->tobj.lock); 188 if (ptr) { 189 *ptr = robj->kptr; 190 } 191 radeon_object_check_tiling(robj, 0, 0); 192 return 0; 193 } 194 195 void radeon_object_kunmap(struct radeon_object *robj) 196 { 197 spin_lock(&robj->tobj.lock); 198 if (robj->kptr == NULL) { 199 spin_unlock(&robj->tobj.lock); 200 return; 201 } 202 robj->kptr = NULL; 203 spin_unlock(&robj->tobj.lock); 204 radeon_object_check_tiling(robj, 0, 0); 205 ttm_bo_kunmap(&robj->kmap); 206 } 207 208 void radeon_object_unref(struct radeon_object **robj) 209 { 210 struct ttm_buffer_object *tobj; 211 212 if ((*robj) == NULL) { 213 return; 214 } 215 tobj = &((*robj)->tobj); 216 ttm_bo_unref(&tobj); 217 if (tobj == NULL) { 218 *robj = NULL; 219 } 220 } 221 222 int radeon_object_mmap(struct radeon_object *robj, uint64_t *offset) 223 { 224 *offset = robj->tobj.addr_space_offset; 225 return 0; 226 } 227 228 int radeon_object_pin(struct radeon_object *robj, uint32_t domain, 229 uint64_t *gpu_addr) 230 { 231 uint32_t flags; 232 uint32_t tmp; 233 int r; 234 235 flags = radeon_object_flags_from_domain(domain); 236 spin_lock(&robj->tobj.lock); 237 if (robj->pin_count) { 238 robj->pin_count++; 239 if (gpu_addr != NULL) { 240 *gpu_addr = robj->gpu_addr; 241 } 242 spin_unlock(&robj->tobj.lock); 243 return 0; 244 } 245 spin_unlock(&robj->tobj.lock); 246 r = radeon_object_reserve(robj, false); 247 if (unlikely(r != 0)) { 248 DRM_ERROR("radeon: failed to reserve object for pinning it.\n"); 249 return r; 250 } 251 tmp = robj->tobj.mem.placement; 252 ttm_flag_masked(&tmp, flags, TTM_PL_MASK_MEM); 253 robj->tobj.proposed_placement = tmp | TTM_PL_FLAG_NO_EVICT | TTM_PL_MASK_CACHING; 254 r = ttm_buffer_object_validate(&robj->tobj, 255 robj->tobj.proposed_placement, 256 false, false); 257 radeon_object_gpu_addr(robj); 258 if (gpu_addr != NULL) { 259 *gpu_addr = robj->gpu_addr; 260 } 261 robj->pin_count = 1; 262 if (unlikely(r != 0)) { 263 DRM_ERROR("radeon: failed to pin object.\n"); 264 } 265 radeon_object_unreserve(robj); 266 return r; 267 } 268 269 void radeon_object_unpin(struct radeon_object *robj) 270 { 271 uint32_t flags; 272 int r; 273 274 spin_lock(&robj->tobj.lock); 275 if (!robj->pin_count) { 276 spin_unlock(&robj->tobj.lock); 277 printk(KERN_WARNING "Unpin not necessary for %p !\n", robj); 278 return; 279 } 280 robj->pin_count--; 281 if (robj->pin_count) { 282 spin_unlock(&robj->tobj.lock); 283 return; 284 } 285 spin_unlock(&robj->tobj.lock); 286 r = radeon_object_reserve(robj, false); 287 if (unlikely(r != 0)) { 288 DRM_ERROR("radeon: failed to reserve object for unpinning it.\n"); 289 return; 290 } 291 flags = robj->tobj.mem.placement; 292 robj->tobj.proposed_placement = flags & ~TTM_PL_FLAG_NO_EVICT; 293 r = ttm_buffer_object_validate(&robj->tobj, 294 robj->tobj.proposed_placement, 295 false, false); 296 if (unlikely(r != 0)) { 297 DRM_ERROR("radeon: failed to unpin buffer.\n"); 298 } 299 radeon_object_unreserve(robj); 300 } 301 302 int radeon_object_wait(struct radeon_object *robj) 303 { 304 int r = 0; 305 306 /* FIXME: should use block reservation instead */ 307 r = radeon_object_reserve(robj, true); 308 if (unlikely(r != 0)) { 309 DRM_ERROR("radeon: failed to reserve object for waiting.\n"); 310 return r; 311 } 312 spin_lock(&robj->tobj.lock); 313 if (robj->tobj.sync_obj) { 314 r = ttm_bo_wait(&robj->tobj, true, true, false); 315 } 316 spin_unlock(&robj->tobj.lock); 317 radeon_object_unreserve(robj); 318 return r; 319 } 320 321 int radeon_object_busy_domain(struct radeon_object *robj, uint32_t *cur_placement) 322 { 323 int r = 0; 324 325 r = radeon_object_reserve(robj, true); 326 if (unlikely(r != 0)) { 327 DRM_ERROR("radeon: failed to reserve object for waiting.\n"); 328 return r; 329 } 330 spin_lock(&robj->tobj.lock); 331 *cur_placement = robj->tobj.mem.mem_type; 332 if (robj->tobj.sync_obj) { 333 r = ttm_bo_wait(&robj->tobj, true, true, true); 334 } 335 spin_unlock(&robj->tobj.lock); 336 radeon_object_unreserve(robj); 337 return r; 338 } 339 340 int radeon_object_evict_vram(struct radeon_device *rdev) 341 { 342 if (rdev->flags & RADEON_IS_IGP) { 343 /* Useless to evict on IGP chips */ 344 return 0; 345 } 346 return ttm_bo_evict_mm(&rdev->mman.bdev, TTM_PL_VRAM); 347 } 348 349 void radeon_object_force_delete(struct radeon_device *rdev) 350 { 351 struct radeon_object *robj, *n; 352 struct drm_gem_object *gobj; 353 354 if (list_empty(&rdev->gem.objects)) { 355 return; 356 } 357 DRM_ERROR("Userspace still has active objects !\n"); 358 list_for_each_entry_safe(robj, n, &rdev->gem.objects, list) { 359 mutex_lock(&rdev->ddev->struct_mutex); 360 gobj = robj->gobj; 361 DRM_ERROR("Force free for (%p,%p,%lu,%lu)\n", 362 gobj, robj, (unsigned long)gobj->size, 363 *((unsigned long *)&gobj->refcount)); 364 list_del_init(&robj->list); 365 radeon_object_unref(&robj); 366 gobj->driver_private = NULL; 367 drm_gem_object_unreference(gobj); 368 mutex_unlock(&rdev->ddev->struct_mutex); 369 } 370 } 371 372 int radeon_object_init(struct radeon_device *rdev) 373 { 374 /* Add an MTRR for the VRAM */ 375 rdev->mc.vram_mtrr = mtrr_add(rdev->mc.aper_base, rdev->mc.aper_size, 376 MTRR_TYPE_WRCOMB, 1); 377 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", 378 rdev->mc.mc_vram_size >> 20, 379 (unsigned long long)rdev->mc.aper_size >> 20); 380 DRM_INFO("RAM width %dbits %cDR\n", 381 rdev->mc.vram_width, rdev->mc.vram_is_ddr ? 'D' : 'S'); 382 return radeon_ttm_init(rdev); 383 } 384 385 void radeon_object_fini(struct radeon_device *rdev) 386 { 387 radeon_ttm_fini(rdev); 388 } 389 390 void radeon_object_list_add_object(struct radeon_object_list *lobj, 391 struct list_head *head) 392 { 393 if (lobj->wdomain) { 394 list_add(&lobj->list, head); 395 } else { 396 list_add_tail(&lobj->list, head); 397 } 398 } 399 400 int radeon_object_list_reserve(struct list_head *head) 401 { 402 struct radeon_object_list *lobj; 403 struct list_head *i; 404 int r; 405 406 list_for_each(i, head) { 407 lobj = list_entry(i, struct radeon_object_list, list); 408 if (!lobj->robj->pin_count) { 409 r = radeon_object_reserve(lobj->robj, true); 410 if (unlikely(r != 0)) { 411 DRM_ERROR("radeon: failed to reserve object.\n"); 412 return r; 413 } 414 } else { 415 } 416 } 417 return 0; 418 } 419 420 void radeon_object_list_unreserve(struct list_head *head) 421 { 422 struct radeon_object_list *lobj; 423 struct list_head *i; 424 425 list_for_each(i, head) { 426 lobj = list_entry(i, struct radeon_object_list, list); 427 if (!lobj->robj->pin_count) { 428 radeon_object_unreserve(lobj->robj); 429 } else { 430 } 431 } 432 } 433 434 int radeon_object_list_validate(struct list_head *head, void *fence) 435 { 436 struct radeon_object_list *lobj; 437 struct radeon_object *robj; 438 struct radeon_fence *old_fence = NULL; 439 struct list_head *i; 440 int r; 441 442 r = radeon_object_list_reserve(head); 443 if (unlikely(r != 0)) { 444 radeon_object_list_unreserve(head); 445 return r; 446 } 447 list_for_each(i, head) { 448 lobj = list_entry(i, struct radeon_object_list, list); 449 robj = lobj->robj; 450 if (!robj->pin_count) { 451 if (lobj->wdomain) { 452 robj->tobj.proposed_placement = 453 radeon_object_flags_from_domain(lobj->wdomain); 454 } else { 455 robj->tobj.proposed_placement = 456 radeon_object_flags_from_domain(lobj->rdomain); 457 } 458 r = ttm_buffer_object_validate(&robj->tobj, 459 robj->tobj.proposed_placement, 460 true, false); 461 if (unlikely(r)) { 462 DRM_ERROR("radeon: failed to validate.\n"); 463 return r; 464 } 465 radeon_object_gpu_addr(robj); 466 } 467 lobj->gpu_offset = robj->gpu_addr; 468 lobj->tiling_flags = robj->tiling_flags; 469 if (fence) { 470 old_fence = (struct radeon_fence *)robj->tobj.sync_obj; 471 robj->tobj.sync_obj = radeon_fence_ref(fence); 472 robj->tobj.sync_obj_arg = NULL; 473 } 474 if (old_fence) { 475 radeon_fence_unref(&old_fence); 476 } 477 } 478 return 0; 479 } 480 481 void radeon_object_list_unvalidate(struct list_head *head) 482 { 483 struct radeon_object_list *lobj; 484 struct radeon_fence *old_fence = NULL; 485 struct list_head *i; 486 487 list_for_each(i, head) { 488 lobj = list_entry(i, struct radeon_object_list, list); 489 old_fence = (struct radeon_fence *)lobj->robj->tobj.sync_obj; 490 lobj->robj->tobj.sync_obj = NULL; 491 if (old_fence) { 492 radeon_fence_unref(&old_fence); 493 } 494 } 495 radeon_object_list_unreserve(head); 496 } 497 498 void radeon_object_list_clean(struct list_head *head) 499 { 500 radeon_object_list_unreserve(head); 501 } 502 503 int radeon_object_fbdev_mmap(struct radeon_object *robj, 504 struct vm_area_struct *vma) 505 { 506 return ttm_fbdev_mmap(vma, &robj->tobj); 507 } 508 509 unsigned long radeon_object_size(struct radeon_object *robj) 510 { 511 return robj->tobj.num_pages << PAGE_SHIFT; 512 } 513 514 int radeon_object_get_surface_reg(struct radeon_object *robj) 515 { 516 struct radeon_device *rdev = robj->rdev; 517 struct radeon_surface_reg *reg; 518 struct radeon_object *old_object; 519 int steal; 520 int i; 521 522 if (!robj->tiling_flags) 523 return 0; 524 525 if (robj->surface_reg >= 0) { 526 reg = &rdev->surface_regs[robj->surface_reg]; 527 i = robj->surface_reg; 528 goto out; 529 } 530 531 steal = -1; 532 for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) { 533 534 reg = &rdev->surface_regs[i]; 535 if (!reg->robj) 536 break; 537 538 old_object = reg->robj; 539 if (old_object->pin_count == 0) 540 steal = i; 541 } 542 543 /* if we are all out */ 544 if (i == RADEON_GEM_MAX_SURFACES) { 545 if (steal == -1) 546 return -ENOMEM; 547 /* find someone with a surface reg and nuke their BO */ 548 reg = &rdev->surface_regs[steal]; 549 old_object = reg->robj; 550 /* blow away the mapping */ 551 DRM_DEBUG("stealing surface reg %d from %p\n", steal, old_object); 552 ttm_bo_unmap_virtual(&old_object->tobj); 553 old_object->surface_reg = -1; 554 i = steal; 555 } 556 557 robj->surface_reg = i; 558 reg->robj = robj; 559 560 out: 561 radeon_set_surface_reg(rdev, i, robj->tiling_flags, robj->pitch, 562 robj->tobj.mem.mm_node->start << PAGE_SHIFT, 563 robj->tobj.num_pages << PAGE_SHIFT); 564 return 0; 565 } 566 567 void radeon_object_clear_surface_reg(struct radeon_object *robj) 568 { 569 struct radeon_device *rdev = robj->rdev; 570 struct radeon_surface_reg *reg; 571 572 if (robj->surface_reg == -1) 573 return; 574 575 reg = &rdev->surface_regs[robj->surface_reg]; 576 radeon_clear_surface_reg(rdev, robj->surface_reg); 577 578 reg->robj = NULL; 579 robj->surface_reg = -1; 580 } 581 582 void radeon_object_set_tiling_flags(struct radeon_object *robj, 583 uint32_t tiling_flags, uint32_t pitch) 584 { 585 robj->tiling_flags = tiling_flags; 586 robj->pitch = pitch; 587 } 588 589 void radeon_object_get_tiling_flags(struct radeon_object *robj, 590 uint32_t *tiling_flags, 591 uint32_t *pitch) 592 { 593 if (tiling_flags) 594 *tiling_flags = robj->tiling_flags; 595 if (pitch) 596 *pitch = robj->pitch; 597 } 598 599 int radeon_object_check_tiling(struct radeon_object *robj, bool has_moved, 600 bool force_drop) 601 { 602 if (!(robj->tiling_flags & RADEON_TILING_SURFACE)) 603 return 0; 604 605 if (force_drop) { 606 radeon_object_clear_surface_reg(robj); 607 return 0; 608 } 609 610 if (robj->tobj.mem.mem_type != TTM_PL_VRAM) { 611 if (!has_moved) 612 return 0; 613 614 if (robj->surface_reg >= 0) 615 radeon_object_clear_surface_reg(robj); 616 return 0; 617 } 618 619 if ((robj->surface_reg >= 0) && !has_moved) 620 return 0; 621 622 return radeon_object_get_surface_reg(robj); 623 } 624 625 void radeon_bo_move_notify(struct ttm_buffer_object *bo, 626 struct ttm_mem_reg *mem) 627 { 628 struct radeon_object *robj = container_of(bo, struct radeon_object, tobj); 629 radeon_object_check_tiling(robj, 0, 1); 630 } 631 632 void radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 633 { 634 struct radeon_object *robj = container_of(bo, struct radeon_object, tobj); 635 radeon_object_check_tiling(robj, 0, 0); 636 } 637