1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <drm/drmP.h> 34 #include "radeon_drm.h" 35 #include "radeon.h" 36 37 struct radeon_object { 38 struct ttm_buffer_object tobj; 39 struct list_head list; 40 struct radeon_device *rdev; 41 struct drm_gem_object *gobj; 42 struct ttm_bo_kmap_obj kmap; 43 unsigned pin_count; 44 uint64_t gpu_addr; 45 void *kptr; 46 bool is_iomem; 47 uint32_t tiling_flags; 48 uint32_t pitch; 49 int surface_reg; 50 }; 51 52 int radeon_ttm_init(struct radeon_device *rdev); 53 void radeon_ttm_fini(struct radeon_device *rdev); 54 55 /* 56 * To exclude mutual BO access we rely on bo_reserve exclusion, as all 57 * function are calling it. 58 */ 59 60 static int radeon_object_reserve(struct radeon_object *robj, bool interruptible) 61 { 62 return ttm_bo_reserve(&robj->tobj, interruptible, false, false, 0); 63 } 64 65 static void radeon_object_unreserve(struct radeon_object *robj) 66 { 67 ttm_bo_unreserve(&robj->tobj); 68 } 69 70 static void radeon_ttm_object_object_destroy(struct ttm_buffer_object *tobj) 71 { 72 struct radeon_object *robj; 73 74 robj = container_of(tobj, struct radeon_object, tobj); 75 list_del_init(&robj->list); 76 radeon_object_clear_surface_reg(robj); 77 kfree(robj); 78 } 79 80 static inline void radeon_object_gpu_addr(struct radeon_object *robj) 81 { 82 /* Default gpu address */ 83 robj->gpu_addr = 0xFFFFFFFFFFFFFFFFULL; 84 if (robj->tobj.mem.mm_node == NULL) { 85 return; 86 } 87 robj->gpu_addr = ((u64)robj->tobj.mem.mm_node->start) << PAGE_SHIFT; 88 switch (robj->tobj.mem.mem_type) { 89 case TTM_PL_VRAM: 90 robj->gpu_addr += (u64)robj->rdev->mc.vram_location; 91 break; 92 case TTM_PL_TT: 93 robj->gpu_addr += (u64)robj->rdev->mc.gtt_location; 94 break; 95 default: 96 DRM_ERROR("Unknown placement %d\n", robj->tobj.mem.mem_type); 97 robj->gpu_addr = 0xFFFFFFFFFFFFFFFFULL; 98 return; 99 } 100 } 101 102 static inline uint32_t radeon_object_flags_from_domain(uint32_t domain) 103 { 104 uint32_t flags = 0; 105 if (domain & RADEON_GEM_DOMAIN_VRAM) { 106 flags |= TTM_PL_FLAG_VRAM | TTM_PL_FLAG_WC | TTM_PL_FLAG_UNCACHED; 107 } 108 if (domain & RADEON_GEM_DOMAIN_GTT) { 109 flags |= TTM_PL_FLAG_TT | TTM_PL_MASK_CACHING; 110 } 111 if (domain & RADEON_GEM_DOMAIN_CPU) { 112 flags |= TTM_PL_FLAG_SYSTEM | TTM_PL_MASK_CACHING; 113 } 114 if (!flags) { 115 flags |= TTM_PL_FLAG_SYSTEM | TTM_PL_MASK_CACHING; 116 } 117 return flags; 118 } 119 120 int radeon_object_create(struct radeon_device *rdev, 121 struct drm_gem_object *gobj, 122 unsigned long size, 123 bool kernel, 124 uint32_t domain, 125 bool interruptible, 126 struct radeon_object **robj_ptr) 127 { 128 struct radeon_object *robj; 129 enum ttm_bo_type type; 130 uint32_t flags; 131 int r; 132 133 if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) { 134 rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping; 135 } 136 if (kernel) { 137 type = ttm_bo_type_kernel; 138 } else { 139 type = ttm_bo_type_device; 140 } 141 *robj_ptr = NULL; 142 robj = kzalloc(sizeof(struct radeon_object), GFP_KERNEL); 143 if (robj == NULL) { 144 return -ENOMEM; 145 } 146 robj->rdev = rdev; 147 robj->gobj = gobj; 148 robj->surface_reg = -1; 149 INIT_LIST_HEAD(&robj->list); 150 151 flags = radeon_object_flags_from_domain(domain); 152 r = ttm_buffer_object_init(&rdev->mman.bdev, &robj->tobj, size, type, flags, 153 0, 0, false, NULL, size, 154 &radeon_ttm_object_object_destroy); 155 if (unlikely(r != 0)) { 156 /* ttm call radeon_ttm_object_object_destroy if error happen */ 157 DRM_ERROR("Failed to allocate TTM object (%ld, 0x%08X, %u)\n", 158 size, flags, 0); 159 return r; 160 } 161 *robj_ptr = robj; 162 if (gobj) { 163 list_add_tail(&robj->list, &rdev->gem.objects); 164 } 165 return 0; 166 } 167 168 int radeon_object_kmap(struct radeon_object *robj, void **ptr) 169 { 170 int r; 171 172 spin_lock(&robj->tobj.lock); 173 if (robj->kptr) { 174 if (ptr) { 175 *ptr = robj->kptr; 176 } 177 spin_unlock(&robj->tobj.lock); 178 return 0; 179 } 180 spin_unlock(&robj->tobj.lock); 181 r = ttm_bo_kmap(&robj->tobj, 0, robj->tobj.num_pages, &robj->kmap); 182 if (r) { 183 return r; 184 } 185 spin_lock(&robj->tobj.lock); 186 robj->kptr = ttm_kmap_obj_virtual(&robj->kmap, &robj->is_iomem); 187 spin_unlock(&robj->tobj.lock); 188 if (ptr) { 189 *ptr = robj->kptr; 190 } 191 radeon_object_check_tiling(robj, 0, 0); 192 return 0; 193 } 194 195 void radeon_object_kunmap(struct radeon_object *robj) 196 { 197 spin_lock(&robj->tobj.lock); 198 if (robj->kptr == NULL) { 199 spin_unlock(&robj->tobj.lock); 200 return; 201 } 202 robj->kptr = NULL; 203 spin_unlock(&robj->tobj.lock); 204 radeon_object_check_tiling(robj, 0, 0); 205 ttm_bo_kunmap(&robj->kmap); 206 } 207 208 void radeon_object_unref(struct radeon_object **robj) 209 { 210 struct ttm_buffer_object *tobj; 211 212 if ((*robj) == NULL) { 213 return; 214 } 215 tobj = &((*robj)->tobj); 216 ttm_bo_unref(&tobj); 217 if (tobj == NULL) { 218 *robj = NULL; 219 } 220 } 221 222 int radeon_object_mmap(struct radeon_object *robj, uint64_t *offset) 223 { 224 *offset = robj->tobj.addr_space_offset; 225 return 0; 226 } 227 228 int radeon_object_pin(struct radeon_object *robj, uint32_t domain, 229 uint64_t *gpu_addr) 230 { 231 uint32_t flags; 232 uint32_t tmp; 233 int r; 234 235 flags = radeon_object_flags_from_domain(domain); 236 spin_lock(&robj->tobj.lock); 237 if (robj->pin_count) { 238 robj->pin_count++; 239 if (gpu_addr != NULL) { 240 *gpu_addr = robj->gpu_addr; 241 } 242 spin_unlock(&robj->tobj.lock); 243 return 0; 244 } 245 spin_unlock(&robj->tobj.lock); 246 r = radeon_object_reserve(robj, false); 247 if (unlikely(r != 0)) { 248 DRM_ERROR("radeon: failed to reserve object for pinning it.\n"); 249 return r; 250 } 251 tmp = robj->tobj.mem.placement; 252 ttm_flag_masked(&tmp, flags, TTM_PL_MASK_MEM); 253 robj->tobj.proposed_placement = tmp | TTM_PL_FLAG_NO_EVICT | TTM_PL_MASK_CACHING; 254 r = ttm_buffer_object_validate(&robj->tobj, 255 robj->tobj.proposed_placement, 256 false, false); 257 radeon_object_gpu_addr(robj); 258 if (gpu_addr != NULL) { 259 *gpu_addr = robj->gpu_addr; 260 } 261 robj->pin_count = 1; 262 if (unlikely(r != 0)) { 263 DRM_ERROR("radeon: failed to pin object.\n"); 264 } 265 radeon_object_unreserve(robj); 266 return r; 267 } 268 269 void radeon_object_unpin(struct radeon_object *robj) 270 { 271 uint32_t flags; 272 int r; 273 274 spin_lock(&robj->tobj.lock); 275 if (!robj->pin_count) { 276 spin_unlock(&robj->tobj.lock); 277 printk(KERN_WARNING "Unpin not necessary for %p !\n", robj); 278 return; 279 } 280 robj->pin_count--; 281 if (robj->pin_count) { 282 spin_unlock(&robj->tobj.lock); 283 return; 284 } 285 spin_unlock(&robj->tobj.lock); 286 r = radeon_object_reserve(robj, false); 287 if (unlikely(r != 0)) { 288 DRM_ERROR("radeon: failed to reserve object for unpinning it.\n"); 289 return; 290 } 291 flags = robj->tobj.mem.placement; 292 robj->tobj.proposed_placement = flags & ~TTM_PL_FLAG_NO_EVICT; 293 r = ttm_buffer_object_validate(&robj->tobj, 294 robj->tobj.proposed_placement, 295 false, false); 296 if (unlikely(r != 0)) { 297 DRM_ERROR("radeon: failed to unpin buffer.\n"); 298 } 299 radeon_object_unreserve(robj); 300 } 301 302 int radeon_object_wait(struct radeon_object *robj) 303 { 304 int r = 0; 305 306 /* FIXME: should use block reservation instead */ 307 r = radeon_object_reserve(robj, true); 308 if (unlikely(r != 0)) { 309 DRM_ERROR("radeon: failed to reserve object for waiting.\n"); 310 return r; 311 } 312 spin_lock(&robj->tobj.lock); 313 if (robj->tobj.sync_obj) { 314 r = ttm_bo_wait(&robj->tobj, true, true, false); 315 } 316 spin_unlock(&robj->tobj.lock); 317 radeon_object_unreserve(robj); 318 return r; 319 } 320 321 int radeon_object_busy_domain(struct radeon_object *robj, uint32_t *cur_placement) 322 { 323 int r = 0; 324 325 r = radeon_object_reserve(robj, true); 326 if (unlikely(r != 0)) { 327 DRM_ERROR("radeon: failed to reserve object for waiting.\n"); 328 return r; 329 } 330 spin_lock(&robj->tobj.lock); 331 *cur_placement = robj->tobj.mem.mem_type; 332 if (robj->tobj.sync_obj) { 333 r = ttm_bo_wait(&robj->tobj, true, true, true); 334 } 335 spin_unlock(&robj->tobj.lock); 336 radeon_object_unreserve(robj); 337 return r; 338 } 339 340 int radeon_object_evict_vram(struct radeon_device *rdev) 341 { 342 if (rdev->flags & RADEON_IS_IGP) { 343 /* Useless to evict on IGP chips */ 344 return 0; 345 } 346 return ttm_bo_evict_mm(&rdev->mman.bdev, TTM_PL_VRAM); 347 } 348 349 void radeon_object_force_delete(struct radeon_device *rdev) 350 { 351 struct radeon_object *robj, *n; 352 struct drm_gem_object *gobj; 353 354 if (list_empty(&rdev->gem.objects)) { 355 return; 356 } 357 DRM_ERROR("Userspace still has active objects !\n"); 358 list_for_each_entry_safe(robj, n, &rdev->gem.objects, list) { 359 mutex_lock(&rdev->ddev->struct_mutex); 360 gobj = robj->gobj; 361 DRM_ERROR("Force free for (%p,%p,%lu,%lu)\n", 362 gobj, robj, (unsigned long)gobj->size, 363 *((unsigned long *)&gobj->refcount)); 364 list_del_init(&robj->list); 365 radeon_object_unref(&robj); 366 gobj->driver_private = NULL; 367 drm_gem_object_unreference(gobj); 368 mutex_unlock(&rdev->ddev->struct_mutex); 369 } 370 } 371 372 int radeon_object_init(struct radeon_device *rdev) 373 { 374 /* Add an MTRR for the VRAM */ 375 rdev->mc.vram_mtrr = mtrr_add(rdev->mc.aper_base, rdev->mc.aper_size, 376 MTRR_TYPE_WRCOMB, 1); 377 DRM_INFO("Detected VRAM RAM=%lluM, BAR=%lluM\n", 378 rdev->mc.mc_vram_size >> 20, 379 (unsigned long long)rdev->mc.aper_size >> 20); 380 DRM_INFO("RAM width %dbits %cDR\n", 381 rdev->mc.vram_width, rdev->mc.vram_is_ddr ? 'D' : 'S'); 382 return radeon_ttm_init(rdev); 383 } 384 385 void radeon_object_fini(struct radeon_device *rdev) 386 { 387 radeon_ttm_fini(rdev); 388 } 389 390 void radeon_object_list_add_object(struct radeon_object_list *lobj, 391 struct list_head *head) 392 { 393 if (lobj->wdomain) { 394 list_add(&lobj->list, head); 395 } else { 396 list_add_tail(&lobj->list, head); 397 } 398 } 399 400 int radeon_object_list_reserve(struct list_head *head) 401 { 402 struct radeon_object_list *lobj; 403 int r; 404 405 list_for_each_entry(lobj, head, list){ 406 if (!lobj->robj->pin_count) { 407 r = radeon_object_reserve(lobj->robj, true); 408 if (unlikely(r != 0)) { 409 DRM_ERROR("radeon: failed to reserve object.\n"); 410 return r; 411 } 412 } else { 413 } 414 } 415 return 0; 416 } 417 418 void radeon_object_list_unreserve(struct list_head *head) 419 { 420 struct radeon_object_list *lobj; 421 422 list_for_each_entry(lobj, head, list) { 423 if (!lobj->robj->pin_count) { 424 radeon_object_unreserve(lobj->robj); 425 } 426 } 427 } 428 429 int radeon_object_list_validate(struct list_head *head, void *fence) 430 { 431 struct radeon_object_list *lobj; 432 struct radeon_object *robj; 433 struct radeon_fence *old_fence = NULL; 434 int r; 435 436 r = radeon_object_list_reserve(head); 437 if (unlikely(r != 0)) { 438 radeon_object_list_unreserve(head); 439 return r; 440 } 441 list_for_each_entry(lobj, head, list) { 442 robj = lobj->robj; 443 if (!robj->pin_count) { 444 if (lobj->wdomain) { 445 robj->tobj.proposed_placement = 446 radeon_object_flags_from_domain(lobj->wdomain); 447 } else { 448 robj->tobj.proposed_placement = 449 radeon_object_flags_from_domain(lobj->rdomain); 450 } 451 r = ttm_buffer_object_validate(&robj->tobj, 452 robj->tobj.proposed_placement, 453 true, false); 454 if (unlikely(r)) { 455 DRM_ERROR("radeon: failed to validate.\n"); 456 return r; 457 } 458 radeon_object_gpu_addr(robj); 459 } 460 lobj->gpu_offset = robj->gpu_addr; 461 lobj->tiling_flags = robj->tiling_flags; 462 if (fence) { 463 old_fence = (struct radeon_fence *)robj->tobj.sync_obj; 464 robj->tobj.sync_obj = radeon_fence_ref(fence); 465 robj->tobj.sync_obj_arg = NULL; 466 } 467 if (old_fence) { 468 radeon_fence_unref(&old_fence); 469 } 470 } 471 return 0; 472 } 473 474 void radeon_object_list_unvalidate(struct list_head *head) 475 { 476 struct radeon_object_list *lobj; 477 struct radeon_fence *old_fence = NULL; 478 479 list_for_each_entry(lobj, head, list) { 480 old_fence = (struct radeon_fence *)lobj->robj->tobj.sync_obj; 481 lobj->robj->tobj.sync_obj = NULL; 482 if (old_fence) { 483 radeon_fence_unref(&old_fence); 484 } 485 } 486 radeon_object_list_unreserve(head); 487 } 488 489 void radeon_object_list_clean(struct list_head *head) 490 { 491 radeon_object_list_unreserve(head); 492 } 493 494 int radeon_object_fbdev_mmap(struct radeon_object *robj, 495 struct vm_area_struct *vma) 496 { 497 return ttm_fbdev_mmap(vma, &robj->tobj); 498 } 499 500 unsigned long radeon_object_size(struct radeon_object *robj) 501 { 502 return robj->tobj.num_pages << PAGE_SHIFT; 503 } 504 505 int radeon_object_get_surface_reg(struct radeon_object *robj) 506 { 507 struct radeon_device *rdev = robj->rdev; 508 struct radeon_surface_reg *reg; 509 struct radeon_object *old_object; 510 int steal; 511 int i; 512 513 if (!robj->tiling_flags) 514 return 0; 515 516 if (robj->surface_reg >= 0) { 517 reg = &rdev->surface_regs[robj->surface_reg]; 518 i = robj->surface_reg; 519 goto out; 520 } 521 522 steal = -1; 523 for (i = 0; i < RADEON_GEM_MAX_SURFACES; i++) { 524 525 reg = &rdev->surface_regs[i]; 526 if (!reg->robj) 527 break; 528 529 old_object = reg->robj; 530 if (old_object->pin_count == 0) 531 steal = i; 532 } 533 534 /* if we are all out */ 535 if (i == RADEON_GEM_MAX_SURFACES) { 536 if (steal == -1) 537 return -ENOMEM; 538 /* find someone with a surface reg and nuke their BO */ 539 reg = &rdev->surface_regs[steal]; 540 old_object = reg->robj; 541 /* blow away the mapping */ 542 DRM_DEBUG("stealing surface reg %d from %p\n", steal, old_object); 543 ttm_bo_unmap_virtual(&old_object->tobj); 544 old_object->surface_reg = -1; 545 i = steal; 546 } 547 548 robj->surface_reg = i; 549 reg->robj = robj; 550 551 out: 552 radeon_set_surface_reg(rdev, i, robj->tiling_flags, robj->pitch, 553 robj->tobj.mem.mm_node->start << PAGE_SHIFT, 554 robj->tobj.num_pages << PAGE_SHIFT); 555 return 0; 556 } 557 558 void radeon_object_clear_surface_reg(struct radeon_object *robj) 559 { 560 struct radeon_device *rdev = robj->rdev; 561 struct radeon_surface_reg *reg; 562 563 if (robj->surface_reg == -1) 564 return; 565 566 reg = &rdev->surface_regs[robj->surface_reg]; 567 radeon_clear_surface_reg(rdev, robj->surface_reg); 568 569 reg->robj = NULL; 570 robj->surface_reg = -1; 571 } 572 573 void radeon_object_set_tiling_flags(struct radeon_object *robj, 574 uint32_t tiling_flags, uint32_t pitch) 575 { 576 robj->tiling_flags = tiling_flags; 577 robj->pitch = pitch; 578 } 579 580 void radeon_object_get_tiling_flags(struct radeon_object *robj, 581 uint32_t *tiling_flags, 582 uint32_t *pitch) 583 { 584 if (tiling_flags) 585 *tiling_flags = robj->tiling_flags; 586 if (pitch) 587 *pitch = robj->pitch; 588 } 589 590 int radeon_object_check_tiling(struct radeon_object *robj, bool has_moved, 591 bool force_drop) 592 { 593 if (!(robj->tiling_flags & RADEON_TILING_SURFACE)) 594 return 0; 595 596 if (force_drop) { 597 radeon_object_clear_surface_reg(robj); 598 return 0; 599 } 600 601 if (robj->tobj.mem.mem_type != TTM_PL_VRAM) { 602 if (!has_moved) 603 return 0; 604 605 if (robj->surface_reg >= 0) 606 radeon_object_clear_surface_reg(robj); 607 return 0; 608 } 609 610 if ((robj->surface_reg >= 0) && !has_moved) 611 return 0; 612 613 return radeon_object_get_surface_reg(robj); 614 } 615 616 void radeon_bo_move_notify(struct ttm_buffer_object *bo, 617 struct ttm_mem_reg *mem) 618 { 619 struct radeon_object *robj = container_of(bo, struct radeon_object, tobj); 620 radeon_object_check_tiling(robj, 0, 1); 621 } 622 623 void radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo) 624 { 625 struct radeon_object *robj = container_of(bo, struct radeon_object, tobj); 626 radeon_object_check_tiling(robj, 0, 0); 627 } 628