1 /* 2 * Copyright 2009 Jerome Glisse. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Jerome Glisse <glisse@freedesktop.org> 29 * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 30 * Dave Airlie 31 */ 32 #include <linux/list.h> 33 #include <drm/drmP.h> 34 #include "radeon_drm.h" 35 #include "radeon.h" 36 37 struct radeon_object { 38 struct ttm_buffer_object tobj; 39 struct list_head list; 40 struct radeon_device *rdev; 41 struct drm_gem_object *gobj; 42 struct ttm_bo_kmap_obj kmap; 43 unsigned pin_count; 44 uint64_t gpu_addr; 45 void *kptr; 46 bool is_iomem; 47 }; 48 49 int radeon_ttm_init(struct radeon_device *rdev); 50 void radeon_ttm_fini(struct radeon_device *rdev); 51 52 /* 53 * To exclude mutual BO access we rely on bo_reserve exclusion, as all 54 * function are calling it. 55 */ 56 57 static int radeon_object_reserve(struct radeon_object *robj, bool interruptible) 58 { 59 return ttm_bo_reserve(&robj->tobj, interruptible, false, false, 0); 60 } 61 62 static void radeon_object_unreserve(struct radeon_object *robj) 63 { 64 ttm_bo_unreserve(&robj->tobj); 65 } 66 67 static void radeon_ttm_object_object_destroy(struct ttm_buffer_object *tobj) 68 { 69 struct radeon_object *robj; 70 71 robj = container_of(tobj, struct radeon_object, tobj); 72 list_del_init(&robj->list); 73 kfree(robj); 74 } 75 76 static inline void radeon_object_gpu_addr(struct radeon_object *robj) 77 { 78 /* Default gpu address */ 79 robj->gpu_addr = 0xFFFFFFFFFFFFFFFFULL; 80 if (robj->tobj.mem.mm_node == NULL) { 81 return; 82 } 83 robj->gpu_addr = ((u64)robj->tobj.mem.mm_node->start) << PAGE_SHIFT; 84 switch (robj->tobj.mem.mem_type) { 85 case TTM_PL_VRAM: 86 robj->gpu_addr += (u64)robj->rdev->mc.vram_location; 87 break; 88 case TTM_PL_TT: 89 robj->gpu_addr += (u64)robj->rdev->mc.gtt_location; 90 break; 91 default: 92 DRM_ERROR("Unknown placement %d\n", robj->tobj.mem.mem_type); 93 robj->gpu_addr = 0xFFFFFFFFFFFFFFFFULL; 94 return; 95 } 96 } 97 98 static inline uint32_t radeon_object_flags_from_domain(uint32_t domain) 99 { 100 uint32_t flags = 0; 101 if (domain & RADEON_GEM_DOMAIN_VRAM) { 102 flags |= TTM_PL_FLAG_VRAM; 103 } 104 if (domain & RADEON_GEM_DOMAIN_GTT) { 105 flags |= TTM_PL_FLAG_TT; 106 } 107 if (domain & RADEON_GEM_DOMAIN_CPU) { 108 flags |= TTM_PL_FLAG_SYSTEM; 109 } 110 if (!flags) { 111 flags |= TTM_PL_FLAG_SYSTEM; 112 } 113 return flags; 114 } 115 116 int radeon_object_create(struct radeon_device *rdev, 117 struct drm_gem_object *gobj, 118 unsigned long size, 119 bool kernel, 120 uint32_t domain, 121 bool interruptible, 122 struct radeon_object **robj_ptr) 123 { 124 struct radeon_object *robj; 125 enum ttm_bo_type type; 126 uint32_t flags; 127 int r; 128 129 if (unlikely(rdev->mman.bdev.dev_mapping == NULL)) { 130 rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping; 131 } 132 if (kernel) { 133 type = ttm_bo_type_kernel; 134 } else { 135 type = ttm_bo_type_device; 136 } 137 *robj_ptr = NULL; 138 robj = kzalloc(sizeof(struct radeon_object), GFP_KERNEL); 139 if (robj == NULL) { 140 return -ENOMEM; 141 } 142 robj->rdev = rdev; 143 robj->gobj = gobj; 144 INIT_LIST_HEAD(&robj->list); 145 146 flags = radeon_object_flags_from_domain(domain); 147 r = ttm_buffer_object_init(&rdev->mman.bdev, &robj->tobj, size, type, flags, 148 0, 0, false, NULL, size, 149 &radeon_ttm_object_object_destroy); 150 if (unlikely(r != 0)) { 151 /* ttm call radeon_ttm_object_object_destroy if error happen */ 152 DRM_ERROR("Failed to allocate TTM object (%ld, 0x%08X, %u)\n", 153 size, flags, 0); 154 return r; 155 } 156 *robj_ptr = robj; 157 if (gobj) { 158 list_add_tail(&robj->list, &rdev->gem.objects); 159 } 160 return 0; 161 } 162 163 int radeon_object_kmap(struct radeon_object *robj, void **ptr) 164 { 165 int r; 166 167 spin_lock(&robj->tobj.lock); 168 if (robj->kptr) { 169 if (ptr) { 170 *ptr = robj->kptr; 171 } 172 spin_unlock(&robj->tobj.lock); 173 return 0; 174 } 175 spin_unlock(&robj->tobj.lock); 176 r = ttm_bo_kmap(&robj->tobj, 0, robj->tobj.num_pages, &robj->kmap); 177 if (r) { 178 return r; 179 } 180 spin_lock(&robj->tobj.lock); 181 robj->kptr = ttm_kmap_obj_virtual(&robj->kmap, &robj->is_iomem); 182 spin_unlock(&robj->tobj.lock); 183 if (ptr) { 184 *ptr = robj->kptr; 185 } 186 return 0; 187 } 188 189 void radeon_object_kunmap(struct radeon_object *robj) 190 { 191 spin_lock(&robj->tobj.lock); 192 if (robj->kptr == NULL) { 193 spin_unlock(&robj->tobj.lock); 194 return; 195 } 196 robj->kptr = NULL; 197 spin_unlock(&robj->tobj.lock); 198 ttm_bo_kunmap(&robj->kmap); 199 } 200 201 void radeon_object_unref(struct radeon_object **robj) 202 { 203 struct ttm_buffer_object *tobj; 204 205 if ((*robj) == NULL) { 206 return; 207 } 208 tobj = &((*robj)->tobj); 209 ttm_bo_unref(&tobj); 210 if (tobj == NULL) { 211 *robj = NULL; 212 } 213 } 214 215 int radeon_object_mmap(struct radeon_object *robj, uint64_t *offset) 216 { 217 *offset = robj->tobj.addr_space_offset; 218 return 0; 219 } 220 221 int radeon_object_pin(struct radeon_object *robj, uint32_t domain, 222 uint64_t *gpu_addr) 223 { 224 uint32_t flags; 225 uint32_t tmp; 226 int r; 227 228 flags = radeon_object_flags_from_domain(domain); 229 spin_lock(&robj->tobj.lock); 230 if (robj->pin_count) { 231 robj->pin_count++; 232 if (gpu_addr != NULL) { 233 *gpu_addr = robj->gpu_addr; 234 } 235 spin_unlock(&robj->tobj.lock); 236 return 0; 237 } 238 spin_unlock(&robj->tobj.lock); 239 r = radeon_object_reserve(robj, false); 240 if (unlikely(r != 0)) { 241 DRM_ERROR("radeon: failed to reserve object for pinning it.\n"); 242 return r; 243 } 244 tmp = robj->tobj.mem.placement; 245 ttm_flag_masked(&tmp, flags, TTM_PL_MASK_MEM); 246 robj->tobj.proposed_placement = tmp | TTM_PL_FLAG_NO_EVICT | TTM_PL_MASK_CACHING; 247 r = ttm_buffer_object_validate(&robj->tobj, 248 robj->tobj.proposed_placement, 249 false, false); 250 radeon_object_gpu_addr(robj); 251 if (gpu_addr != NULL) { 252 *gpu_addr = robj->gpu_addr; 253 } 254 robj->pin_count = 1; 255 if (unlikely(r != 0)) { 256 DRM_ERROR("radeon: failed to pin object.\n"); 257 } 258 radeon_object_unreserve(robj); 259 return r; 260 } 261 262 void radeon_object_unpin(struct radeon_object *robj) 263 { 264 uint32_t flags; 265 int r; 266 267 spin_lock(&robj->tobj.lock); 268 if (!robj->pin_count) { 269 spin_unlock(&robj->tobj.lock); 270 printk(KERN_WARNING "Unpin not necessary for %p !\n", robj); 271 return; 272 } 273 robj->pin_count--; 274 if (robj->pin_count) { 275 spin_unlock(&robj->tobj.lock); 276 return; 277 } 278 spin_unlock(&robj->tobj.lock); 279 r = radeon_object_reserve(robj, false); 280 if (unlikely(r != 0)) { 281 DRM_ERROR("radeon: failed to reserve object for unpinning it.\n"); 282 return; 283 } 284 flags = robj->tobj.mem.placement; 285 robj->tobj.proposed_placement = flags & ~TTM_PL_FLAG_NO_EVICT; 286 r = ttm_buffer_object_validate(&robj->tobj, 287 robj->tobj.proposed_placement, 288 false, false); 289 if (unlikely(r != 0)) { 290 DRM_ERROR("radeon: failed to unpin buffer.\n"); 291 } 292 radeon_object_unreserve(robj); 293 } 294 295 int radeon_object_wait(struct radeon_object *robj) 296 { 297 int r = 0; 298 299 /* FIXME: should use block reservation instead */ 300 r = radeon_object_reserve(robj, true); 301 if (unlikely(r != 0)) { 302 DRM_ERROR("radeon: failed to reserve object for waiting.\n"); 303 return r; 304 } 305 spin_lock(&robj->tobj.lock); 306 if (robj->tobj.sync_obj) { 307 r = ttm_bo_wait(&robj->tobj, true, false, false); 308 } 309 spin_unlock(&robj->tobj.lock); 310 radeon_object_unreserve(robj); 311 return r; 312 } 313 314 int radeon_object_evict_vram(struct radeon_device *rdev) 315 { 316 if (rdev->flags & RADEON_IS_IGP) { 317 /* Useless to evict on IGP chips */ 318 return 0; 319 } 320 return ttm_bo_evict_mm(&rdev->mman.bdev, TTM_PL_VRAM); 321 } 322 323 void radeon_object_force_delete(struct radeon_device *rdev) 324 { 325 struct radeon_object *robj, *n; 326 struct drm_gem_object *gobj; 327 328 if (list_empty(&rdev->gem.objects)) { 329 return; 330 } 331 DRM_ERROR("Userspace still has active objects !\n"); 332 list_for_each_entry_safe(robj, n, &rdev->gem.objects, list) { 333 mutex_lock(&rdev->ddev->struct_mutex); 334 gobj = robj->gobj; 335 DRM_ERROR("Force free for (%p,%p,%lu,%lu)\n", 336 gobj, robj, (unsigned long)gobj->size, 337 *((unsigned long *)&gobj->refcount)); 338 list_del_init(&robj->list); 339 radeon_object_unref(&robj); 340 gobj->driver_private = NULL; 341 drm_gem_object_unreference(gobj); 342 mutex_unlock(&rdev->ddev->struct_mutex); 343 } 344 } 345 346 int radeon_object_init(struct radeon_device *rdev) 347 { 348 return radeon_ttm_init(rdev); 349 } 350 351 void radeon_object_fini(struct radeon_device *rdev) 352 { 353 radeon_ttm_fini(rdev); 354 } 355 356 void radeon_object_list_add_object(struct radeon_object_list *lobj, 357 struct list_head *head) 358 { 359 if (lobj->wdomain) { 360 list_add(&lobj->list, head); 361 } else { 362 list_add_tail(&lobj->list, head); 363 } 364 } 365 366 int radeon_object_list_reserve(struct list_head *head) 367 { 368 struct radeon_object_list *lobj; 369 struct list_head *i; 370 int r; 371 372 list_for_each(i, head) { 373 lobj = list_entry(i, struct radeon_object_list, list); 374 if (!lobj->robj->pin_count) { 375 r = radeon_object_reserve(lobj->robj, true); 376 if (unlikely(r != 0)) { 377 DRM_ERROR("radeon: failed to reserve object.\n"); 378 return r; 379 } 380 } else { 381 } 382 } 383 return 0; 384 } 385 386 void radeon_object_list_unreserve(struct list_head *head) 387 { 388 struct radeon_object_list *lobj; 389 struct list_head *i; 390 391 list_for_each(i, head) { 392 lobj = list_entry(i, struct radeon_object_list, list); 393 if (!lobj->robj->pin_count) { 394 radeon_object_unreserve(lobj->robj); 395 } else { 396 } 397 } 398 } 399 400 int radeon_object_list_validate(struct list_head *head, void *fence) 401 { 402 struct radeon_object_list *lobj; 403 struct radeon_object *robj; 404 struct radeon_fence *old_fence = NULL; 405 struct list_head *i; 406 uint32_t flags; 407 int r; 408 409 r = radeon_object_list_reserve(head); 410 if (unlikely(r != 0)) { 411 radeon_object_list_unreserve(head); 412 return r; 413 } 414 list_for_each(i, head) { 415 lobj = list_entry(i, struct radeon_object_list, list); 416 robj = lobj->robj; 417 if (lobj->wdomain) { 418 flags = radeon_object_flags_from_domain(lobj->wdomain); 419 flags |= TTM_PL_FLAG_TT; 420 } else { 421 flags = radeon_object_flags_from_domain(lobj->rdomain); 422 flags |= TTM_PL_FLAG_TT; 423 flags |= TTM_PL_FLAG_VRAM; 424 } 425 if (!robj->pin_count) { 426 robj->tobj.proposed_placement = flags | TTM_PL_MASK_CACHING; 427 r = ttm_buffer_object_validate(&robj->tobj, 428 robj->tobj.proposed_placement, 429 true, false); 430 if (unlikely(r)) { 431 radeon_object_list_unreserve(head); 432 DRM_ERROR("radeon: failed to validate.\n"); 433 return r; 434 } 435 radeon_object_gpu_addr(robj); 436 } 437 lobj->gpu_offset = robj->gpu_addr; 438 if (fence) { 439 old_fence = (struct radeon_fence *)robj->tobj.sync_obj; 440 robj->tobj.sync_obj = radeon_fence_ref(fence); 441 robj->tobj.sync_obj_arg = NULL; 442 } 443 if (old_fence) { 444 radeon_fence_unref(&old_fence); 445 } 446 } 447 return 0; 448 } 449 450 void radeon_object_list_unvalidate(struct list_head *head) 451 { 452 struct radeon_object_list *lobj; 453 struct radeon_fence *old_fence = NULL; 454 struct list_head *i; 455 456 list_for_each(i, head) { 457 lobj = list_entry(i, struct radeon_object_list, list); 458 old_fence = (struct radeon_fence *)lobj->robj->tobj.sync_obj; 459 lobj->robj->tobj.sync_obj = NULL; 460 if (old_fence) { 461 radeon_fence_unref(&old_fence); 462 } 463 } 464 radeon_object_list_unreserve(head); 465 } 466 467 void radeon_object_list_clean(struct list_head *head) 468 { 469 radeon_object_list_unreserve(head); 470 } 471 472 int radeon_object_fbdev_mmap(struct radeon_object *robj, 473 struct vm_area_struct *vma) 474 { 475 return ttm_fbdev_mmap(vma, &robj->tobj); 476 } 477 478 unsigned long radeon_object_size(struct radeon_object *robj) 479 { 480 return robj->tobj.num_pages << PAGE_SHIFT; 481 } 482